├── data └── .gitkeep ├── logs └── .gitkeep ├── recipe1m ├── __init__.py ├── models │ ├── __init__.py │ ├── criterions │ │ ├── __init__.py │ │ ├── pairwise.py │ │ ├── trijoint.py │ │ └── triplet.py │ ├── metrics │ │ ├── __init__.py │ │ ├── utils.py │ │ └── trijoint.py │ ├── networks │ │ ├── __init__.py │ │ └── trijoint.py │ ├── factory.py │ └── trijoint.py ├── visu │ ├── __init__.py │ ├── ingrs_count.py │ ├── old_top5.py │ ├── calculate_mean_features.py │ ├── mean_to_images.py │ ├── ingrs_to_images.py │ ├── old_ingrs_to_img_by_class.py │ ├── modality_to_modality_top5.py │ ├── make_menu.py │ ├── remove_ingrs.py │ ├── ingrs_to_images_per_class.py │ ├── old_tsne.py │ └── old_top5_old.py ├── datasets │ ├── __init__.py │ ├── factory.py │ ├── batch_sampler.py │ └── recipe1m.py ├── optimizers │ ├── __init__.py │ ├── factory.py │ └── trijoint.py ├── options │ ├── pairwise.yaml │ ├── pairwise_plus.yaml │ ├── lifted_struct.yaml │ ├── max.yaml │ ├── semi_hard.yaml │ ├── avg_nosem.yaml │ ├── triplet.yaml │ ├── avg.yaml │ ├── adamine.yaml │ └── abstract.yaml ├── extract.py └── api.py ├── images ├── model.png └── task.png ├── demo_web ├── images │ ├── loading.gif │ ├── logos.png │ └── walle.jpg ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ └── glyphicons-halflings-regular.woff2 ├── css │ ├── custom.css │ └── bootstrap-theme.min.css ├── js │ ├── npm.js │ └── custom.js └── index.html ├── requirements.txt ├── .gitignore └── README.md /data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /recipe1m/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /recipe1m/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /recipe1m/visu/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /recipe1m/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /recipe1m/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /recipe1m/models/criterions/__init__.py: -------------------------------------------------------------------------------- 1 | from .trijoint import Trijoint -------------------------------------------------------------------------------- /recipe1m/models/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .trijoint import Trijoint -------------------------------------------------------------------------------- /recipe1m/models/networks/__init__.py: -------------------------------------------------------------------------------- 1 | from .trijoint import Trijoint -------------------------------------------------------------------------------- /images/model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cadene/recipe1m.bootstrap.pytorch/HEAD/images/model.png -------------------------------------------------------------------------------- /images/task.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cadene/recipe1m.bootstrap.pytorch/HEAD/images/task.png -------------------------------------------------------------------------------- /demo_web/images/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cadene/recipe1m.bootstrap.pytorch/HEAD/demo_web/images/loading.gif -------------------------------------------------------------------------------- /demo_web/images/logos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cadene/recipe1m.bootstrap.pytorch/HEAD/demo_web/images/logos.png -------------------------------------------------------------------------------- /demo_web/images/walle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cadene/recipe1m.bootstrap.pytorch/HEAD/demo_web/images/walle.jpg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch>=1.0 2 | torchvision 3 | lmdb 4 | pretrainedmodels 5 | bootstrap.pytorch 6 | tqdm 7 | werkzeug 8 | -------------------------------------------------------------------------------- /demo_web/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cadene/recipe1m.bootstrap.pytorch/HEAD/demo_web/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /demo_web/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cadene/recipe1m.bootstrap.pytorch/HEAD/demo_web/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /demo_web/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cadene/recipe1m.bootstrap.pytorch/HEAD/demo_web/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /demo_web/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cadene/recipe1m.bootstrap.pytorch/HEAD/demo_web/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /recipe1m/optimizers/factory.py: -------------------------------------------------------------------------------- 1 | from bootstrap.lib.options import Options 2 | from .trijoint import Trijoint 3 | 4 | def factory(model, engine=None): 5 | 6 | if Options()['optimizer']['name'] == 'trijoint_fixed_fine_tune': 7 | optimizer = Trijoint(Options()['optimizer'], model, engine) 8 | else: 9 | raise ValueError() 10 | 11 | return optimizer 12 | 13 | -------------------------------------------------------------------------------- /recipe1m/options/pairwise.yaml: -------------------------------------------------------------------------------- 1 | __include__: abstract.yaml 2 | exp: 3 | dir: logs/recipe1m/pairwise 4 | dataset: 5 | freq_mismatch: 0.80 6 | model: 7 | with_classif: True 8 | criterion: 9 | name: trijoint 10 | weight_classif: 0.01 11 | keep_background: False 12 | retrieval_strategy: 13 | name: pairwise_pytorch # quadruplet, triplet, pairwise, or pairwise_pytorch 14 | margin: 0.1 -------------------------------------------------------------------------------- /recipe1m/models/factory.py: -------------------------------------------------------------------------------- 1 | from bootstrap.lib.options import Options 2 | from .trijoint import Trijoint 3 | 4 | def factory(engine=None): 5 | 6 | if Options()['model.name'] == 'trijoint': 7 | model = Trijoint( 8 | Options()['model'], 9 | Options()['dataset.nb_classes'], 10 | engine.dataset.keys(), 11 | engine) 12 | else: 13 | raise ValueError() 14 | 15 | return model 16 | 17 | -------------------------------------------------------------------------------- /recipe1m/options/pairwise_plus.yaml: -------------------------------------------------------------------------------- 1 | __include__: abstract.yaml 2 | exp: 3 | dir: logs/recipe1m/pairwise_plus 4 | dataset: 5 | freq_mismatch: 0.80 6 | model: 7 | with_classif: True 8 | criterion: 9 | name: trijoint 10 | weight_classif: 0.01 11 | keep_background: False 12 | retrieval_strategy: 13 | name: pairwise # quadruplet, triplet, pairwise, or pairwise_pytorch 14 | pos_margin: 0.3 15 | neg_margin: 0.9 -------------------------------------------------------------------------------- /demo_web/css/custom.css: -------------------------------------------------------------------------------- 1 | .btn-file { 2 | position: relative; 3 | overflow: hidden; 4 | } 5 | .btn-file input[type=file] { 6 | position: absolute; 7 | top: 0; 8 | right: 0; 9 | min-width: 100%; 10 | min-height: 100%; 11 | font-size: 100px; 12 | text-align: right; 13 | filter: alpha(opacity=0); 14 | opacity: 0; 15 | outline: none; 16 | background: white; 17 | cursor: inherit; 18 | display: block; 19 | } 20 | 21 | #vqa-visual{ 22 | width: 300px; 23 | } -------------------------------------------------------------------------------- /demo_web/js/npm.js: -------------------------------------------------------------------------------- 1 | // This file is autogenerated via the `commonjs` Grunt task. You can require() this file in a CommonJS environment. 2 | require('../../js/transition.js') 3 | require('../../js/alert.js') 4 | require('../../js/button.js') 5 | require('../../js/carousel.js') 6 | require('../../js/collapse.js') 7 | require('../../js/dropdown.js') 8 | require('../../js/modal.js') 9 | require('../../js/tooltip.js') 10 | require('../../js/popover.js') 11 | require('../../js/scrollspy.js') 12 | require('../../js/tab.js') 13 | require('../../js/affix.js') -------------------------------------------------------------------------------- /recipe1m/options/lifted_struct.yaml: -------------------------------------------------------------------------------- 1 | __include__: abstract.yaml 2 | exp: 3 | dir: logs/recipe1m/lifted_struct 4 | dataset: 5 | freq_mismatch: 0.0 6 | model: 7 | with_classif: False 8 | criterion: 9 | name: trijoint 10 | keep_background: False 11 | retrieval_strategy: 12 | name: triplet # quadruplet, triplet, pairwise, or pairwise_pytorch 13 | margin: 0.3 14 | sampling: max_negative # random (outdated), max_negative, or prob_negative 15 | nb_samples: 9999 16 | aggregation: valid # mean, valid 17 | substrategy: 18 | - LIFT 19 | substrategy_weights: 20 | - 1.0 -------------------------------------------------------------------------------- /recipe1m/options/max.yaml: -------------------------------------------------------------------------------- 1 | __include__: abstract.yaml 2 | exp: 3 | dir: logs/recipe1m/max 4 | dataset: 5 | freq_mismatch: 0.0 6 | model: 7 | with_classif: False 8 | criterion: 9 | name: trijoint 10 | keep_background: False 11 | retrieval_strategy: 12 | name: triplet # quadruplet, triplet, pairwise, or pairwise_pytorch 13 | margin: 0.3 14 | sampling: max_negative # random (outdated), max_negative, or prob_negative 15 | nb_samples: 1 16 | aggregation: mean # mean, valid 17 | substrategy: 18 | - IRR 19 | - RII 20 | substrategy_weights: 21 | - 1.0 22 | - 1.0 -------------------------------------------------------------------------------- /recipe1m/options/semi_hard.yaml: -------------------------------------------------------------------------------- 1 | __include__: abstract.yaml 2 | exp: 3 | dir: logs/recipe1m/semi_hard 4 | dataset: 5 | freq_mismatch: 0.0 6 | model: 7 | with_classif: False 8 | criterion: 9 | name: trijoint 10 | keep_background: False 11 | retrieval_strategy: 12 | name: triplet # quadruplet, triplet, pairwise, or pairwise_pytorch 13 | margin: 0.3 14 | sampling: semi_hard # random (outdated), max_negative, or prob_negative 15 | nb_samples: 9999 16 | aggregation: valid # mean, valid 17 | substrategy: 18 | - IRR 19 | - RII 20 | substrategy_weights: 21 | - 1.0 22 | - 1.0 -------------------------------------------------------------------------------- /recipe1m/options/avg_nosem.yaml: -------------------------------------------------------------------------------- 1 | __include__: abstract.yaml 2 | exp: 3 | dir: logs/recipe1m/avg_nosem 4 | dataset: 5 | freq_mismatch: 0.0 6 | model: 7 | with_classif: False 8 | criterion: 9 | name: trijoint 10 | keep_background: False 11 | retrieval_strategy: 12 | name: triplet # quadruplet, triplet, pairwise, or pairwise_pytorch 13 | margin: 0.3 14 | sampling: max_negative # random (outdated), max_negative, or prob_negative 15 | nb_samples: 9999 16 | aggregation: mean # mean, valid (adamine) 17 | substrategy: 18 | - IRR 19 | - RII 20 | substrategy_weights: 21 | - 1.0 22 | - 1.0 23 | -------------------------------------------------------------------------------- /recipe1m/options/triplet.yaml: -------------------------------------------------------------------------------- 1 | __include__: abstract.yaml 2 | exp: 3 | dir: logs/recipe1m/triplet 4 | dataset: 5 | freq_mismatch: 0.0 6 | model: 7 | with_classif: False 8 | criterion: 9 | name: trijoint 10 | keep_background: False 11 | retrieval_strategy: 12 | name: triplet # quadruplet, triplet, pairwise, or pairwise_pytorch 13 | margin: 0.3 14 | sampling: max_negative # random (outdated), max_negative, or prob_negative 15 | nb_samples: 9999 16 | aggregation: valid # mean, valid 17 | substrategy: 18 | - IRR 19 | - RII 20 | - SIRR 21 | - SRII 22 | substrategy_weights: 23 | - 1.0 24 | - 1.0 25 | - 0.1 26 | - 0.1 -------------------------------------------------------------------------------- /recipe1m/options/avg.yaml: -------------------------------------------------------------------------------- 1 | __include__: abstract.yaml 2 | exp: 3 | dir: logs/recipe1m/avg 4 | dataset: 5 | freq_mismatch: 0.0 6 | model: 7 | with_classif: False 8 | criterion: 9 | name: trijoint 10 | keep_background: False 11 | retrieval_strategy: 12 | name: triplet # quadruplet, triplet, pairwise, or pairwise_pytorch 13 | margin: 0.3 14 | sampling: max_negative # random (outdated), max_negative, or prob_negative 15 | nb_samples: 9999 16 | aggregation: mean # mean, valid (adamine) 17 | substrategy: 18 | - IRR 19 | - RII 20 | - SIRR 21 | - SRII 22 | substrategy_weights: 23 | - 1.0 24 | - 1.0 25 | - 0.1 26 | - 0.1 27 | -------------------------------------------------------------------------------- /recipe1m/options/adamine.yaml: -------------------------------------------------------------------------------- 1 | __include__: abstract.yaml 2 | exp: 3 | dir: logs/recipe1m/adamine 4 | dataset: 5 | freq_mismatch: 0.0 6 | model: 7 | with_classif: False 8 | criterion: 9 | name: trijoint 10 | keep_background: False 11 | retrieval_strategy: 12 | name: triplet # quadruplet, triplet, pairwise, or pairwise_pytorch 13 | margin: 0.3 14 | sampling: max_negative # random (outdated), max_negative, or prob_negative 15 | nb_samples: 9999 16 | aggregation: valid # mean, valid (adamine) 17 | substrategy: 18 | - IRR 19 | - RII 20 | - SIRR 21 | - SRII 22 | substrategy_weights: 23 | - 1.0 24 | - 1.0 25 | - 0.1 26 | - 0.1 27 | -------------------------------------------------------------------------------- /recipe1m/datasets/factory.py: -------------------------------------------------------------------------------- 1 | from bootstrap.lib.options import Options 2 | from .recipe1m import Recipe1M 3 | 4 | def factory(engine=None): 5 | dataset = {} 6 | 7 | if Options()['dataset']['name'] == 'recipe1m': 8 | 9 | if Options()['dataset'].get('train_split', None): 10 | dataset['train'] = factory_recipe1m(Options()['dataset']['train_split']) 11 | 12 | if Options()['dataset'].get('eval_split', None): 13 | dataset['eval'] = factory_recipe1m(Options()['dataset']['eval_split']) 14 | else: 15 | raise ValueError() 16 | 17 | return dataset 18 | 19 | 20 | def factory_recipe1m(split): 21 | dataset = Recipe1M( 22 | Options()['dataset']['dir'], 23 | split, 24 | batch_size=Options()['dataset']['batch_size'], 25 | nb_threads=Options()['dataset']['nb_threads'], 26 | freq_mismatch=Options()['dataset']['freq_mismatch'], 27 | batch_sampler=Options()['dataset']['batch_sampler'], 28 | image_from=Options()['dataset']['image_from']) 29 | return dataset -------------------------------------------------------------------------------- /recipe1m/models/criterions/pairwise.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | 7 | class Pairwise(nn.Module): 8 | 9 | def __init__(self, opt): 10 | super(Pairwise, self).__init__() 11 | self.alpha_pos = opt['retrieval_strategy']['pos_margin'] 12 | self.alpha_neg = opt['retrieval_strategy']['neg_margin'] 13 | 14 | def forward(self, input1, input2, target): 15 | # target should be 1 for matched samples or -1 for not matched ones 16 | distances = self.dist(input1, input2) 17 | target = target.squeeze(1) 18 | cost = target * distances 19 | 20 | cost[target > 0] -= self.alpha_pos 21 | cost[target < 0] += self.alpha_neg 22 | cost[cost < 0] = 0 23 | 24 | out = {} 25 | out['bad_pairs'] = (cost == 0).float().sum() / cost.numel() 26 | out['loss'] = cost.mean() 27 | return out 28 | 29 | def dist(self, input1, input2): 30 | input1 = nn.functional.normalize(input1) 31 | input2 = nn.functional.normalize(input2) 32 | return 1 - torch.mul(input1, input2).sum(1) 33 | -------------------------------------------------------------------------------- /recipe1m/models/trijoint.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from bootstrap.lib.options import Options 5 | from bootstrap.datasets import transforms 6 | from bootstrap.models.model import Model 7 | 8 | from . import networks 9 | from . import criterions 10 | from . import metrics 11 | 12 | class Trijoint(Model): 13 | 14 | def __init__(self, 15 | opt, 16 | nb_classes, 17 | modes=['train', 'eval'], 18 | engine=None, 19 | cuda_tf=transforms.ToCuda): 20 | super(Trijoint, self).__init__(engine, cuda_tf=cuda_tf) 21 | 22 | self.network = networks.Trijoint( 23 | opt['network'], 24 | nb_classes, 25 | with_classif=opt['with_classif']) 26 | 27 | self.criterions = {} 28 | self.metrics = {} 29 | 30 | if 'train' in modes: 31 | self.criterions['train'] = criterions.Trijoint( 32 | opt['criterion'], 33 | nb_classes, 34 | opt['network']['dim_emb'], 35 | with_classif=opt['with_classif'], 36 | engine=engine) 37 | 38 | self.metrics['train'] = metrics.Trijoint( 39 | opt['metric'], 40 | with_classif=opt['with_classif'], 41 | engine=engine, 42 | mode='train') 43 | 44 | if 'eval' in modes: 45 | self.metrics['eval'] = metrics.Trijoint( 46 | opt['metric'], 47 | with_classif=opt['with_classif'], 48 | engine=engine, 49 | mode='eval') -------------------------------------------------------------------------------- /recipe1m/visu/ingrs_count.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from torch.autograd import Variable 8 | from PIL import Image 9 | from tqdm import tqdm 10 | import bootstrap.lib.utils as utils 11 | 12 | def main(): 13 | 14 | Logger('.') 15 | 16 | split = 'train' 17 | dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' 18 | path_opts = os.path.join(dir_exp, 'options.yaml') 19 | dir_extract = os.path.join(dir_exp, 'extract_count', split) 20 | path_ingrs_count = os.path.join(dir_extract, 'ingrs.pth') 21 | 22 | Options(path_opts) 23 | utils.set_random_seed(Options()['misc']['seed']) 24 | 25 | dataset = factory(split) 26 | 27 | if not os.path.isfile(path_ingrs_count): 28 | ingrs_count = {} 29 | os.system('mkdir -p '+dir_extract) 30 | 31 | for i in tqdm(range(len(dataset.recipes_dataset))): 32 | item = dataset.recipes_dataset[i] 33 | for ingr in item['ingrs']['interim']: 34 | if ingr not in ingrs_count: 35 | ingrs_count[ingr] = 1 36 | else: 37 | ingrs_count[ingr] += 1 38 | 39 | torch.save(ingrs_count, path_ingrs_count) 40 | else: 41 | ingrs_count = torch.load(path_ingrs_count) 42 | 43 | import ipdb; ipdb.set_trace() 44 | sort = sorted(ingrs_count, key=ingrs_count.get) 45 | import ipdb; ipdb.set_trace() 46 | 47 | Logger()('End') 48 | 49 | 50 | # python -m recipe1m.visu.top5 51 | if __name__ == '__main__': 52 | main() -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Boostrap.pytorch 2 | !.gitkeep 3 | 4 | # Apple 5 | .DS_Store 6 | ._.DS_Store 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | # lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | .hypothesis/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | .static_storage/ 63 | .media/ 64 | local_settings.py 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | -------------------------------------------------------------------------------- /recipe1m/visu/old_top5.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from recipe1m.models.networks.trijoint import Trijoint 8 | 9 | def main(): 10 | classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] 11 | nb_points = 100 12 | split = 'test' 13 | dir_exp = 'logs/recipe1m/trijoint/2017-12-14-15-04-51' 14 | path_opts = os.path.join(dir_exp, 'options.yaml') 15 | dir_extract = os.path.join(dir_exp, 'extract', split) 16 | dir_img = os.path.join(dir_extract, 'image') 17 | dir_rcp = os.path.join(dir_extract, 'recipe') 18 | path_model = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') 19 | 20 | dir_visu = os.path.join(dir_exp, 'visu', 'top5') 21 | 22 | #Options(path_opts) 23 | Options.load_from_yaml(path_opts) 24 | dataset = factory(split) 25 | 26 | network = Trijoint() 27 | network.eval() 28 | model_state = torch.load(path_model) 29 | network.load_state_dict(model_state['network']) 30 | 31 | list_idx = torch.randperm(len(dataset)) 32 | 33 | img_embs = [] 34 | rcp_embs = [] 35 | for i in range(nb_points): 36 | idx = list_idx[i] 37 | path_img = os.path.join(dir_img, '{}.pth'.format(idx)) 38 | path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) 39 | img_embs.append(torch.load(path_img)) 40 | rcp_embs.append(torch.load(path_rcp)) 41 | 42 | img_embs = torch.stack(img_embs, 0) 43 | rcp_embs = torch.stack(rcp_embs, 0) 44 | 45 | dist = fast_distance(img_embs, rcp_embs) 46 | 47 | im2recipe_ids = np.argsort(dist.numpy(), axis=0) 48 | recipe2im_ids = np.argsort(dist.numpy(), axis=1) 49 | 50 | import ipdb; ipdb.set_trace() 51 | 52 | 53 | 54 | def fast_distance(A,B): 55 | # A and B must have norm 1 for this to work for the ranking 56 | return torch.mm(A,B.t()) * -1 57 | 58 | # python -m recipe1m.visu.top5 59 | if __name__ == '__main__': 60 | main() -------------------------------------------------------------------------------- /recipe1m/models/metrics/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from bootstrap.lib.options import Options 4 | 5 | def accuracy(output, target, topk=(1,), ignore_index=None): 6 | """Computes the precision@k for the specified values of k""" 7 | 8 | if ignore_index is not None: 9 | target_mask = (target != ignore_index) 10 | target = target[target_mask] 11 | output_mask = target_mask.unsqueeze(1) 12 | output_mask = output_mask.expand_as(output) 13 | output = output[output_mask] 14 | output = output.view(-1, output_mask.size(1)) 15 | 16 | maxk = max(topk) 17 | batch_size = target.size(0) 18 | 19 | _, pred = output.topk(maxk, 1, True, True) 20 | pred = pred.t() 21 | correct = pred.eq(target.view(1, -1).expand_as(pred)) 22 | 23 | res = [] 24 | for k in topk: 25 | correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) 26 | res.append(correct_k.mul_(100.0 / batch_size)[0]) 27 | return res 28 | 29 | def save_activation(identifier, activation): 30 | retrieval_dir = os.path.join(Options()['model']['metric']['retrieval_dir'], 31 | os.path.basename(Options()['exp']['dir'])) 32 | if not os.path.exists(retrieval_dir): 33 | os.makedirs(retrieval_dir) 34 | file_path = os.path.join(retrieval_dir, '{}.pth'.format(identifier)) 35 | torch.save(activation, file_path) 36 | 37 | def load_activation(identifier): 38 | retrieval_dir = os.path.join(Options()['model']['metric']['retrieval_dir'], 39 | os.path.basename(Options()['exp']['dir'])) 40 | if not os.path.exists(retrieval_dir): 41 | os.makedirs(retrieval_dir) 42 | file_path = os.path.join(retrieval_dir, '{}.pth'.format(identifier)) 43 | return torch.load(file_path) 44 | 45 | def delete_activation(identifier): 46 | retrieval_dir = os.path.join(Options()['model']['metric']['retrieval_dir'], 47 | os.path.basename(Options()['exp']['dir'])) 48 | file_path = os.path.join(retrieval_dir, '{}.pth'.format(identifier)) 49 | if os.path.isfile(file_path): 50 | os.remove(file_path) 51 | -------------------------------------------------------------------------------- /recipe1m/options/abstract.yaml: -------------------------------------------------------------------------------- 1 | exp: 2 | dir: logs/recipe1m/default 3 | resume: # best, last, or empty (from scratch) 4 | dataset: 5 | import: recipe1m.datasets.factory 6 | name: recipe1m 7 | dir: data/recipe1m 8 | train_split: train 9 | eval_split: val 10 | nb_classes: 1048 11 | database: lmdb 12 | image_from: database # or pil_loader 13 | batch_size: 100 # = optimizer.batch_size or inferior 14 | batch_sampler: triplet_classif # random or triplet_classif 15 | nb_threads: 4 16 | debug: False 17 | model: 18 | import: recipe1m.models.factory 19 | name: trijoint 20 | network: 21 | name: trijoint 22 | path_ingrs: data/recipe1m/text/vocab.pkl 23 | dim_image_out: 2048 24 | with_ingrs: True 25 | dim_ingr_out: 300 26 | with_instrs: True 27 | dim_instr_in: 1024 28 | dim_instr_out: 1024 29 | dim_emb: 1024 30 | activations: 31 | - tanh 32 | - normalize 33 | criterion: __NotImplemented__ 34 | metric: 35 | name: trijoint 36 | retrieval_dir: /tmp/recipe1m 37 | nb_bags: 10 38 | nb_matchs_per_bag: 1000 39 | optimizer: 40 | import: recipe1m.optimizers.factory 41 | name: trijoint_fixed_fine_tune 42 | switch_epoch: 20 43 | lr: 0.0001 44 | #switch_step: 50000 45 | batch_size_factor: # TODO remove? 46 | clip_grad: 8. 47 | engine: 48 | name: logger 49 | nb_epochs: 80 50 | print_freq: 10 51 | debug: False 52 | saving_criteria: 53 | #- train_epoch.loss:min 54 | - eval_epoch.metric.med_im2recipe_mean:min 55 | - eval_epoch.metric.recall_at_1_im2recipe_mean:max 56 | misc: 57 | cuda: True 58 | seed: 1338 59 | logs_name: 60 | view: 61 | - logs:train_epoch.loss 62 | - logs:train_epoch.bad_pairs 63 | - logs:eval_epoch.metric.med_im2recipe_mean 64 | - logs:eval_epoch.metric.recall_at_1_im2recipe_mean 65 | - logs:eval_epoch.metric.recall_at_5_im2recipe_mean 66 | - logs:eval_epoch.metric.recall_at_10_im2recipe_mean 67 | - logs:eval_epoch.metric.med_recipe2im_mean 68 | - logs:eval_epoch.metric.recall_at_1_recipe2im_mean 69 | - logs:eval_epoch.metric.recall_at_5_recipe2im_mean 70 | - logs:eval_epoch.metric.recall_at_10_recipe2im_mean 71 | - logs:optimizer.is_optimizer_recipe&image 72 | - logs:optimizer.total_norm 73 | -------------------------------------------------------------------------------- /recipe1m/visu/calculate_mean_features.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from bootstrap.models.factory import factory as model_factory 8 | from torch.autograd import Variable 9 | from PIL import Image 10 | from tqdm import tqdm 11 | import bootstrap.lib.utils as utils 12 | 13 | def main(): 14 | 15 | Logger('.') 16 | 17 | #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] 18 | split = 'test' 19 | dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' 20 | path_opts = os.path.join(dir_exp, 'options.yaml') 21 | dir_extract = os.path.join(dir_exp, 'extract_mean_features', split) 22 | dir_img = os.path.join(dir_extract, 'image') 23 | dir_rcp = os.path.join(dir_extract, 'recipe') 24 | path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') 25 | 26 | Options.load_from_yaml(path_opts) 27 | utils.set_random_seed(Options()['misc']['seed']) 28 | 29 | Logger()('Load dataset...') 30 | dataset = factory(split) 31 | 32 | Logger()('Load model...') 33 | model = model_factory() 34 | model_state = torch.load(path_model_ckpt) 35 | model.load_state_dict(model_state) 36 | model.set_mode(split) 37 | 38 | if not os.path.isdir(dir_extract): 39 | Logger()('Create extract_dir {}'.format(dir_extract)) 40 | os.system('mkdir -p '+dir_extract) 41 | 42 | mean_ingrs = torch.zeros(model.network.recipe_embedding.dim_ingr_out*2) # bi LSTM 43 | mean_instrs = torch.zeros(model.network.recipe_embedding.dim_instr_out) 44 | 45 | for i in tqdm(range(len(dataset))): 46 | item = dataset[i] 47 | batch = dataset.items_tf()([item]) 48 | 49 | batch = model.prepare_batch(batch) 50 | out_ingrs = model.network.recipe_embedding.forward_ingrs(batch['recipe']['ingrs']) 51 | out_instrs = model.network.recipe_embedding.forward_instrs(batch['recipe']['instrs']) 52 | 53 | mean_ingrs += out_ingrs.data.cpu().squeeze(0) 54 | mean_instrs += out_instrs.data.cpu().squeeze(0) 55 | 56 | mean_ingrs /= len(dataset) 57 | mean_instrs /= len(dataset) 58 | 59 | path_ingrs = os.path.join(dir_extract, 'ingrs.pth') 60 | path_instrs = os.path.join(dir_extract, 'instrs.pth') 61 | 62 | torch.save(mean_ingrs, path_ingrs) 63 | torch.save(mean_instrs, path_instrs) 64 | 65 | Logger()('End') 66 | 67 | 68 | # python -m recipe1m.visu.top5 69 | if __name__ == '__main__': 70 | main() -------------------------------------------------------------------------------- /recipe1m/extract.py: -------------------------------------------------------------------------------- 1 | """ 2 | # How to use extract.py 3 | 4 | ``` 5 | $ python -m recipe1m.extract -o logs/recipe1m/adamine/options.yaml \ 6 | --dataset.train_split \ 7 | --dataset.eval_split test \ 8 | --exp.resume best_eval_epoch.metric.med_im2recipe_mean \ 9 | --misc.logs_name extract_test 10 | ``` 11 | """ 12 | 13 | import os 14 | import torch 15 | import torch.backends.cudnn as cudnn 16 | import bootstrap.lib.utils as utils 17 | import bootstrap.engines as engines 18 | import bootstrap.models as models 19 | import bootstrap.datasets as datasets 20 | import bootstrap.views as views 21 | from bootstrap.lib.logger import Logger 22 | from bootstrap.lib.options import Options 23 | from bootstrap.run import init_logs_options_files 24 | 25 | def extract(path_opts=None): 26 | Options(path_opts) 27 | utils.set_random_seed(Options()['misc']['seed']) 28 | 29 | assert Options()['dataset']['eval_split'] is not None, 'eval_split must be set' 30 | assert Options()['dataset']['train_split'] is None, 'train_split must be None' 31 | 32 | init_logs_options_files(Options()['exp']['dir'], Options()['exp']['resume']) 33 | 34 | Logger().log_dict('options', Options(), should_print=True) 35 | Logger()(os.uname()) 36 | if torch.cuda.is_available(): 37 | cudnn.benchmark = True 38 | Logger()('Available GPUs: {}'.format(utils.available_gpu_ids())) 39 | 40 | engine = engines.factory() 41 | engine.dataset = datasets.factory(engine) 42 | engine.model = models.factory(engine) 43 | engine.view = views.factory(engine) 44 | 45 | # init extract directory 46 | dir_extract = os.path.join(Options()['exp']['dir'], 'extract', Options()['dataset']['eval_split']) 47 | os.system('mkdir -p '+dir_extract) 48 | path_img_embs = os.path.join(dir_extract, 'image_emdeddings.pth') 49 | path_rcp_embs = os.path.join(dir_extract, 'recipe_emdeddings.pth') 50 | img_embs = torch.FloatTensor(len(engine.dataset['eval']), Options()['model.network.dim_emb']) 51 | rcp_embs = torch.FloatTensor(len(engine.dataset['eval']), Options()['model.network.dim_emb']) 52 | 53 | def save_embeddings(module, input, out): 54 | nonlocal img_embs 55 | nonlocal rcp_embs 56 | batch = input[0] # tuple of len=1 57 | for j, idx in enumerate(batch['recipe']['index']): 58 | # path_image = os.path.join(dir_image, '{}.pth'.format(idx)) 59 | # path_recipe = os.path.join(dir_recipe, '{}.pth'.format(idx)) 60 | # torch.save(out['image_embedding'][j].data.cpu(), path_image) 61 | # torch.save(out['recipe_embedding'][j].data.cpu(), path_recipe) 62 | img_embs[idx] = out['image_embedding'][j].data.cpu() 63 | rcp_embs[idx] = out['recipe_embedding'][j].data.cpu() 64 | 65 | engine.model.register_forward_hook(save_embeddings) 66 | engine.resume() 67 | engine.eval() 68 | 69 | Logger()('Saving image embeddings to {} ...'.format(path_img_embs)) 70 | torch.save(img_embs, path_img_embs) 71 | 72 | Logger()('Saving recipe embeddings to {} ...'.format(path_rcp_embs)) 73 | torch.save(rcp_embs, path_rcp_embs) 74 | 75 | 76 | if __name__ == '__main__': 77 | from bootstrap.run import main 78 | main(run=extract) -------------------------------------------------------------------------------- /recipe1m/models/criterions/trijoint.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from bootstrap.lib.logger import Logger 4 | from bootstrap.lib.options import Options 5 | from .triplet import Triplet 6 | from .pairwise import Pairwise 7 | 8 | class Trijoint(nn.Module): 9 | 10 | def __init__(self, opt, nb_classes, dim_emb, with_classif=False, engine=None): 11 | super(Trijoint, self).__init__() 12 | self.with_classif = with_classif 13 | if self.with_classif: 14 | self.weight_classif = opt['weight_classif'] 15 | if self.weight_classif == 0: 16 | Logger()('You should use "--model.with_classif False"', Logger.ERROR) 17 | self.weight_retrieval = 1 - 2 * opt['weight_classif'] 18 | 19 | self.keep_background = opt.get('keep_background', False) 20 | if self.keep_background: 21 | # http://pytorch.org/docs/master/nn.html?highlight=crossentropy#torch.nn.CrossEntropyLoss 22 | self.ignore_index = -100 23 | else: 24 | self.ignore_index = 0 25 | 26 | Logger()('ignore_index={}'.format(self.ignore_index)) 27 | if self.with_classif: 28 | self.criterion_image_classif = nn.CrossEntropyLoss(ignore_index=self.ignore_index) 29 | self.criterion_recipe_classif = nn.CrossEntropyLoss(ignore_index=self.ignore_index) 30 | 31 | self.retrieval_strategy = opt['retrieval_strategy.name'] 32 | 33 | if self.retrieval_strategy == 'triplet': 34 | self.criterion_retrieval = Triplet( 35 | opt, 36 | nb_classes, 37 | dim_emb, 38 | engine) 39 | 40 | elif self.retrieval_strategy == 'pairwise': 41 | self.criterion_retrieval = Pairwise(opt) 42 | 43 | elif self.retrieval_strategy == 'pairwise_pytorch': 44 | self.criterion_retrieval = nn.CosineEmbeddingLoss() 45 | 46 | else: 47 | raise ValueError('Unknown loss ({})'.format(self.retrieval_strategy)) 48 | 49 | def forward(self, net_out, batch): 50 | if self.retrieval_strategy == 'triplet': 51 | out = self.criterion_retrieval(net_out['image_embedding'], 52 | net_out['recipe_embedding'], 53 | batch['match'], 54 | batch['image']['class_id'], 55 | batch['recipe']['class_id']) 56 | elif self.retrieval_strategy == 'pairwise': 57 | out = self.criterion_retrieval(net_out['image_embedding'], 58 | net_out['recipe_embedding'], 59 | batch['match']) 60 | elif self.retrieval_strategy == 'pairwise_pytorch': 61 | out = {} 62 | out['loss'] = self.criterion_retrieval(net_out['image_embedding'], 63 | net_out['recipe_embedding'], 64 | batch['match']) 65 | 66 | if self.with_classif: 67 | out['image_classif'] = self.criterion_image_classif(net_out['image_classif'], 68 | batch['image']['class_id'].squeeze(1)) 69 | out['recipe_classif'] = self.criterion_recipe_classif(net_out['recipe_classif'], 70 | batch['recipe']['class_id'].squeeze(1)) 71 | out['loss'] *= self.weight_retrieval 72 | out['loss'] += out['image_classif'] * self.weight_classif 73 | out['loss'] += out['recipe_classif'] * self.weight_classif 74 | 75 | return out 76 | -------------------------------------------------------------------------------- /recipe1m/optimizers/trijoint.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from bootstrap.lib.options import Options 3 | from bootstrap.lib.logger import Logger 4 | from torch.nn.utils.clip_grad import clip_grad_norm_ 5 | 6 | class Trijoint(torch.optim.Optimizer): 7 | 8 | def __init__(self, opt, model, engine=None): 9 | self.model = model 10 | self.lr = opt['lr'] 11 | self.switch_epoch = opt['switch_epoch'] 12 | self.clip_grad = opt.get('clip_grad', False) 13 | self.optimizers = {} 14 | self.optimizers['recipe'] = torch.optim.Adam(self.model.network.get_parameters_recipe(), self.lr) 15 | self.optimizers['image'] = torch.optim.Adam(self.model.network.get_parameters_image(), self.lr) 16 | self.current_optimizer_name = 'recipe' 17 | self.epoch = 0 18 | self._activate_model() 19 | if engine: 20 | engine.register_hook('train_on_start_epoch', self._auto_fixed_fine_tune) 21 | if self.clip_grad: 22 | engine.register_hook('train_on_print', self.print_total_norm) 23 | 24 | def state_dict(self): 25 | state = {} 26 | state['optimizers'] = {} 27 | for key, value in self.optimizers.items(): 28 | state['optimizers'][key] = value.state_dict() 29 | state['attributs'] = { 30 | 'current_optimizer_name': self.current_optimizer_name, 31 | 'epoch': self.epoch 32 | } 33 | return state 34 | 35 | def load_state_dict(self, state_dict): 36 | for key, _ in self.optimizers.items(): 37 | value = state_dict['optimizers'][key] 38 | if len(value['state']) != 0: # bug pytorch?? 39 | self.optimizers[key].load_state_dict(value) 40 | if 'attributs' in state_dict: 41 | for key, value in state_dict['attributs'].items(): 42 | setattr(self, key, value) 43 | self._activate_model() 44 | 45 | def zero_grad(self): 46 | for name in self.current_optimizer_name.split('&'): 47 | self.optimizers[name].zero_grad() 48 | 49 | def step(self, closure=None): 50 | if self.clip_grad: 51 | self.clip_grad_norm() 52 | for name in self.current_optimizer_name.split('&'): 53 | self.optimizers[name].step(closure) 54 | 55 | def clip_grad_norm(self): 56 | params = [] 57 | for k in self.optimizers: 58 | for group in self.optimizers[k].param_groups: 59 | for p in group['params']: 60 | params.append(p) 61 | self.total_norm = clip_grad_norm_(params, self.clip_grad) 62 | Logger().log_value('optimizer.total_norm', self.total_norm, should_print=False) 63 | 64 | def print_total_norm(self): 65 | Logger()('{} total_norm: {:.6f}'.format(' '*len('train'), self.total_norm)) 66 | 67 | def _activate_model(self): 68 | optim_name = self.current_optimizer_name 69 | activate_recipe = (optim_name == 'recipe') or (optim_name == 'recipe&image') 70 | activate_image = (optim_name == 'image') or (optim_name == 'recipe&image') 71 | for p_dict in self.model.network.get_parameters_recipe(): 72 | for p in p_dict['params']: 73 | p.requires_grad = activate_recipe 74 | for p in self.model.network.get_parameters_image(): 75 | p.requires_grad = activate_image 76 | 77 | def _auto_fixed_fine_tune(self): 78 | if self.current_optimizer_name == 'recipe' and self.epoch == self.switch_epoch: 79 | self.current_optimizer_name = 'recipe&image' 80 | self._activate_model() 81 | Logger()('Switched to optimizer '+self.current_optimizer_name) 82 | 83 | Logger().log_value('optimizer.is_optimizer_recipe&image', 84 | int(self.current_optimizer_name == 'recipe&image'), 85 | should_print=False) 86 | self.epoch += 1 87 | -------------------------------------------------------------------------------- /recipe1m/visu/mean_to_images.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from bootstrap.models.factory import factory as model_factory 8 | from torch.autograd import Variable 9 | from PIL import Image 10 | from tqdm import tqdm 11 | import bootstrap.lib.utils as utils 12 | 13 | def main(): 14 | 15 | Logger('.') 16 | 17 | 18 | #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] 19 | nb_points = 1000 20 | split = 'test' 21 | dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' 22 | path_opts = os.path.join(dir_exp, 'options.yaml') 23 | dir_extract = os.path.join(dir_exp, 'extract', split) 24 | dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split) 25 | dir_img = os.path.join(dir_extract, 'image') 26 | dir_rcp = os.path.join(dir_extract, 'recipe') 27 | path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') 28 | 29 | dir_visu = os.path.join(dir_exp, 'visu', 'mean_to_image') 30 | os.system('mkdir -p '+dir_visu) 31 | 32 | #Options(path_opts) 33 | Options.load_from_yaml(path_opts) 34 | utils.set_random_seed(Options()['misc']['seed']) 35 | 36 | dataset = factory(split) 37 | 38 | Logger()('Load model...') 39 | model = model_factory() 40 | model_state = torch.load(path_model_ckpt) 41 | model.load_state_dict(model_state) 42 | model.set_mode(split) 43 | 44 | #emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs']) 45 | list_idx = torch.randperm(len(dataset)) 46 | 47 | Logger()('Load embeddings...') 48 | img_embs = [] 49 | rcp_embs = [] 50 | for i in range(nb_points): 51 | idx = list_idx[i] 52 | path_img = os.path.join(dir_img, '{}.pth'.format(idx)) 53 | path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) 54 | if not os.path.isfile(path_img): 55 | Logger()('No such file: {}'.format(path_img)) 56 | continue 57 | if not os.path.isfile(path_rcp): 58 | Logger()('No such file: {}'.format(path_rcp)) 59 | continue 60 | img_embs.append(torch.load(path_img)) 61 | rcp_embs.append(torch.load(path_rcp)) 62 | 63 | img_embs = torch.stack(img_embs, 0) 64 | rcp_embs = torch.stack(rcp_embs, 0) 65 | 66 | 67 | Logger()('Load means') 68 | path_ingrs = os.path.join(dir_extract_mean, 'ingrs.pth') 69 | path_instrs = os.path.join(dir_extract_mean, 'instrs.pth') 70 | 71 | mean_ingrs = torch.load(path_ingrs) 72 | mean_instrs = torch.load(path_instrs) 73 | 74 | mean_ingrs = Variable(mean_ingrs.unsqueeze(0).cuda(), requires_grad=False) 75 | mean_instrs = Variable(mean_instrs.unsqueeze(0).cuda(), requires_grad=False) 76 | 77 | Logger()('Forward ingredient...') 78 | ingr_emb = model.network.recipe_embedding.forward_ingrs_instrs(mean_ingrs, mean_instrs) 79 | ingr_emb = ingr_emb.data.cpu() 80 | ingr_emb = ingr_emb.expand_as(img_embs) 81 | 82 | Logger()('Fast distance...') 83 | dist = fast_distance(img_embs, ingr_emb)[:, 0] 84 | 85 | sorted_img_ids = np.argsort(dist.numpy()) 86 | 87 | Logger()('Load/save images...') 88 | for i in range(20): 89 | img_id = sorted_img_ids[i] 90 | img_id = int(img_id) 91 | 92 | path_img_from = dataset[img_id]['image']['path'] 93 | path_img_to = os.path.join(dir_visu, 'image_top_{}.png'.format(i+1)) 94 | img = Image.open(path_img_from) 95 | img.save(path_img_to) 96 | #os.system('cp {} {}'.format(path_img_from, path_img_to)) 97 | 98 | Logger()('End') 99 | 100 | 101 | def fast_distance(A,B): 102 | # A and B must have norm 1 for this to work for the ranking 103 | return torch.mm(A,B.t()) * -1 104 | 105 | # python -m recipe1m.visu.top5 106 | if __name__ == '__main__': 107 | main() -------------------------------------------------------------------------------- /demo_web/js/custom.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function () { 2 | 3 | // Image processing 4 | 5 | $(document).on('change', '.btn-file :file', function() { 6 | var input = $(this), 7 | label = input.val().replace(/\\/g, '/').replace(/.*\//, ''); 8 | input.trigger('fileselect', [label]); 9 | }); 10 | 11 | $('.btn-file :file').on('fileselect', function(event, label) { 12 | var input = $(this).parents('.input-group').find(':text'), 13 | log = label; 14 | 15 | if( input.length ) { 16 | input.val(log); 17 | } else { 18 | if( log ) alert(log); 19 | } 20 | 21 | }); 22 | 23 | function readURL(input) { 24 | if (input.files && input.files[0]) { 25 | var reader = new FileReader(); 26 | 27 | reader.onload = function (e) { 28 | $('#adamine-image').attr('src', e.target.result); 29 | } 30 | 31 | reader.readAsDataURL(input.files[0]); 32 | } 33 | } 34 | 35 | $("#imgInp").change(function(){ 36 | readURL(this); 37 | }); 38 | 39 | // Send Image 40 | 41 | $(document).ajaxStart(function () { 42 | $('#loading').show(); 43 | $('#adamine-recipe').hide(); 44 | }).ajaxStop(function () { 45 | $('#loading').hide(); 46 | $('#adamine-recipe').show(); 47 | }); 48 | 49 | var formBasic = function () { 50 | var formData = $("#formBasic").serialize(); 51 | var data = { 52 | image: $('#adamine-image').attr('src'), 53 | mode: 'all' 54 | } 55 | //question : $('#adamine-question').val()} 56 | $.ajax({ 57 | 58 | type: 'post', 59 | data: data, 60 | dataType: 'json', 61 | url: 'http://edwards:3456', // your global ip address and port 62 | 63 | error: function () { 64 | alert("There was an error processing this page."); 65 | return false; 66 | }, 67 | 68 | complete: function (output) { 69 | if ('responseJSON' in output) { 70 | var ul = $(''); 71 | 72 | for (i=0; i < output.responseJSON.length; i++) { 73 | var li = $('
  • '); 74 | var out = output.responseJSON[i] 75 | 76 | li.append($('

    class: ' + out['class_name'] + '

    ')); 77 | li.append($('

    title: ' + out['title'] + '

    ')); 78 | li.append($('')); 79 | 80 | ingrs = [] 81 | for (j=0; j < out['ingredients'].length; j++) { 82 | ingrs.push(out['ingredients'][j]['text']) 83 | } 84 | li.append($('

    ingredients: ' + ingrs.join(', ') + '

    ')); 85 | 86 | instrs = [] 87 | for (j=0; j < out['instructions'].length; j++) { 88 | instrs.push(out['instructions'][j]['text']) 89 | } 90 | li.append($('

    instructions: ' + instrs.join('\n') + '

    ')); 91 | 92 | ul.append(li); 93 | } 94 | 95 | $('#adamine-recipe').html(ul); 96 | console.log(output.responseJSON); 97 | 98 | } else if ('responseText' in output) { 99 | alert(output.responseText); 100 | console.log(output.responseText); 101 | 102 | } else { 103 | alert('Something wrong happend!'); 104 | console.log(output) 105 | } 106 | } 107 | }); 108 | return false; 109 | }; 110 | 111 | $("#basic-submit").on("click", function (e) { 112 | e.preventDefault(); 113 | formBasic(); 114 | }); 115 | }); -------------------------------------------------------------------------------- /recipe1m/api.py: -------------------------------------------------------------------------------- 1 | """ 2 | # How to use api.py 3 | 4 | ``` 5 | $ python -m recipe1m.api -o logs/recipe1m/adamine/options.yaml \ 6 | --dataset.train_split \ 7 | --dataset.eval_split test \ 8 | --exp.resume best_eval_epoch.metric.med_im2recipe_mean \ 9 | --dataset.eval_split test \ 10 | --misc.logs_name api 11 | ``` 12 | """ 13 | # TODO: add ip and port as cli? 14 | # TODO: save the image and the results 15 | 16 | import os 17 | import re 18 | import json 19 | import base64 20 | import numpy as np 21 | import torch 22 | import torch.backends.cudnn as cudnn 23 | import bootstrap.lib.utils as utils 24 | import bootstrap.engines as engines 25 | import bootstrap.models as models 26 | import bootstrap.datasets as datasets 27 | from PIL import Image 28 | from io import BytesIO 29 | from glob import glob 30 | from werkzeug.wrappers import Request, Response 31 | from werkzeug.serving import run_simple 32 | from bootstrap.lib.logger import Logger 33 | from bootstrap.lib.options import Options 34 | from bootstrap.run import init_logs_options_files 35 | from bootstrap.run import main 36 | from .models.metrics.trijoint import fast_distance 37 | 38 | 39 | def decode_image(strb64): 40 | strb64 = re.sub('^data:image/.+;base64,', '', strb64) 41 | pil_img = Image.open(BytesIO(base64.b64decode(strb64))) 42 | return pil_img 43 | 44 | def encode_image(pil_img): 45 | buffer_ = BytesIO() 46 | pil_img.save(buffer_, format='PNG') 47 | img_str = base64.b64encode(buffer_.getvalue()).decode() 48 | img_str = 'data:image/png;base64,'+img_str 49 | return img_str 50 | 51 | def load_img(path): 52 | with open(path, 'rb') as f: 53 | with Image.open(f) as img: 54 | return img.convert('RGB') 55 | 56 | def process_image(pil_img, mode='recipe', top=5): 57 | tensor = engine.dataset['eval'].images_dataset.image_tf(pil_img) 58 | item = {'data': tensor} 59 | batch = engine.dataset['eval'].items_tf()([item]) 60 | batch = engine.model.prepare_batch(batch) 61 | 62 | if mode == 'recipe': 63 | embs = rcp_embs 64 | elif mode == 'image': 65 | embs = img_embs 66 | elif mode == 'all': 67 | embs = all_embs 68 | 69 | with torch.no_grad(): 70 | img_emb = engine.model.network.image_embedding(batch) 71 | img_emb = img_emb.data.cpu() 72 | distances = fast_distance(img_emb, embs) 73 | values, ids = distances[0].sort() 74 | 75 | out = [] 76 | for i in range(top): 77 | idx = ids[i].item() 78 | if mode == 'all': 79 | idx = all_ids[idx] 80 | item = engine.dataset['eval'][idx] 81 | 82 | info = {} 83 | info['class_name'] = item['recipe']['class_name'] 84 | info['ingredients'] = item['recipe']['layer1']['ingredients'] 85 | info['instructions'] = item['recipe']['layer1']['instructions'] 86 | info['url'] = item['recipe']['layer1']['url'] 87 | info['title'] = item['recipe']['layer1']['title'] 88 | info['path_img'] = item['image']['path'] 89 | info['img_strb64'] = encode_image(load_img(item['image']['path'])) 90 | out.append(info) 91 | 92 | return out 93 | 94 | @Request.application 95 | def application(request): 96 | utils.set_random_seed(Options()['misc']['seed']) 97 | 98 | if 'image' not in request.form: 99 | return Response('"image" POST field is missing') 100 | if 'mode' not in request.form: 101 | return Response('"mode" POST field is missing') 102 | if 'top' not in request.form: 103 | return Response('"top" POST field is missing') 104 | 105 | if request.form['mode'] not in ['recipe', 'image', 'all']: 106 | return Response('"mode" must be equals to ' + ' | '.join(['recipe', 'image', 'all'])) 107 | 108 | pil_img = decode_image(request.form['image']) 109 | out = process_image(pil_img, 110 | mode=request.form['mode'], 111 | top=int(request.form['top'])) 112 | out = json.dumps(out) 113 | response = Response(out) 114 | response.headers.add('Access-Control-Allow-Origin', '*') 115 | response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,PATCH') 116 | response.headers.add('Access-Control-Allow-Headers', 'Content-Type, Authorization') 117 | response.headers.add('X-XSS-Protection', '0') 118 | return response 119 | 120 | def api(path_opts=None): 121 | global engine 122 | global img_embs 123 | global rcp_embs 124 | global all_embs 125 | global all_ids 126 | 127 | Options(path_opts) 128 | utils.set_random_seed(Options()['misc']['seed']) 129 | 130 | assert Options()['dataset']['eval_split'] is not None, 'eval_split must be set' 131 | assert Options()['dataset']['train_split'] is None, 'train_split must be None' 132 | 133 | init_logs_options_files(Options()['exp']['dir'], Options()['exp']['resume']) 134 | 135 | Logger().log_dict('options', Options(), should_print=True) 136 | Logger()(os.uname()) 137 | if torch.cuda.is_available(): 138 | cudnn.benchmark = True 139 | Logger()('Available GPUs: {}'.format(utils.available_gpu_ids())) 140 | 141 | engine = engines.factory() 142 | engine.dataset = datasets.factory(engine) 143 | engine.model = models.factory(engine) 144 | engine.model.eval() 145 | engine.resume() 146 | 147 | dir_extract = os.path.join(Options()['exp']['dir'], 'extract', Options()['dataset']['eval_split']) 148 | path_img_embs = os.path.join(dir_extract, 'image_emdeddings.pth') 149 | path_rcp_embs = os.path.join(dir_extract, 'recipe_emdeddings.pth') 150 | img_embs = torch.load(path_img_embs) 151 | rcp_embs = torch.load(path_rcp_embs) 152 | all_embs = torch.cat([img_embs, rcp_embs], dim=0) 153 | all_ids = list(range(img_embs.shape[0])) + list(range(rcp_embs.shape[0])) 154 | 155 | my_local_ip = '132.227.204.160' # localhost | 192.168.0.41 (hostname --ip-address) 156 | my_local_port = 3456 # 8080 | 3456 157 | run_simple(my_local_ip, my_local_port, application) 158 | 159 | 160 | if __name__ == '__main__': 161 | main(run=api) -------------------------------------------------------------------------------- /recipe1m/visu/ingrs_to_images.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from bootstrap.models.factory import factory as model_factory 8 | from torch.autograd import Variable 9 | from PIL import Image 10 | from tqdm import tqdm 11 | import bootstrap.lib.utils as utils 12 | 13 | def main(): 14 | 15 | Logger('.') 16 | 17 | 18 | #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] 19 | nb_points = 1000 20 | split = 'test' 21 | dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' 22 | path_opts = os.path.join(dir_exp, 'options.yaml') 23 | dir_extract = os.path.join(dir_exp, 'extract', split) 24 | dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split) 25 | dir_img = os.path.join(dir_extract, 'image') 26 | dir_rcp = os.path.join(dir_extract, 'recipe') 27 | path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') 28 | 29 | is_mean = True 30 | ingrs_list = ['tomato']#['tomato', 'salad', 'onion', 'chicken'] 31 | 32 | 33 | #Options(path_opts) 34 | Options.load_from_yaml(path_opts) 35 | utils.set_random_seed(Options()['misc']['seed']) 36 | 37 | dataset = factory(split) 38 | 39 | Logger()('Load model...') 40 | model = model_factory() 41 | model_state = torch.load(path_model_ckpt) 42 | model.load_state_dict(model_state) 43 | model.eval() 44 | 45 | if not os.path.isdir(dir_extract): 46 | os.system('mkdir -p '+dir_rcp) 47 | os.system('mkdir -p '+dir_img) 48 | 49 | for i in tqdm(range(len(dataset))): 50 | item = dataset[i] 51 | batch = dataset.items_tf()([item]) 52 | 53 | if model.is_cuda: 54 | batch = model.cuda_tf()(batch) 55 | 56 | is_volatile = (model.mode not in ['train', 'trainval']) 57 | batch = model.variable_tf(volatile=is_volatile)(batch) 58 | 59 | out = model.network(batch) 60 | 61 | path_image = os.path.join(dir_img, '{}.pth'.format(i)) 62 | path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i)) 63 | torch.save(out['image_embedding'][0].data.cpu(), path_image) 64 | torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe) 65 | 66 | 67 | 68 | # b = dataset.make_batch_loader().__iter__().__next__() 69 | # import ipdb; ipdb.set_trace() 70 | 71 | ingrs = torch.LongTensor(1, len(ingrs_list)) 72 | for i, ingr_name in enumerate(ingrs_list): 73 | ingrs[0, i] = dataset.recipes_dataset.ingrname_to_ingrid[ingr_name] 74 | 75 | input_ = { 76 | 'recipe': { 77 | 'ingrs': { 78 | 'data': Variable(ingrs.cuda(), requires_grad=False), 79 | 'lengths': [ingrs.size(1)] 80 | }, 81 | 'instrs': { 82 | 'data': Variable(torch.FloatTensor(1, 1, 1024).fill_(0).cuda(), requires_grad=False), 83 | 'lengths': [1] 84 | } 85 | } 86 | } 87 | 88 | #emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs']) 89 | list_idx = torch.randperm(len(dataset)) 90 | # nb_points = list_idx.size(0) 91 | 92 | Logger()('Load embeddings...') 93 | img_embs = [] 94 | rcp_embs = [] 95 | for i in range(nb_points): 96 | idx = list_idx[i] 97 | path_img = os.path.join(dir_img, '{}.pth'.format(idx)) 98 | path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) 99 | if not os.path.isfile(path_img): 100 | Logger()('No such file: {}'.format(path_img)) 101 | continue 102 | if not os.path.isfile(path_rcp): 103 | Logger()('No such file: {}'.format(path_rcp)) 104 | continue 105 | img_embs.append(torch.load(path_img)) 106 | rcp_embs.append(torch.load(path_rcp)) 107 | 108 | img_embs = torch.stack(img_embs, 0) 109 | rcp_embs = torch.stack(rcp_embs, 0) 110 | 111 | Logger()('Load mean embeddings') 112 | 113 | path_ingrs = os.path.join(dir_extract_mean, 'ingrs.pth') 114 | path_instrs = os.path.join(dir_extract_mean, 'instrs.pth') 115 | 116 | mean_ingrs = torch.load(path_ingrs) 117 | mean_instrs = torch.load(path_instrs) 118 | 119 | Logger()('Forward ingredient...') 120 | #ingr_emb = model.network.recipe_embedding(input_['recipe']) 121 | ingr_emb = model.network.recipe_embedding.forward_one_ingr( 122 | input_['recipe']['ingrs'], 123 | emb_instrs=mean_instrs.unsqueeze(0)) 124 | 125 | ingr_emb = ingr_emb.data.cpu() 126 | ingr_emb = ingr_emb.expand_as(img_embs) 127 | 128 | Logger()('Fast distance...') 129 | dist = fast_distance(img_embs, ingr_emb)[:, 0] 130 | 131 | sorted_ids = np.argsort(dist.numpy()) 132 | 133 | dir_visu = os.path.join(dir_exp, 'visu', 'ingrs_to_image_nb_points:{}_instrs:{}_mean:{}'.format(nb_points, '-'.join(ingrs_list), is_mean)) 134 | os.system('mkdir -p '+dir_visu) 135 | 136 | Logger()('Load/save images to {}...'.format(dir_visu)) 137 | for i in range(20): 138 | idx = int(sorted_ids[i]) 139 | item_id = list_idx[idx] 140 | item = dataset[item_id] 141 | path_img_from = item['image']['path'] 142 | ingrs = [ingr.replace('/', '\'') for ingr in item['recipe']['ingrs']['interim']] 143 | cname = item['recipe']['class_name'] 144 | path_img_to = os.path.join(dir_visu, 'image_top:{}_cname:{}.png'.format(i+1, cname)) 145 | img = Image.open(path_img_from) 146 | img.save(path_img_to) 147 | #os.system('cp {} {}'.format(path_img_from, path_img_to)) 148 | 149 | 150 | Logger()('End') 151 | 152 | 153 | 154 | 155 | def fast_distance(A,B): 156 | # A and B must have norm 1 for this to work for the ranking 157 | return torch.mm(A,B.t()) * -1 158 | 159 | # python -m recipe1m.visu.top5 160 | if __name__ == '__main__': 161 | main() -------------------------------------------------------------------------------- /recipe1m/visu/old_ingrs_to_img_by_class.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from bootstrap.models.factory import factory as model_factory 8 | from torch.autograd import Variable 9 | from PIL import Image 10 | from tqdm import tqdm 11 | import bootstrap.lib.utils as utils 12 | 13 | def main(): 14 | 15 | Logger('.') 16 | 17 | 18 | #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] 19 | nb_points = 1000 20 | split = 'test' 21 | dir_exp = 'logs/recipe1m/trijoint/2017-12-14-15-04-51' 22 | path_opts = os.path.join(dir_exp, 'options.yaml') 23 | dir_extract = os.path.join(dir_exp, 'extract', split) 24 | dir_img = os.path.join(dir_extract, 'image') 25 | dir_rcp = os.path.join(dir_extract, 'recipe') 26 | path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') 27 | 28 | 29 | 30 | #Options(path_opts) 31 | Options.load_from_yaml(path_opts) 32 | utils.set_random_seed(Options()['misc']['seed']) 33 | 34 | dataset = factory(split) 35 | 36 | Logger()('Load model...') 37 | model = model_factory() 38 | model_state = torch.load(path_model_ckpt) 39 | model.load_state_dict(model_state) 40 | model.set_mode(split) 41 | 42 | if not os.path.isdir(dir_extract): 43 | os.system('mkdir -p '+dir_rcp) 44 | os.system('mkdir -p '+dir_img) 45 | 46 | for i in tqdm(range(len(dataset))): 47 | item = dataset[i] 48 | batch = dataset.items_tf()([item]) 49 | 50 | if model.is_cuda: 51 | batch = model.cuda_tf()(batch) 52 | 53 | is_volatile = (model.mode not in ['train', 'trainval']) 54 | batch = model.variable_tf(volatile=is_volatile)(batch) 55 | 56 | out = model.network(batch) 57 | 58 | path_image = os.path.join(dir_img, '{}.pth'.format(i)) 59 | path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i)) 60 | torch.save(out['image_embedding'][0].data.cpu(), path_image) 61 | torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe) 62 | 63 | 64 | # b = dataset.make_batch_loader().__iter__().__next__() 65 | # class_name = 'pizza' 66 | # ingrs = torch.LongTensor(1, 2) 67 | # ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['mushrooms'] 68 | # ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['mushroom'] 69 | 70 | class_name = 'hamburger' 71 | ingrs = torch.LongTensor(1, 2) 72 | ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['mushroom'] 73 | ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['mushrooms'] 74 | 75 | 76 | #ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['tomato'] 77 | #ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['salad'] 78 | #ingrs[0, 2] = dataset.recipes_dataset.ingrname_to_ingrid['onion'] 79 | #ingrs[0, 3] = dataset.recipes_dataset.ingrname_to_ingrid['chicken'] 80 | 81 | input_ = { 82 | 'recipe': { 83 | 'ingrs': { 84 | 'data': Variable(ingrs.cuda(), requires_grad=False), 85 | 'lengths': [ingrs.size(1)] 86 | }, 87 | 'instrs': { 88 | 'data': Variable(torch.FloatTensor(1, 1, 1024).fill_(0).cuda(), requires_grad=False), 89 | 'lengths': [1] 90 | } 91 | } 92 | } 93 | 94 | #emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs']) 95 | #list_idx = torch.randperm(len(dataset)) 96 | 97 | indices_by_class = dataset._make_indices_by_class() 98 | class_id = dataset.cname_to_cid[class_name] 99 | list_idx = torch.Tensor(indices_by_class[class_id]) 100 | rand_idx = torch.randperm(list_idx.size(0)) 101 | list_idx = list_idx[rand_idx] 102 | 103 | list_idx = list_idx.view(-1).int() 104 | 105 | img_embs = [] 106 | rcp_embs = [] 107 | 108 | if nb_points > list_idx.size(0): 109 | nb_points = list_idx.size(0) 110 | 111 | Logger()('Load {} embeddings...'.format(nb_points)) 112 | for i in range(nb_points): 113 | idx = list_idx[i] 114 | path_img = os.path.join(dir_img, '{}.pth'.format(idx)) 115 | path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) 116 | if not os.path.isfile(path_img): 117 | Logger()('No such file: {}'.format(path_img)) 118 | continue 119 | if not os.path.isfile(path_rcp): 120 | Logger()('No such file: {}'.format(path_rcp)) 121 | continue 122 | img_embs.append(torch.load(path_img)) 123 | rcp_embs.append(torch.load(path_rcp)) 124 | 125 | img_embs = torch.stack(img_embs, 0) 126 | rcp_embs = torch.stack(rcp_embs, 0) 127 | 128 | Logger()('Forward ingredient...') 129 | #ingr_emb = model.network.recipe_embedding(input_['recipe']) 130 | ingr_emb = model.network.recipe_embedding.forward_one_ingr(input_['recipe']['ingrs']) 131 | 132 | ingr_emb = ingr_emb.data.cpu() 133 | ingr_emb = ingr_emb.expand_as(img_embs) 134 | 135 | Logger()('Fast distance...') 136 | dist = fast_distance(img_embs, ingr_emb)[:, 0] 137 | 138 | sorted_ids = np.argsort(dist.numpy()) 139 | 140 | dir_visu = os.path.join(dir_exp, 'visu', 'ingrs_to_image_by_class_{}'.format(class_name)) 141 | os.system('mkdir -p '+dir_visu) 142 | 143 | Logger()('Load/save images...') 144 | for i in range(20): 145 | idx = int(sorted_ids[i]) 146 | item_id = list_idx[idx] 147 | item = dataset[item_id] 148 | Logger()(item['recipe']['class_name']) 149 | Logger()(item['image']['class_name']) 150 | path_img_from = item['image']['path'] 151 | path_img_to = os.path.join(dir_visu, 'image_top_{}.png'.format(i+1)) 152 | img = Image.open(path_img_from) 153 | img.save(path_img_to) 154 | #os.system('cp {} {}'.format(path_img_from, path_img_to)) 155 | 156 | 157 | Logger()('End') 158 | 159 | 160 | 161 | 162 | def fast_distance(A,B): 163 | # A and B must have norm 1 for this to work for the ranking 164 | return torch.mm(A,B.t()) * -1 165 | 166 | # python -m recipe1m.visu.top5 167 | if __name__ == '__main__': 168 | main() -------------------------------------------------------------------------------- /recipe1m/visu/modality_to_modality_top5.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from bootstrap.models.factory import factory as model_factory 8 | from torch.autograd import Variable 9 | from PIL import Image 10 | from tqdm import tqdm 11 | import bootstrap.lib.utils as utils 12 | 13 | def main(): 14 | 15 | Logger('.') 16 | 17 | import argparse 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('modality_to_modality', help='foo help', default='recipe_to_image') 20 | args = parser.parse_args() 21 | 22 | 23 | #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] 24 | nb_points = 1000 25 | modality_to_modality = args.modality_to_modality#'image_to_image' 26 | print(modality_to_modality) 27 | split = 'test' 28 | dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' 29 | path_opts = os.path.join(dir_exp, 'options.yaml') 30 | dir_extract = os.path.join(dir_exp, 'extract', split) 31 | dir_img = os.path.join(dir_extract, 'image') 32 | dir_rcp = os.path.join(dir_extract, 'recipe') 33 | path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') 34 | 35 | Options.load_from_yaml(path_opts) 36 | Options()['misc']['seed'] = 11 37 | utils.set_random_seed(Options()['misc']['seed']) 38 | 39 | dataset = factory(split) 40 | 41 | Logger()('Load model...') 42 | model = model_factory() 43 | model_state = torch.load(path_model_ckpt) 44 | model.load_state_dict(model_state) 45 | model.eval() 46 | 47 | if not os.path.isdir(dir_extract): 48 | os.system('mkdir -p '+dir_rcp) 49 | os.system('mkdir -p '+dir_img) 50 | 51 | for i in tqdm(range(len(dataset))): 52 | item = dataset[i] 53 | batch = dataset.items_tf()([item]) 54 | 55 | if model.is_cuda: 56 | batch = model.cuda_tf()(batch) 57 | 58 | is_volatile = (model.mode not in ['train', 'trainval']) 59 | batch = model.variable_tf(volatile=is_volatile)(batch) 60 | 61 | out = model.network(batch) 62 | 63 | path_image = os.path.join(dir_img, '{}.pth'.format(i)) 64 | path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i)) 65 | torch.save(out['image_embedding'][0].data.cpu(), path_image) 66 | torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe) 67 | 68 | 69 | 70 | indices_by_class = dataset._make_indices_by_class() 71 | 72 | # class_name = classes[0] # TODO 73 | # class_id = dataset.cname_to_cid[class_name] 74 | # list_idx = torch.Tensor(indices_by_class[class_id]) 75 | # rand_idx = torch.randperm(list_idx.size(0)) 76 | # list_idx = list_idx[rand_idx] 77 | # list_idx = list_idx.view(-1).int() 78 | list_idx = torch.randperm(len(dataset)) 79 | 80 | #nb_points = list_idx.size(0) 81 | 82 | dir_visu = os.path.join(dir_exp, 'visu', '{}_top20_seed:{}'.format(modality_to_modality, Options()['misc']['seed'])) 83 | os.system('rm -rf '+dir_visu) 84 | os.system('mkdir -p '+dir_visu) 85 | 86 | Logger()('Load embeddings...') 87 | img_embs = [] 88 | rcp_embs = [] 89 | for i in range(nb_points): 90 | idx = list_idx[i] 91 | #idx = i 92 | path_img = os.path.join(dir_img, '{}.pth'.format(idx)) 93 | path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) 94 | if not os.path.isfile(path_img): 95 | Logger()('No such file: {}'.format(path_img)) 96 | continue 97 | if not os.path.isfile(path_rcp): 98 | Logger()('No such file: {}'.format(path_rcp)) 99 | continue 100 | img_embs.append(torch.load(path_img)) 101 | rcp_embs.append(torch.load(path_rcp)) 102 | 103 | img_embs = torch.stack(img_embs, 0) 104 | rcp_embs = torch.stack(rcp_embs, 0) 105 | 106 | # Logger()('Forward ingredient...') 107 | # #ingr_emb = model.network.recipe_embedding(input_['recipe']) 108 | # ingr_emb = model.network.recipe_embedding.forward_one_ingr( 109 | # input_['recipe']['ingrs'], 110 | # emb_instrs=mean_instrs.unsqueeze(0)) 111 | 112 | # ingr_emb = ingr_emb.data.cpu() 113 | # ingr_emb = ingr_emb.expand_as(img_embs) 114 | 115 | 116 | Logger()('Fast distance...') 117 | 118 | if modality_to_modality == 'image_to_recipe': 119 | dist = fast_distance(img_embs, rcp_embs) 120 | elif modality_to_modality == 'recipe_to_image': 121 | dist = fast_distance(rcp_embs, img_embs) 122 | elif modality_to_modality == 'recipe_to_recipe': 123 | dist = fast_distance(rcp_embs, rcp_embs) 124 | elif modality_to_modality == 'image_to_image': 125 | dist = fast_distance(img_embs, img_embs) 126 | 127 | dist=dist[:, 0] 128 | sorted_ids = np.argsort(dist.numpy()) 129 | 130 | Logger()('Load/save images in {}...'.format(dir_visu)) 131 | for i in range(20): 132 | idx = int(sorted_ids[i]) 133 | item_id = list_idx[idx] 134 | #item_id = idx 135 | item = dataset[item_id] 136 | write_img_rcp(dir_visu, item, top=i) 137 | #os.system('cp {} {}'.format(path_img_from, path_img_to)) 138 | 139 | 140 | Logger()('End') 141 | 142 | def write_img_rcp(dir_visu, item, top=1): 143 | dir_visu = os.path.join(dir_visu, 'top{}_class:{}_item:{}'.format(top, item['recipe']['class_name'].replace(' ', '_'), item['index'])) 144 | path_rcp = dir_visu + '_rcp.txt' 145 | path_img = dir_visu + '_img.png' 146 | #os.system('mkdir -p '+dir_fig_i) 147 | 148 | s = [item['recipe']['layer1']['title']] 149 | s += [d['text'] for d in item['recipe']['layer1']['ingredients']] 150 | s += [d['text'] for d in item['recipe']['layer1']['instructions']] 151 | 152 | with open(path_rcp, 'w') as f: 153 | f.write('\n'.join(s)) 154 | 155 | path_img_from = item['image']['path'] 156 | img = Image.open(path_img_from) 157 | img.save(path_img) 158 | 159 | # for j in range(5): 160 | # id_img = recipe2im[i,j] 161 | # path_img_load = X_img['path'][id_img] 162 | # class_name = X_img['class_name'][id_img] 163 | # class_name = class_name.replace(' ', '-') 164 | # if id_img == i: 165 | # path_img_save = os.path.join(dir_fig_i, 'img_{}_{}_found.png'.format(j, class_name)) 166 | # else: 167 | # path_img_save = os.path.join(dir_fig_i, 'img_{}_{}.png'.format(j, class_name)) 168 | # I = load_image(path_img_load, crop_size=500) 169 | # I.save(path_img_save) 170 | 171 | def fast_distance(A,B): 172 | # A and B must have norm 1 for this to work for the ranking 173 | return torch.mm(A,B.t()) * -1 174 | 175 | # python -m recipe1m.visu.top5 176 | if __name__ == '__main__': 177 | main() -------------------------------------------------------------------------------- /recipe1m/visu/make_menu.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from bootstrap.models.factory import factory as model_factory 8 | from torch.autograd import Variable 9 | from PIL import Image 10 | from tqdm import tqdm 11 | import bootstrap.lib.utils as utils 12 | 13 | def main(): 14 | 15 | Logger('.') 16 | 17 | #classes = ['hamburger'] 18 | #nb_points = 19 | split = 'test' 20 | class_name = None#'potato salad' 21 | modality_to_modality = 'recipe_to_image' 22 | dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' 23 | path_opts = os.path.join(dir_exp, 'options.yaml') 24 | dir_extract = os.path.join(dir_exp, 'extract', split) 25 | dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split) 26 | dir_img = os.path.join(dir_extract, 'image') 27 | dir_rcp = os.path.join(dir_extract, 'recipe') 28 | path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') 29 | 30 | 31 | 32 | #is_mean = True 33 | #ingrs_list = ['carotte', 'salad', 'tomato']#['avocado'] 34 | 35 | #Options(path_opts) 36 | Options(path_opts) 37 | Options()['misc']['seed'] = 11 38 | utils.set_random_seed(Options()['misc']['seed']) 39 | 40 | chosen_item_id = 51259 41 | dataset = factory(split) 42 | if class_name: 43 | class_id = dataset.cname_to_cid[class_name] 44 | indices_by_class = dataset._make_indices_by_class() 45 | nb_points = len(indices_by_class[class_id]) 46 | list_idx = torch.Tensor(indices_by_class[class_id]) 47 | rand_idx = torch.randperm(list_idx.size(0)) 48 | list_idx = list_idx[rand_idx] 49 | list_idx = list_idx.view(-1).int() 50 | dir_visu = os.path.join(dir_exp, 'visu', 'remove_ingrs_item:{}_nb_points:{}_class:{}'.format(chosen_item_id, nb_points, class_name.replace(' ', '_'))) 51 | else: 52 | nb_points = 1000 53 | list_idx = torch.randperm(len(dataset)) 54 | dir_visu = os.path.join(dir_exp, 'visu', 'remove_ingrs_item:{}_nb_points:{}_removed'.format(chosen_item_id, nb_points)) 55 | 56 | 57 | # for i in range(20): 58 | # item_id = list_idx[i] 59 | # item = dataset[item_id] 60 | # write_img_rcp(dir_visu, item, top=i) 61 | 62 | Logger()('Load model...') 63 | model = model_factory() 64 | model_state = torch.load(path_model_ckpt) 65 | model.load_state_dict(model_state) 66 | model.eval() 67 | 68 | item = dataset[chosen_item_id] 69 | 70 | # from tqdm import tqdm 71 | # ids = [] 72 | # for i in tqdm(range(len(dataset.recipes_dataset))): 73 | # item = dataset.recipes_dataset[i]#23534] 74 | # if 'broccoli' in item['ingrs']['interim']: 75 | # print('broccoli', i) 76 | # ids.append(i) 77 | 78 | # # if 'mushroom' in item['ingrs']['interim']: 79 | # # print('mushroom', i) 80 | # # break 81 | 82 | import ipdb; ipdb.set_trace() 83 | 84 | 85 | 86 | # input_ = { 87 | # 'recipe': { 88 | # 'ingrs': { 89 | # 'data': item['recipe']['ingrs']['data'], 90 | # 'lengths': item['recipe']['ingrs']['lengths'] 91 | # }, 92 | # 'instrs': { 93 | # 'data': item['recipe']['instrs']['data'], 94 | # 'lengths': item['recipe']['instrs']['lengths'] 95 | # } 96 | # } 97 | # } 98 | 99 | instrs = torch.FloatTensor(6,1024) 100 | instrs[0] = item['recipe']['instrs']['data'][0] 101 | instrs[1] = item['recipe']['instrs']['data'][1] 102 | instrs[2] = item['recipe']['instrs']['data'][3] 103 | instrs[3] = item['recipe']['instrs']['data'][4] 104 | instrs[4] = item['recipe']['instrs']['data'][6] 105 | instrs[5] = item['recipe']['instrs']['data'][7] 106 | 107 | ingrs = torch.LongTensor([612,585,844,3087,144,188,1]) 108 | 109 | input_ = { 110 | 'recipe': { 111 | 'ingrs': { 112 | 'data': ingrs, 113 | 'lengths': ingrs.size(0) 114 | }, 115 | 'instrs': { 116 | 'data': instrs, 117 | 'lengths': instrs.size(0) 118 | } 119 | } 120 | } 121 | 122 | batch = dataset.items_tf()([input_]) 123 | batch = model.prepare_batch(batch) 124 | out = model.network.recipe_embedding(batch['recipe']) 125 | 126 | # path_rcp = os.path.join(dir_rcp, '{}.pth'.format(23534)) 127 | # rcp_emb = torch.load(path_rcp) 128 | 129 | 130 | Logger()('Load embeddings...') 131 | img_embs = [] 132 | for i in range(nb_points): 133 | try: 134 | idx = list_idx[i] 135 | except: 136 | import ipdb; ipdb.set_trace() 137 | #idx = i 138 | path_img = os.path.join(dir_img, '{}.pth'.format(idx)) 139 | if not os.path.isfile(path_img): 140 | Logger()('No such file: {}'.format(path_img)) 141 | continue 142 | img_embs.append(torch.load(path_img)) 143 | 144 | img_embs = torch.stack(img_embs, 0) 145 | 146 | Logger()('Fast distance...') 147 | 148 | dist = fast_distance(out.data.cpu().expand_as(img_embs), img_embs) 149 | dist = dist[0, :] 150 | sorted_ids = np.argsort(dist.numpy()) 151 | 152 | os.system('rm -rf '+dir_visu) 153 | os.system('mkdir -p '+dir_visu) 154 | 155 | Logger()('Load/save images in {}...'.format(dir_visu)) 156 | write_img_rcp(dir_visu, item, top=0, begin_with='query') 157 | for i in range(20): 158 | idx = int(sorted_ids[i]) 159 | item_id = list_idx[idx] 160 | #item_id = idx 161 | item = dataset[item_id] 162 | write_img_rcp(dir_visu, item, top=i, begin_with='nn') 163 | 164 | Logger()('End') 165 | 166 | def write_img_rcp(dir_visu, item, top=1, begin_with=''): 167 | dir_visu = os.path.join(dir_visu, begin_with+'_top{}_class:{}_item:{}'.format(top, item['recipe']['class_name'].replace(' ', '_'), item['index'])) 168 | path_rcp = dir_visu + '_rcp.txt' 169 | path_img = dir_visu + '_img.png' 170 | #os.system('mkdir -p '+dir_fig_i) 171 | 172 | s = [item['recipe']['layer1']['title']] 173 | s += ['\nIngredients raw'] 174 | s += [d['text'] for d in item['recipe']['layer1']['ingredients']] 175 | s += ['\nIngredients interim'] 176 | s += ['{}: {}'.format(item['recipe']['ingrs']['data'][idx], d) for idx, d in enumerate(item['recipe']['ingrs']['interim'])] 177 | s += ['\nInstructions raw'] 178 | s += [d['text'] for d in item['recipe']['layer1']['instructions']] 179 | 180 | with open(path_rcp, 'w') as f: 181 | f.write('\n'.join(s)) 182 | 183 | path_img_from = item['image']['path'] 184 | img = Image.open(path_img_from) 185 | img.save(path_img) 186 | 187 | 188 | def fast_distance(A,B): 189 | # A and B must have norm 1 for this to work for the ranking 190 | return torch.mm(A,B.t()) * -1 191 | 192 | # python -m recipe1m.visu.top5 193 | if __name__ == '__main__': 194 | main() -------------------------------------------------------------------------------- /recipe1m/visu/remove_ingrs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from bootstrap.models.factory import factory as model_factory 8 | from torch.autograd import Variable 9 | from PIL import Image 10 | from tqdm import tqdm 11 | import bootstrap.lib.utils as utils 12 | 13 | def main(): 14 | 15 | Logger('.') 16 | 17 | #classes = ['hamburger'] 18 | #nb_points = 19 | split = 'test' 20 | class_name = None#'potato salad' 21 | modality_to_modality = 'recipe_to_image' 22 | dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' 23 | path_opts = os.path.join(dir_exp, 'options.yaml') 24 | dir_extract = os.path.join(dir_exp, 'extract', split) 25 | dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split) 26 | dir_img = os.path.join(dir_extract, 'image') 27 | dir_rcp = os.path.join(dir_extract, 'recipe') 28 | path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') 29 | 30 | 31 | 32 | #is_mean = True 33 | #ingrs_list = ['carotte', 'salad', 'tomato']#['avocado'] 34 | 35 | #Options(path_opts) 36 | Options(path_opts) 37 | Options()['misc']['seed'] = 11 38 | utils.set_random_seed(Options()['misc']['seed']) 39 | 40 | chosen_item_id = 51259 41 | dataset = factory(split) 42 | if class_name: 43 | class_id = dataset.cname_to_cid[class_name] 44 | indices_by_class = dataset._make_indices_by_class() 45 | nb_points = len(indices_by_class[class_id]) 46 | list_idx = torch.Tensor(indices_by_class[class_id]) 47 | rand_idx = torch.randperm(list_idx.size(0)) 48 | list_idx = list_idx[rand_idx] 49 | list_idx = list_idx.view(-1).int() 50 | dir_visu = os.path.join(dir_exp, 'visu', 'remove_ingrs_item:{}_nb_points:{}_class:{}'.format(chosen_item_id, nb_points, class_name.replace(' ', '_'))) 51 | else: 52 | nb_points = 1000 53 | list_idx = torch.randperm(len(dataset)) 54 | dir_visu = os.path.join(dir_exp, 'visu', 'remove_ingrs_item:{}_nb_points:{}_removed'.format(chosen_item_id, nb_points)) 55 | 56 | 57 | # for i in range(20): 58 | # item_id = list_idx[i] 59 | # item = dataset[item_id] 60 | # write_img_rcp(dir_visu, item, top=i) 61 | 62 | Logger()('Load model...') 63 | model = model_factory() 64 | model_state = torch.load(path_model_ckpt) 65 | model.load_state_dict(model_state) 66 | model.eval() 67 | 68 | item = dataset[chosen_item_id] 69 | 70 | # from tqdm import tqdm 71 | # ids = [] 72 | # for i in tqdm(range(len(dataset.recipes_dataset))): 73 | # item = dataset.recipes_dataset[i]#23534] 74 | # if 'broccoli' in item['ingrs']['interim']: 75 | # print('broccoli', i) 76 | # ids.append(i) 77 | 78 | # # if 'mushroom' in item['ingrs']['interim']: 79 | # # print('mushroom', i) 80 | # # break 81 | 82 | # import ipdb; ipdb.set_trace() 83 | 84 | 85 | 86 | # input_ = { 87 | # 'recipe': { 88 | # 'ingrs': { 89 | # 'data': item['recipe']['ingrs']['data'], 90 | # 'lengths': item['recipe']['ingrs']['lengths'] 91 | # }, 92 | # 'instrs': { 93 | # 'data': item['recipe']['instrs']['data'], 94 | # 'lengths': item['recipe']['instrs']['lengths'] 95 | # } 96 | # } 97 | # } 98 | 99 | instrs = torch.FloatTensor(6,1024) 100 | instrs[0] = item['recipe']['instrs']['data'][0] 101 | instrs[1] = item['recipe']['instrs']['data'][1] 102 | instrs[2] = item['recipe']['instrs']['data'][3] 103 | instrs[3] = item['recipe']['instrs']['data'][4] 104 | instrs[4] = item['recipe']['instrs']['data'][6] 105 | instrs[5] = item['recipe']['instrs']['data'][7] 106 | 107 | ingrs = torch.LongTensor([612,585,844,3087,144,188,1]) 108 | 109 | input_ = { 110 | 'recipe': { 111 | 'ingrs': { 112 | 'data': ingrs, 113 | 'lengths': ingrs.size(0) 114 | }, 115 | 'instrs': { 116 | 'data': instrs, 117 | 'lengths': instrs.size(0) 118 | } 119 | } 120 | } 121 | 122 | batch = dataset.items_tf()([input_]) 123 | batch = model.prepare_batch(batch) 124 | out = model.network.recipe_embedding(batch['recipe']) 125 | 126 | # path_rcp = os.path.join(dir_rcp, '{}.pth'.format(23534)) 127 | # rcp_emb = torch.load(path_rcp) 128 | 129 | 130 | Logger()('Load embeddings...') 131 | img_embs = [] 132 | for i in range(nb_points): 133 | try: 134 | idx = list_idx[i] 135 | except: 136 | import ipdb; ipdb.set_trace() 137 | #idx = i 138 | path_img = os.path.join(dir_img, '{}.pth'.format(idx)) 139 | if not os.path.isfile(path_img): 140 | Logger()('No such file: {}'.format(path_img)) 141 | continue 142 | img_embs.append(torch.load(path_img)) 143 | 144 | img_embs = torch.stack(img_embs, 0) 145 | 146 | Logger()('Fast distance...') 147 | 148 | dist = fast_distance(out.data.cpu().expand_as(img_embs), img_embs) 149 | dist = dist[0, :] 150 | sorted_ids = np.argsort(dist.numpy()) 151 | 152 | os.system('rm -rf '+dir_visu) 153 | os.system('mkdir -p '+dir_visu) 154 | 155 | Logger()('Load/save images in {}...'.format(dir_visu)) 156 | write_img_rcp(dir_visu, item, top=0, begin_with='query') 157 | for i in range(20): 158 | idx = int(sorted_ids[i]) 159 | item_id = list_idx[idx] 160 | #item_id = idx 161 | item = dataset[item_id] 162 | write_img_rcp(dir_visu, item, top=i, begin_with='nn') 163 | 164 | Logger()('End') 165 | 166 | def write_img_rcp(dir_visu, item, top=1, begin_with=''): 167 | dir_visu = os.path.join(dir_visu, begin_with+'_top{}_class:{}_item:{}'.format(top, item['recipe']['class_name'].replace(' ', '_'), item['index'])) 168 | path_rcp = dir_visu + '_rcp.txt' 169 | path_img = dir_visu + '_img.png' 170 | #os.system('mkdir -p '+dir_fig_i) 171 | 172 | s = [item['recipe']['layer1']['title']] 173 | s += ['\nIngredients raw'] 174 | s += [d['text'] for d in item['recipe']['layer1']['ingredients']] 175 | s += ['\nIngredients interim'] 176 | s += ['{}: {}'.format(item['recipe']['ingrs']['data'][idx], d) for idx, d in enumerate(item['recipe']['ingrs']['interim'])] 177 | s += ['\nInstructions raw'] 178 | s += [d['text'] for d in item['recipe']['layer1']['instructions']] 179 | 180 | with open(path_rcp, 'w') as f: 181 | f.write('\n'.join(s)) 182 | 183 | path_img_from = item['image']['path'] 184 | img = Image.open(path_img_from) 185 | img.save(path_img) 186 | 187 | 188 | def fast_distance(A,B): 189 | # A and B must have norm 1 for this to work for the ranking 190 | return torch.mm(A,B.t()) * -1 191 | 192 | # python -m recipe1m.visu.top5 193 | if __name__ == '__main__': 194 | main() -------------------------------------------------------------------------------- /recipe1m/visu/ingrs_to_images_per_class.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from bootstrap.lib.logger import Logger 5 | from bootstrap.lib.options import Options 6 | from recipe1m.datasets.factory import factory 7 | from bootstrap.models.factory import factory as model_factory 8 | from torch.autograd import Variable 9 | from PIL import Image 10 | from tqdm import tqdm 11 | import bootstrap.lib.utils as utils 12 | 13 | def main(): 14 | 15 | Logger('.') 16 | 17 | 18 | 19 | #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] 20 | nb_points = 1000 21 | split = 'test' 22 | class_name = 'pizza' 23 | dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' 24 | path_opts = os.path.join(dir_exp, 'options.yaml') 25 | dir_extract = os.path.join(dir_exp, 'extract', split) 26 | dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split) 27 | dir_img = os.path.join(dir_extract, 'image') 28 | dir_rcp = os.path.join(dir_extract, 'recipe') 29 | path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') 30 | 31 | is_mean = True 32 | ingrs_list = ['fresh_strawberries']#['avocado'] 33 | 34 | 35 | Options(path_opts) 36 | Options()['misc']['seed'] = 2 37 | utils.set_random_seed(Options()['misc']['seed']) 38 | 39 | dataset = factory(split) 40 | 41 | Logger()('Load model...') 42 | model = model_factory() 43 | model_state = torch.load(path_model_ckpt) 44 | model.load_state_dict(model_state) 45 | model.eval() 46 | 47 | if not os.path.isdir(dir_extract): 48 | os.system('mkdir -p '+dir_rcp) 49 | os.system('mkdir -p '+dir_img) 50 | 51 | for i in tqdm(range(len(dataset))): 52 | item = dataset[i] 53 | batch = dataset.items_tf()([item]) 54 | 55 | if model.is_cuda: 56 | batch = model.cuda_tf()(batch) 57 | 58 | is_volatile = (model.mode not in ['train', 'trainval']) 59 | batch = model.variable_tf(volatile=is_volatile)(batch) 60 | 61 | out = model.network(batch) 62 | 63 | path_image = os.path.join(dir_img, '{}.pth'.format(i)) 64 | path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i)) 65 | torch.save(out['image_embedding'][0].data.cpu(), path_image) 66 | torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe) 67 | 68 | 69 | 70 | # b = dataset.make_batch_loader().__iter__().__next__() 71 | # import ipdb; ipdb.set_trace() 72 | 73 | ingrs = torch.LongTensor(1, len(ingrs_list)) 74 | for i, ingr_name in enumerate(ingrs_list): 75 | ingrs[0, i] = dataset.recipes_dataset.ingrname_to_ingrid[ingr_name] 76 | 77 | input_ = { 78 | 'recipe': { 79 | 'ingrs': { 80 | 'data': Variable(ingrs.cuda(), requires_grad=False), 81 | 'lengths': [ingrs.size(1)] 82 | }, 83 | 'instrs': { 84 | 'data': Variable(torch.FloatTensor(1, 1, 1024).fill_(0).cuda(), requires_grad=False), 85 | 'lengths': [1] 86 | } 87 | } 88 | } 89 | 90 | #emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs']) 91 | #list_idx = torch.randperm(len(dataset)) 92 | 93 | indices_by_class = dataset._make_indices_by_class() 94 | 95 | #import ipdb; ipdb.set_trace() 96 | 97 | class_id = dataset.cname_to_cid[class_name] 98 | list_idx = torch.Tensor(indices_by_class[class_id]) 99 | rand_idx = torch.randperm(list_idx.size(0)) 100 | list_idx = list_idx[rand_idx] 101 | list_idx = list_idx.view(-1).int() 102 | 103 | nb_points = list_idx.size(0) 104 | 105 | dir_visu = os.path.join(dir_exp, 'visu', 'ingrs_to_image_nb_points:{}_class:{}_instrs:{}_mean:{}_v2'.format(nb_points, class_name, '-'.join(ingrs_list), is_mean)) 106 | os.system('mkdir -p '+dir_visu) 107 | 108 | Logger()('Load embeddings...') 109 | img_embs = [] 110 | rcp_embs = [] 111 | for i in range(nb_points): 112 | idx = list_idx[i] 113 | path_img = os.path.join(dir_img, '{}.pth'.format(idx)) 114 | path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) 115 | if not os.path.isfile(path_img): 116 | Logger()('No such file: {}'.format(path_img)) 117 | continue 118 | if not os.path.isfile(path_rcp): 119 | Logger()('No such file: {}'.format(path_rcp)) 120 | continue 121 | img_embs.append(torch.load(path_img)) 122 | rcp_embs.append(torch.load(path_rcp)) 123 | 124 | img_embs = torch.stack(img_embs, 0) 125 | rcp_embs = torch.stack(rcp_embs, 0) 126 | 127 | Logger()('Load mean embeddings') 128 | 129 | path_ingrs = os.path.join(dir_extract_mean, 'ingrs.pth') 130 | path_instrs = os.path.join(dir_extract_mean, 'instrs.pth') 131 | 132 | mean_ingrs = torch.load(path_ingrs) 133 | mean_instrs = torch.load(path_instrs) 134 | 135 | Logger()('Forward ingredient...') 136 | #ingr_emb = model.network.recipe_embedding(input_['recipe']) 137 | ingr_emb = model.network.recipe_embedding.forward_one_ingr( 138 | input_['recipe']['ingrs'], 139 | emb_instrs=mean_instrs.unsqueeze(0)) 140 | 141 | ingr_emb = ingr_emb.data.cpu() 142 | ingr_emb = ingr_emb.expand_as(img_embs) 143 | 144 | Logger()('Fast distance...') 145 | dist = fast_distance(img_embs, ingr_emb)[:, 0] 146 | 147 | sorted_ids = np.argsort(dist.numpy()) 148 | 149 | Logger()('Load/save images in {}...'.format(dir_visu)) 150 | for i in range(20): 151 | idx = int(sorted_ids[i]) 152 | item_id = list_idx[idx] 153 | item = dataset[item_id] 154 | # path_img_from = item['image']['path'] 155 | # ingrs = [ingr.replace('/', '\'') for ingr in item['recipe']['ingrs']['interim']] 156 | # cname = item['recipe']['class_name'] 157 | # path_img_to = os.path.join(dir_visu, 'image_top:{}_ingrs:{}_cname:{}.png'.format(i+1, '-'.join(ingrs), cname)) 158 | # img = Image.open(path_img_from) 159 | # img.save(path_img_to) 160 | #os.system('cp {} {}'.format(path_img_from, path_img_to)) 161 | 162 | write_img_rcp(dir_visu, item, top=i, begin_with='nn') 163 | 164 | Logger()('End') 165 | 166 | def write_img_rcp(dir_visu, item, top=1, begin_with=''): 167 | dir_visu = os.path.join(dir_visu, begin_with+'_top{}_class:{}_item:{}'.format(top, item['recipe']['class_name'].replace(' ', '_'), item['index'])) 168 | path_rcp = dir_visu + '_rcp.txt' 169 | path_img = dir_visu + '_img.png' 170 | #os.system('mkdir -p '+dir_fig_i) 171 | 172 | s = [item['recipe']['layer1']['title']] 173 | s += ['\nIngredients raw'] 174 | s += [d['text'] for d in item['recipe']['layer1']['ingredients']] 175 | s += ['\nIngredients interim'] 176 | s += ['{}: {}'.format(item['recipe']['ingrs']['data'][idx], d) for idx, d in enumerate(item['recipe']['ingrs']['interim'])] 177 | s += ['\nInstructions raw'] 178 | s += [d['text'] for d in item['recipe']['layer1']['instructions']] 179 | 180 | with open(path_rcp, 'w') as f: 181 | f.write('\n'.join(s)) 182 | 183 | path_img_from = item['image']['path'] 184 | img = Image.open(path_img_from) 185 | img.save(path_img) 186 | 187 | 188 | Logger()('End') 189 | 190 | 191 | 192 | 193 | def fast_distance(A,B): 194 | # A and B must have norm 1 for this to work for the ranking 195 | return torch.mm(A,B.t()) * -1 196 | 197 | # python -m recipe1m.visu.top5 198 | if __name__ == '__main__': 199 | main() -------------------------------------------------------------------------------- /demo_web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Recipe1M Demo Adamine 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 24 | 25 | 26 | 27 | 54 | 55 | 56 |
    57 |
    58 |

    Welcome to the demo page

    59 |

    60 |

    61 | Github » 62 | Paper » 63 |

    64 |
    65 |
    66 | 67 | 88 | 89 | 90 | 91 |
    92 |

    API

    93 | 94 |
    95 |

    1. Upload a foodie picture

    96 |
    97 |
    98 | 99 |
    100 | 101 |
    102 | 103 | Browse... 104 | 105 | 106 | 107 |
    108 |
    109 |
    110 |
    111 |
    112 |
    113 | 114 |
    115 |
    116 |
    117 |
    118 |
    119 |

    2. Receive the recipe

    120 | 121 |
    Adamine is waiting for your image.
    122 | 123 |
    124 |
    125 | 126 |
    127 | 155 | 156 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | -------------------------------------------------------------------------------- /recipe1m/datasets/batch_sampler.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import torch 3 | import numpy as np 4 | from torch.utils.data.sampler import Sampler 5 | #from torch.utils.data.sampler import SequentialSampler 6 | from torch.utils.data.sampler import RandomSampler 7 | #from torch.utils.data.sampler import SubsetRandomSampler 8 | #from torch.utils.data.sampler import WeightedRandomSampler 9 | from torch.utils.data.sampler import BatchSampler 10 | 11 | from bootstrap.lib.options import Options 12 | 13 | class RandomSamplerValues(Sampler): 14 | """Samples elements randomly, without replacement. 15 | 16 | Arguments: 17 | data_source (Dataset): dataset to sample from 18 | 19 | Example: 20 | >>> list(RandomSamplerValues(range(10,20))) 21 | [15, 16, 10, 17, 11, 12, 14, 13, 18, 19] 22 | 23 | >>> list(RandomSampler(range(10,20))) 24 | [0, 4, 9, 2, 6, 1, 3, 5, 7, 8] 25 | """ 26 | 27 | def __init__(self, data_source): 28 | self.data_source = data_source 29 | 30 | def __iter__(self): 31 | if Options()['dataset'].get("debug", False): 32 | generator = iter(list(range(len(self.data_source)))) 33 | else: 34 | generator = iter(torch.randperm(len(self.data_source)).long()) 35 | 36 | for value in generator: 37 | yield self.data_source[value] 38 | 39 | def __len__(self): 40 | return len(self.data_source) 41 | 42 | 43 | class BatchSamplerClassif(object): 44 | """Randomly samples indices from a list of indices grouped by class, without replacement. 45 | BatchSamplerClassif wraps a list of BatchSampler, one for each class (besides background). 46 | 47 | Arguments: 48 | indices_by_class (list of list of int): 49 | batch_size (int): nb of indices in a batch returned by the BatchSampler 50 | nb_indices_same_class (int): nb of indices from the same class returned after a class sampling 51 | 52 | Example: 53 | >>> list(BatchSamplerClassif([list(range(3)), list(range(10,14)), list(range(20,25))], 4, 2)) 54 | [[10, 13, 20, 22], [0, 1, 21, 24]] 55 | """ 56 | 57 | def __init__(self, indices_by_class, batch_size, nb_indices_same_class): 58 | if batch_size % nb_indices_same_class != 0: 59 | raise ValueError('batch_size of BatchSamplerClassif ({}) must be divisible by nb_indices_same_class ({})'.format( 60 | batch_size, nb_indices_same_class)) 61 | 62 | self.indices_by_class = indices_by_class 63 | self.batch_size = batch_size 64 | self.nb_indices_same_class = nb_indices_same_class 65 | 66 | self.batch_sampler_by_class = [] 67 | for indices in indices_by_class: 68 | self.batch_sampler_by_class.append( 69 | BatchSampler(RandomSamplerValues(indices), 70 | self.nb_indices_same_class, 71 | True)) 72 | 73 | def _make_nb_samples_by_class(self): 74 | """ Note that nb_samples != nb_indices 75 | In fact, if nb_indices = 9 and nb_indices_same_class = 2, 76 | then nb_samples = 4 77 | """ 78 | return [len(sampler) for sampler in self.batch_sampler_by_class] 79 | 80 | def __iter__(self): 81 | 82 | nb_samples_by_class = torch.Tensor(self._make_nb_samples_by_class()) 83 | gen_by_class = [sampler.__iter__() for sampler in self.batch_sampler_by_class] 84 | 85 | for i in range(len(self)): 86 | batch = [] 87 | nb_samples = self.batch_size // self.nb_indices_same_class 88 | for j in range(nb_samples): 89 | # Class sampling 90 | if Options()['dataset'].get("debug", False): 91 | idx = np.random.multinomial(1,(nb_samples_by_class / sum(nb_samples_by_class)).numpy()).argmax() 92 | else: 93 | idx = torch.multinomial(nb_samples_by_class, 94 | 1, # num_samples 95 | False)[0] #replacement 96 | 97 | nb_samples_by_class[idx] -= 1 98 | batch += gen_by_class[idx].__next__() 99 | yield batch 100 | 101 | def __len__(self): 102 | """ 103 | Count the real number of indices using nb_samples 104 | 105 | Note that the "false" number of indices is: 106 | >>> nb_total_indices = sum([len(indices) for indices in self.indices_by_class]) 107 | """ 108 | nb_possible_indices = sum(self._make_nb_samples_by_class()) * self.nb_indices_same_class 109 | return nb_possible_indices // self.batch_size 110 | 111 | 112 | class BatchSamplerTripletClassif(object): 113 | """Wraps BatchSampler for items associated to background and BatchSamplerClassif for items with classes. 114 | 115 | Args: 116 | indices_by_class (list of list of int): 117 | batch_size (int): Size of mini-batch. 118 | pc_noclassif (float): Percentage of items associated to background in the batch 119 | nb_indices_same_class (int): nb of indices from the same class returned after a class sampling 120 | 121 | Warning: `indices_by_class` assumes that the list in position 0 contains 122 | indices associated to items without classes (background) 123 | 124 | Warning: `pc_noclassif` is used to calculate the number of items associated to classes in the batch, 125 | the latter must be a multiple of `nb_indices_same_class`. 126 | 127 | Example: 128 | >>> list(BatchSamplerTripletClassif([ 129 | list(range(8)), # indices of background 130 | list(range(10,14)), # class 1 131 | list(range(20,25)), # class 2 132 | list(range(30,36))], # class 3 133 | 4, # batch_size 134 | pc_noclassif=0.5, 135 | nb_indices_same_class=2)) 136 | [[13, 12, 2, 5], [31, 32, 4, 0], [33, 30, 6, 3], [23, 22, 7, 1]] 137 | """ 138 | 139 | def __init__(self, indices_by_class, batch_size, pc_noclassif=0.5, nb_indices_same_class=2): 140 | self.indices_by_class = copy.copy(indices_by_class) 141 | self.indices_no_class = self.indices_by_class.pop(0) 142 | self.batch_size = batch_size 143 | self.pc_noclassif = pc_noclassif 144 | self.nb_indices_same_class = nb_indices_same_class 145 | 146 | self.batch_size_classif = round((1 - self.pc_noclassif) * self.batch_size) 147 | self.batch_size_noclassif = self.batch_size - self.batch_size_classif 148 | 149 | # Batch Sampler NoClassif 150 | self.batch_sampler_noclassif = BatchSampler( 151 | RandomSamplerValues(self.indices_no_class), 152 | self.batch_size_noclassif, 153 | True) 154 | 155 | # Batch Sampler Classif 156 | self.batch_sampler_classif = BatchSamplerClassif( 157 | RandomSamplerValues(self.indices_by_class), 158 | self.batch_size_classif, 159 | self.nb_indices_same_class) 160 | 161 | def __iter__(self): 162 | gen_classif = self.batch_sampler_classif.__iter__() 163 | gen_noclassif = self.batch_sampler_noclassif.__iter__() 164 | for i in range(len(self)): 165 | batch = [] 166 | batch += gen_classif.__next__() 167 | batch += gen_noclassif.__next__() 168 | yield batch 169 | 170 | def __len__(self): 171 | return min([len(self.batch_sampler_classif), 172 | len(self.batch_sampler_noclassif)]) 173 | 174 | 175 | if __name__ == '__main__': 176 | 177 | batch_sampler = BatchSamplerClassif([ 178 | list(range(3)), # indices of class 1 179 | list(range(10,14)), # class 2 180 | list(range(20,25))], # class 3 181 | 4, # batch_size 182 | 2) # nb_indices_same_class 183 | print(list(batch_sampler)) 184 | 185 | batch_sampler = BatchSamplerTripletClassif([ 186 | list(range(8)), # indices of background 187 | list(range(10,14)), # class 1 188 | list(range(20,25)), # class 2 189 | list(range(30,36))], # class 3 190 | 4, # batch_size 191 | pc_noclassif=0.5, 192 | nb_indices_same_class=2) 193 | print(list(batch_sampler)) 194 | -------------------------------------------------------------------------------- /recipe1m/models/networks/trijoint.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pretrainedmodels 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import pickle 7 | from bootstrap.lib.logger import Logger 8 | from bootstrap.lib.options import Options 9 | 10 | class ResNet(nn.Module): 11 | 12 | def __init__(self): 13 | super(ResNet, self).__init__() 14 | self.resnet = pretrainedmodels.resnet50(num_classes=1000, pretrained='imagenet') 15 | self.dim_out = self.resnet.last_linear.in_features 16 | self.resnet.last_linear = None 17 | 18 | def forward(self, x): 19 | x = self.resnet.conv1(x) 20 | x = self.resnet.bn1(x) 21 | x = self.resnet.relu(x) 22 | x = self.resnet.maxpool(x) 23 | 24 | x = self.resnet.layer1(x) 25 | x = self.resnet.layer2(x) 26 | x = self.resnet.layer3(x) 27 | x = self.resnet.layer4(x) 28 | 29 | x = self.resnet.avgpool(x) 30 | x = x.view(x.size(0), -1) 31 | return x 32 | 33 | 34 | class ImageEmbedding(nn.Module): 35 | 36 | def __init__(self, opt): 37 | super(ImageEmbedding, self).__init__() 38 | self.dim_emb = opt['dim_emb'] 39 | self.activations = opt.get('activations', None) 40 | # modules 41 | self.convnet = ResNet() 42 | self.fc = nn.Linear(self.convnet.dim_out, self.dim_emb) 43 | 44 | def forward(self, image): 45 | x = self.convnet(image['data']) 46 | x = self.fc(x) 47 | if self.activations is not None: 48 | for name in self.activations: 49 | x = nn.functional.__dict__[name](x) 50 | return x 51 | 52 | 53 | class RecipeEmbedding(nn.Module): 54 | 55 | def __init__(self, opt): 56 | super(RecipeEmbedding, self).__init__() 57 | self.path_ingrs = opt['path_ingrs'] 58 | self.dim_ingr_out = opt['dim_ingr_out'] # 2048 59 | self.dim_instr_in = opt['dim_instr_in'] 60 | self.dim_instr_out = opt['dim_instr_out'] 61 | self.with_ingrs = opt['with_ingrs'] 62 | self.with_instrs = opt['with_instrs'] 63 | self.dim_emb = opt['dim_emb'] 64 | self.activations = opt.get('activations', None) 65 | # modules 66 | if self.with_ingrs: 67 | self._make_emb_ingrs() 68 | self.rnn_ingrs = nn.LSTM(self.dim_ingr_in, self.dim_ingr_out, 69 | bidirectional=True, batch_first=True) 70 | if self.with_instrs: 71 | self.rnn_instrs = nn.LSTM(self.dim_instr_in, self.dim_instr_out, 72 | bidirectional=False, batch_first=True) 73 | self.fusion = 'cat' 74 | self.dim_recipe = 0 75 | if self.with_ingrs: 76 | self.dim_recipe += 2*self.dim_ingr_out 77 | if self.with_instrs: 78 | self.dim_recipe += self.dim_instr_out 79 | if self.dim_recipe == 0: 80 | Logger()('Ingredients or/and instructions must be embedded "--model.network.with_{ingrs,instrs} True"', Logger.ERROR) 81 | 82 | self.fc = nn.Linear(self.dim_recipe, self.dim_emb) 83 | 84 | def forward_ingrs_instrs(self, ingrs_out=None, instrs_out=None): 85 | if self.with_ingrs and self.with_instrs: 86 | if self.fusion == 'cat': 87 | fusion_out = torch.cat([ingrs_out, instrs_out], 1) 88 | else: 89 | raise ValueError() 90 | x = self.fc(fusion_out) 91 | elif self.with_ingrs: 92 | x = self.fc(ingrs_out) 93 | elif self.with_instrs: 94 | x = self.fc(instrs_out) 95 | 96 | if self.activations is not None: 97 | for name in self.activations: 98 | x = nn.functional.__dict__[name](x) 99 | return x 100 | 101 | def forward(self, recipe): 102 | if self.with_ingrs: 103 | ingrs_out = self.forward_ingrs(recipe['ingrs']) 104 | else: 105 | ingrs_out = None 106 | 107 | if self.with_instrs: 108 | instrs_out = self.forward_instrs(recipe['instrs']) 109 | else: 110 | instrs_out = None 111 | 112 | x = self.forward_ingrs_instrs(ingrs_out, instrs_out) 113 | return x 114 | 115 | def _make_emb_ingrs(self): 116 | with open(self.path_ingrs, 'rb') as fobj: 117 | data = pickle.load(fobj) 118 | 119 | self.nb_ingrs = data[0].size(0) 120 | self.dim_ingr_in = data[0].size(1) 121 | self.emb_ingrs = nn.Embedding(self.nb_ingrs, self.dim_ingr_in) 122 | 123 | state_dict = {} 124 | state_dict['weight'] = data[0] 125 | self.emb_ingrs.load_state_dict(state_dict) 126 | 127 | # idx+1 because 0 is padding 128 | # data[1] contains idx_to_name_ingrs (look in datasets.Recipes(HDF5)) 129 | #self.idx_to_name_ingrs = {idx+1:name for idx, name in enumerate(data[1])} 130 | 131 | # def _process_lengths(self, tensor): 132 | # max_length = tensor.data.size(1) 133 | # lengths = list(max_length - tensor.data.eq(0).sum(1).sequeeze()) 134 | # return lengths 135 | 136 | def _sort_by_lengths(self, ingrs, lengths): 137 | sorted_ids = sorted(range(len(lengths)), 138 | key=lambda k: lengths[k], 139 | reverse=True) 140 | sorted_lengths = sorted(lengths, reverse=True) 141 | unsorted_ids = sorted(range(len(lengths)), 142 | key=lambda k: sorted_ids[k]) 143 | sorted_ids = torch.LongTensor(sorted_ids) 144 | unsorted_ids = torch.LongTensor(unsorted_ids) 145 | if ingrs.is_cuda: 146 | sorted_ids = sorted_ids.cuda() 147 | unsorted_ids = unsorted_ids.cuda() 148 | ingrs = ingrs[sorted_ids] 149 | return ingrs, sorted_lengths, unsorted_ids 150 | 151 | def forward_ingrs(self, ingrs): 152 | # TODO: to put in dataloader 153 | #lengths = self._process_lengths(ingrs) 154 | sorted_ingrs, sorted_lengths, unsorted_ids = self._sort_by_lengths( 155 | ingrs['data'], ingrs['lengths']) 156 | 157 | emb_out = self.emb_ingrs(sorted_ingrs) 158 | pack_out = nn.utils.rnn.pack_padded_sequence(emb_out, 159 | sorted_lengths, batch_first=True) 160 | 161 | rnn_out, (hn, cn) = self.rnn_ingrs(pack_out) 162 | batch_size = hn.size(1) 163 | hn = hn.transpose(0,1) 164 | hn = hn.contiguous() 165 | hn = hn.view(batch_size, self.dim_ingr_out*2) 166 | #hn = torch.cat(hn, 2) # because bidirectional 167 | #hn = hn.squeeze(0) 168 | hn = hn[unsorted_ids] 169 | return hn 170 | 171 | def forward_instrs(self, instrs): 172 | # TODO: to put in dataloader 173 | sorted_instrs, sorted_lengths, unsorted_ids = self._sort_by_lengths( 174 | instrs['data'], instrs['lengths']) 175 | pack_out = nn.utils.rnn.pack_padded_sequence(sorted_instrs, 176 | sorted_lengths, batch_first=True) 177 | 178 | rnn_out, (hn, cn) = self.rnn_instrs(sorted_instrs) 179 | hn = hn.squeeze(0) 180 | hn = hn[unsorted_ids] 181 | return hn 182 | 183 | def forward_one_ingr(self, ingrs, emb_instrs=None): 184 | emb_ingr = self.forward_ingrs(ingrs) 185 | if emb_instrs is None: 186 | emb_instrs = torch.zeros(1,self.dim_instr_out) 187 | if emb_ingr.is_cuda: 188 | emb_instrs = emb_instrs.cuda() 189 | 190 | fusion_out = torch.cat([emb_ingr, emb_instrs], 1) 191 | x = self.fc(fusion_out) 192 | 193 | if self.activations is not None: 194 | for name in self.activations: 195 | x = nn.functional.__dict__[name](x) 196 | 197 | return x 198 | 199 | 200 | class Trijoint(nn.Module): 201 | 202 | def __init__(self, opt, nb_classes, with_classif=False): 203 | super(Trijoint, self).__init__() 204 | self.dim_emb = opt['dim_emb'] 205 | self.nb_classes = nb_classes 206 | self.with_classif = with_classif 207 | # modules 208 | self.image_embedding = ImageEmbedding(opt) 209 | self.recipe_embedding = RecipeEmbedding(opt) 210 | 211 | if self.with_classif: 212 | self.linear_classif = nn.Linear(self.dim_emb, self.nb_classes) 213 | 214 | def get_parameters_recipe(self): 215 | params = [] 216 | params.append({'params': self.recipe_embedding.parameters()}) 217 | if self.with_classif: 218 | params.append({'params': self.linear_classif.parameters()}) 219 | params.append({'params': self.image_embedding.fc.parameters()}) 220 | return params 221 | 222 | def get_parameters_image(self): 223 | return self.image_embedding.convnet.parameters() 224 | 225 | def forward(self, batch): 226 | out = {} 227 | out['image_embedding'] = self.image_embedding(batch['image']) 228 | out['recipe_embedding'] = self.recipe_embedding(batch['recipe']) 229 | 230 | if self.with_classif: 231 | out['image_classif'] = self.linear_classif(out['image_embedding']) 232 | out['recipe_classif'] = self.linear_classif(out['recipe_embedding']) 233 | 234 | return out 235 | -------------------------------------------------------------------------------- /recipe1m/models/metrics/trijoint.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | import torch 4 | import numpy as np 5 | import torch.nn as nn 6 | from torch.autograd import Variable 7 | from bootstrap.lib.logger import Logger 8 | from bootstrap.lib.options import Options 9 | from . import utils 10 | 11 | class Trijoint(nn.Module): 12 | 13 | def __init__(self, opt, with_classif=False, engine=None, mode='train'): 14 | super(Trijoint, self).__init__() 15 | self.mode = mode 16 | self.with_classif = with_classif 17 | self.engine = engine 18 | # Attributs to process 1000*10 matchs 19 | # for the retrieval evaluation procedure 20 | self.nb_bags_retrieval = opt['nb_bags'] 21 | self.nb_matchs_per_bag = opt['nb_matchs_per_bag'] 22 | self.nb_matchs_expected = self.nb_bags_retrieval * self.nb_matchs_per_bag 23 | self.nb_matchs_saved = 0 24 | 25 | if opt.get('keep_background', False): 26 | self.ignore_index = None 27 | else: 28 | self.ignore_index = 0 29 | 30 | self.identifiers = {'image': [], 'recipe': []} 31 | 32 | if engine and self.mode == 'eval': 33 | self.split = engine.dataset[mode].split 34 | engine.register_hook('eval_on_end_epoch', self.calculate_metrics) 35 | 36 | def forward(self, cri_out, net_out, batch): 37 | out = {} 38 | if self.with_classif: 39 | # Accuracy 40 | [out['acc_image']] = utils.accuracy(net_out['image_classif'].detach().cpu(), 41 | batch['image']['class_id'].detach().squeeze().cpu(), 42 | topk=(1,), 43 | ignore_index=self.ignore_index) 44 | [out['acc_recipe']] = utils.accuracy(net_out['recipe_classif'].detach().cpu(), 45 | batch['recipe']['class_id'].detach().squeeze().cpu(), 46 | topk=(1,), 47 | ignore_index=self.ignore_index) 48 | if self.engine and self.mode == 'eval': 49 | # Retrieval 50 | batch_size = len(batch['image']['index']) 51 | for i in range(batch_size): 52 | if self.nb_matchs_saved == self.nb_matchs_expected: 53 | continue 54 | if batch['match'].data[i][0] == -1: 55 | continue 56 | 57 | identifier = '{}_img_{}'.format(self.split, batch['image']['index'][i]) 58 | utils.save_activation(identifier, net_out['image_embedding'][i].detach().cpu()) 59 | self.identifiers['image'].append(identifier) 60 | 61 | identifier = '{}_rcp_{}'.format(self.split, batch['recipe']['index'][i]) 62 | utils.save_activation(identifier, net_out['recipe_embedding'][i].detach().cpu()) 63 | self.identifiers['recipe'].append(identifier) 64 | 65 | self.nb_matchs_saved += 1 66 | 67 | return out 68 | 69 | def calculate_metrics(self): 70 | final_nb_bags = math.floor(self.nb_matchs_saved / self.nb_matchs_per_bag) 71 | final_matchs_left = self.nb_matchs_saved % self.nb_matchs_per_bag 72 | 73 | if final_nb_bags < self.nb_bags_retrieval: 74 | log_level = Logger.ERROR if self.split == 'test' else Logger.WARNING 75 | Logger().log_message('Insufficient matchs ({} saved), {} bags instead of {}'.format( 76 | self.nb_matchs_saved, final_nb_bags, self.nb_bags_retrieval), log_level=log_level) 77 | 78 | Logger().log_message('Computing retrieval ranking for {} x {} matchs'.format(final_nb_bags, 79 | self.nb_matchs_per_bag)) 80 | list_med_im2recipe = [] 81 | list_med_recipe2im = [] 82 | list_recall_at_1_im2recipe = [] 83 | list_recall_at_5_im2recipe = [] 84 | list_recall_at_10_im2recipe = [] 85 | list_recall_at_1_recipe2im = [] 86 | list_recall_at_5_recipe2im = [] 87 | list_recall_at_10_recipe2im = [] 88 | 89 | for i in range(final_nb_bags): 90 | nb_identifiers_image = self.nb_matchs_per_bag 91 | nb_identifiers_recipe = self.nb_matchs_per_bag 92 | 93 | distances = np.zeros((nb_identifiers_image, nb_identifiers_recipe), dtype=float) 94 | 95 | # load 96 | im_matrix = None 97 | rc_matrix = None 98 | for j in range(self.nb_matchs_per_bag): 99 | index = j + i * self.nb_matchs_per_bag 100 | 101 | identifier_image = self.identifiers['image'][index] 102 | activation_image = utils.load_activation(identifier_image) 103 | if im_matrix is None: 104 | im_matrix = torch.zeros(nb_identifiers_image, activation_image.size(0)) 105 | im_matrix[j] = activation_image 106 | 107 | identifier_recipe = self.identifiers['recipe'][index] 108 | activation_recipe = utils.load_activation(identifier_recipe) 109 | if rc_matrix is None: 110 | rc_matrix = torch.zeros(nb_identifiers_recipe, activation_recipe.size(0)) 111 | rc_matrix[j] = activation_recipe 112 | 113 | #im_matrix = im_matrix.cuda() 114 | #rc_matrix = rc_matrix.cuda() 115 | 116 | distances = fast_distance(im_matrix, rc_matrix) 117 | #distances[i][j] = torch.dist(activation_image.data, activation_recipe.data, p=2) 118 | 119 | im2recipe = np.argsort(distances.numpy(), axis=0) 120 | recipe2im = np.argsort(distances.numpy(), axis=1) 121 | 122 | recall_at_1_recipe2im = 0 123 | recall_at_5_recipe2im = 0 124 | recall_at_10_recipe2im = 0 125 | recall_at_1_im2recipe = 0 126 | recall_at_5_im2recipe = 0 127 | recall_at_10_im2recipe = 0 128 | med_rank_im2recipe = [] 129 | med_rank_recipe2im = [] 130 | 131 | for i in range(nb_identifiers_image): 132 | pos_im2recipe = im2recipe[:,i].tolist().index(i) 133 | pos_recipe2im = recipe2im[i,:].tolist().index(i) 134 | 135 | if pos_im2recipe == 0: 136 | recall_at_1_im2recipe += 1 137 | if pos_im2recipe <= 4: 138 | recall_at_5_im2recipe += 1 139 | if pos_im2recipe <= 9: 140 | recall_at_10_im2recipe += 1 141 | 142 | if pos_recipe2im == 0: 143 | recall_at_1_recipe2im += 1 144 | if pos_recipe2im <= 4: 145 | recall_at_5_recipe2im += 1 146 | if pos_recipe2im <= 9: 147 | recall_at_10_recipe2im += 1 148 | 149 | med_rank_im2recipe.append(pos_im2recipe) 150 | med_rank_recipe2im.append(pos_recipe2im) 151 | 152 | list_med_im2recipe.append(np.median(med_rank_im2recipe)) 153 | list_med_recipe2im.append(np.median(med_rank_recipe2im)) 154 | list_recall_at_1_im2recipe.append(recall_at_1_im2recipe / nb_identifiers_image) 155 | list_recall_at_5_im2recipe.append(recall_at_5_im2recipe / nb_identifiers_image) 156 | list_recall_at_10_im2recipe.append(recall_at_10_im2recipe / nb_identifiers_image) 157 | list_recall_at_1_recipe2im.append(recall_at_1_recipe2im / nb_identifiers_image) 158 | list_recall_at_5_recipe2im.append(recall_at_5_recipe2im / nb_identifiers_image) 159 | list_recall_at_10_recipe2im.append(recall_at_10_recipe2im / nb_identifiers_image) 160 | 161 | out = {} 162 | out['med_im2recipe_mean'] = np.mean(list_med_im2recipe) 163 | out['med_recipe2im_mean'] = np.mean(list_med_recipe2im) 164 | out['recall_at_1_im2recipe_mean'] = np.mean(list_recall_at_1_im2recipe) 165 | out['recall_at_5_im2recipe_mean'] = np.mean(list_recall_at_5_im2recipe) 166 | out['recall_at_10_im2recipe_mean'] = np.mean(list_recall_at_10_im2recipe) 167 | out['recall_at_1_recipe2im_mean'] = np.mean(list_recall_at_1_recipe2im) 168 | out['recall_at_5_recipe2im_mean'] = np.mean(list_recall_at_5_recipe2im) 169 | out['recall_at_10_recipe2im_mean'] = np.mean(list_recall_at_10_recipe2im) 170 | 171 | out['med_im2recipe_std'] = np.std(list_med_im2recipe) 172 | out['med_recipe2im_std'] = np.std(list_med_recipe2im) 173 | out['recall_at_1_im2recipe_std'] = np.std(list_recall_at_1_im2recipe) 174 | out['recall_at_5_im2recipe_std'] = np.std(list_recall_at_5_im2recipe) 175 | out['recall_at_10_im2recipe_std'] = np.std(list_recall_at_10_im2recipe) 176 | out['recall_at_1_recipe2im_std'] = np.std(list_recall_at_1_recipe2im) 177 | out['recall_at_5_recipe2im_std'] = np.std(list_recall_at_5_recipe2im) 178 | out['recall_at_10_recipe2im_std'] = np.std(list_recall_at_10_recipe2im) 179 | 180 | for key, value in out.items(): 181 | Logger().log_value('{}_epoch.metric.{}'.format(self.mode,key), float(value), should_print=True) 182 | 183 | #self.outputs_epoch['total_samples'] = nb_identifiers_image 184 | 185 | for identifier_image in self.identifiers['image']: 186 | utils.delete_activation(identifier_image) 187 | 188 | for identifier_recipe in self.identifiers['recipe']: 189 | utils.delete_activation(identifier_recipe) 190 | 191 | self.identifiers = {'image': [], 'recipe': []} 192 | self.nb_matchs_saved = 0 193 | 194 | # mAP ? 195 | # ConfusionMatrix ? 196 | 197 | def fast_distance(A,B): 198 | # A and B must have norm 1 for this to work for the ranking 199 | return torch.mm(A,B.t()) * -1 200 | 201 | def euclidean_distance_fast(A,B): 202 | n = A.size(0) 203 | ZA = (A * A).sum(1) 204 | ZB = (B * B).sum(1) 205 | 206 | ZA = ZA.expand(n,n) 207 | ZB = ZB.expand(n,n).t() 208 | 209 | D = torch.mm(B, A.t()) 210 | D.mul_(-2) 211 | D.add_(ZA).add_(ZB) 212 | D.sqrt_() 213 | D.t_() 214 | return D 215 | 216 | def euclidean_distance_slow(A,B): 217 | n = A.size(0) 218 | D = torch.zeros(n,n) 219 | for i in range(n): 220 | for j in range(n): 221 | D[i,j] = torch.dist(A[i], B[j]) 222 | return D 223 | -------------------------------------------------------------------------------- /recipe1m/visu/old_tsne.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import torch 4 | from sklearn.manifold import TSNE 5 | import argparse 6 | import scipy.io as sio 7 | import numpy as np 8 | import bootstrap.datasets.transforms as transforms 9 | import torchvision.transforms as viztransforms 10 | from bootstrap.lib.logger import Logger 11 | from PIL import Image 12 | 13 | from scipy.misc import imsave 14 | 15 | import matplotlib as mpl 16 | mpl.use('Agg') 17 | import matplotlib.pyplot as plt 18 | plt.ioff() #http://matplotlib.org/faq/usage_faq.html (interactive mode) 19 | 20 | 21 | 22 | def stack(): 23 | return transforms.Compose([ 24 | transforms.ListDictsToDictLists(), 25 | transforms.StackTensors() 26 | ]) 27 | 28 | def load_image(path, crop_size=50): 29 | with open(path, 'rb') as f: 30 | with Image.open(f) as img: 31 | img_rgb = img.convert('RGB') 32 | img_rgb = viztransforms.Scale(crop_size)(img_rgb) 33 | img_rgb = viztransforms.CenterCrop(crop_size)(img_rgb) 34 | img_rgb = viztransforms.ToTensor()(img_rgb) 35 | img_rgb = img_rgb.transpose(0,2) 36 | img_rgb = img_rgb.transpose(0,1) 37 | img_rgb = img_rgb.numpy() * 255 38 | return img_rgb 39 | 40 | 41 | parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') 42 | parser.add_argument('--perplexity', type=float, default=5.0, nargs='+', help='') 43 | parser.add_argument('--exaggeration', type=float, default=12.0, nargs='+', help='') 44 | 45 | def main(): 46 | global args 47 | args = parser.parse_args() 48 | 49 | #dir_root = '/home/carvalho/experiments/im2recipe.pytorch/logs/lmdb/2017_10_10_23_54_31_517_anm_IRR1.0_RII1.0_SIRR0.1_SRII0.1_80epochs' 50 | dir_root = '/home/carvalho/experiments/im2recipe.pytorch/logs/lmdb/2017_10_06_08_10_47_631_anm_IRR_RII_80epochs' 51 | dir_embs = os.path.join(dir_root, 'embeddings_train') 52 | dir_img_items = os.path.join(dir_embs, 'img') 53 | dir_rcp_items = os.path.join(dir_embs, 'rcp') 54 | #dir_img_items = os.path.join(dir_img_embs, '2017_10_10_23_54_31_517_anm_IRR1.0_RII1.0_SIRR0.1_SRII0.1_80epochs') 55 | #dir_rcp_items = os.path.join(dir_rcp_embs, '2017_10_10_23_54_31_517_anm_IRR1.0_RII1.0_SIRR0.1_SRII0.1_80epochs') 56 | path_load_img_out = os.path.join(dir_embs, 'load_img_out.pth') 57 | path_load_rcp_out = os.path.join(dir_embs, 'load_rcp_out.pth') 58 | 59 | dir_visu = os.path.join(dir_root, 'visualizations') 60 | dir_fig = os.path.join(dir_visu, 'fig_train_6') 61 | os.system('mkdir -p '+dir_fig) 62 | 63 | Logger(dir_fig) 64 | Logger()('Begin') 65 | 66 | rcp_items = [] 67 | img_items = [] 68 | if not os.path.isfile(path_load_img_out): 69 | Logger()('Loading embeddings...') 70 | 71 | for filename in os.listdir(dir_img_items): 72 | idx = int(filename.split('.')[0].split('_')[-1]) 73 | # 74 | path_item = os.path.join(dir_img_items, filename) 75 | img_item = torch.load(path_item) 76 | img_item['idx'] = idx 77 | img_items.append(img_item) 78 | # 79 | path_item = path_item.replace('img', 'rcp') 80 | rcp_item = torch.load(path_item) 81 | rcp_item['idx'] = idx 82 | rcp_items.append(rcp_item) 83 | 84 | Logger()('Stacking image items...') 85 | X_img = stack()(img_items) 86 | torch.save(X_img, path_load_img_out) 87 | 88 | Logger()('Stacking rcpipe items...') 89 | X_rcp = stack()(rcp_items) 90 | torch.save(X_rcp, path_load_rcp_out) 91 | else: 92 | X_img = torch.load(path_load_img_out) 93 | X_rcp = torch.load(path_load_rcp_out) 94 | 95 | 96 | # all_classes = list(set(X_img['class_name'])) 97 | # nb_items_by_class = {class_name:0 for class_name in all_classes} 98 | # for i in range(len(X_img['index'])): 99 | # class_name = X_img['class_name'][i] 100 | # nb_items_by_class[class_name] += 1 101 | 102 | # for key, value in sorted(nb_items_by_class.items(), key=lambda item: item[1]): 103 | # print("{}: {}".format(key, value)) 104 | # import ipdb;ipdb.set_trace() 105 | 106 | #print(set(X_img['class_name'])) 107 | #return 108 | 109 | X_new = { 110 | #'path':[], 111 | 'class_name':[], 112 | 'index':[], 113 | 'data':[], 114 | 'class_id':[], 115 | 'type':[], 116 | 'idx': [] 117 | } 118 | 119 | # tiramisu 120 | # bread salad 121 | # pork chop 122 | #classes = ['pork chop', 'strawberry pie', 'cheddar cheese', 'greek salad', 'curry chicken'] 123 | #classes = ['bell pepper', 'chocolate banana', 'celery root', 'fruit salad', 'pasta sauce'] 124 | #classes = ['chocolate banana', 'lemon pepper', 'fruit salad', 'pasta sauce'] 125 | 126 | classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] 127 | #classes = ['sweet potato', 'pizza', 'chocolate chip', 'crock pot', 'peanut butter'] 128 | #classes = ['crock pot', 'peanut butter'] 129 | colors = ['crimson', 'darkgreen', 'navy', 'darkorange', 'deeppink'] 130 | class_color = {class_name: colors[i] for i, class_name in enumerate(classes)} 131 | 132 | Logger()(classes) 133 | 134 | nb_points = 80 135 | total_nb_points = len(classes)*nb_points*2 136 | 137 | def filter(X, X_new, classes, nb_points): 138 | for c in classes: 139 | idx = 0 140 | for i in range(len(X['index'])): 141 | if c == X['class_name'][i]: 142 | if 'path' in X: 143 | X_new['type'].append('img') 144 | else: 145 | X_new['type'].append('rcp') 146 | 147 | X_new['class_name'].append(X['class_name'][i]) 148 | X_new['index'].append(X['index'][i]) 149 | X_new['data'].append(X['data'][i]) 150 | X_new['class_id'].append(X['class_id'][i]) 151 | X_new['idx'].append(X['idx'][i]) 152 | 153 | idx += 1 154 | if idx >= nb_points: 155 | break 156 | if idx < nb_points: 157 | Logger()('Warning: classe {} has {} items'.format(c, idx)) 158 | return X_new 159 | 160 | X_new = filter(X_img, X_new, classes, nb_points) 161 | X_new = filter(X_rcp, X_new, classes, nb_points) 162 | 163 | 164 | def shuffle(X): 165 | length = int(len(X['index'])/2) 166 | indexes = list(torch.randperm(length)) 167 | X_new = {} 168 | for key in ['class_name', 'index', 'data', 'class_id', 'type', 'idx']: 169 | X_new[key] = [] 170 | # shuffle img 171 | for idx in indexes: 172 | X_new[key].append(X[key][idx]) 173 | # shuffle rcp 174 | for idx in indexes: 175 | X_new[key].append(X[key][idx+length]) 176 | return X_new 177 | 178 | X = shuffle(X_new) 179 | X['data'] = transforms.StackTensors()(X['data']) 180 | X['data'] = X['data'].numpy() 181 | 182 | print(X['data'].shape) 183 | 184 | for perplexity in range(0,100,2):#args.perplexity: 185 | for exaggeration in range(1,10,2):#args.exaggeration: 186 | 187 | path_tsne_out = os.path.join(dir_fig, 'ckpt_perplexity,{}_exaggeration,{}.pth'.format(perplexity, exaggeration)) 188 | if True or not os.path.isfile(path_tsne_out): 189 | 190 | Logger()('Calculating TSNE...') 191 | X_embedded = TSNE(n_components=2, 192 | perplexity=perplexity, 193 | early_exaggeration=exaggeration, 194 | learning_rate=100.0, 195 | n_iter=5000, 196 | n_iter_without_progress=300, 197 | min_grad_norm=1e-07, 198 | metric='euclidean', 199 | init='random',#'random', 200 | verbose=0, 201 | random_state=None, 202 | method='exact',#'barnes_hut', 203 | angle=0.5).fit_transform(X['data']) 204 | 205 | torch.save(torch.from_numpy(X_embedded), path_tsne_out) 206 | else: 207 | X_embedded = torch.load(path_tsne_out).numpy() 208 | 209 | Logger()('Painting...') 210 | # set min point to 0 and scale 211 | X_embedded = X_embedded - np.min(X_embedded) 212 | X_embedded = X_embedded / np.max(X_embedded) 213 | 214 | X['tsne'] = [] 215 | for i in range(total_nb_points): 216 | X['tsne'].append(X_embedded[i]) 217 | 218 | 219 | 220 | # X_img_per_class = {} 221 | # X_rcp_per_class = {} 222 | # for c in classes: 223 | # X_img_per_class[c] = np.zeros((nb_points,2)) 224 | # X_rcp_per_class[c] = np.zeros((nb_points,2)) 225 | 226 | # idx_img = 0 227 | # idx_rcp = 0 228 | # for i in range(len(X['idx'])): 229 | # if c == X['class_name'][i]: 230 | # if X['type'][i] == 'img': 231 | # X_img_per_class[c][idx_img] = X_embedded[i] 232 | # idx_img += 1 233 | # else: 234 | # X_rcp_per_class[c][idx_rcp] = X_embedded[i] 235 | # idx_rcp += 1 236 | 237 | # import ipdb; ipdb.set_trace() 238 | 239 | fig = plt.figure(figsize=(20,20)) 240 | #ax = plt.subplot(111) 241 | 242 | for i in range(total_nb_points): 243 | if X['type'][i] == 'img': 244 | marker = '+' 245 | else: 246 | marker = '.' 247 | class_name = X['class_name'][i] 248 | color = class_color[class_name] 249 | x = X['tsne'][i][0] 250 | y = X['tsne'][i][1] 251 | plt.scatter(x, y, color=color, marker=marker, label=class_name, s=1000) 252 | 253 | nb_img_points = int(total_nb_points/2) 254 | for i in range(nb_img_points): 255 | class_name = X['class_name'][i] 256 | color = class_color[class_name] 257 | img_x = X['tsne'][i][0] 258 | img_y = X['tsne'][i][1] 259 | rcp_x = X['tsne'][nb_img_points+i][0] 260 | rcp_y = X['tsne'][nb_img_points+i][1] 261 | plt.plot([img_x, rcp_x], [img_y, rcp_y], '-', color=color, lw=2) 262 | 263 | #plt.grid(True) 264 | #plt.xticks([x/10 for x in range(0,11)]) 265 | #plt.yticks([x/10 for x in range(0,11)]) 266 | #plt.legend() 267 | 268 | plt.yticks([]) 269 | plt.xticks([]) 270 | 271 | path_fig = os.path.join(dir_fig, 'fig_perplexity,{}_exaggeration,{}.png'.format(perplexity, exaggeration)) 272 | fig.savefig(path_fig) 273 | Logger()('Saved fig to '+path_fig) 274 | 275 | plt.show() 276 | 277 | Logger()('End') 278 | 279 | 280 | if __name__ == '__main__': 281 | main() 282 | # try: 283 | # main() 284 | # except: 285 | # try: 286 | # Logger()(traceback.format_exc(), Logger.ERROR) 287 | # except: 288 | # pass 289 | # pass -------------------------------------------------------------------------------- /recipe1m/visu/old_top5_old.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import os 4 | import scipy.io as sio 5 | import numpy as np 6 | 7 | import matplotlib as mpl 8 | mpl.use('Agg') 9 | import matplotlib.pyplot as plt 10 | plt.ioff() #http://matplotlib.org/faq/usage_faq.html (interactive mode) 11 | 12 | import torch 13 | import torchvision.transforms as viztransforms 14 | import bootstrap.datasets.transforms as transforms 15 | from bootstrap.lib.logger import Logger 16 | from bootstrap.lib.options import Options 17 | from PIL import Image 18 | from scipy.misc import imsave 19 | 20 | def stack(): 21 | return transforms.Compose([ 22 | transforms.ListDictsToDictLists(), 23 | transforms.StackTensors() 24 | ]) 25 | 26 | def load_image(path, crop_size=50): 27 | with open(path, 'rb') as f: 28 | with Image.open(f) as img: 29 | img_rgb = img.convert('RGB') 30 | img_rgb = viztransforms.Scale(crop_size)(img_rgb) 31 | img_rgb = viztransforms.CenterCrop(crop_size)(img_rgb) 32 | #img_rgb = viztransforms.ToTensor()(img_rgb) 33 | # img_rgb = img_rgb.transpose(0,2) 34 | # img_rgb = img_rgb.transpose(0,1) 35 | # img_rgb = img_rgb.numpy() * 255 36 | return img_rgb 37 | 38 | 39 | parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') 40 | parser.add_argument('--perplexity', type=float, default=5.0, nargs='+', help='') 41 | parser.add_argument('--exaggeration', type=float, default=12.0, nargs='+', help='') 42 | 43 | 44 | def fast_distance(A,B): 45 | # A and B must have norm 1 for this to work for the ranking 46 | return torch.mm(A,B.t()) * -1 47 | 48 | def load_embs(dir_root): 49 | dir_embs = os.path.join(dir_root, 'embeddings_test') 50 | dir_img_items = os.path.join(dir_embs, 'img') 51 | dir_rcp_items = os.path.join(dir_embs, 'rcp') 52 | path_load_img_out = os.path.join(dir_embs, 'load_img_out.pth') 53 | path_load_rcp_out = os.path.join(dir_embs, 'load_rcp_out.pth') 54 | 55 | if not os.path.isfile(path_load_img_out): 56 | Logger()('Loading embeddings...') 57 | rcp_items = [] 58 | img_items = [] 59 | for filename in os.listdir(dir_img_items): 60 | idx = int(filename.split('.')[0].split('_')[-1]) 61 | # 62 | path_item = os.path.join(dir_img_items, filename) 63 | img_item = torch.load(path_item) 64 | img_item['idx'] = idx 65 | img_items.append(img_item) 66 | # 67 | path_item = path_item.replace('img', 'rcp') 68 | rcp_item = torch.load(path_item) 69 | rcp_item['idx'] = idx 70 | rcp_items.append(rcp_item) 71 | 72 | Logger()('Stacking image items...') 73 | X_img = stack()(img_items) 74 | torch.save(X_img, path_load_img_out) 75 | 76 | Logger()('Stacking rcpipe items...') 77 | X_rcp = stack()(rcp_items) 78 | torch.save(X_rcp, path_load_rcp_out) 79 | else: 80 | X_img = torch.load(path_load_img_out) 81 | X_rcp = torch.load(path_load_rcp_out) 82 | return X_img, X_rcp 83 | 84 | def filter_img(X, classes, nb_points): 85 | X_new = { 86 | 'path':[], 87 | 'class_name':[], 88 | 'index':[], 89 | 'data':[], 90 | 'class_id':[], 91 | 'idx': [] 92 | } 93 | idx = 0 94 | for c in classes: 95 | for i in range(len(X['index'])): 96 | if c == X['class_name'][i]: 97 | X_new['path'].append(X['path'][i]) 98 | X_new['class_name'].append(X['class_name'][i]) 99 | X_new['index'].append(X['index'][i]) 100 | X_new['data'].append(X['data'][i]) 101 | X_new['class_id'].append(X['class_id'][i]) 102 | X_new['idx'].append(X['idx'][i]) 103 | idx += 1 104 | if idx >= nb_points: 105 | break 106 | # if idx < nb_points: 107 | # Logger()('Warning: classe {} has {} items'.format(c, idx)) 108 | X_new['data'] = transforms.StackTensors()(X_new['data']) 109 | return X_new 110 | 111 | def filter_rcp(X, classes, nb_points): 112 | X_new = { 113 | 'url':[], 114 | 'ingredients':[], 115 | 'instructions':[], 116 | 'title':[], 117 | 'class_name':[], 118 | 'index':[], 119 | 'data':[], 120 | 'class_id':[], 121 | 'idx': [] 122 | } 123 | idx = 0 124 | for c in classes: 125 | for i in range(len(X['index'])): 126 | if c == X['class_name'][i]: 127 | X_new['url'].append(X['url'][i]) 128 | X_new['ingredients'].append(X['ingredients'][i]) 129 | X_new['instructions'].append(X['instructions'][i]) 130 | X_new['title'].append(X['title'][i]) 131 | X_new['class_name'].append(X['class_name'][i]) 132 | X_new['index'].append(X['index'][i]) 133 | X_new['data'].append(X['data'][i]) 134 | X_new['class_id'].append(X['class_id'][i]) 135 | X_new['idx'].append(X['idx'][i]) 136 | idx += 1 137 | if idx >= nb_points: 138 | break 139 | # if idx < nb_points: 140 | # Logger()('Warning: classe {} has {} items'.format(c, idx)) 141 | X_new['data'] = transforms.StackTensors()(X_new['data']) 142 | return X_new 143 | 144 | 145 | def main(): 146 | global args 147 | args = parser.parse_args() 148 | 149 | Logger('.') 150 | Logger()('Begin') 151 | 152 | classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] 153 | classes = ['ice cream'] 154 | nb_points = 10000 155 | 156 | 157 | dir_root_triplet = '/home/carvalho/experiments/im2recipe.pytorch/logs/lmdb/2017_10_06_08_10_47_631_anm_IRR_RII_80epochs' 158 | 159 | dir_fig_triplet = os.path.join(dir_root_triplet,'visualization_top5') 160 | os.system('rm -rf '+dir_fig_triplet) 161 | os.system('mkdir -p '+dir_fig_triplet) 162 | 163 | X_img_triplet, X_rcp_triplet = load_embs(dir_root_triplet) 164 | 165 | classes = list(set(X_img_triplet['class_name'])) 166 | new_classes = [] 167 | for c in classes: 168 | if c != 'background': 169 | new_classes.append(c) 170 | classes=new_classes 171 | # nb_items_by_class = {class_name:0 for class_name in all_classes} 172 | # for i in range(len(X_img_triplet['index'])): 173 | # class_name = X_img_triplet['class_name'][i] 174 | # nb_items_by_class[class_name] += 1 175 | 176 | # for key, value in sorted(nb_items_by_class.items(), key=lambda item: item[1]): 177 | # print("{}: {}".format(key, value)) 178 | # import ipdb;ipdb.set_trace() 179 | 180 | X_img_triplet = filter_img(X_img_triplet, classes, nb_points) 181 | X_rcp_triplet = filter_rcp(X_rcp_triplet, classes, nb_points) 182 | 183 | distances_triplet = fast_distance(X_img_triplet['data'], X_rcp_triplet['data']) 184 | 185 | im2recipe_triplet = np.argsort(distances_triplet.numpy(), axis=0) 186 | recipe2im_triplet = np.argsort(distances_triplet.numpy(), axis=1) 187 | 188 | 189 | 190 | 191 | 192 | dir_root_tri_sem = '/home/carvalho/experiments/im2recipe.pytorch/logs/lmdb/2017_10_10_23_54_31_517_anm_IRR1.0_RII1.0_SIRR0.1_SRII0.1_80epochs' 193 | 194 | dir_fig_tri_sem = os.path.join(dir_root_tri_sem,'visualization_top5') 195 | os.system('rm -rf '+dir_fig_tri_sem) 196 | os.system('mkdir -p '+dir_fig_tri_sem) 197 | 198 | X_img_tri_sem, X_rcp_tri_sem = load_embs(dir_root_tri_sem) 199 | X_img_tri_sem = filter_img(X_img_tri_sem, classes, nb_points) 200 | X_rcp_tri_sem = filter_rcp(X_rcp_tri_sem, classes, nb_points) 201 | 202 | distances_tri_sem = fast_distance(X_img_tri_sem['data'], X_rcp_tri_sem['data']) 203 | 204 | im2recipe_tri_sem = np.argsort(distances_tri_sem.numpy(), axis=0) 205 | recipe2im_tri_sem = np.argsort(distances_tri_sem.numpy(), axis=1) 206 | 207 | 208 | for i in range(500000): 209 | 210 | 211 | 212 | 213 | # triplet_sem nb_same class > triplet nb_same_claa 214 | 215 | 216 | # if triplet less good 217 | 218 | # if top5 tri_sem has 5 classes 219 | 220 | # if top5 triplet has less than 3 classes 221 | 222 | # if tri_sem first rank 223 | 224 | pos_tri_sem = None 225 | for j in range(5): 226 | id_img = recipe2im_tri_sem[i,j] 227 | if i == id_img: 228 | pos_tri_sem = j 229 | 230 | if pos_tri_sem is None: 231 | continue 232 | 233 | pos_triplet = None 234 | for j in range(5): 235 | id_img = recipe2im_triplet[i,j] 236 | if i == id_img: 237 | pos_triplet = j 238 | 239 | if pos_triplet is None: 240 | continue 241 | 242 | # if triplet_sem better than triplet 243 | if pos_tri_sem > pos_triplet: 244 | continue 245 | 246 | if pos_tri_sem != 0: 247 | continue 248 | 249 | class_name = X_rcp_tri_sem['class_name'][i] 250 | nb_same_class_tri_sem = 0 251 | for j in range(5): 252 | id_img = recipe2im_tri_sem[i,j] 253 | if class_name == X_img_tri_sem['class_name'][id_img]: 254 | nb_same_class_tri_sem += 1 255 | 256 | class_name = X_rcp_triplet['class_name'][i] 257 | nb_same_class_triplet = 0 258 | for j in range(5): 259 | id_img = recipe2im_triplet[i,j] 260 | if class_name == X_img_triplet['class_name'][id_img]: 261 | nb_same_class_triplet += 1 262 | 263 | if nb_same_class_tri_sem <= nb_same_class_triplet: 264 | continue 265 | 266 | if nb_same_class_tri_sem != 5: 267 | continue 268 | 269 | 270 | # if recipe2im_tri_sem[i,0] != i: 271 | # continue 272 | 273 | # if recipe2im_triplet[i,0] == i: 274 | # continue 275 | 276 | # class_name = X_rcp_tri_sem['class_name'][i] 277 | # nb_same_class = 0 278 | # for j in range(5): 279 | # id_img = recipe2im_triplet[i,j] 280 | # if class_name == X_img_triplet['class_name'][id_img]: 281 | # nb_same_class += 1 282 | 283 | # if nb_same_class == 5: 284 | # continue 285 | 286 | Logger()('{} found'.format(i)) 287 | print(nb_same_class_tri_sem, nb_same_class_triplet) 288 | print(pos_tri_sem, pos_triplet) 289 | 290 | write_img_rcp(dir_fig_tri_sem, i, X_img_tri_sem, X_rcp_tri_sem, recipe2im_tri_sem) 291 | write_img_rcp(dir_fig_triplet, i, X_img_triplet, X_rcp_triplet, recipe2im_triplet) 292 | 293 | 294 | Logger()('End') 295 | 296 | 297 | def write_img_rcp(dir_fig, i, X_img, X_rcp, recipe2im): 298 | dir_fig_i = os.path.join(dir_fig, 'fig_{}_{}'.format(i, X_rcp['class_name'][i].replace(' ', '_'))) 299 | os.system('mkdir -p '+dir_fig_i) 300 | 301 | s = [X_rcp['title'][i]] 302 | s += [d['text'] for d in X_rcp['ingredients'][i]] 303 | s += [d['text'] for d in X_rcp['instructions'][i]] 304 | 305 | path_rcp = os.path.join(dir_fig_i, 'rcp.txt') 306 | with open(path_rcp, 'w') as f: 307 | f.write('\n'.join(s)) 308 | 309 | for j in range(5): 310 | id_img = recipe2im[i,j] 311 | path_img_load = X_img['path'][id_img] 312 | class_name = X_img['class_name'][id_img] 313 | class_name = class_name.replace(' ', '-') 314 | if id_img == i: 315 | path_img_save = os.path.join(dir_fig_i, 'img_{}_{}_found.png'.format(j, class_name)) 316 | else: 317 | path_img_save = os.path.join(dir_fig_i, 'img_{}_{}.png'.format(j, class_name)) 318 | I = load_image(path_img_load, crop_size=500) 319 | I.save(path_img_save) 320 | 321 | if __name__ == '__main__': 322 | main() 323 | # try: 324 | # main() 325 | # except: 326 | # try: 327 | # Logger()(traceback.format_exc(), Logger.ERROR) 328 | # except: 329 | # pass 330 | # pass -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # recipe1m.bootstrap.pytorch 2 | 3 | We are a [Machine Learning research team](https://mlia.lip6.fr/members) from Sorbonne University. Our goal for this project was to create a cross-modal retrieval system trained on the biggest dataset of cooking recipes. This kind of systems is able to retrieve the corresponding recipe given an image (food selfie), and the corresponding image from the recipe. 4 | 5 | It was also the occasion to compare several state-of-the-art metric learning loss functions in a new context. This first analysis gave us some idea on how to improve the generalization of our model. Following this, we wrote two research papers on a new model, called Adamine after Adaptive Mining, that add structure in the retrieval space: 6 | 7 | - [Cross-Modal Retrieval in the Cooking Context: Learning Semantic Text-Image Embeddings (ACM SIGIR2018)](https://arxiv.org/abs/1804.11146) 8 | - [Images & Recipes: Retrieval in the cooking context (IEEE ICDE2018, DECOR workshop)](https://arxiv.org/abs/1805.00900) 9 | 10 | 11 | ### Summary: 12 | 13 | * [Introduction](#introduction) 14 | * [Recipe-to-Image retrieval task](#recipe-to-image-retrieval-task) 15 | * [Quick insight about AdaMine](#quick-insight-about-our-adamine) 16 | * [Installation](#installation) 17 | * [Install python3](#install-python3) 18 | * [Clone & requirements](#clone-requirements) 19 | * [Download dataset](#download-dataset) 20 | * [Quick start](#quick-start) 21 | * [Train a model](#train-a-model-on-the-train-val-sets) 22 | * [Evaluate a model](#evaluate-a-model-on-the-test-set) 23 | * [Available (pretrained) models](#available-pretrained-models) 24 | * [PWC](#pwc) 25 | * [PWC++ (Ours)](#pwc-ours) 26 | * [VSE](#vse) 27 | * [VSE++](#vse-1) 28 | * [AdaMine_avg (Ours)](#adamine_avg-ours) 29 | * [AdaMine (Ours)](#adamine-ours) 30 | * [Lifted structure](#lifted-structure) 31 | * [Documentation](#documentation) 32 | * [Useful commands](#useful-commands) 33 | * [Compare experiments](#compare-experiments) 34 | * [Use a specific GPU](#use-a-specific-gpu) 35 | * [Overwrite an option](#overwrite-an-option) 36 | * [Resume training](#resume-training) 37 | * [Evaluate with 10k setup](#evaluate-with-10k-setup) 38 | * [API](#api) 39 | * [Extract your own image features](#extract-your-own-image-features) 40 | * [Citation](#citation) 41 | * [Acknowledgment](#acknowledgment) 42 | 43 | 44 | ## Introduction 45 | 46 | ### Recipe-to-Image retrieval task 47 | 48 |

    49 | 50 |

    51 | 52 | Given a list of ingredients and a sequence of cooking instructions, the goal is to train a statistical model to retrieve the associated image. For each recipe, the top row indicates the top 5 images retrieved by our AdaMine model, and the bottom row, by a strong baseline. 53 | 54 | ### Quick insight about AdaMine 55 | 56 |

    57 | 58 |

    59 | 60 | Features embedding 61 | 62 | - The list of ingredients is embedded using a bi-LSTM. 63 | - The sequence of instructions is embedded using a hierarchical LSTM (a LSTM to embed sentences word-by-word, a second LSTM to embed the outputs of the first one). 64 | - Both ingredients and instructions representations are concatenated and embedded once again. 65 | - The image is embedded using a ResNet101. 66 | 67 | Metric learning: 68 | 69 | - The cross-modal (texts and images) retrieval space is learned through a joint retrieval and classification loss. 70 | - Aligning items according to a retrieval task allows capturing the fine-grained semantics of items. 71 | - Aligning items according to class meta-data (ex: hamburger, pizza, cocktail, ice-cream) allows capturing the high-level semantic information. 72 | - Both retrieval and classification losses are based on a triplet loss (VSE), which is improved by our proposed Adaptive Mining (AdaMine) strategy for efficient negative sampling. 73 | 74 | Negative sampling strategy 75 | 76 | - The classic triplet loss strategy takes all negative samples into account to calculate the error. However, this tends to produce a vanishing gradient. 77 | - The recent (VSE++) strategy only takes the hard negative sample. It is usually efficient, but does not allow the model to converge on this dataset. 78 | - Our AdaMine strategy takes into account informative samples only (i.e., non-zero loss). It corresponds to a smooth curriculum learning, starting with the classic strategy and ending with the hard samples, but without the burden of switching between strategies. AdaMine also controls the trade-off between the retrieval and classification losses along the training. 79 | 80 | 81 | ## Installation 82 | 83 | ### 1. Install python 3 84 | 85 | We don't provide support for python 2. We advise you to install python 3 with [Anaconda](https://www.continuum.io/downloads). Then, you can create an environment. 86 | 87 | ``` 88 | conda create --name recipe1m python=3.7 89 | source activate recipe1m 90 | ``` 91 | 92 | ### 2. Clone & requirements 93 | 94 | We use a [high level framework](https://github.com/Cadene/bootstrap.pytorch.git) to be able to focus on the model instead of boilerplate code. 95 | 96 | ``` 97 | cd $HOME 98 | git clone https://github.com/Cadene/recipe1m.bootstrap.pytorch.git 99 | cd recipe1m.bootstrap.pytorch 100 | pip install -r requirements.txt 101 | ``` 102 | 103 | ### 3. Download dataset 104 | 105 | Please, create an account on http://im2recipe.csail.mit.edu/ and agree to the terms of use. This dataset was made for research and not for commercial use. 106 | 107 | ``` 108 | mkdir data/recip1m 109 | cd data/recip1m 110 | tar -xvf data_lmdb.tar 111 | rm data_lmdb.tar 112 | tar -xzvf recipe1M.tar.gz 113 | rm recipe1M.tar.gz 114 | tar -xzvf text.tar.gz 115 | rm text.tar.gz 116 | cd text 117 | ``` 118 | 119 | Note: Features extracted from resnet50 are included in data_lmdb. 120 | 121 | 122 | ## Quick start 123 | 124 | ### Train a model on the train/val sets 125 | 126 | The [boostrap/run.py](https://github.com/Cadene/bootstrap.pytorch/blob/master/bootstrap/run.py) file load the options contained in a yaml file, create the corresponding experiment directory (in logs/recipe1m) and start the training procedure. 127 | 128 | For instance, you can train our best model by running: 129 | ``` 130 | python -m bootstrap.run -o recipe1m/options/adamine.yaml 131 | ``` 132 | Then, several files are going to be created: 133 | - options.yaml (copy of options) 134 | - logs.txt (history of print) 135 | - logs.json (batchs and epochs statistics) 136 | - view.html (learning curves) 137 | - ckpt_last_engine.pth.tar (checkpoints of last epoch) 138 | - ckpt_last_model.pth.tar 139 | - ckpt_last_optimizer.pth.tar 140 | - ckpt_best_eval_epoch.metric.recall_at_1_im2recipe_mean_engine.pth.tar (checkpoints of best epoch) 141 | - ckpt_best_eval_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar 142 | - ckpt_best_eval_epoch.metric.recall_at_1_im2recipe_mean_optimizer.pth.tar 143 | 144 | Many loss functions are available in the `recipe1m/options` directory. 145 | 146 | ### Evaluate a model on the test set 147 | 148 | At the end of the training procedure, you can evaluate your model on the testing set. In this example, [boostrap/run.py](https://github.com/Cadene/bootstrap.pytorch/blob/master/bootstrap/run.py) load the options from your experiment directory, resume the best checkpoint on the validation set and start an evaluation on the testing set instead of the validation set while skipping the training set (train_split is empty). 149 | ``` 150 | python -m bootstrap.run \ 151 | -o logs/recipe1m/adamine/options.yaml \ 152 | --exp.resume best_eval_epoch.metric.recall_at_1_im2recipe_mean \ 153 | --dataset.train_split \ 154 | --dataset.eval_split test 155 | ``` 156 | 157 | Note: by default, the model is evaluated on the 1k setup; more info on the 10k setup [here]() 158 | 159 | 160 | ## Available (pretrained) models 161 | 162 | ### PWC 163 | 164 | Pairwise loss [[paper]](http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf) 165 | ``` 166 | python -m bootstrap.run -o recipe1m/options/pairwise.yaml 167 | ``` 168 | 169 | ### PWC++ (Ours) 170 | 171 | Pairwise with positive and negative margins loss 172 | ``` 173 | python -m bootstrap.run -o recipe1m/options/pairwise_plus.yaml 174 | ``` 175 | 176 | ### VSE 177 | 178 | Triplet loss (VSE) [[paper]](http://www.jmlr.org/papers/volume10/weinberger09a/weinberger09a.pdf) 179 | ``` 180 | python -m bootstrap.run -o recipe1m/options/avg_nosem.yaml 181 | ``` 182 | 183 | ### VSE++ 184 | 185 | Triplet loss with hard negative mining [[paper]](https://arxiv.org/abs/1707.05612) 186 | ``` 187 | python -m bootstrap.run -o recipe1m/options/max.yaml 188 | ``` 189 | 190 | ### AdaMine_avg (Ours) 191 | 192 | Triplet loss with semantic loss 193 | ``` 194 | python -m bootstrap.run -o recipe1m/options/avg.yaml 195 | ``` 196 | 197 | ### AdaMine (Ours) 198 | 199 | Triplet loss with semantic loss and adaptive sampling 200 | ``` 201 | python -m bootstrap.run -o recipe1m/options/adamine.yaml 202 | ``` 203 | 204 | Features from testing set: 205 | 206 | ``` 207 | cd logs/recipe1m 208 | wget http://data.lip6.fr/cadene/im2recipe/logs/adamine.tar.gz 209 | tar -xzvf adamine.tar.gz 210 | ``` 211 | 212 | ### Lifted structure 213 | Lifted structure loss [[paper]](https://arxiv.org/abs/1511.06452) 214 | ``` 215 | python -m bootstrap.run -o recipe1m/options/lifted_struct.yaml 216 | ``` 217 | 218 | 219 | 220 | ## Documentation 221 | 222 | ``` 223 | TODO 224 | ``` 225 | 226 | 227 | 228 | ## Useful commands 229 | 230 | ### Compare experiments 231 | 232 | ``` 233 | python -m bootstrap.compare -d \ 234 | logs/recipe1m/adamine \ 235 | logs/recipe1m/avg \ 236 | -k eval_epoch.metric.recall_at_1_im2recipe_mean max 237 | ``` 238 | 239 | Results: 240 | ``` 241 | ## eval_epoch.metric.recall_at_1_im2recipe_mean 242 | 243 | Place Method Score Epoch 244 | ------- -------- ------- ------- 245 | 1 adamine 0.3827 76 246 | 2 avg 0.3201 51 247 | ``` 248 | 249 | ### Use a specific GPU 250 | 251 | ``` 252 | CUDA_VISIBLE_DEVICES=0 python -m boostrap.run -o options/recipe1m/adamine.yaml 253 | ``` 254 | 255 | ### Overwrite an option 256 | 257 | The boostrap.pytorch framework makes it easy to overwrite a hyperparameter. In this example, I run an experiment with a non-default learning rate. Thus, I also overwrite the experiment directory path: 258 | ``` 259 | python -m bootstrap.run -o recipe1m/options/adamine.yaml \ 260 | --optimizer.lr 0.0003 \ 261 | --exp.dir logs/recipe1m/adamine_lr,0.0003 262 | ``` 263 | 264 | ### Resume training 265 | 266 | If a problem occurs, it is easy to resume the last epoch by specifying the options file from the experiment directory while overwritting the `exp.resume` option (default is None): 267 | ``` 268 | python -m bootstrap.run -o logs/recipe1m/adamine/options.yaml \ 269 | --exp.resume last 270 | ``` 271 | 272 | ### Evaluate with the 10k setup 273 | 274 | Just as with the [1k setup](#evaluate-a-model-on-the-test-set), we load the best checkpoint. This time we also overwrite some options. The metrics will be displayed on your terminal at the end of the evaluation. 275 | 276 | ``` 277 | python -m bootstrap.run \ 278 | -o logs/recipe1m/adamine/options.yaml \ 279 | --exp.resume best_eval_epoch.metric.recall_at_1_im2recipe_mean \ 280 | --dataset.train_split \ 281 | --dataset.eval_split test \ 282 | --model.metric.nb_bags 5 \ 283 | --model.metric.nb_matchs_per_bag 10000 284 | ``` 285 | 286 | Note: Metrics can be stored in a json file by adding the `--misc.logs_name eval,test10k` option. It will create a `logs_eval,test10k.json` in your experiment directory. 287 | 288 | ### API 289 | 290 | ``` 291 | TODO 292 | ``` 293 | 294 | ### Extract your own image features 295 | 296 | ``` 297 | TODO 298 | ``` 299 | 300 | 301 | 302 | ## Citation 303 | 304 | ``` 305 | @inproceddings{carvalho2018cross, 306 | title={Cross-Modal Retrieval in the Cooking Context: Learning Semantic Text-Image Embeddings}, 307 | author={Carvalho, Micael and Cad{\`e}ne, R{\'e}mi and Picard, David and Soulier, Laure and Thome, Nicolas and Cord, Matthieu}, 308 | booktitle={The ACM conference on Research and Development in Information Retrieval (SIGIR)}, 309 | year={2018}, 310 | url={https://arxiv.org/abs/1804.11146} 311 | } 312 | ``` 313 | 314 | 315 | ## Acknowledgment 316 | 317 | Special thanks to the authors of [im2recipe](http://im2recipe.csail.mit.edu) who developped Recip1M, the dataset used in this research project. 318 | -------------------------------------------------------------------------------- /recipe1m/models/criterions/triplet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | import torch.nn as nn 4 | import scipy.linalg as la 5 | from bootstrap.lib.logger import Logger 6 | from bootstrap.lib.options import Options 7 | 8 | class Triplet(nn.Module): 9 | 10 | def __init__(self, opt, nb_classes, dim_emb, engine=None): 11 | self.alpha = opt['retrieval_strategy']['margin'] 12 | self.sampling = opt['retrieval_strategy']['sampling'] 13 | self.nb_samples = opt['retrieval_strategy'].get('nb_samples', 1) 14 | self.substrategy = opt['retrieval_strategy'].get('substrategy', ['IRR']) 15 | self.aggregation = opt['retrieval_strategy'].get('aggregation', 'mean') 16 | self.id_background = opt['retrieval_strategy'].get('id_background', 0) 17 | self.nb_classes = nb_classes 18 | self.dim_emb = dim_emb 19 | 20 | if 'substrategy_weights' in opt['retrieval_strategy']: 21 | self.substrategy_weights = opt['retrieval_strategy']['substrategy_weights'] 22 | if len(self.substrategy) > len(self.substrategy_weights): 23 | Logger()('Incorrect number of items in substrategy_weights (expected {}, got {})'.format( 24 | len(self.substrategy), len(self.substrategy_weights)), Logger.ERROR) 25 | elif len(self.substrategy) < len(self.substrategy_weights): 26 | Logger()('Higher number of items in substrategy_weights than expected ({}, got {}). Discarding exceeding values'.format( 27 | len(self.substrategy), len(self.substrategy_weights)), Logger.WARNING) 28 | else: 29 | Logger()('No substrategy_weights provided, automatically setting all items to 1', Logger.WARNING) 30 | self.substrategy_weights = [1.0] * len(self.substrategy) 31 | 32 | if engine: 33 | engine.register_hook('train_on_end_epoch', self.reset_barycenters) 34 | 35 | def calculate_cost(self, cost, enable_naive=True): 36 | if self.sampling == 'max_negative': 37 | ans,_ = torch.sort(cost, dim=1, descending=True) 38 | elif self.sampling == 'semi_hard': 39 | noalpha = cost - self.alpha 40 | mask = (noalpha <= 0) 41 | noalpha.masked_scatter_(mask, noalpha.max().expand_as(mask)) 42 | ans, __argmax = torch.sort(noalpha, dim=1, descending=True) 43 | ans += self.alpha 44 | elif self.sampling == 'prob_negative': 45 | indexes = torch.multinomial(cost, cost.size(1)) 46 | ans = torch.gather(cost, 1, indexes.detach()) 47 | elif self.sampling == 'random': 48 | if enable_naive: 49 | Logger()('Random triplet strategy is outdated and does not work with non-square matrices :(', Logger.ERROR) 50 | indexes = la.hankel(np.roll(np.arange(cost.size(0)),-1), np.arange(cost.size(1))) # anti-circular matrix 51 | indexes = cost.data.new(indexes.tolist()).long() 52 | ans = torch.gather(cost, 1, indexes.detach()) 53 | else: 54 | Logger()('Random triplet strategy not allowed with this configuration', Logger.ERROR) 55 | else: 56 | Logger()('Unknown substrategy {}.'.format(self.sampling), Logger.ERROR) 57 | 58 | return ans[:,:self.nb_samples] 59 | 60 | def add_cost(self, name, cost, bad_pairs, losses): 61 | invalid_pairs = (cost == 0).float().sum() 62 | bad_pairs['bad_pairs_{}'.format(name)] = invalid_pairs / cost.numel() 63 | if self.aggregation == 'mean': 64 | losses['loss_{}'.format(name)] = cost.mean() * self.substrategy_weights[self.substrategy.index(name)] 65 | elif self.aggregation == 'valid': 66 | valid_pairs = cost.numel() - invalid_pairs 67 | losses['loss_{}'.format(name)] = cost.sum() * self.substrategy_weights[self.substrategy.index(name)] / valid_pairs 68 | else: 69 | Logger()('Unknown aggregation strategy {}.'.format(self.aggregation), Logger.ERROR) 70 | 71 | def reset_barycenters(self, force=True, base_variable=None): 72 | if len(set(['RBB', 'IBB']).intersection(self.substrategy)) > 0: 73 | if not hasattr(self, 'barycenters') or force: 74 | if base_variable is None: 75 | base_variable = self.barycenters 76 | self.barycenters = base_variable.data.new(self.nb_classes, self.dim_emb) 77 | self.barycenters[:,:] = 0 78 | self.counters = base_variable.data.new(self.nb_classes) 79 | self.counters[:] = 0 80 | 81 | def semantic_unimodal(self, distances, class1): 82 | return self.semantic_multimodal(distances, class1, class1) 83 | 84 | def semantic_multimodal(self, distances, class1, class2, erase_diagonal=True): 85 | class1_matrix = class1.squeeze(1).repeat(class1.size(0), 1) 86 | class2_matrix = class2.squeeze(1).repeat(class2.size(0), 1).t() 87 | matrix_mask = ((class1_matrix != 0) + (class2_matrix != 0)) == 2 88 | 89 | same_class = torch.eq(class1_matrix, class2_matrix) 90 | anti_class = same_class.clone() 91 | 92 | anti_class = anti_class == 0 # get the dissimilar classes 93 | if erase_diagonal: 94 | same_class[range(same_class.size(0)),range(same_class.size(1))] = 0 # erase instance-instance pairs 95 | new_dimension = matrix_mask.int().sum(1).max().item() 96 | same_class = torch.masked_select(same_class, matrix_mask).view(new_dimension, new_dimension) 97 | anti_class = torch.masked_select(anti_class, matrix_mask).view(new_dimension, new_dimension) 98 | mdistances = torch.masked_select(distances, matrix_mask).view(new_dimension, new_dimension) 99 | 100 | same_class[same_class.cumsum(dim=1) > 1] = 0 # erasing extra positives 101 | pos_samples = torch.masked_select(mdistances, same_class) # only the first one 102 | min_neg_samples = anti_class.int().sum(1).min().item() # selecting max negatives possible 103 | anti_class[anti_class.cumsum(dim=1) > min_neg_samples] = 0 # erasing extra negatives 104 | neg_samples = torch.masked_select(mdistances, anti_class).view(new_dimension, min_neg_samples) 105 | 106 | cost = pos_samples.unsqueeze(1) - neg_samples + self.alpha 107 | cost[cost < 0] = 0 # hinge 108 | return cost 109 | 110 | def __call__(self, input1, input2, target, class1, class2): 111 | bad_pairs = {} 112 | losses = {} 113 | 114 | # Detect and treat unbalanced batch 115 | size1 = class1.size(0) 116 | size2 = class2.size(0) 117 | if size1 > size2: 118 | exceeding_input = input1[size2:,:] # Set exceeding samples apart 119 | exceeding_class = class1[size2:,:] # Set exceeding samples apart 120 | exceeding_type = 1 121 | class1 = class1[:size2] # Remove exceeding samples 122 | input1 = input1[:size2,:] # Remove exceeding samples 123 | size1 = class1.size(0) 124 | Logger()('Size of input1 automatically reduced to balance batch (from {} to {})'.format(size1, class1.size(0))) 125 | if target.size(0) > size1: 126 | target = target[:size1] 127 | elif size2 > size1: 128 | exceeding_input = input2[size1:,:] # Set exceeding samples apart 129 | exceeding_class = class2[size1:,:] # Set exceeding samples apart 130 | exceeding_type = 2 131 | class2 = class2[:size1] # Remove exceeding samples 132 | input2 = input2[:size1,:] # Remove exceeding samples 133 | size2 = class2.size(0) 134 | Logger()('Size of input2 automatically reduced to balance batch (from {} to {})'.format(size2, class2.size(0))) 135 | if target.size(0) > size2: 136 | target = target[:size2] 137 | else: 138 | exceeding_type = 0 139 | 140 | # Prepare instance samples (matched pairs) 141 | matches = target.squeeze(1) == 1 # To support -1 or 0 as mismatch 142 | instance_input1 = input1[matches].view(matches.sum().int().item(), input1.size(1)) 143 | instance_class1 = class1[matches] 144 | instance_input2 = input2[matches].view(matches.sum().int().item(), input2.size(1)) 145 | instance_class2 = class2[matches] 146 | 147 | # Prepare semantic samples (class != 0) 148 | valid_input1 = class1.squeeze(1) != 0 149 | valid_input2 = class2.squeeze(1) != 0 150 | semantic_input1 = input1[valid_input1].view(valid_input1.sum().int().item(), input1.size(1)) 151 | semantic_class1 = class1[valid_input1] 152 | semantic_input2 = input2[valid_input2].view(valid_input2.sum().int().item(), input2.size(1)) 153 | semantic_class2 = class2[valid_input2] 154 | 155 | # Augmented semantic samples (unmatched and class != 0) 156 | extra_input1 = (matches == 0) + valid_input1 == 2 157 | extra_input2 = (matches == 0) + valid_input2 == 2 158 | augmented_input1 = input1[extra_input1].view(extra_input1.sum().int().item(), input1.size(1)) 159 | augmented_class1 = class1[extra_input1] 160 | augmented_input2 = input2[extra_input2].view(extra_input2.sum().int().item(), input2.size(1)) 161 | augmented_class2 = class2[extra_input2] 162 | 163 | # Instance-based triplets 164 | if len(set(['IRR', 'RII', 'IRI', 'RIR', 'LIFT']).intersection(self.substrategy)) > 0: 165 | distances = self.dist(instance_input1, instance_input2) 166 | if 'IRR' in self.substrategy: 167 | cost = distances.diag().unsqueeze(1) - distances + self.alpha # all triplets 168 | cost[cost < 0] = 0 # hinge 169 | cost[range(cost.size(0)),range(cost.size(1))] = 0 # erase pos-pos pairs 170 | self.add_cost('IRR', self.calculate_cost(cost), bad_pairs, losses) 171 | if 'RII' in self.substrategy: 172 | cost = distances.diag().unsqueeze(0) - distances + self.alpha # all triplets 173 | cost[cost < 0] = 0 # hinge 174 | cost[range(cost.size(0)),range(cost.size(1))] = 0 # erase pos-pos pairs 175 | self.add_cost('RII', self.calculate_cost(cost.t()), bad_pairs, losses) 176 | if 'IRI' in self.substrategy: 177 | distances_image = self.dist(instance_input1, instance_input1) 178 | cost = distances.diag().unsqueeze(1) - distances_image + self.alpha # all triplets 179 | cost[cost < 0] = 0 # hinge 180 | cost[range(cost.size(0)),range(cost.size(1))] = 0 # erase pos-pos pairs 181 | self.add_cost('IRI', self.calculate_cost(cost), bad_pairs, losses) 182 | if 'RIR' in self.substrategy: 183 | distances_recipe = self.dist(instance_input2, instance_input2) 184 | cost = distances.diag().unsqueeze(0) - distances_recipe + self.alpha # all triplets 185 | cost[cost < 0] = 0 # hinge 186 | cost[range(cost.size(0)),range(cost.size(1))] = 0 # erase pos-pos pairs 187 | self.add_cost('RIR', self.calculate_cost(cost), bad_pairs, losses) 188 | # Lifted, instance-based triplet 189 | if 'LIFT' in self.substrategy: 190 | distances_mexp = (self.alpha - distances).exp() 191 | sum0 = distances_mexp.sum(0) 192 | sum1 = distances_mexp.sum(1) 193 | negdiag = torch.log(sum0 + sum1 - 2*distances_mexp.diag()) # see equation 4 on the paper, this is the left side : https://arxiv.org/pdf/1511.06452.pdf 194 | cost = distances.diag() + negdiag 195 | cost[cost < 0] = 0 # hinge 196 | cost = cost.pow(2).sum() / 2*distances.diag().numel() 197 | self.add_cost('LIFT', cost, bad_pairs, losses) 198 | 199 | # Semantic-based triplets 200 | if len(set(['SIRR', 'SRII']).intersection(self.substrategy)) > 0: 201 | distances = self.dist(semantic_input1, semantic_input2) 202 | if 'SIRR' in self.substrategy: 203 | cost = self.semantic_multimodal(distances, semantic_class1, semantic_class2) 204 | self.add_cost('SIRR', self.calculate_cost(cost), bad_pairs, losses) 205 | 206 | if 'SRII' in self.substrategy: 207 | cost = self.semantic_multimodal(distances.t(), semantic_class2, semantic_class1) 208 | self.add_cost('SRII', self.calculate_cost(cost), bad_pairs, losses) 209 | 210 | if 'SIII' in self.substrategy: 211 | cost = self.semantic_unimodal(self.dist(semantic_input1, semantic_input1), semantic_class1) 212 | self.add_cost('SIII', self.calculate_cost(cost), bad_pairs, losses) 213 | 214 | if 'SRRR' in self.substrategy: 215 | cost = self.semantic_unimodal(self.dist(semantic_input2, semantic_input2), semantic_class2) 216 | self.add_cost('SRRR', self.calculate_cost(cost), bad_pairs, losses) 217 | 218 | out = {} 219 | if len(bad_pairs.keys()) > 0: 220 | total_bad_pairs = input1.data.new([0]) 221 | for key in bad_pairs.keys(): 222 | total_bad_pairs += bad_pairs[key] 223 | out[key] = bad_pairs[key] 224 | total_bad_pairs = total_bad_pairs / len(bad_pairs.keys()) 225 | out['bad_pairs'] = total_bad_pairs 226 | else: 227 | out['bad_pairs'] = input1.data.new([0]) 228 | 229 | total_loss = input1.data.new([0]) 230 | if len(losses.keys()) > 0: 231 | for key in losses.keys(): 232 | total_loss += losses[key] 233 | out[key] = losses[key] 234 | out['loss'] = total_loss / len(losses.keys()) 235 | else: 236 | out['loss'] = input1.data.new([0]) 237 | 238 | return out 239 | 240 | def dist(self, input_1, input_2): 241 | input_1 = nn.functional.normalize(input_1) 242 | input_2 = nn.functional.normalize(input_2) 243 | return 1 - torch.mm(input_1, input_2.t()) 244 | -------------------------------------------------------------------------------- /recipe1m/datasets/recipe1m.py: -------------------------------------------------------------------------------- 1 | import os 2 | import lmdb 3 | import pickle 4 | import torch 5 | import torch.utils.data as data 6 | 7 | from PIL import Image 8 | 9 | from bootstrap.lib.options import Options 10 | from bootstrap.lib.logger import Logger 11 | from bootstrap.datasets import utils 12 | from bootstrap.datasets import transforms 13 | 14 | from .batch_sampler import BatchSamplerTripletClassif 15 | from bootstrap.lib.options import Options 16 | 17 | def default_items_tf(): 18 | return transforms.Compose([ 19 | transforms.ListDictsToDictLists(), 20 | transforms.PadTensors(value=0), 21 | transforms.StackTensors() 22 | ]) 23 | 24 | 25 | class Images(data.Dataset): 26 | 27 | def __init__(self, dir_data, split, batch_size, nb_threads, items_tf=default_items_tf): 28 | super(Dataset, self).__init__() 29 | self.dir_data = dir_data 30 | self.split = split 31 | self.batch_size = batch_size 32 | self.nb_threads = nb_threads 33 | self.items_tf = default_items_tf 34 | 35 | def make_batch_loader(self, shuffle=True): 36 | # allways shuffle even for valset/testset 37 | # see testing procedure 38 | 39 | if Options()['dataset'].get("debug", False): 40 | return data.DataLoader(self, 41 | batch_size=self.batch_size, 42 | num_workers=self.nb_threads, 43 | shuffle=False, 44 | pin_memory=True, 45 | collate_fn=self.items_tf(), 46 | drop_last=True) # Removing last batch if not full (quick fix accuracy calculation with class 0 only) 47 | else: 48 | return data.DataLoader(self, 49 | batch_size=self.batch_size, 50 | num_workers=self.nb_threads, 51 | shuffle=shuffle, 52 | pin_memory=True, 53 | collate_fn=self.items_tf(), 54 | drop_last=True) # Removing last batch if not full (quick fix accuracy calculation with class 0 only) 55 | 56 | class DatasetLMDB(Dataset): 57 | 58 | def __init__(self, dir_data, split, batch_size, nb_threads): 59 | super(DatasetLMDB, self).__init__(dir_data, split, batch_size, nb_threads) 60 | self.dir_lmdb = os.path.join(self.dir_data, 'data_lmdb') 61 | 62 | self.path_envs = {} 63 | self.path_envs['ids'] = os.path.join(self.dir_lmdb, split, 'ids.lmdb') 64 | self.path_envs['numims'] = os.path.join(self.dir_lmdb, split, 'numims.lmdb') 65 | self.path_envs['impos'] = os.path.join(self.dir_lmdb, split, 'impos.lmdb') 66 | self.path_envs['ingrs'] = os.path.join(self.dir_lmdb, split, 'ingrs.lmdb') 67 | self.path_envs['ilens'] = os.path.join(self.dir_lmdb, split, 'ilens.lmdb') 68 | self.path_envs['classes'] = os.path.join(self.dir_lmdb, split, 'classes.lmdb') 69 | self.path_envs['rlens'] = os.path.join(self.dir_lmdb, split, 'rlens.lmdb') 70 | self.path_envs['rbps'] = os.path.join(self.dir_lmdb, split, 'rbps.lmdb') 71 | self.path_envs['numims'] = os.path.join(self.dir_lmdb, split, 'numims.lmdb') 72 | # len(stvecs) train == 2163024 73 | self.path_envs['stvecs'] = os.path.join(self.dir_lmdb, split, 'stvecs.lmdb') 74 | # len(imnames) train == 383687 75 | self.path_envs['imnames'] = os.path.join(self.dir_lmdb, split, 'imnames.lmdb') 76 | self.path_envs['ims'] = os.path.join(self.dir_lmdb, split, 'ims.lmdb') 77 | 78 | self.envs = {} 79 | self.envs['ids'] = lmdb.open(self.path_envs['ids']) 80 | self.envs['classes'] = lmdb.open(self.path_envs['classes']) 81 | 82 | self.txns = {} 83 | self.txns['ids'] = self.envs['ids'].begin(write=False, buffers=True) 84 | self.txns['classes'] = self.envs['classes'].begin(write=False, buffers=True) 85 | 86 | self.nb_recipes = self.envs['ids'].stat()['entries'] 87 | 88 | self.path_pkl = os.path.join(self.dir_data, 'classes1M.pkl') 89 | #https://github.com/torralba-lab/im2recipe/blob/master/pyscripts/bigrams.py#L176 90 | with open(self.path_pkl, 'rb') as f: 91 | _ = pickle.load(f) # load the first line/object 92 | self.classes = pickle.load(f) # load the second line/object 93 | 94 | self.cname_to_cid = {v:k for k,v in self.classes.items()} 95 | 96 | def encode(self, value): 97 | return pickle.dumps(value) 98 | 99 | def decode(self, bytes_value): 100 | return pickle.loads(bytes_value) 101 | 102 | def get(self, index, env_name): 103 | buf = self.txns[env_name].get(self.encode(index)) 104 | value = self.decode(bytes(buf)) 105 | return value 106 | 107 | def _load_class(self, index): 108 | class_id = self.get(index, 'classes') - 1 # lua to python 109 | return torch.LongTensor([class_id]), self.classes[class_id] 110 | 111 | def __len__(self): 112 | return self.nb_recipes 113 | 114 | def true_nb_images(self): 115 | return self.envs['imnames'].stat()['entries'] 116 | 117 | 118 | class Images(DatasetLMDB): 119 | 120 | def __init__(self, dir_data, split, batch_size, nb_threads, image_from='database', image_tf=utils.default_image_tf(256, 224)): 121 | super(Images, self).__init__(dir_data, split, batch_size, nb_threads) 122 | self.image_tf = image_tf 123 | self.dir_img = os.path.join(dir_data, 'recipe1M', 'images') 124 | 125 | self.envs['numims'] = lmdb.open(self.path_envs['numims']) 126 | self.envs['impos'] = lmdb.open(self.path_envs['impos']) 127 | self.envs['imnames'] = lmdb.open(self.path_envs['imnames']) 128 | 129 | self.txns['numims'] = self.envs['numims'].begin(write=False, buffers=True) 130 | self.txns['impos'] = self.envs['impos'].begin(write=False, buffers=True) 131 | self.txns['imnames'] = self.envs['imnames'].begin(write=False, buffers=True) 132 | 133 | self.image_from = image_from 134 | if self.image_from == 'database': 135 | self.envs['ims'] = lmdb.open(self.path_envs['ims']) 136 | self.txns['ims'] = self.envs['ims'].begin(write=False, buffers=True) 137 | 138 | def __getitem__(self, index): 139 | item = self.get_image(index) 140 | return item 141 | 142 | def format_path_img(self, raw_path): 143 | # "recipe1M/images/train/6/b/d/c/6bdca6e490.jpg" 144 | basename = os.path.basename(raw_path) 145 | path_img = os.path.join(self.dir_img, 146 | self.split, 147 | basename[0], 148 | basename[1], 149 | basename[2], 150 | basename[3], 151 | basename) 152 | return path_img 153 | 154 | def get_image(self, index): 155 | item = {} 156 | item['data'], item['index'], item['path'] = self._load_image_data(index) 157 | item['class_id'], item['class_name'] = self._load_class(index) 158 | return item 159 | 160 | def _pil_loader(self, path): 161 | # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) 162 | with open(path, 'rb') as f: 163 | with Image.open(f) as img: 164 | return img.convert('RGB') 165 | 166 | def _load_image_data(self, index): 167 | # select random image from list of images for that sample 168 | nb_images = self.get(index, 'numims') 169 | if Options()['dataset'].get("debug", False): 170 | im_idx = 0 171 | else: 172 | im_idx = torch.randperm(nb_images)[0] 173 | index_img = self.get(index, 'impos')[im_idx] - 1 # lua to python 174 | 175 | path_img = self.format_path_img(self.get(index_img, 'imnames')) 176 | 177 | if self.image_from == 'pil_loader': 178 | image_data = self._pil_loader(path_img) 179 | elif self.image_from == 'database': 180 | image_data = self.get(index_img, 'ims') 181 | 182 | if self.image_tf is not None: 183 | image_data = self.image_tf(image_data) 184 | 185 | return image_data, index_img, path_img 186 | 187 | 188 | class Recipes(DatasetLMDB): 189 | 190 | def __init__(self, dir_data, split, batch_size, nb_threads): 191 | super(Recipes, self).__init__(dir_data, split, batch_size, nb_threads) 192 | self.path_ingrs = Options()['model']['network']['path_ingrs'] 193 | with open(self.path_ingrs, 'rb') as fobj: 194 | data = pickle.load(fobj) 195 | # idx+1 because 0 is padding 196 | # https://github.com/torralba-lab/im2recipe/blob/master/pyscripts/mk_dataset.py#L98 197 | self.ingrid_to_ingrname = {idx+2:name for idx, name in enumerate(data[1])} 198 | self.ingrid_to_ingrname[1] = '' 199 | self.ingrname_to_ingrid = {v:k for k,v in self.ingrid_to_ingrname.items()} 200 | 201 | # ~added for visu 202 | import json 203 | self.path_layer1 = os.path.join(dir_data, 'recipe1M', 'layer1.json') 204 | with open(self.path_layer1, 'r') as f: 205 | self.layer1 = json.load(f) 206 | self.layer1 = {data['id']:data for data in self.layer1} 207 | self.envs['ids'] = lmdb.open(self.path_envs['ids']) 208 | # ~end 209 | 210 | self.envs['ingrs'] = lmdb.open(self.path_envs['ingrs']) 211 | self.envs['rbps'] = lmdb.open(self.path_envs['rbps']) 212 | self.envs['rlens'] = lmdb.open(self.path_envs['rlens']) 213 | # not save length 214 | self.envs['stvecs'] = lmdb.open(self.path_envs['stvecs']) 215 | 216 | self.txns['ingrs'] = self.envs['ingrs'].begin(write=False, buffers=True) 217 | self.txns['rbps'] = self.envs['rbps'].begin(write=False, buffers=True) 218 | self.txns['rlens'] = self.envs['rlens'].begin(write=False, buffers=True) 219 | self.txns['stvecs'] = self.envs['stvecs'].begin(write=False, buffers=True) 220 | 221 | def __getitem__(self, index): 222 | item = self.get_recipe(index) 223 | return item 224 | 225 | def get_recipe(self, index): 226 | item = {} 227 | item['class_id'], item['class_name'] = self._load_class(index) 228 | item['ingrs'] = self._load_ingrs(index) 229 | item['instrs'] = self._load_instrs(index) 230 | item['index'] = index 231 | # ~added for visu 232 | item['ids'] = self.get(index, 'ids') 233 | item['layer1'] = self.layer1[item['ids']] 234 | # ~end 235 | return item 236 | 237 | def _load_ingrs(self, index): 238 | ingrs = {} 239 | ingrs['data'] = torch.LongTensor(self.get(index, 'ingrs')) 240 | max_length = ingrs['data'].size(0) 241 | ingrs['lengths'] = max_length - ingrs['data'].eq(0).sum(0).item() 242 | ingrs['interim'] = self.data_to_words_ingrs(ingrs['data'], ingrs['lengths']) 243 | return ingrs 244 | 245 | def data_to_words_ingrs(self, data, lengths): 246 | words = [self.ingrid_to_ingrname[data[i].item()] for i in range(lengths)] 247 | return words 248 | 249 | def _load_instrs(self, index): 250 | index_stv = self.get(index, 'rbps') - 1 # -1 cause indexing lua to python 251 | rlen = self.get(index, 'rlens') 252 | stvec_size = self.get(index_stv, 'stvecs').size(0) 253 | instrs = {} 254 | instrs['data'] = torch.zeros(rlen, stvec_size) 255 | for i in range(rlen): 256 | instrs['data'][i] = self.get(index_stv+i, 'stvecs') # -1 cause indexing lua to python 257 | max_length = instrs['data'].size(0) 258 | instrs['lengths'] = max_length - instrs['data'][:,0].eq(0).sum(0).item() 259 | return instrs 260 | 261 | 262 | class Recipe1M(DatasetLMDB): 263 | 264 | def __init__(self, dir_data, split, batch_size=100, nb_threads=4, freq_mismatch=0., 265 | batch_sampler='triplet_classif', 266 | image_from='dataset', image_tf=utils.default_image_tf(256, 224)): 267 | super(Recipe1M, self).__init__(dir_data, split, batch_size, nb_threads) 268 | self.images_dataset = Images(dir_data, split, batch_size, nb_threads, image_from=image_from, image_tf=image_tf) 269 | self.recipes_dataset = Recipes(dir_data, split, batch_size, nb_threads) 270 | self.freq_mismatch = freq_mismatch 271 | self.batch_sampler = batch_sampler 272 | 273 | #self.indices_by_class = self._make_indices_by_class() 274 | if self.split == 'train' and self.batch_sampler == 'triplet_classif': 275 | self.indices_by_class = self._make_indices_by_class() 276 | 277 | def _make_indices_by_class(self): 278 | Logger()('Calculate indices by class...') 279 | indices_by_class = [[] for class_id in range(len(self.classes))] 280 | for index in range(len(self.recipes_dataset)): 281 | class_id = self._load_class(index)[0][0] # bcause (class_id, class_name) and class_id is a Tensor 282 | indices_by_class[class_id].append(index) 283 | Logger()('Done!') 284 | return indices_by_class 285 | 286 | def make_batch_loader(self, shuffle=True): 287 | if self.split in ['val', 'test'] or self.batch_sampler == 'random': 288 | if Options()['dataset'].get("debug", False): 289 | batch_loader = super(Recipe1M, self).make_batch_loader(shuffle=False) 290 | else: 291 | batch_loader = super(Recipe1M, self).make_batch_loader(shuffle=shuffle) 292 | Logger()('Dataset will be sampled with "random" batch_sampler.') 293 | elif self.batch_sampler == 'triplet_classif': 294 | batch_sampler = BatchSamplerTripletClassif( 295 | self.indices_by_class, 296 | self.batch_size, 297 | pc_noclassif=0.5, 298 | nb_indices_same_class=2) 299 | batch_loader = data.DataLoader(self, 300 | num_workers=self.nb_threads, 301 | batch_sampler=batch_sampler, 302 | pin_memory=True, 303 | collate_fn=self.items_tf()) 304 | Logger()('Dataset will be sampled with "triplet_classif" batch_sampler.') 305 | else: 306 | raise ValueError() 307 | return batch_loader 308 | 309 | def __getitem__(self, index): 310 | #ids = self.data['ids'][index] 311 | item = {} 312 | item['index'] = index 313 | item['recipe'] = self.recipes_dataset[index] 314 | 315 | if self.freq_mismatch > 0: 316 | is_match = torch.rand(1)[0] > self.freq_mismatch 317 | else: 318 | is_match = True 319 | 320 | if is_match: 321 | item['image'] = self.images_dataset[index] 322 | item['match'] = torch.FloatTensor([1]) 323 | else: 324 | n_index = int(torch.rand(1)[0] * len(self)) 325 | item['image'] = self.images_dataset[n_index] 326 | item['match'] = torch.FloatTensor([-1]) 327 | return item 328 | 329 | # python -m recipe1m.datasets.recipe1m 330 | if __name__ == '__main__': 331 | 332 | Logger(Options()['logs']['dir'])('lol') 333 | 334 | 335 | -------------------------------------------------------------------------------- /demo_web/css/bootstrap-theme.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap v3.3.7 (http://getbootstrap.com) 3 | * Copyright 2011-2016 Twitter, Inc. 4 | * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) 5 | */.btn-danger,.btn-default,.btn-info,.btn-primary,.btn-success,.btn-warning{text-shadow:0 -1px 0 rgba(0,0,0,.2);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 1px rgba(0,0,0,.075)}.btn-danger.active,.btn-danger:active,.btn-default.active,.btn-default:active,.btn-info.active,.btn-info:active,.btn-primary.active,.btn-primary:active,.btn-success.active,.btn-success:active,.btn-warning.active,.btn-warning:active{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)}.btn-danger.disabled,.btn-danger[disabled],.btn-default.disabled,.btn-default[disabled],.btn-info.disabled,.btn-info[disabled],.btn-primary.disabled,.btn-primary[disabled],.btn-success.disabled,.btn-success[disabled],.btn-warning.disabled,.btn-warning[disabled],fieldset[disabled] .btn-danger,fieldset[disabled] .btn-default,fieldset[disabled] .btn-info,fieldset[disabled] .btn-primary,fieldset[disabled] .btn-success,fieldset[disabled] .btn-warning{-webkit-box-shadow:none;box-shadow:none}.btn-danger .badge,.btn-default .badge,.btn-info .badge,.btn-primary .badge,.btn-success .badge,.btn-warning .badge{text-shadow:none}.btn.active,.btn:active{background-image:none}.btn-default{text-shadow:0 1px 0 #fff;background-image:-webkit-linear-gradient(top,#fff 0,#e0e0e0 100%);background-image:-o-linear-gradient(top,#fff 0,#e0e0e0 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fff),to(#e0e0e0));background-image:linear-gradient(to bottom,#fff 0,#e0e0e0 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe0e0e0', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#dbdbdb;border-color:#ccc}.btn-default:focus,.btn-default:hover{background-color:#e0e0e0;background-position:0 -15px}.btn-default.active,.btn-default:active{background-color:#e0e0e0;border-color:#dbdbdb}.btn-default.disabled,.btn-default.disabled.active,.btn-default.disabled.focus,.btn-default.disabled:active,.btn-default.disabled:focus,.btn-default.disabled:hover,.btn-default[disabled],.btn-default[disabled].active,.btn-default[disabled].focus,.btn-default[disabled]:active,.btn-default[disabled]:focus,.btn-default[disabled]:hover,fieldset[disabled] .btn-default,fieldset[disabled] .btn-default.active,fieldset[disabled] .btn-default.focus,fieldset[disabled] .btn-default:active,fieldset[disabled] .btn-default:focus,fieldset[disabled] .btn-default:hover{background-color:#e0e0e0;background-image:none}.btn-primary{background-image:-webkit-linear-gradient(top,#337ab7 0,#265a88 100%);background-image:-o-linear-gradient(top,#337ab7 0,#265a88 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#265a88));background-image:linear-gradient(to bottom,#337ab7 0,#265a88 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff265a88', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#245580}.btn-primary:focus,.btn-primary:hover{background-color:#265a88;background-position:0 -15px}.btn-primary.active,.btn-primary:active{background-color:#265a88;border-color:#245580}.btn-primary.disabled,.btn-primary.disabled.active,.btn-primary.disabled.focus,.btn-primary.disabled:active,.btn-primary.disabled:focus,.btn-primary.disabled:hover,.btn-primary[disabled],.btn-primary[disabled].active,.btn-primary[disabled].focus,.btn-primary[disabled]:active,.btn-primary[disabled]:focus,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary,fieldset[disabled] .btn-primary.active,fieldset[disabled] .btn-primary.focus,fieldset[disabled] .btn-primary:active,fieldset[disabled] .btn-primary:focus,fieldset[disabled] .btn-primary:hover{background-color:#265a88;background-image:none}.btn-success{background-image:-webkit-linear-gradient(top,#5cb85c 0,#419641 100%);background-image:-o-linear-gradient(top,#5cb85c 0,#419641 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5cb85c),to(#419641));background-image:linear-gradient(to bottom,#5cb85c 0,#419641 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5cb85c', endColorstr='#ff419641', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#3e8f3e}.btn-success:focus,.btn-success:hover{background-color:#419641;background-position:0 -15px}.btn-success.active,.btn-success:active{background-color:#419641;border-color:#3e8f3e}.btn-success.disabled,.btn-success.disabled.active,.btn-success.disabled.focus,.btn-success.disabled:active,.btn-success.disabled:focus,.btn-success.disabled:hover,.btn-success[disabled],.btn-success[disabled].active,.btn-success[disabled].focus,.btn-success[disabled]:active,.btn-success[disabled]:focus,.btn-success[disabled]:hover,fieldset[disabled] .btn-success,fieldset[disabled] .btn-success.active,fieldset[disabled] .btn-success.focus,fieldset[disabled] .btn-success:active,fieldset[disabled] .btn-success:focus,fieldset[disabled] .btn-success:hover{background-color:#419641;background-image:none}.btn-info{background-image:-webkit-linear-gradient(top,#5bc0de 0,#2aabd2 100%);background-image:-o-linear-gradient(top,#5bc0de 0,#2aabd2 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5bc0de),to(#2aabd2));background-image:linear-gradient(to bottom,#5bc0de 0,#2aabd2 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2aabd2', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#28a4c9}.btn-info:focus,.btn-info:hover{background-color:#2aabd2;background-position:0 -15px}.btn-info.active,.btn-info:active{background-color:#2aabd2;border-color:#28a4c9}.btn-info.disabled,.btn-info.disabled.active,.btn-info.disabled.focus,.btn-info.disabled:active,.btn-info.disabled:focus,.btn-info.disabled:hover,.btn-info[disabled],.btn-info[disabled].active,.btn-info[disabled].focus,.btn-info[disabled]:active,.btn-info[disabled]:focus,.btn-info[disabled]:hover,fieldset[disabled] .btn-info,fieldset[disabled] .btn-info.active,fieldset[disabled] .btn-info.focus,fieldset[disabled] .btn-info:active,fieldset[disabled] .btn-info:focus,fieldset[disabled] .btn-info:hover{background-color:#2aabd2;background-image:none}.btn-warning{background-image:-webkit-linear-gradient(top,#f0ad4e 0,#eb9316 100%);background-image:-o-linear-gradient(top,#f0ad4e 0,#eb9316 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f0ad4e),to(#eb9316));background-image:linear-gradient(to bottom,#f0ad4e 0,#eb9316 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff0ad4e', endColorstr='#ffeb9316', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#e38d13}.btn-warning:focus,.btn-warning:hover{background-color:#eb9316;background-position:0 -15px}.btn-warning.active,.btn-warning:active{background-color:#eb9316;border-color:#e38d13}.btn-warning.disabled,.btn-warning.disabled.active,.btn-warning.disabled.focus,.btn-warning.disabled:active,.btn-warning.disabled:focus,.btn-warning.disabled:hover,.btn-warning[disabled],.btn-warning[disabled].active,.btn-warning[disabled].focus,.btn-warning[disabled]:active,.btn-warning[disabled]:focus,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning,fieldset[disabled] .btn-warning.active,fieldset[disabled] .btn-warning.focus,fieldset[disabled] .btn-warning:active,fieldset[disabled] .btn-warning:focus,fieldset[disabled] .btn-warning:hover{background-color:#eb9316;background-image:none}.btn-danger{background-image:-webkit-linear-gradient(top,#d9534f 0,#c12e2a 100%);background-image:-o-linear-gradient(top,#d9534f 0,#c12e2a 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9534f),to(#c12e2a));background-image:linear-gradient(to bottom,#d9534f 0,#c12e2a 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9534f', endColorstr='#ffc12e2a', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#b92c28}.btn-danger:focus,.btn-danger:hover{background-color:#c12e2a;background-position:0 -15px}.btn-danger.active,.btn-danger:active{background-color:#c12e2a;border-color:#b92c28}.btn-danger.disabled,.btn-danger.disabled.active,.btn-danger.disabled.focus,.btn-danger.disabled:active,.btn-danger.disabled:focus,.btn-danger.disabled:hover,.btn-danger[disabled],.btn-danger[disabled].active,.btn-danger[disabled].focus,.btn-danger[disabled]:active,.btn-danger[disabled]:focus,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger,fieldset[disabled] .btn-danger.active,fieldset[disabled] .btn-danger.focus,fieldset[disabled] .btn-danger:active,fieldset[disabled] .btn-danger:focus,fieldset[disabled] .btn-danger:hover{background-color:#c12e2a;background-image:none}.img-thumbnail,.thumbnail{-webkit-box-shadow:0 1px 2px rgba(0,0,0,.075);box-shadow:0 1px 2px rgba(0,0,0,.075)}.dropdown-menu>li>a:focus,.dropdown-menu>li>a:hover{background-color:#e8e8e8;background-image:-webkit-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-o-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f5f5f5),to(#e8e8e8));background-image:linear-gradient(to bottom,#f5f5f5 0,#e8e8e8 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#ffe8e8e8', GradientType=0);background-repeat:repeat-x}.dropdown-menu>.active>a,.dropdown-menu>.active>a:focus,.dropdown-menu>.active>a:hover{background-color:#2e6da4;background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2e6da4));background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2e6da4', GradientType=0);background-repeat:repeat-x}.navbar-default{background-image:-webkit-linear-gradient(top,#fff 0,#f8f8f8 100%);background-image:-o-linear-gradient(top,#fff 0,#f8f8f8 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fff),to(#f8f8f8));background-image:linear-gradient(to bottom,#fff 0,#f8f8f8 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff8f8f8', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-radius:4px;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 5px rgba(0,0,0,.075);box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 5px rgba(0,0,0,.075)}.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.open>a{background-image:-webkit-linear-gradient(top,#dbdbdb 0,#e2e2e2 100%);background-image:-o-linear-gradient(top,#dbdbdb 0,#e2e2e2 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#dbdbdb),to(#e2e2e2));background-image:linear-gradient(to bottom,#dbdbdb 0,#e2e2e2 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdbdbdb', endColorstr='#ffe2e2e2', GradientType=0);background-repeat:repeat-x;-webkit-box-shadow:inset 0 3px 9px rgba(0,0,0,.075);box-shadow:inset 0 3px 9px rgba(0,0,0,.075)}.navbar-brand,.navbar-nav>li>a{text-shadow:0 1px 0 rgba(255,255,255,.25)}.navbar-inverse{background-image:-webkit-linear-gradient(top,#3c3c3c 0,#222 100%);background-image:-o-linear-gradient(top,#3c3c3c 0,#222 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#3c3c3c),to(#222));background-image:linear-gradient(to bottom,#3c3c3c 0,#222 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff3c3c3c', endColorstr='#ff222222', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-radius:4px}.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.open>a{background-image:-webkit-linear-gradient(top,#080808 0,#0f0f0f 100%);background-image:-o-linear-gradient(top,#080808 0,#0f0f0f 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#080808),to(#0f0f0f));background-image:linear-gradient(to bottom,#080808 0,#0f0f0f 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff080808', endColorstr='#ff0f0f0f', GradientType=0);background-repeat:repeat-x;-webkit-box-shadow:inset 0 3px 9px rgba(0,0,0,.25);box-shadow:inset 0 3px 9px rgba(0,0,0,.25)}.navbar-inverse .navbar-brand,.navbar-inverse .navbar-nav>li>a{text-shadow:0 -1px 0 rgba(0,0,0,.25)}.navbar-fixed-bottom,.navbar-fixed-top,.navbar-static-top{border-radius:0}@media (max-width:767px){.navbar .navbar-nav .open .dropdown-menu>.active>a,.navbar .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar .navbar-nav .open .dropdown-menu>.active>a:hover{color:#fff;background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2e6da4));background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2e6da4', GradientType=0);background-repeat:repeat-x}}.alert{text-shadow:0 1px 0 rgba(255,255,255,.2);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.25),0 1px 2px rgba(0,0,0,.05);box-shadow:inset 0 1px 0 rgba(255,255,255,.25),0 1px 2px rgba(0,0,0,.05)}.alert-success{background-image:-webkit-linear-gradient(top,#dff0d8 0,#c8e5bc 100%);background-image:-o-linear-gradient(top,#dff0d8 0,#c8e5bc 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#dff0d8),to(#c8e5bc));background-image:linear-gradient(to bottom,#dff0d8 0,#c8e5bc 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdff0d8', endColorstr='#ffc8e5bc', GradientType=0);background-repeat:repeat-x;border-color:#b2dba1}.alert-info{background-image:-webkit-linear-gradient(top,#d9edf7 0,#b9def0 100%);background-image:-o-linear-gradient(top,#d9edf7 0,#b9def0 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9edf7),to(#b9def0));background-image:linear-gradient(to bottom,#d9edf7 0,#b9def0 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9edf7', endColorstr='#ffb9def0', GradientType=0);background-repeat:repeat-x;border-color:#9acfea}.alert-warning{background-image:-webkit-linear-gradient(top,#fcf8e3 0,#f8efc0 100%);background-image:-o-linear-gradient(top,#fcf8e3 0,#f8efc0 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fcf8e3),to(#f8efc0));background-image:linear-gradient(to bottom,#fcf8e3 0,#f8efc0 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffcf8e3', endColorstr='#fff8efc0', GradientType=0);background-repeat:repeat-x;border-color:#f5e79e}.alert-danger{background-image:-webkit-linear-gradient(top,#f2dede 0,#e7c3c3 100%);background-image:-o-linear-gradient(top,#f2dede 0,#e7c3c3 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f2dede),to(#e7c3c3));background-image:linear-gradient(to bottom,#f2dede 0,#e7c3c3 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2dede', endColorstr='#ffe7c3c3', GradientType=0);background-repeat:repeat-x;border-color:#dca7a7}.progress{background-image:-webkit-linear-gradient(top,#ebebeb 0,#f5f5f5 100%);background-image:-o-linear-gradient(top,#ebebeb 0,#f5f5f5 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#ebebeb),to(#f5f5f5));background-image:linear-gradient(to bottom,#ebebeb 0,#f5f5f5 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffebebeb', endColorstr='#fff5f5f5', GradientType=0);background-repeat:repeat-x}.progress-bar{background-image:-webkit-linear-gradient(top,#337ab7 0,#286090 100%);background-image:-o-linear-gradient(top,#337ab7 0,#286090 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#286090));background-image:linear-gradient(to bottom,#337ab7 0,#286090 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff286090', GradientType=0);background-repeat:repeat-x}.progress-bar-success{background-image:-webkit-linear-gradient(top,#5cb85c 0,#449d44 100%);background-image:-o-linear-gradient(top,#5cb85c 0,#449d44 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5cb85c),to(#449d44));background-image:linear-gradient(to bottom,#5cb85c 0,#449d44 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5cb85c', endColorstr='#ff449d44', GradientType=0);background-repeat:repeat-x}.progress-bar-info{background-image:-webkit-linear-gradient(top,#5bc0de 0,#31b0d5 100%);background-image:-o-linear-gradient(top,#5bc0de 0,#31b0d5 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5bc0de),to(#31b0d5));background-image:linear-gradient(to bottom,#5bc0de 0,#31b0d5 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff31b0d5', GradientType=0);background-repeat:repeat-x}.progress-bar-warning{background-image:-webkit-linear-gradient(top,#f0ad4e 0,#ec971f 100%);background-image:-o-linear-gradient(top,#f0ad4e 0,#ec971f 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f0ad4e),to(#ec971f));background-image:linear-gradient(to bottom,#f0ad4e 0,#ec971f 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff0ad4e', endColorstr='#ffec971f', GradientType=0);background-repeat:repeat-x}.progress-bar-danger{background-image:-webkit-linear-gradient(top,#d9534f 0,#c9302c 100%);background-image:-o-linear-gradient(top,#d9534f 0,#c9302c 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9534f),to(#c9302c));background-image:linear-gradient(to bottom,#d9534f 0,#c9302c 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9534f', endColorstr='#ffc9302c', GradientType=0);background-repeat:repeat-x}.progress-bar-striped{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.list-group{border-radius:4px;-webkit-box-shadow:0 1px 2px rgba(0,0,0,.075);box-shadow:0 1px 2px rgba(0,0,0,.075)}.list-group-item.active,.list-group-item.active:focus,.list-group-item.active:hover{text-shadow:0 -1px 0 #286090;background-image:-webkit-linear-gradient(top,#337ab7 0,#2b669a 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2b669a 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2b669a));background-image:linear-gradient(to bottom,#337ab7 0,#2b669a 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2b669a', GradientType=0);background-repeat:repeat-x;border-color:#2b669a}.list-group-item.active .badge,.list-group-item.active:focus .badge,.list-group-item.active:hover .badge{text-shadow:none}.panel{-webkit-box-shadow:0 1px 2px rgba(0,0,0,.05);box-shadow:0 1px 2px rgba(0,0,0,.05)}.panel-default>.panel-heading{background-image:-webkit-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-o-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f5f5f5),to(#e8e8e8));background-image:linear-gradient(to bottom,#f5f5f5 0,#e8e8e8 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#ffe8e8e8', GradientType=0);background-repeat:repeat-x}.panel-primary>.panel-heading{background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2e6da4));background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2e6da4', GradientType=0);background-repeat:repeat-x}.panel-success>.panel-heading{background-image:-webkit-linear-gradient(top,#dff0d8 0,#d0e9c6 100%);background-image:-o-linear-gradient(top,#dff0d8 0,#d0e9c6 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#dff0d8),to(#d0e9c6));background-image:linear-gradient(to bottom,#dff0d8 0,#d0e9c6 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdff0d8', endColorstr='#ffd0e9c6', GradientType=0);background-repeat:repeat-x}.panel-info>.panel-heading{background-image:-webkit-linear-gradient(top,#d9edf7 0,#c4e3f3 100%);background-image:-o-linear-gradient(top,#d9edf7 0,#c4e3f3 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9edf7),to(#c4e3f3));background-image:linear-gradient(to bottom,#d9edf7 0,#c4e3f3 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9edf7', endColorstr='#ffc4e3f3', GradientType=0);background-repeat:repeat-x}.panel-warning>.panel-heading{background-image:-webkit-linear-gradient(top,#fcf8e3 0,#faf2cc 100%);background-image:-o-linear-gradient(top,#fcf8e3 0,#faf2cc 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fcf8e3),to(#faf2cc));background-image:linear-gradient(to bottom,#fcf8e3 0,#faf2cc 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffcf8e3', endColorstr='#fffaf2cc', GradientType=0);background-repeat:repeat-x}.panel-danger>.panel-heading{background-image:-webkit-linear-gradient(top,#f2dede 0,#ebcccc 100%);background-image:-o-linear-gradient(top,#f2dede 0,#ebcccc 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f2dede),to(#ebcccc));background-image:linear-gradient(to bottom,#f2dede 0,#ebcccc 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2dede', endColorstr='#ffebcccc', GradientType=0);background-repeat:repeat-x}.well{background-image:-webkit-linear-gradient(top,#e8e8e8 0,#f5f5f5 100%);background-image:-o-linear-gradient(top,#e8e8e8 0,#f5f5f5 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#e8e8e8),to(#f5f5f5));background-image:linear-gradient(to bottom,#e8e8e8 0,#f5f5f5 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffe8e8e8', endColorstr='#fff5f5f5', GradientType=0);background-repeat:repeat-x;border-color:#dcdcdc;-webkit-box-shadow:inset 0 1px 3px rgba(0,0,0,.05),0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 3px rgba(0,0,0,.05),0 1px 0 rgba(255,255,255,.1)} 6 | /*# sourceMappingURL=bootstrap-theme.min.css.map */ --------------------------------------------------------------------------------