├── .gitignore ├── LICENSE ├── cls.py ├── data ├── README.md ├── re2 │ ├── combine.py │ ├── corpus.json │ ├── data.json │ ├── generate.py │ ├── get_hints.py │ ├── hints.json │ ├── templates.json │ ├── turk.py │ ├── turk │ │ ├── Batch_2843440_batch_results.csv │ │ └── Batch_2845506_batch_results.csv │ ├── turk_data.csv │ └── turk_data_2.csv └── shapeworld │ ├── generate.py │ └── spatial_jda.py ├── exp ├── cls_bird_ex │ ├── run.sh │ ├── train.err │ └── train.out ├── cls_bird_hint │ ├── run.sh │ ├── train.err │ └── train.out ├── cls_bird_sim │ ├── run.sh │ ├── train.err │ └── train.out ├── cls_ex │ ├── run.sh │ ├── train.err │ └── train.out ├── cls_gold │ ├── run.sh │ ├── train_gold.err │ └── train_gold.out ├── cls_hint │ ├── run.sh │ ├── train.err │ └── train.out ├── cls_joint │ ├── run.sh │ ├── train.err │ └── train.out ├── cls_sim │ ├── run.sh │ ├── train.err │ └── train.out ├── cls_vis │ ├── run.sh │ ├── vis.err │ ├── vis.out │ ├── vis.tar.gz │ └── vis │ │ ├── 0 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png │ │ ├── 1 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png │ │ ├── 2 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png │ │ ├── 3 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png │ │ ├── 4 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png │ │ ├── 5 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png │ │ ├── 6 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png │ │ ├── 7 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png │ │ ├── 8 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png │ │ └── 9 │ │ ├── desc.txt │ │ ├── ex_0.png │ │ ├── ex_1.png │ │ ├── ex_2.png │ │ ├── ex_3.png │ │ └── input.png ├── pbd_ex │ ├── run.sh │ ├── train.err │ └── train.out ├── pbd_hint │ ├── run.sh │ ├── train.err │ └── train.out ├── pbd_hint_eval │ ├── eval.err │ ├── eval.out │ ├── eval_gold.err │ ├── eval_gold.out │ └── run.sh ├── pbd_joint │ ├── run.sh │ ├── train.err │ └── train.out ├── pbd_reval │ ├── run.sh │ ├── train.err │ └── train.out ├── pbd_reval_eval │ ├── eval.err │ ├── eval.out │ ├── eval_gold.err │ ├── eval_gold.out │ └── run.sh ├── pbd_rhint │ ├── run.sh │ ├── train.err │ └── train.out ├── pbd_rhint_eval │ ├── eval.err │ ├── eval.out │ ├── eval_gold.err │ ├── eval_gold.out │ └── run.sh ├── pbd_vis │ ├── eval.err │ ├── eval.out │ └── run.sh ├── rl_ex │ ├── run.sh │ ├── train.err │ └── train.out ├── rl_ex_eval │ ├── eval.err │ ├── eval.out │ └── run.sh ├── rl_hint │ ├── run.sh │ ├── train.err │ └── train.out ├── rl_hint_eval │ ├── eval.err │ ├── eval.out │ └── run.sh ├── rl_scratch │ ├── run.sh │ ├── train.err │ └── train.out └── rl_scratch_eval │ ├── eval.err │ ├── eval.out │ └── run.sh ├── extras └── featurize.py ├── misc ├── __init__.py ├── array.py └── util.py ├── models.py ├── net.py ├── old ├── main.py ├── main.py.bak ├── models.py ├── net.py └── util.py ├── pbd.py ├── rl.py ├── rl_models.py └── tasks ├── __init__.py ├── birds.py ├── minicraft2.py ├── nav ├── __init__.py ├── library.py └── loading.py ├── regex2.py └── shapes.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | *.swp 104 | exp/*/model.* 105 | exp/*/checkpoint 106 | -------------------------------------------------------------------------------- /cls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import models 4 | from models import ClsModel, SimModel 5 | 6 | import gflags 7 | import os 8 | import shutil 9 | import sys 10 | 11 | FLAGS = gflags.FLAGS 12 | gflags.DEFINE_boolean("train", False, "do a training run") 13 | gflags.DEFINE_boolean("test", False, "evaluate on held-out concepts") 14 | gflags.DEFINE_boolean("test_same", False, "evaluate on training concepts") 15 | gflags.DEFINE_boolean("vis", False, "generate output visualizations") 16 | gflags.DEFINE_integer("n_epochs", 0, "number of epochs to run for") 17 | gflags.DEFINE_integer("n_batch", 100, "batch size") 18 | gflags.DEFINE_boolean("augment", False, "data augmentation") 19 | gflags.DEFINE_string("task", "shapes", "which task to use") 20 | gflags.DEFINE_string("model", "cls", "which model to use") 21 | models._set_flags() 22 | 23 | def main(): 24 | if FLAGS.task == "shapes": 25 | from tasks import shapes 26 | task = shapes.ShapeworldTask() 27 | elif FLAGS.task == "birds": 28 | from tasks import birds 29 | task = birds.BirdsTask() 30 | else: 31 | assert False 32 | 33 | if FLAGS.model == "cls": 34 | model = ClsModel(task) 35 | elif FLAGS.model == "sim": 36 | model = SimModel(task) 37 | else: 38 | assert False 39 | 40 | if FLAGS.train: 41 | for i_epoch in range(FLAGS.n_epochs): 42 | e_loss = 0. 43 | for i_batch in range(100): 44 | batch = task.sample_train(FLAGS.n_batch, augment=FLAGS.augment) 45 | b_loss = model.train(batch) 46 | e_loss += b_loss 47 | 48 | batch = task.sample_train(FLAGS.n_batch, augment=FLAGS.augment) 49 | e_acc = model.predict(batch) 50 | 51 | v_batch = task.sample_val(same=False) 52 | e_v_acc = model.predict(v_batch) 53 | 54 | vs_batch = task.sample_val(same=True) 55 | e_vs_acc = model.predict(vs_batch) 56 | 57 | print("[iter] %d" % i_epoch) 58 | print("[loss] %01.4f" % e_loss) 59 | print("[trn_acc] %01.4f" % e_acc) 60 | print("[val_acc] %01.4f" % e_v_acc) 61 | print("[val_same_acc] %01.4f" % e_vs_acc) 62 | print("[val_mean_acc] %01.4f" % ((e_v_acc + e_vs_acc) / 2)) 63 | print 64 | 65 | if i_epoch % 10 == 0: 66 | model.save() 67 | 68 | if FLAGS.test: 69 | v_batch = task.sample_val(same=False) 70 | e_v_acc = model.predict(v_batch) 71 | 72 | t_batch = task.sample_test(same=False) 73 | e_t_acc = model.predict(t_batch) 74 | 75 | print("[FINAL val_acc] %01.4f" % e_v_acc) 76 | print("[FINAL tst_acc] %01.4f" % e_t_acc) 77 | 78 | if FLAGS.test_same: 79 | v_batch = task.sample_val(same=True) 80 | e_v_acc = model.predict(v_batch) 81 | 82 | t_batch = task.sample_test(same=True) 83 | e_t_acc = model.predict(t_batch) 84 | 85 | print("[FINAL val_same_acc] %01.4f" % e_v_acc) 86 | print("[FINAL tst_same_acc] %01.4f" % e_t_acc) 87 | 88 | if FLAGS.vis: 89 | v_batch = task.sample_val(same=True) 90 | preds, labels, hyps = model.predict(v_batch, debug=True) 91 | if os.path.exists("vis"): 92 | shutil.rmtree("vis") 93 | os.mkdir("vis") 94 | for i in range(10): 95 | task.visualize(v_batch[i], hyps[i], preds[i], "vis/%d" % i) 96 | 97 | if __name__ == "__main__": 98 | argv = FLAGS(sys.argv) 99 | main() 100 | -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | ShapeWorld data can be downloaded at http://people.eecs.berkeley.edu/~jda/data/shapeworld.tar.gz 2 | 3 | Unfortunately, Berkeley seems to have deleted the version of the dataset used in 4 | the original paper. Jesse Mu created a new version, which can be downloaded 5 | [here](https://github.com/jayelm/lsl/tree/master/shapeworld) and gives similar 6 | results; [his paper](https://arxiv.org/abs/1911.02683) reports accuracies for 7 | both L^3 and the (better) LSL model. 8 | 9 | If you want to generate a new version of the dataset yourself, code can be found 10 | in the `ShapeWorld` directory here. To use it, download the 11 | [ShapeWorld](https://github.com/AlexKuhnle/ShapeWorld) repository (I think you 12 | need to go back to [this 13 | commit](https://github.com/AlexKuhnle/ShapeWorld/tree/da452fc49f0a0dd18517153bf0f47ce7588065d7)), 14 | place `spatial_jda.py` in `shapeworld/datasets/agreement`, then call 15 | `generate.py`. 16 | 17 | I'm still figuring out how to distribute the navigation data; for now, email me 18 | at jda@mit.edu. 19 | -------------------------------------------------------------------------------- /data/re2/combine.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import json 4 | import re 5 | import numpy as np 6 | 7 | START = "<" 8 | STOP = ">" 9 | SEP = "@" 10 | 11 | random = np.random.RandomState(0) 12 | 13 | N_EX = 5 14 | 15 | with open("data.json") as data_f: 16 | data = json.load(data_f) 17 | 18 | with open("templates.json") as template_f: 19 | templates = json.load(template_f) 20 | 21 | with open("hints.json") as hint_f: 22 | hints = json.load(hint_f) 23 | hints = {int(k): v for k, v in hints.items()} 24 | 25 | annotations = [] 26 | 27 | for i, example in enumerate(data): 28 | t_before = example["before"] 29 | t_before = t_before.replace("[aeiou]", "V").replace("[^aeiou]", "C") 30 | re_before = t_before 31 | letters_before = t_before.split(")(")[1].replace(".", "").replace("V", "").replace("C", "") 32 | letters_before = " ".join(letters_before) 33 | t_before = re.sub("[a-z]+", "l", t_before) 34 | t_after = example["after"][2:-2] 35 | re_after = t_after 36 | letters_after = t_after.replace("\\2", "") 37 | letters_after = " ".join(letters_after) 38 | t_after = re.sub("[a-z]+", "l", t_after) 39 | template_key = t_before + SEP + t_after 40 | 41 | if template_key not in templates: 42 | continue 43 | 44 | aug_hints = [] 45 | for template in templates[template_key]: 46 | aug_hint = template.replace("BEFORE", letters_before).replace("AFTER", letters_after) 47 | aug_hint = [START] + aug_hint.split() + [STOP] 48 | aug_hints.append(aug_hint) 49 | 50 | if i in hints: 51 | hint = hints[i] 52 | else: 53 | hint = "" 54 | hint = [START] + hint.split() + [STOP] 55 | 56 | re_hint = START + re_before + SEP + re_after + STOP 57 | 58 | ex = [] 59 | for inp, out in example["examples"]: 60 | inp = START + inp + STOP 61 | out = START + out + STOP 62 | ex.append((inp, out)) 63 | 64 | annotations.append({ 65 | "examples": ex, 66 | "re": re_hint, 67 | "hint": hint, 68 | "hints_aug": aug_hints 69 | }) 70 | 71 | train = annotations[:3000] 72 | val = annotations[3000:3500] 73 | test = annotations[3500:4000] 74 | 75 | for datum in val: 76 | del datum["examples"][N_EX+1:] 77 | 78 | for datum in test: 79 | del datum["examples"][N_EX+1:] 80 | 81 | corpus = { 82 | "train": train, 83 | "val": val, 84 | "test": test 85 | } 86 | 87 | with open("corpus.json", "w") as corpus_f: 88 | json.dump(corpus, corpus_f) 89 | -------------------------------------------------------------------------------- /data/re2/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import json 4 | import numpy as np 5 | import re 6 | 7 | N_VAL = 500 8 | N_TEST = 500 9 | 10 | random = np.random.RandomState(0) 11 | 12 | word_re = re.compile("^[a-z]+$") 13 | words = [] 14 | with open("/usr/share/dict/words") as words_f: 15 | for line in words_f: 16 | word = line.strip() 17 | if word_re.match(word): 18 | words.append(word) 19 | 20 | chars = "." 21 | vowels = "[aeiou]" 22 | consonants = "[^aeiou]" 23 | letters = [chr(i) for i in range(ord("a"), ord("z"))] 24 | classes = [chars, vowels, consonants] 25 | 26 | N_EX = 30 27 | 28 | def sample_block(size): 29 | out = "" 30 | for i in range(size): 31 | use_letter = random.randint(2) 32 | if use_letter: 33 | out += random.choice(letters) 34 | else: 35 | out += random.choice(classes) 36 | return out 37 | 38 | def sample_replace(size): 39 | out = "" 40 | for i in range(size): 41 | use_letter = random.randint(3) 42 | if use_letter: 43 | out += random.choice(letters) 44 | else: 45 | out += "\\2" 46 | return out 47 | 48 | def sample(): 49 | anchor_start = random.randint(2) 50 | anchor_end = not anchor_start and random.randint(2) 51 | 52 | start_len = random.randint(1) 53 | match_len = random.randint(2) + 1 54 | rep_len = random.randint(3) 55 | end_len = random.randint(1) 56 | 57 | start = "^" if anchor_start else "" 58 | start += sample_block(start_len) 59 | 60 | match = sample_block(match_len) 61 | replace = sample_replace(rep_len) 62 | 63 | end = sample_block(end_len) 64 | if anchor_end: 65 | end += "$" 66 | 67 | before = "(%s)(%s)(%s)" % (start, match, end) 68 | after = "\\1%s\\3" % replace 69 | 70 | return before, after 71 | 72 | seen = set() 73 | data = [] 74 | while len(data) < 5000: 75 | before, after = sample() 76 | if (before, after) in seen: 77 | continue 78 | seen.add((before, after)) 79 | matches = [] 80 | order = list(range(len(words))) 81 | random.shuffle(order) 82 | attempts = 0 83 | success = True 84 | while len(matches) < N_EX: 85 | attempts += 1 86 | if attempts > 100: 87 | success = False 88 | break 89 | j = order.pop() 90 | word = words[j] 91 | sub = re.sub(before, after, word) 92 | if sub == word and len(matches) < N_EX / 2: 93 | continue 94 | #if sub == word: 95 | # continue 96 | matches.append((word, sub)) 97 | 98 | random.shuffle(matches) 99 | 100 | if not success: 101 | continue 102 | 103 | print len(data) 104 | 105 | data.append({"before": before, "after": after, "examples": matches}) 106 | 107 | #print before, after 108 | #for match in matches: 109 | # print " ", match 110 | 111 | random.shuffle(data) 112 | 113 | #train_data = data[:-(N_VAL+N_TEST)] 114 | #val_data = data[-(N_VAL+N_TEST):-N_TEST] 115 | #test_data = data[-N_TEST:] 116 | # 117 | #dataset = { 118 | # "train": train_data, 119 | # "val": val_data, 120 | # "test": test_data 121 | #} 122 | 123 | dataset = data 124 | 125 | with open("data.json", "w") as data_f: 126 | json.dump(dataset, data_f) 127 | -------------------------------------------------------------------------------- /data/re2/get_hints.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | from collections import defaultdict 4 | import csv 5 | import json 6 | from nltk.tokenize import word_tokenize 7 | import os 8 | import re 9 | 10 | templates = defaultdict(list) 11 | hints = {} 12 | 13 | exclude = ["'", '"', "`", ".", "\\"] 14 | 15 | template_to_pat = defaultdict(set) 16 | 17 | with open("data.json") as data_f: 18 | data = json.load(data_f) 19 | 20 | for name in os.listdir("turk"): 21 | path = os.path.join("turk", name) 22 | with open(path) as turk_f: 23 | reader = csv.DictReader(turk_f) 24 | for row in reader: 25 | datum_id = int(row["Input.id"]) 26 | hint = row["Answer.hint_text"] 27 | words = word_tokenize(hint) 28 | hint = " ".join(words) 29 | hint = hint.lower() 30 | for e in exclude: 31 | hint = hint.replace(e, "") 32 | hint = hint.replace("-", " - ") 33 | hint = hint.replace("+", " + ") 34 | hint = hint.replace("/", " / ") 35 | hint = re.sub(r"\s+", " ", hint) 36 | 37 | datum = data[datum_id] 38 | special_b = datum["before"].split(")(")[1] 39 | special_b = special_b.replace(".", "").replace("[^aeiou]", "").replace("[aeiou]", "") 40 | special_a = datum["after"][2:-2] 41 | special_a = special_a.replace("\\2", "") 42 | 43 | a_exploded = " ".join(special_a) 44 | b_exploded = " ".join(special_b) 45 | 46 | pattern_before = datum["before"].replace("[aeiou]", "V").replace("[^aeiou]", "C") 47 | pattern_before = re.sub("[a-z]", "l", pattern_before) 48 | pattern_after = datum["after"][2:-2] 49 | pattern_after = re.sub("[a-z]+", "l", pattern_after) 50 | 51 | template = hint 52 | 53 | if special_a: 54 | template = re.sub(r"\b%s\b" % special_a, "AFTER", template) 55 | template = re.sub(r"\b%s\b" % a_exploded, "AFTER", template) 56 | hint = re.sub(r"\b%s\b" % special_a, a_exploded, hint) 57 | if special_b: 58 | template = re.sub(r"\b%s\b" % special_b, "BEFORE", template) 59 | template = re.sub(r"\b%s\b" % b_exploded, "BEFORE", template) 60 | hint = re.sub(r"\b%s\b" % special_b, b_exploded, hint) 61 | 62 | assert datum_id not in hints 63 | hints[datum_id] = hint 64 | 65 | if re.search(r"\b[b-r,t-z]\b", template): 66 | # bad template 67 | continue 68 | 69 | pattern = pattern_before + "@" + pattern_after 70 | templates[pattern].append(template) 71 | template_to_pat[template].add(pattern) 72 | 73 | #print (pattern_before, pattern_after), hint 74 | 75 | with open("hints.json", "w") as hint_f: 76 | json.dump(hints, hint_f) 77 | 78 | with open("templates.json", "w") as template_f: 79 | json.dump(templates, template_f) 80 | 81 | #for template, pats in template_to_pat.items(): 82 | # print len(pats), template, pats 83 | -------------------------------------------------------------------------------- /data/re2/turk.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import json 4 | import re 5 | 6 | N_EX = 8 7 | 8 | with open("data.json") as data_f: 9 | data = json.load(data_f) 10 | 11 | train_data = data["train"] 12 | #label_data = train_data[:10] 13 | #label_data = train_data[:1000] 14 | label_data = train_data 15 | 16 | with open("turk_data_3.csv", "w") as turk_f: 17 | header_labels = ["id"] 18 | header_labels += ["before_%d" % i for i in range(N_EX)] 19 | header_labels += ["after_%d" % i for i in range(N_EX)] 20 | header_labels += ["test_before", "test_after"] 21 | header_labels += ["hint"] 22 | print >>turk_f, ",".join(header_labels) 23 | 24 | #for i, datum in enumerate(label_data): 25 | for i in range(2000, 3000): 26 | datum = label_data[i] 27 | ref_before, ref_after = zip(*datum["examples"][:N_EX]) 28 | test_before, test_after = datum["examples"][N_EX] 29 | hints = [] 30 | pat = datum["before"] 31 | sub = datum["after"] 32 | before = [re.sub(pat, "\\1\\2\\3", b) for b in ref_before] 33 | after = [re.sub(pat, "" + sub + "", b) for b in ref_before] 34 | after_check = [re.sub(pat, sub, b) for b in ref_before] 35 | assert after_check == list(ref_after) 36 | if "(^)" in pat: 37 | hints.append("look at the beginning of the word") 38 | if "$" in pat: 39 | hints.append("look at the end of the word") 40 | if "[aeiou]" in pat: 41 | hints.append("look at vowels") 42 | if "[^aeiou]" in pat: 43 | hints.append("look at consonants") 44 | if "\\2\\2" in sub: 45 | hints.append("look for repitition") 46 | if len(hints) > 0: 47 | hint = "
  • ".join(hints) 48 | else: 49 | hint = "(no hint for this problem)" 50 | 51 | line = ",".join([str(i)] + before + after + [test_before, test_after, hint]) 52 | print >>turk_f, line 53 | -------------------------------------------------------------------------------- /data/shapeworld/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from collections import defaultdict 4 | import numpy as np 5 | import os 6 | from shapeworld import dataset 7 | import json 8 | import itertools 9 | 10 | #N_CAPTIONS = 5000 11 | #N_TRAIN = 4000 12 | #N_VAL = 500 13 | #N_TEST = 500 14 | 15 | N_CAPTIONS = 3000 16 | N_TRAIN = 2000 17 | N_VAL = 500 18 | N_TEST = 500 19 | 20 | #N_CAPTIONS = 100 21 | #N_TRAIN = 50 22 | #N_VAL = 25 23 | #N_TEST = 25 24 | 25 | assert N_TRAIN + N_VAL + N_TEST == N_CAPTIONS 26 | 27 | WIDTH = 64 28 | HEIGHT = 64 29 | CHANNELS = 3 30 | EXAMPLES = 4 31 | 32 | DATASET = dataset(dtype="agreement", name="spatial_jda") 33 | random = np.random.RandomState(0) 34 | 35 | all_captions = {} 36 | while len(all_captions) < N_CAPTIONS: 37 | if len(all_captions) % 500 == 0: 38 | print("%d / %d captions" % (len(all_captions), N_CAPTIONS)) 39 | 40 | DATASET.world_generator.sample_values(mode="train") 41 | DATASET.world_captioner.sample_values(mode="train", correct=True) 42 | while True: 43 | world = DATASET.world_generator() 44 | if world is None: 45 | continue 46 | caption = DATASET.world_captioner(entities=world.entities) 47 | if caption is None: 48 | continue 49 | break 50 | realized, = DATASET.caption_realizer.realize(captions=[caption]) 51 | realized = tuple(realized) 52 | all_captions[realized] = caption 53 | 54 | captions = list(sorted(all_captions.keys())) 55 | random.shuffle(captions) 56 | train_captions = captions[:N_TRAIN] 57 | val_captions = captions[N_TRAIN:N_TRAIN+N_VAL] 58 | test_captions = captions[N_TRAIN+N_VAL:] 59 | caption_data = all_captions 60 | 61 | def generate(name, captions, n_examples): 62 | mappings = defaultdict(list) 63 | for i_scene in range(n_examples * 20): 64 | #for _ in range(n_examples * 5): 65 | DATASET.world_generator.sample_values(mode="train") 66 | world = DATASET.world_generator() 67 | if world is None: 68 | continue 69 | for key in captions: 70 | caption = caption_data[key] 71 | agree = caption.agreement(entities=world.entities) > 0 72 | #print(agree) 73 | #print(world.entities) 74 | if not agree: 75 | continue 76 | mappings[key].append(world) 77 | 78 | if i_scene % 1000 == 0: 79 | print("%d / %d scenes" % (i_scene, n_examples * 20)) 80 | 81 | for key in mappings: 82 | print(" ".join(key), len(mappings[key])) 83 | 84 | total_scenes = sum(len(l) for l in mappings.values()) 85 | assert total_scenes > n_examples * 6 86 | 87 | examples = np.zeros((n_examples, EXAMPLES, WIDTH, HEIGHT, CHANNELS)) 88 | inputs = np.zeros((n_examples, WIDTH, HEIGHT, CHANNELS)) 89 | labels = np.zeros((n_examples,)) 90 | hints = [] 91 | 92 | i_example = 0 93 | while i_example < n_examples: 94 | key = captions[random.randint(len(captions))] 95 | 96 | worlds = mappings[key] 97 | if len(worlds) < EXAMPLES + 1: 98 | continue 99 | for i_world in range(EXAMPLES): 100 | world = worlds.pop() 101 | examples[i_example, i_world, ...] = world.get_array() 102 | 103 | if random.randint(2) == 0: 104 | world = worlds.pop() 105 | inputs[i_example, ...] = world.get_array() 106 | labels[i_example] = 1 107 | else: 108 | while True: 109 | other_key = captions[random.randint(len(captions))] 110 | if len(mappings[other_key]) > 0: 111 | other_world = mappings[other_key].pop() 112 | break 113 | inputs[i_example, ...] = other_world.get_array() 114 | labels[i_example] = 0 115 | hints.append(" ".join(key)) 116 | 117 | if i_example % 500 == 0: 118 | print("%d / %d examples" % (i_example, n_examples)) 119 | 120 | i_example += 1 121 | 122 | print("\n\n") 123 | 124 | np.save(os.path.join(name, "examples.npy"), examples) 125 | np.save(os.path.join(name, "inputs.npy"), inputs) 126 | np.save(os.path.join(name, "labels.npy"), labels) 127 | with open(os.path.join(name, "hints.json"), "w") as hint_f: 128 | json.dump(hints, hint_f) 129 | 130 | generate("train", train_captions, 9000) 131 | generate("val", val_captions, 500) 132 | generate("test", test_captions, 500) 133 | generate("val_same", train_captions, 500) 134 | generate("test_same", train_captions, 500) 135 | 136 | #generate("train", train_captions, 100) 137 | #generate("val", val_captions, 100) 138 | #generate("test", test_captions, 100) 139 | -------------------------------------------------------------------------------- /data/shapeworld/spatial_jda.py: -------------------------------------------------------------------------------- 1 | from shapeworld.dataset import CaptionAgreementDataset 2 | from shapeworld.generators import GenericGenerator 3 | from shapeworld.captioners import AttributesTypeCaptioner, SpatialRelationCaptioner, ExistentialCaptioner 4 | 5 | 6 | class SpatialJdaDataset(CaptionAgreementDataset): 7 | 8 | dataset_name = 'spatial_jda' 9 | 10 | def __init__(self, validation_combinations, test_combinations, caption_size, words, language=None): 11 | world_generator = GenericGenerator( 12 | entity_counts=[4, 5], 13 | collision_tolerance=0.0, 14 | boundary_tolerance=0.0, 15 | validation_combinations=validation_combinations, 16 | test_combinations=test_combinations, 17 | max_provoke_collision_rate=0.0 18 | ) 19 | world_captioner = ExistentialCaptioner( 20 | restrictor_captioner=AttributesTypeCaptioner( 21 | trivial_acceptance_rate=1.0 22 | ), 23 | body_captioner=SpatialRelationCaptioner( 24 | reference_captioner=AttributesTypeCaptioner(), 25 | relations=('x-rel', 'y-rel') 26 | ) 27 | ) 28 | super(SpatialJdaDataset, self).__init__( 29 | world_generator=world_generator, 30 | world_captioner=world_captioner, 31 | caption_size=caption_size, 32 | words=words, 33 | language=language 34 | ) 35 | 36 | 37 | dataset = SpatialJdaDataset 38 | SpatialJdaDataset.default_config = dict( 39 | #validation_combinations=[['square', 'red', 'solid'], ['triangle', 'green', 'solid'], ['circle', 'blue', 'solid']], 40 | #test_combinations=[['rectangle', 'yellow', 'solid'], ['cross', 'magenta', 'solid'], ['ellipse', 'cyan', 'solid']], 41 | validation_combinations=None, 42 | test_combinations=None, 43 | caption_size=12, 44 | words=['.', 'a', 'above', 'an', 'below', 'black', 'blue', 'circle', 'cross', 'cyan', 'ellipse', 'green', 'is', 'left', 'magenta', 'of', 'pentagon', 'rectangle', 'red', 'right', 'semicircle', 'shape', 'square', 'the', 'to', 'triangle', 'white', 'yellow'] 45 | ) 46 | -------------------------------------------------------------------------------- /exp/cls_bird_ex/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../cls.py \ 4 | --task=birds \ 5 | --model=sim \ 6 | --learning_rate 0.001 \ 7 | --train \ 8 | --n_epochs=10 \ 9 | --test \ 10 | > train.out \ 11 | 2> train.err 12 | 13 | -------------------------------------------------------------------------------- /exp/cls_bird_ex/train.err: -------------------------------------------------------------------------------- 1 | 2017-10-04 12:12:44.221191: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-10-04 12:12:44.221217: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-10-04 12:12:44.221221: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-10-04 12:12:44.221229: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-10-04 12:12:44.221232: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-10-04 12:12:46.232760: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-10-04 12:12:46.233189: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-10-04 12:12:46.233201: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-10-04 12:12:46.233205: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-10-04 12:12:46.233212: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | -------------------------------------------------------------------------------- /exp/cls_bird_ex/train.out: -------------------------------------------------------------------------------- 1 | [iter] 0 2 | [loss] 5942.4493 3 | [trn_acc] 0.6000 4 | [val_acc] 0.6000 5 | [val_same_acc] 0.6000 6 | [val_mean_acc] 0.6000 7 | 8 | [iter] 1 9 | [loss] 5869.8213 10 | [trn_acc] 0.8300 11 | [val_acc] 0.8000 12 | [val_same_acc] 0.8000 13 | [val_mean_acc] 0.8000 14 | 15 | [iter] 2 16 | [loss] 5834.8512 17 | [trn_acc] 0.9000 18 | [val_acc] 0.9000 19 | [val_same_acc] 0.9000 20 | [val_mean_acc] 0.9000 21 | 22 | [iter] 3 23 | [loss] 5791.9795 24 | [trn_acc] 0.9200 25 | [val_acc] 1.0000 26 | [val_same_acc] 1.0000 27 | [val_mean_acc] 1.0000 28 | 29 | [iter] 4 30 | [loss] 5776.5085 31 | [trn_acc] 0.9300 32 | [val_acc] 1.0000 33 | [val_same_acc] 1.0000 34 | [val_mean_acc] 1.0000 35 | 36 | [iter] 5 37 | [loss] 5767.6663 38 | [trn_acc] 0.9900 39 | [val_acc] 1.0000 40 | [val_same_acc] 1.0000 41 | [val_mean_acc] 1.0000 42 | 43 | [iter] 6 44 | [loss] 5754.3891 45 | [trn_acc] 0.9500 46 | [val_acc] 1.0000 47 | [val_same_acc] 1.0000 48 | [val_mean_acc] 1.0000 49 | 50 | [iter] 7 51 | [loss] 5769.9242 52 | [trn_acc] 0.9600 53 | [val_acc] 1.0000 54 | [val_same_acc] 1.0000 55 | [val_mean_acc] 1.0000 56 | 57 | [iter] 8 58 | [loss] 5756.5457 59 | [trn_acc] 0.9900 60 | [val_acc] 1.0000 61 | [val_same_acc] 1.0000 62 | [val_mean_acc] 1.0000 63 | 64 | [iter] 9 65 | [loss] 5761.4098 66 | [trn_acc] 0.9600 67 | [val_acc] 1.0000 68 | [val_same_acc] 1.0000 69 | [val_mean_acc] 1.0000 70 | 71 | [FINAL val_acc] 1.0000 72 | [FINAL tst_acc] 0.9800 73 | -------------------------------------------------------------------------------- /exp/cls_bird_hint/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../cls.py \ 4 | --task=birds \ 5 | --learning_rate 0.0001 \ 6 | --predict_hyp=true \ 7 | --infer_hyp=true \ 8 | --train \ 9 | --n_epochs=30 \ 10 | --test \ 11 | > train.out \ 12 | 2> train.err 13 | 14 | -------------------------------------------------------------------------------- /exp/cls_bird_hint/train.err: -------------------------------------------------------------------------------- 1 | 2017-10-04 11:24:36.813683: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-10-04 11:24:36.813708: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-10-04 11:24:36.813712: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-10-04 11:24:36.813714: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-10-04 11:24:36.813717: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-10-04 11:24:38.821444: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-10-04 11:24:38.821863: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-10-04 11:24:38.821875: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-10-04 11:24:38.821879: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-10-04 11:24:38.821886: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | 2017-10-04 11:24:39.475582: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 2811 get requests, put_count=2471 evicted_count=1000 eviction_rate=0.404694 and unsatisfied allocation rate=0.512273 17 | 2017-10-04 11:24:39.475609: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 100 to 110 18 | 2017-10-04 11:24:42.007597: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 2779 get requests, put_count=3093 evicted_count=1000 eviction_rate=0.323311 and unsatisfied allocation rate=0.255128 19 | 2017-10-04 11:24:42.007619: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 256 to 281 20 | 2017-10-04 11:24:47.501118: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 15443 get requests, put_count=15407 evicted_count=1000 eviction_rate=0.0649056 and unsatisfied allocation rate=0.0709059 21 | 2017-10-04 11:24:47.501144: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 655 to 720 22 | 2017-10-04 11:28:26.114656: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 1778243 get requests, put_count=1778216 evicted_count=3000 eviction_rate=0.00168708 and unsatisfied allocation rate=0.00177366 23 | 2017-10-04 11:36:08.517655: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 7055682 get requests, put_count=7055710 evicted_count=13000 eviction_rate=0.00184248 and unsatisfied allocation rate=0.00185652 24 | -------------------------------------------------------------------------------- /exp/cls_bird_hint/train.out: -------------------------------------------------------------------------------- 1 | [iter] 0 2 | [loss] 17989.5978 3 | [trn_acc] 0.5000 4 | [val_acc] 0.5200 5 | [val_same_acc] 0.5200 6 | [val_mean_acc] 0.5200 7 | 8 | [iter] 1 9 | [loss] 8263.7306 10 | [trn_acc] 0.7400 11 | [val_acc] 0.7200 12 | [val_same_acc] 0.7200 13 | [val_mean_acc] 0.7200 14 | 15 | [iter] 2 16 | [loss] 7786.5046 17 | [trn_acc] 0.6500 18 | [val_acc] 0.5200 19 | [val_same_acc] 0.5200 20 | [val_mean_acc] 0.5200 21 | 22 | [iter] 3 23 | [loss] 7209.2556 24 | [trn_acc] 0.7800 25 | [val_acc] 0.7400 26 | [val_same_acc] 0.7400 27 | [val_mean_acc] 0.7400 28 | 29 | [iter] 4 30 | [loss] 6962.1927 31 | [trn_acc] 0.8000 32 | [val_acc] 0.7800 33 | [val_same_acc] 0.7800 34 | [val_mean_acc] 0.7800 35 | 36 | [iter] 5 37 | [loss] 6724.4684 38 | [trn_acc] 0.7600 39 | [val_acc] 0.6800 40 | [val_same_acc] 0.6800 41 | [val_mean_acc] 0.6800 42 | 43 | [iter] 6 44 | [loss] 6357.8601 45 | [trn_acc] 0.7800 46 | [val_acc] 0.7800 47 | [val_same_acc] 0.7800 48 | [val_mean_acc] 0.7800 49 | 50 | [iter] 7 51 | [loss] 5979.6020 52 | [trn_acc] 0.7600 53 | [val_acc] 0.7400 54 | [val_same_acc] 0.7400 55 | [val_mean_acc] 0.7400 56 | 57 | [iter] 8 58 | [loss] 5672.1243 59 | [trn_acc] 0.7500 60 | [val_acc] 0.8000 61 | [val_same_acc] 0.8000 62 | [val_mean_acc] 0.8000 63 | 64 | [iter] 9 65 | [loss] 5470.1555 66 | [trn_acc] 0.7600 67 | [val_acc] 0.8000 68 | [val_same_acc] 0.8000 69 | [val_mean_acc] 0.8000 70 | 71 | [iter] 10 72 | [loss] 5229.9220 73 | [trn_acc] 0.7700 74 | [val_acc] 0.8000 75 | [val_same_acc] 0.8000 76 | [val_mean_acc] 0.8000 77 | 78 | [iter] 11 79 | [loss] 5104.8033 80 | [trn_acc] 0.8400 81 | [val_acc] 0.7400 82 | [val_same_acc] 0.7400 83 | [val_mean_acc] 0.7400 84 | 85 | [iter] 12 86 | [loss] 4905.1615 87 | [trn_acc] 0.7800 88 | [val_acc] 0.8000 89 | [val_same_acc] 0.8000 90 | [val_mean_acc] 0.8000 91 | 92 | [iter] 13 93 | [loss] 4851.6408 94 | [trn_acc] 0.8000 95 | [val_acc] 0.7800 96 | [val_same_acc] 0.7800 97 | [val_mean_acc] 0.7800 98 | 99 | [iter] 14 100 | [loss] 4784.5258 101 | [trn_acc] 0.8000 102 | [val_acc] 0.8000 103 | [val_same_acc] 0.8000 104 | [val_mean_acc] 0.8000 105 | 106 | [iter] 15 107 | [loss] 4668.3424 108 | [trn_acc] 0.7900 109 | [val_acc] 0.7400 110 | [val_same_acc] 0.7400 111 | [val_mean_acc] 0.7400 112 | 113 | [iter] 16 114 | [loss] 4576.9065 115 | [trn_acc] 0.8700 116 | [val_acc] 0.8200 117 | [val_same_acc] 0.8200 118 | [val_mean_acc] 0.8200 119 | 120 | [iter] 17 121 | [loss] 4531.2519 122 | [trn_acc] 0.7600 123 | [val_acc] 0.7800 124 | [val_same_acc] 0.7800 125 | [val_mean_acc] 0.7800 126 | 127 | [iter] 18 128 | [loss] 4449.1543 129 | [trn_acc] 0.8500 130 | [val_acc] 0.8200 131 | [val_same_acc] 0.8200 132 | [val_mean_acc] 0.8200 133 | 134 | [iter] 19 135 | [loss] 4386.0174 136 | [trn_acc] 0.8700 137 | [val_acc] 0.7000 138 | [val_same_acc] 0.7000 139 | [val_mean_acc] 0.7000 140 | 141 | [iter] 20 142 | [loss] 4284.9040 143 | [trn_acc] 0.8700 144 | [val_acc] 0.8400 145 | [val_same_acc] 0.8400 146 | [val_mean_acc] 0.8400 147 | 148 | [iter] 21 149 | [loss] 4279.6747 150 | [trn_acc] 0.8100 151 | [val_acc] 0.8200 152 | [val_same_acc] 0.8200 153 | [val_mean_acc] 0.8200 154 | 155 | [iter] 22 156 | [loss] 4247.2744 157 | [trn_acc] 0.8300 158 | [val_acc] 0.7800 159 | [val_same_acc] 0.7800 160 | [val_mean_acc] 0.7800 161 | 162 | [iter] 23 163 | [loss] 4205.0676 164 | [trn_acc] 0.7200 165 | [val_acc] 0.8200 166 | [val_same_acc] 0.8200 167 | [val_mean_acc] 0.8200 168 | 169 | [iter] 24 170 | [loss] 4147.7798 171 | [trn_acc] 0.8400 172 | [val_acc] 0.7400 173 | [val_same_acc] 0.7400 174 | [val_mean_acc] 0.7400 175 | 176 | [iter] 25 177 | [loss] 4110.5965 178 | [trn_acc] 0.8300 179 | [val_acc] 0.8000 180 | [val_same_acc] 0.8000 181 | [val_mean_acc] 0.8000 182 | 183 | [iter] 26 184 | [loss] 4095.1002 185 | [trn_acc] 0.8400 186 | [val_acc] 0.7800 187 | [val_same_acc] 0.7800 188 | [val_mean_acc] 0.7800 189 | 190 | [iter] 27 191 | [loss] 4001.1533 192 | [trn_acc] 0.8600 193 | [val_acc] 0.8600 194 | [val_same_acc] 0.8600 195 | [val_mean_acc] 0.8600 196 | 197 | [iter] 28 198 | [loss] 4004.8759 199 | [trn_acc] 0.8000 200 | [val_acc] 0.7200 201 | [val_same_acc] 0.7200 202 | [val_mean_acc] 0.7200 203 | 204 | [iter] 29 205 | [loss] 3947.5688 206 | [trn_acc] 0.8800 207 | [val_acc] 0.7800 208 | [val_same_acc] 0.7800 209 | [val_mean_acc] 0.7800 210 | 211 | [FINAL val_acc] 0.7800 212 | [FINAL tst_acc] 0.9000 213 | -------------------------------------------------------------------------------- /exp/cls_bird_sim/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../cls.py \ 4 | --task=birds \ 5 | --model=sim \ 6 | --learning_rate 0.001 \ 7 | --train \ 8 | --n_epochs=10 \ 9 | --test \ 10 | > train.out \ 11 | 2> train.err 12 | 13 | -------------------------------------------------------------------------------- /exp/cls_bird_sim/train.err: -------------------------------------------------------------------------------- 1 | 2017-10-04 12:13:53.960558: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-10-04 12:13:53.960583: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-10-04 12:13:53.960587: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-10-04 12:13:53.960589: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-10-04 12:13:53.960592: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-10-04 12:13:55.965049: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-10-04 12:13:55.965452: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-10-04 12:13:55.965463: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-10-04 12:13:55.965466: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-10-04 12:13:55.965473: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | -------------------------------------------------------------------------------- /exp/cls_bird_sim/train.out: -------------------------------------------------------------------------------- 1 | [iter] 0 2 | [loss] 5942.4493 3 | [trn_acc] 0.6000 4 | [val_acc] 0.6000 5 | [val_same_acc] 0.6000 6 | [val_mean_acc] 0.6000 7 | 8 | [iter] 1 9 | [loss] 5869.8213 10 | [trn_acc] 0.8300 11 | [val_acc] 0.8000 12 | [val_same_acc] 0.8000 13 | [val_mean_acc] 0.8000 14 | 15 | [iter] 2 16 | [loss] 5834.8512 17 | [trn_acc] 0.9000 18 | [val_acc] 0.9000 19 | [val_same_acc] 0.9000 20 | [val_mean_acc] 0.9000 21 | 22 | [iter] 3 23 | [loss] 5791.9794 24 | [trn_acc] 0.9200 25 | [val_acc] 1.0000 26 | [val_same_acc] 1.0000 27 | [val_mean_acc] 1.0000 28 | 29 | [iter] 4 30 | [loss] 5776.5085 31 | [trn_acc] 0.9300 32 | [val_acc] 1.0000 33 | [val_same_acc] 1.0000 34 | [val_mean_acc] 1.0000 35 | 36 | [iter] 5 37 | [loss] 5767.6663 38 | [trn_acc] 0.9900 39 | [val_acc] 1.0000 40 | [val_same_acc] 1.0000 41 | [val_mean_acc] 1.0000 42 | 43 | [iter] 6 44 | [loss] 5754.3891 45 | [trn_acc] 0.9500 46 | [val_acc] 1.0000 47 | [val_same_acc] 1.0000 48 | [val_mean_acc] 1.0000 49 | 50 | [iter] 7 51 | [loss] 5769.9242 52 | [trn_acc] 0.9600 53 | [val_acc] 1.0000 54 | [val_same_acc] 1.0000 55 | [val_mean_acc] 1.0000 56 | 57 | [iter] 8 58 | [loss] 5756.5457 59 | [trn_acc] 0.9900 60 | [val_acc] 1.0000 61 | [val_same_acc] 1.0000 62 | [val_mean_acc] 1.0000 63 | 64 | [iter] 9 65 | [loss] 5761.4099 66 | [trn_acc] 0.9600 67 | [val_acc] 1.0000 68 | [val_same_acc] 1.0000 69 | [val_mean_acc] 1.0000 70 | 71 | [FINAL val_acc] 1.0000 72 | [FINAL tst_acc] 0.9800 73 | -------------------------------------------------------------------------------- /exp/cls_ex/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export PYTHONPATH=".:../../../shapeworld" 4 | 5 | python -u ../../cls.py \ 6 | --learning_rate 0.0001 \ 7 | --predict_hyp=false \ 8 | --infer_hyp=false \ 9 | --train \ 10 | --n_epochs=53 \ 11 | --test \ 12 | --test_same \ 13 | --augment=true \ 14 | > train.out \ 15 | 2> train.err 16 | 17 | -------------------------------------------------------------------------------- /exp/cls_ex/train.err: -------------------------------------------------------------------------------- 1 | 2017-09-23 14:22:44.605525: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-09-23 14:22:44.605550: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-09-23 14:22:44.605554: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-09-23 14:22:44.605556: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-09-23 14:22:44.605559: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-09-23 14:22:46.608843: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-09-23 14:22:46.609526: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-09-23 14:22:46.609538: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-09-23 14:22:46.609541: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-09-23 14:22:46.609548: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | -------------------------------------------------------------------------------- /exp/cls_ex/train.out: -------------------------------------------------------------------------------- 1 | [iter] 0 2 | [loss] 129.1383 3 | [trn_acc] 0.5800 4 | [val_acc] 0.4940 5 | [val_same_acc] 0.4760 6 | [val_mean_acc] 0.4850 7 | 8 | [iter] 1 9 | [loss] 71.9788 10 | [trn_acc] 0.5600 11 | [val_acc] 0.5340 12 | [val_same_acc] 0.4920 13 | [val_mean_acc] 0.5130 14 | 15 | [iter] 2 16 | [loss] 66.4642 17 | [trn_acc] 0.6600 18 | [val_acc] 0.5640 19 | [val_same_acc] 0.5360 20 | [val_mean_acc] 0.5500 21 | 22 | [iter] 3 23 | [loss] 56.0357 24 | [trn_acc] 0.7600 25 | [val_acc] 0.5440 26 | [val_same_acc] 0.5820 27 | [val_mean_acc] 0.5630 28 | 29 | [iter] 4 30 | [loss] 50.8504 31 | [trn_acc] 0.6200 32 | [val_acc] 0.5360 33 | [val_same_acc] 0.5600 34 | [val_mean_acc] 0.5480 35 | 36 | [iter] 5 37 | [loss] 48.6963 38 | [trn_acc] 0.7900 39 | [val_acc] 0.5600 40 | [val_same_acc] 0.5980 41 | [val_mean_acc] 0.5790 42 | 43 | [iter] 6 44 | [loss] 44.5170 45 | [trn_acc] 0.7600 46 | [val_acc] 0.5640 47 | [val_same_acc] 0.5680 48 | [val_mean_acc] 0.5660 49 | 50 | [iter] 7 51 | [loss] 42.6294 52 | [trn_acc] 0.7800 53 | [val_acc] 0.5500 54 | [val_same_acc] 0.5920 55 | [val_mean_acc] 0.5710 56 | 57 | [iter] 8 58 | [loss] 42.3114 59 | [trn_acc] 0.8200 60 | [val_acc] 0.5460 61 | [val_same_acc] 0.5820 62 | [val_mean_acc] 0.5640 63 | 64 | [iter] 9 65 | [loss] 39.4681 66 | [trn_acc] 0.8300 67 | [val_acc] 0.5240 68 | [val_same_acc] 0.5540 69 | [val_mean_acc] 0.5390 70 | 71 | [iter] 10 72 | [loss] 36.9068 73 | [trn_acc] 0.6600 74 | [val_acc] 0.6060 75 | [val_same_acc] 0.5860 76 | [val_mean_acc] 0.5960 77 | 78 | [iter] 11 79 | [loss] 38.3862 80 | [trn_acc] 0.7500 81 | [val_acc] 0.5280 82 | [val_same_acc] 0.5560 83 | [val_mean_acc] 0.5420 84 | 85 | [iter] 12 86 | [loss] 35.2492 87 | [trn_acc] 0.8800 88 | [val_acc] 0.5620 89 | [val_same_acc] 0.5860 90 | [val_mean_acc] 0.5740 91 | 92 | [iter] 13 93 | [loss] 35.4934 94 | [trn_acc] 0.9000 95 | [val_acc] 0.5640 96 | [val_same_acc] 0.5860 97 | [val_mean_acc] 0.5750 98 | 99 | [iter] 14 100 | [loss] 35.0367 101 | [trn_acc] 0.8700 102 | [val_acc] 0.5520 103 | [val_same_acc] 0.5480 104 | [val_mean_acc] 0.5500 105 | 106 | [iter] 15 107 | [loss] 33.8650 108 | [trn_acc] 0.8200 109 | [val_acc] 0.5800 110 | [val_same_acc] 0.5820 111 | [val_mean_acc] 0.5810 112 | 113 | [iter] 16 114 | [loss] 32.7650 115 | [trn_acc] 0.8600 116 | [val_acc] 0.5960 117 | [val_same_acc] 0.5860 118 | [val_mean_acc] 0.5910 119 | 120 | [iter] 17 121 | [loss] 32.5319 122 | [trn_acc] 0.8300 123 | [val_acc] 0.5240 124 | [val_same_acc] 0.5420 125 | [val_mean_acc] 0.5330 126 | 127 | [iter] 18 128 | [loss] 31.9664 129 | [trn_acc] 0.8600 130 | [val_acc] 0.5780 131 | [val_same_acc] 0.5900 132 | [val_mean_acc] 0.5840 133 | 134 | [iter] 19 135 | [loss] 30.6675 136 | [trn_acc] 0.9000 137 | [val_acc] 0.5740 138 | [val_same_acc] 0.5940 139 | [val_mean_acc] 0.5840 140 | 141 | [iter] 20 142 | [loss] 29.7720 143 | [trn_acc] 0.8800 144 | [val_acc] 0.6020 145 | [val_same_acc] 0.5860 146 | [val_mean_acc] 0.5940 147 | 148 | [iter] 21 149 | [loss] 28.4020 150 | [trn_acc] 0.8800 151 | [val_acc] 0.6160 152 | [val_same_acc] 0.5920 153 | [val_mean_acc] 0.6040 154 | 155 | [iter] 22 156 | [loss] 29.1965 157 | [trn_acc] 0.8700 158 | [val_acc] 0.5560 159 | [val_same_acc] 0.5780 160 | [val_mean_acc] 0.5670 161 | 162 | [iter] 23 163 | [loss] 28.4645 164 | [trn_acc] 0.8600 165 | [val_acc] 0.6100 166 | [val_same_acc] 0.6040 167 | [val_mean_acc] 0.6070 168 | 169 | [iter] 24 170 | [loss] 27.9859 171 | [trn_acc] 0.8900 172 | [val_acc] 0.5720 173 | [val_same_acc] 0.5760 174 | [val_mean_acc] 0.5740 175 | 176 | [iter] 25 177 | [loss] 27.8851 178 | [trn_acc] 0.9000 179 | [val_acc] 0.5880 180 | [val_same_acc] 0.5700 181 | [val_mean_acc] 0.5790 182 | 183 | [iter] 26 184 | [loss] 26.7233 185 | [trn_acc] 0.9200 186 | [val_acc] 0.5600 187 | [val_same_acc] 0.5700 188 | [val_mean_acc] 0.5650 189 | 190 | [iter] 27 191 | [loss] 25.8744 192 | [trn_acc] 0.9400 193 | [val_acc] 0.5520 194 | [val_same_acc] 0.5700 195 | [val_mean_acc] 0.5610 196 | 197 | [iter] 28 198 | [loss] 25.6765 199 | [trn_acc] 0.9600 200 | [val_acc] 0.5880 201 | [val_same_acc] 0.6080 202 | [val_mean_acc] 0.5980 203 | 204 | [iter] 29 205 | [loss] 25.9578 206 | [trn_acc] 0.9100 207 | [val_acc] 0.5960 208 | [val_same_acc] 0.6000 209 | [val_mean_acc] 0.5980 210 | 211 | [iter] 30 212 | [loss] 25.4068 213 | [trn_acc] 0.9600 214 | [val_acc] 0.5880 215 | [val_same_acc] 0.6080 216 | [val_mean_acc] 0.5980 217 | 218 | [iter] 31 219 | [loss] 24.6175 220 | [trn_acc] 0.8700 221 | [val_acc] 0.5480 222 | [val_same_acc] 0.5740 223 | [val_mean_acc] 0.5610 224 | 225 | [iter] 32 226 | [loss] 24.0660 227 | [trn_acc] 0.8500 228 | [val_acc] 0.5940 229 | [val_same_acc] 0.6100 230 | [val_mean_acc] 0.6020 231 | 232 | [iter] 33 233 | [loss] 24.3847 234 | [trn_acc] 0.9200 235 | [val_acc] 0.5720 236 | [val_same_acc] 0.6160 237 | [val_mean_acc] 0.5940 238 | 239 | [iter] 34 240 | [loss] 23.2592 241 | [trn_acc] 0.9100 242 | [val_acc] 0.5500 243 | [val_same_acc] 0.5720 244 | [val_mean_acc] 0.5610 245 | 246 | [iter] 35 247 | [loss] 23.5982 248 | [trn_acc] 0.9200 249 | [val_acc] 0.5420 250 | [val_same_acc] 0.5660 251 | [val_mean_acc] 0.5540 252 | 253 | [iter] 36 254 | [loss] 24.4046 255 | [trn_acc] 0.9200 256 | [val_acc] 0.5980 257 | [val_same_acc] 0.5780 258 | [val_mean_acc] 0.5880 259 | 260 | [iter] 37 261 | [loss] 22.9960 262 | [trn_acc] 0.8800 263 | [val_acc] 0.6220 264 | [val_same_acc] 0.6000 265 | [val_mean_acc] 0.6110 266 | 267 | [iter] 38 268 | [loss] 22.3939 269 | [trn_acc] 0.9000 270 | [val_acc] 0.6040 271 | [val_same_acc] 0.6080 272 | [val_mean_acc] 0.6060 273 | 274 | [iter] 39 275 | [loss] 23.5954 276 | [trn_acc] 0.9300 277 | [val_acc] 0.5720 278 | [val_same_acc] 0.5940 279 | [val_mean_acc] 0.5830 280 | 281 | [iter] 40 282 | [loss] 22.5681 283 | [trn_acc] 0.9100 284 | [val_acc] 0.5820 285 | [val_same_acc] 0.6160 286 | [val_mean_acc] 0.5990 287 | 288 | [iter] 41 289 | [loss] 21.8148 290 | [trn_acc] 0.8600 291 | [val_acc] 0.6240 292 | [val_same_acc] 0.6200 293 | [val_mean_acc] 0.6220 294 | 295 | [iter] 42 296 | [loss] 22.0357 297 | [trn_acc] 0.9300 298 | [val_acc] 0.5560 299 | [val_same_acc] 0.5720 300 | [val_mean_acc] 0.5640 301 | 302 | [iter] 43 303 | [loss] 20.9007 304 | [trn_acc] 0.9300 305 | [val_acc] 0.5380 306 | [val_same_acc] 0.5680 307 | [val_mean_acc] 0.5530 308 | 309 | [iter] 44 310 | [loss] 21.3347 311 | [trn_acc] 0.9200 312 | [val_acc] 0.6080 313 | [val_same_acc] 0.6020 314 | [val_mean_acc] 0.6050 315 | 316 | [iter] 45 317 | [loss] 20.2569 318 | [trn_acc] 0.9100 319 | [val_acc] 0.6040 320 | [val_same_acc] 0.5740 321 | [val_mean_acc] 0.5890 322 | 323 | [iter] 46 324 | [loss] 20.5533 325 | [trn_acc] 0.9300 326 | [val_acc] 0.5760 327 | [val_same_acc] 0.5780 328 | [val_mean_acc] 0.5770 329 | 330 | [iter] 47 331 | [loss] 22.0645 332 | [trn_acc] 0.9500 333 | [val_acc] 0.5700 334 | [val_same_acc] 0.5680 335 | [val_mean_acc] 0.5690 336 | 337 | [iter] 48 338 | [loss] 20.5115 339 | [trn_acc] 0.9100 340 | [val_acc] 0.5660 341 | [val_same_acc] 0.5900 342 | [val_mean_acc] 0.5780 343 | 344 | [iter] 49 345 | [loss] 20.4833 346 | [trn_acc] 0.8900 347 | [val_acc] 0.5600 348 | [val_same_acc] 0.5720 349 | [val_mean_acc] 0.5660 350 | 351 | [iter] 50 352 | [loss] 20.1416 353 | [trn_acc] 0.9400 354 | [val_acc] 0.5840 355 | [val_same_acc] 0.5980 356 | [val_mean_acc] 0.5910 357 | 358 | [iter] 51 359 | [loss] 19.7317 360 | [trn_acc] 0.9400 361 | [val_acc] 0.5900 362 | [val_same_acc] 0.5860 363 | [val_mean_acc] 0.5880 364 | 365 | [iter] 52 366 | [loss] 20.5963 367 | [trn_acc] 0.9400 368 | [val_acc] 0.6180 369 | [val_same_acc] 0.6300 370 | [val_mean_acc] 0.6240 371 | 372 | [FINAL val_acc] 0.6180 373 | [FINAL tst_acc] 0.6400 374 | [FINAL val_same_acc] 0.6300 375 | [FINAL tst_same_acc] 0.6320 376 | -------------------------------------------------------------------------------- /exp/cls_gold/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export PYTHONPATH=".:../../../shapeworld" 4 | 5 | python -u ../../cls.py \ 6 | --learning_rate=0.0001 \ 7 | --train \ 8 | --n_epochs=45 \ 9 | --predict_hyp=true \ 10 | --infer_hyp=true \ 11 | --test \ 12 | --test_same \ 13 | --use_true_hyp=true \ 14 | --augment \ 15 | > train_gold.out \ 16 | 2> train_gold.err 17 | -------------------------------------------------------------------------------- /exp/cls_gold/train_gold.err: -------------------------------------------------------------------------------- 1 | 2017-10-07 12:25:51.699860: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-10-07 12:25:51.699884: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-10-07 12:25:51.699887: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-10-07 12:25:51.699890: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-10-07 12:25:51.699893: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-10-07 12:25:53.696778: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-10-07 12:25:53.697181: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-10-07 12:25:53.697930: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-10-07 12:25:53.697938: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-10-07 12:25:53.697947: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | 2017-10-07 12:25:54.303448: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 3367 get requests, put_count=3005 evicted_count=1000 eviction_rate=0.332779 and unsatisfied allocation rate=0.434214 17 | 2017-10-07 12:25:54.303495: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 100 to 110 18 | 2017-10-07 12:25:56.506234: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 4623 get requests, put_count=4646 evicted_count=1000 eviction_rate=0.215239 and unsatisfied allocation rate=0.21631 19 | 2017-10-07 12:25:56.506258: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 256 to 281 20 | 2017-10-07 12:27:44.518296: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 1208015 get requests, put_count=1208016 evicted_count=2000 eviction_rate=0.00165561 and unsatisfied allocation rate=0.00169948 21 | -------------------------------------------------------------------------------- /exp/cls_gold/train_gold.out: -------------------------------------------------------------------------------- 1 | [iter] 0 2 | [loss] 2343.9908 3 | [trn_acc] 0.5700 4 | [val_acc] 0.5340 5 | [val_same_acc] 0.5820 6 | [val_mean_acc] 0.5580 7 | 8 | [iter] 1 9 | [loss] 1097.3443 10 | [trn_acc] 0.5800 11 | [val_acc] 0.6240 12 | [val_same_acc] 0.5500 13 | [val_mean_acc] 0.5870 14 | 15 | [iter] 2 16 | [loss] 975.2225 17 | [trn_acc] 0.6600 18 | [val_acc] 0.6620 19 | [val_same_acc] 0.6560 20 | [val_mean_acc] 0.6590 21 | 22 | [iter] 3 23 | [loss] 913.4021 24 | [trn_acc] 0.6700 25 | [val_acc] 0.6420 26 | [val_same_acc] 0.6820 27 | [val_mean_acc] 0.6620 28 | 29 | [iter] 4 30 | [loss] 866.0761 31 | [trn_acc] 0.6900 32 | [val_acc] 0.6680 33 | [val_same_acc] 0.6980 34 | [val_mean_acc] 0.6830 35 | 36 | [iter] 5 37 | [loss] 831.8925 38 | [trn_acc] 0.7300 39 | [val_acc] 0.7080 40 | [val_same_acc] 0.6980 41 | [val_mean_acc] 0.7030 42 | 43 | [iter] 6 44 | [loss] 808.2402 45 | [trn_acc] 0.6600 46 | [val_acc] 0.7140 47 | [val_same_acc] 0.7300 48 | [val_mean_acc] 0.7220 49 | 50 | [iter] 7 51 | [loss] 780.4193 52 | [trn_acc] 0.6700 53 | [val_acc] 0.7240 54 | [val_same_acc] 0.7080 55 | [val_mean_acc] 0.7160 56 | 57 | [iter] 8 58 | [loss] 762.2854 59 | [trn_acc] 0.7100 60 | [val_acc] 0.7500 61 | [val_same_acc] 0.7360 62 | [val_mean_acc] 0.7430 63 | 64 | [iter] 9 65 | [loss] 744.4249 66 | [trn_acc] 0.7800 67 | [val_acc] 0.7260 68 | [val_same_acc] 0.7320 69 | [val_mean_acc] 0.7290 70 | 71 | [iter] 10 72 | [loss] 734.4964 73 | [trn_acc] 0.8100 74 | [val_acc] 0.7340 75 | [val_same_acc] 0.7200 76 | [val_mean_acc] 0.7270 77 | 78 | [iter] 11 79 | [loss] 718.9864 80 | [trn_acc] 0.7600 81 | [val_acc] 0.7240 82 | [val_same_acc] 0.7400 83 | [val_mean_acc] 0.7320 84 | 85 | [iter] 12 86 | [loss] 707.6658 87 | [trn_acc] 0.7700 88 | [val_acc] 0.7500 89 | [val_same_acc] 0.7440 90 | [val_mean_acc] 0.7470 91 | 92 | [iter] 13 93 | [loss] 691.7978 94 | [trn_acc] 0.7100 95 | [val_acc] 0.7260 96 | [val_same_acc] 0.7180 97 | [val_mean_acc] 0.7220 98 | 99 | [iter] 14 100 | [loss] 684.6654 101 | [trn_acc] 0.7300 102 | [val_acc] 0.7360 103 | [val_same_acc] 0.7340 104 | [val_mean_acc] 0.7350 105 | 106 | [iter] 15 107 | [loss] 674.0890 108 | [trn_acc] 0.8000 109 | [val_acc] 0.7720 110 | [val_same_acc] 0.7620 111 | [val_mean_acc] 0.7670 112 | 113 | [iter] 16 114 | [loss] 667.6520 115 | [trn_acc] 0.7400 116 | [val_acc] 0.7520 117 | [val_same_acc] 0.7440 118 | [val_mean_acc] 0.7480 119 | 120 | [iter] 17 121 | [loss] 658.0706 122 | [trn_acc] 0.7600 123 | [val_acc] 0.7700 124 | [val_same_acc] 0.7460 125 | [val_mean_acc] 0.7580 126 | 127 | [iter] 18 128 | [loss] 648.3880 129 | [trn_acc] 0.7800 130 | [val_acc] 0.7620 131 | [val_same_acc] 0.7360 132 | [val_mean_acc] 0.7490 133 | 134 | [iter] 19 135 | [loss] 643.9387 136 | [trn_acc] 0.8000 137 | [val_acc] 0.7580 138 | [val_same_acc] 0.7380 139 | [val_mean_acc] 0.7480 140 | 141 | [iter] 20 142 | [loss] 634.8131 143 | [trn_acc] 0.8200 144 | [val_acc] 0.7700 145 | [val_same_acc] 0.7320 146 | [val_mean_acc] 0.7510 147 | 148 | [iter] 21 149 | [loss] 630.5602 150 | [trn_acc] 0.7900 151 | [val_acc] 0.7700 152 | [val_same_acc] 0.7380 153 | [val_mean_acc] 0.7540 154 | 155 | [iter] 22 156 | [loss] 616.4227 157 | [trn_acc] 0.8400 158 | [val_acc] 0.7640 159 | [val_same_acc] 0.7780 160 | [val_mean_acc] 0.7710 161 | 162 | [iter] 23 163 | [loss] 612.3155 164 | [trn_acc] 0.8200 165 | [val_acc] 0.7520 166 | [val_same_acc] 0.7760 167 | [val_mean_acc] 0.7640 168 | 169 | [iter] 24 170 | [loss] 610.0698 171 | [trn_acc] 0.8900 172 | [val_acc] 0.7640 173 | [val_same_acc] 0.7620 174 | [val_mean_acc] 0.7630 175 | 176 | [iter] 25 177 | [loss] 598.8083 178 | [trn_acc] 0.7900 179 | [val_acc] 0.7800 180 | [val_same_acc] 0.7560 181 | [val_mean_acc] 0.7680 182 | 183 | [iter] 26 184 | [loss] 596.4248 185 | [trn_acc] 0.8400 186 | [val_acc] 0.7820 187 | [val_same_acc] 0.7560 188 | [val_mean_acc] 0.7690 189 | 190 | [iter] 27 191 | [loss] 590.9931 192 | [trn_acc] 0.8100 193 | [val_acc] 0.7740 194 | [val_same_acc] 0.7640 195 | [val_mean_acc] 0.7690 196 | 197 | [iter] 28 198 | [loss] 581.8700 199 | [trn_acc] 0.8000 200 | [val_acc] 0.7820 201 | [val_same_acc] 0.7560 202 | [val_mean_acc] 0.7690 203 | 204 | [iter] 29 205 | [loss] 578.6030 206 | [trn_acc] 0.8800 207 | [val_acc] 0.7980 208 | [val_same_acc] 0.7700 209 | [val_mean_acc] 0.7840 210 | 211 | [iter] 30 212 | [loss] 572.9467 213 | [trn_acc] 0.7700 214 | [val_acc] 0.7880 215 | [val_same_acc] 0.7640 216 | [val_mean_acc] 0.7760 217 | 218 | [iter] 31 219 | [loss] 570.2735 220 | [trn_acc] 0.8800 221 | [val_acc] 0.7860 222 | [val_same_acc] 0.7700 223 | [val_mean_acc] 0.7780 224 | 225 | [iter] 32 226 | [loss] 562.6251 227 | [trn_acc] 0.8400 228 | [val_acc] 0.7800 229 | [val_same_acc] 0.7520 230 | [val_mean_acc] 0.7660 231 | 232 | [iter] 33 233 | [loss] 555.5009 234 | [trn_acc] 0.8400 235 | [val_acc] 0.7780 236 | [val_same_acc] 0.7700 237 | [val_mean_acc] 0.7740 238 | 239 | [iter] 34 240 | [loss] 549.5878 241 | [trn_acc] 0.9000 242 | [val_acc] 0.7980 243 | [val_same_acc] 0.7620 244 | [val_mean_acc] 0.7800 245 | 246 | [iter] 35 247 | [loss] 548.5479 248 | [trn_acc] 0.8500 249 | [val_acc] 0.7960 250 | [val_same_acc] 0.7700 251 | [val_mean_acc] 0.7830 252 | 253 | [iter] 36 254 | [loss] 540.4464 255 | [trn_acc] 0.8100 256 | [val_acc] 0.7520 257 | [val_same_acc] 0.7640 258 | [val_mean_acc] 0.7580 259 | 260 | [iter] 37 261 | [loss] 537.4079 262 | [trn_acc] 0.8400 263 | [val_acc] 0.7880 264 | [val_same_acc] 0.7420 265 | [val_mean_acc] 0.7650 266 | 267 | [iter] 38 268 | [loss] 532.3928 269 | [trn_acc] 0.8300 270 | [val_acc] 0.8080 271 | [val_same_acc] 0.7520 272 | [val_mean_acc] 0.7800 273 | 274 | [iter] 39 275 | [loss] 529.0230 276 | [trn_acc] 0.8000 277 | [val_acc] 0.7940 278 | [val_same_acc] 0.7780 279 | [val_mean_acc] 0.7860 280 | 281 | [iter] 40 282 | [loss] 521.9027 283 | [trn_acc] 0.8600 284 | [val_acc] 0.7500 285 | [val_same_acc] 0.7420 286 | [val_mean_acc] 0.7460 287 | 288 | [iter] 41 289 | [loss] 520.1489 290 | [trn_acc] 0.8300 291 | [val_acc] 0.7800 292 | [val_same_acc] 0.7580 293 | [val_mean_acc] 0.7690 294 | 295 | [iter] 42 296 | [loss] 514.7426 297 | [trn_acc] 0.8100 298 | [val_acc] 0.7880 299 | [val_same_acc] 0.7580 300 | [val_mean_acc] 0.7730 301 | 302 | [iter] 43 303 | [loss] 520.1346 304 | [trn_acc] 0.8200 305 | [val_acc] 0.7880 306 | [val_same_acc] 0.7900 307 | [val_mean_acc] 0.7890 308 | 309 | [iter] 44 310 | [loss] 503.8756 311 | [trn_acc] 0.9100 312 | [val_acc] 0.8040 313 | [val_same_acc] 0.7680 314 | [val_mean_acc] 0.7860 315 | 316 | [FINAL val_acc] 0.8040 317 | [FINAL tst_acc] 0.7720 318 | [FINAL val_same_acc] 0.7680 319 | [FINAL tst_same_acc] 0.7820 320 | -------------------------------------------------------------------------------- /exp/cls_hint/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export PYTHONPATH=".:../../../shapeworld" 4 | 5 | python -u ../../cls.py \ 6 | --learning_rate 0.0001 \ 7 | --predict_hyp=true \ 8 | --infer_hyp=true \ 9 | --train \ 10 | --n_epochs=36 \ 11 | --test \ 12 | --test_same \ 13 | --n_sample_hyps=10 \ 14 | --use_true_hyp=false \ 15 | --augment=true \ 16 | > train.out \ 17 | 2> train.err 18 | 19 | -------------------------------------------------------------------------------- /exp/cls_hint/train.err: -------------------------------------------------------------------------------- 1 | 2017-10-04 12:14:42.838345: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-10-04 12:14:42.838370: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-10-04 12:14:42.838374: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-10-04 12:14:42.838377: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-10-04 12:14:42.838380: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-10-04 12:14:44.831417: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-10-04 12:14:44.831809: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-10-04 12:14:44.831820: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-10-04 12:14:44.831824: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-10-04 12:14:44.831831: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | 2017-10-04 12:14:45.442926: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 3371 get requests, put_count=3014 evicted_count=1000 eviction_rate=0.331785 and unsatisfied allocation rate=0.432216 17 | 2017-10-04 12:14:45.442950: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 100 to 110 18 | 2017-10-04 12:14:47.592444: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 4614 get requests, put_count=4619 evicted_count=1000 eviction_rate=0.216497 and unsatisfied allocation rate=0.220633 19 | 2017-10-04 12:14:47.592471: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 256 to 281 20 | 2017-10-04 12:18:05.208342: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 1079022 get requests, put_count=1079024 evicted_count=2000 eviction_rate=0.00185353 and unsatisfied allocation rate=0.00190172 21 | -------------------------------------------------------------------------------- /exp/cls_hint/train.out: -------------------------------------------------------------------------------- 1 | [iter] 0 2 | [loss] 2344.3659 3 | [trn_acc] 0.5200 4 | [val_acc] 0.5060 5 | [val_same_acc] 0.4920 6 | [val_mean_acc] 0.4990 7 | 8 | [iter] 1 9 | [loss] 1097.5105 10 | [trn_acc] 0.5600 11 | [val_acc] 0.5380 12 | [val_same_acc] 0.5080 13 | [val_mean_acc] 0.5230 14 | 15 | [iter] 2 16 | [loss] 975.2169 17 | [trn_acc] 0.6300 18 | [val_acc] 0.5660 19 | [val_same_acc] 0.5340 20 | [val_mean_acc] 0.5500 21 | 22 | [iter] 3 23 | [loss] 913.5962 24 | [trn_acc] 0.6600 25 | [val_acc] 0.5760 26 | [val_same_acc] 0.5760 27 | [val_mean_acc] 0.5760 28 | 29 | [iter] 4 30 | [loss] 866.0199 31 | [trn_acc] 0.7400 32 | [val_acc] 0.6380 33 | [val_same_acc] 0.6200 34 | [val_mean_acc] 0.6290 35 | 36 | [iter] 5 37 | [loss] 831.8363 38 | [trn_acc] 0.7900 39 | [val_acc] 0.6700 40 | [val_same_acc] 0.6140 41 | [val_mean_acc] 0.6420 42 | 43 | [iter] 6 44 | [loss] 807.9379 45 | [trn_acc] 0.7500 46 | [val_acc] 0.6580 47 | [val_same_acc] 0.6440 48 | [val_mean_acc] 0.6510 49 | 50 | [iter] 7 51 | [loss] 780.2131 52 | [trn_acc] 0.7100 53 | [val_acc] 0.6740 54 | [val_same_acc] 0.6360 55 | [val_mean_acc] 0.6550 56 | 57 | [iter] 8 58 | [loss] 762.1251 59 | [trn_acc] 0.8400 60 | [val_acc] 0.6640 61 | [val_same_acc] 0.6680 62 | [val_mean_acc] 0.6660 63 | 64 | [iter] 9 65 | [loss] 744.2238 66 | [trn_acc] 0.8300 67 | [val_acc] 0.6600 68 | [val_same_acc] 0.6360 69 | [val_mean_acc] 0.6480 70 | 71 | [iter] 10 72 | [loss] 734.2637 73 | [trn_acc] 0.8100 74 | [val_acc] 0.6820 75 | [val_same_acc] 0.6180 76 | [val_mean_acc] 0.6500 77 | 78 | [iter] 11 79 | [loss] 719.5335 80 | [trn_acc] 0.8100 81 | [val_acc] 0.7100 82 | [val_same_acc] 0.6700 83 | [val_mean_acc] 0.6900 84 | 85 | [iter] 12 86 | [loss] 707.7236 87 | [trn_acc] 0.8200 88 | [val_acc] 0.6940 89 | [val_same_acc] 0.6180 90 | [val_mean_acc] 0.6560 91 | 92 | [iter] 13 93 | [loss] 692.3785 94 | [trn_acc] 0.8900 95 | [val_acc] 0.6760 96 | [val_same_acc] 0.6520 97 | [val_mean_acc] 0.6640 98 | 99 | [iter] 14 100 | [loss] 685.3195 101 | [trn_acc] 0.7600 102 | [val_acc] 0.6640 103 | [val_same_acc] 0.6620 104 | [val_mean_acc] 0.6630 105 | 106 | [iter] 15 107 | [loss] 674.4640 108 | [trn_acc] 0.8400 109 | [val_acc] 0.6820 110 | [val_same_acc] 0.6740 111 | [val_mean_acc] 0.6780 112 | 113 | [iter] 16 114 | [loss] 668.0843 115 | [trn_acc] 0.7900 116 | [val_acc] 0.7000 117 | [val_same_acc] 0.6540 118 | [val_mean_acc] 0.6770 119 | 120 | [iter] 17 121 | [loss] 658.2875 122 | [trn_acc] 0.8700 123 | [val_acc] 0.6640 124 | [val_same_acc] 0.6640 125 | [val_mean_acc] 0.6640 126 | 127 | [iter] 18 128 | [loss] 648.9586 129 | [trn_acc] 0.8800 130 | [val_acc] 0.6740 131 | [val_same_acc] 0.6420 132 | [val_mean_acc] 0.6580 133 | 134 | [iter] 19 135 | [loss] 644.3238 136 | [trn_acc] 0.8400 137 | [val_acc] 0.7040 138 | [val_same_acc] 0.6660 139 | [val_mean_acc] 0.6850 140 | 141 | [iter] 20 142 | [loss] 634.9471 143 | [trn_acc] 0.8200 144 | [val_acc] 0.7000 145 | [val_same_acc] 0.6740 146 | [val_mean_acc] 0.6870 147 | 148 | [iter] 21 149 | [loss] 630.9394 150 | [trn_acc] 0.8200 151 | [val_acc] 0.6860 152 | [val_same_acc] 0.6560 153 | [val_mean_acc] 0.6710 154 | 155 | [iter] 22 156 | [loss] 616.9236 157 | [trn_acc] 0.8400 158 | [val_acc] 0.6680 159 | [val_same_acc] 0.6860 160 | [val_mean_acc] 0.6770 161 | 162 | [iter] 23 163 | [loss] 613.1513 164 | [trn_acc] 0.7900 165 | [val_acc] 0.6800 166 | [val_same_acc] 0.6760 167 | [val_mean_acc] 0.6780 168 | 169 | [iter] 24 170 | [loss] 610.0148 171 | [trn_acc] 0.8700 172 | [val_acc] 0.6740 173 | [val_same_acc] 0.6900 174 | [val_mean_acc] 0.6820 175 | 176 | [iter] 25 177 | [loss] 599.6041 178 | [trn_acc] 0.8500 179 | [val_acc] 0.6920 180 | [val_same_acc] 0.6580 181 | [val_mean_acc] 0.6750 182 | 183 | [iter] 26 184 | [loss] 597.2351 185 | [trn_acc] 0.8400 186 | [val_acc] 0.7000 187 | [val_same_acc] 0.6560 188 | [val_mean_acc] 0.6780 189 | 190 | [iter] 27 191 | [loss] 591.8149 192 | [trn_acc] 0.8700 193 | [val_acc] 0.6800 194 | [val_same_acc] 0.6980 195 | [val_mean_acc] 0.6890 196 | 197 | [iter] 28 198 | [loss] 583.2320 199 | [trn_acc] 0.8600 200 | [val_acc] 0.6820 201 | [val_same_acc] 0.6840 202 | [val_mean_acc] 0.6830 203 | 204 | [iter] 29 205 | [loss] 579.4193 206 | [trn_acc] 0.8500 207 | [val_acc] 0.7000 208 | [val_same_acc] 0.6560 209 | [val_mean_acc] 0.6780 210 | 211 | [iter] 30 212 | [loss] 574.1136 213 | [trn_acc] 0.8500 214 | [val_acc] 0.6780 215 | [val_same_acc] 0.7000 216 | [val_mean_acc] 0.6890 217 | 218 | [iter] 31 219 | [loss] 571.0754 220 | [trn_acc] 0.8600 221 | [val_acc] 0.6780 222 | [val_same_acc] 0.6800 223 | [val_mean_acc] 0.6790 224 | 225 | [iter] 32 226 | [loss] 563.4866 227 | [trn_acc] 0.8500 228 | [val_acc] 0.6960 229 | [val_same_acc] 0.6800 230 | [val_mean_acc] 0.6880 231 | 232 | [iter] 33 233 | [loss] 555.8092 234 | [trn_acc] 0.8500 235 | [val_acc] 0.6800 236 | [val_same_acc] 0.6680 237 | [val_mean_acc] 0.6740 238 | 239 | [iter] 34 240 | [loss] 551.0594 241 | [trn_acc] 0.9400 242 | [val_acc] 0.6780 243 | [val_same_acc] 0.6540 244 | [val_mean_acc] 0.6660 245 | 246 | [iter] 35 247 | [loss] 549.0658 248 | [trn_acc] 0.9000 249 | [val_acc] 0.7220 250 | [val_same_acc] 0.7040 251 | [val_mean_acc] 0.7130 252 | 253 | [FINAL val_acc] 0.7220 254 | [FINAL tst_acc] 0.7220 255 | [FINAL val_same_acc] 0.7040 256 | [FINAL tst_same_acc] 0.6740 257 | -------------------------------------------------------------------------------- /exp/cls_joint/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export PYTHONPATH=".:../../../shapeworld" 4 | 5 | # surprisingly, data augmentation hurts here 6 | 7 | python -u ../../cls.py \ 8 | --learning_rate 0.0001 \ 9 | --predict_hyp=true \ 10 | --infer_hyp=false \ 11 | --train \ 12 | --n_epochs=28 \ 13 | --test \ 14 | --test_same \ 15 | --augment=true \ 16 | > train.out \ 17 | 2> train.err 18 | 19 | -------------------------------------------------------------------------------- /exp/cls_joint/train.err: -------------------------------------------------------------------------------- 1 | 2017-09-23 15:18:08.915742: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-09-23 15:18:08.915767: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-09-23 15:18:08.915771: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-09-23 15:18:08.915774: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-09-23 15:18:08.915776: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-09-23 15:18:10.926612: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-09-23 15:18:10.927020: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-09-23 15:18:10.927031: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-09-23 15:18:10.927035: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-09-23 15:18:10.927042: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | 2017-09-23 15:18:11.529010: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 3164 get requests, put_count=2932 evicted_count=1000 eviction_rate=0.341064 and unsatisfied allocation rate=0.420986 17 | 2017-09-23 15:18:11.529035: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 100 to 110 18 | 2017-09-23 15:18:14.293381: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 6473 get requests, put_count=6566 evicted_count=1000 eviction_rate=0.1523 and unsatisfied allocation rate=0.143674 19 | 2017-09-23 15:18:14.293396: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 256 to 281 20 | -------------------------------------------------------------------------------- /exp/cls_joint/train.out: -------------------------------------------------------------------------------- 1 | [iter] 0 2 | [loss] 2635.1620 3 | [trn_acc] 0.5200 4 | [val_acc] 0.4880 5 | [val_same_acc] 0.4980 6 | [val_mean_acc] 0.4930 7 | 8 | [iter] 1 9 | [loss] 1225.9759 10 | [trn_acc] 0.5200 11 | [val_acc] 0.4980 12 | [val_same_acc] 0.4780 13 | [val_mean_acc] 0.4880 14 | 15 | [iter] 2 16 | [loss] 1067.5509 17 | [trn_acc] 0.6400 18 | [val_acc] 0.5380 19 | [val_same_acc] 0.5060 20 | [val_mean_acc] 0.5220 21 | 22 | [iter] 3 23 | [loss] 1026.5322 24 | [trn_acc] 0.7000 25 | [val_acc] 0.5140 26 | [val_same_acc] 0.5360 27 | [val_mean_acc] 0.5250 28 | 29 | [iter] 4 30 | [loss] 989.9436 31 | [trn_acc] 0.5900 32 | [val_acc] 0.5360 33 | [val_same_acc] 0.5540 34 | [val_mean_acc] 0.5450 35 | 36 | [iter] 5 37 | [loss] 963.5533 38 | [trn_acc] 0.7400 39 | [val_acc] 0.5660 40 | [val_same_acc] 0.5580 41 | [val_mean_acc] 0.5620 42 | 43 | [iter] 6 44 | [loss] 934.8146 45 | [trn_acc] 0.6000 46 | [val_acc] 0.5540 47 | [val_same_acc] 0.5220 48 | [val_mean_acc] 0.5380 49 | 50 | [iter] 7 51 | [loss] 901.3414 52 | [trn_acc] 0.8000 53 | [val_acc] 0.5640 54 | [val_same_acc] 0.5700 55 | [val_mean_acc] 0.5670 56 | 57 | [iter] 8 58 | [loss] 865.0788 59 | [trn_acc] 0.7600 60 | [val_acc] 0.5720 61 | [val_same_acc] 0.5580 62 | [val_mean_acc] 0.5650 63 | 64 | [iter] 9 65 | [loss] 828.4952 66 | [trn_acc] 0.8700 67 | [val_acc] 0.5940 68 | [val_same_acc] 0.5840 69 | [val_mean_acc] 0.5890 70 | 71 | [iter] 10 72 | [loss] 805.1672 73 | [trn_acc] 0.6900 74 | [val_acc] 0.6040 75 | [val_same_acc] 0.5900 76 | [val_mean_acc] 0.5970 77 | 78 | [iter] 11 79 | [loss] 775.9832 80 | [trn_acc] 0.8400 81 | [val_acc] 0.6000 82 | [val_same_acc] 0.5700 83 | [val_mean_acc] 0.5850 84 | 85 | [iter] 12 86 | [loss] 755.6879 87 | [trn_acc] 0.8100 88 | [val_acc] 0.5400 89 | [val_same_acc] 0.5620 90 | [val_mean_acc] 0.5510 91 | 92 | [iter] 13 93 | [loss] 738.0169 94 | [trn_acc] 0.8800 95 | [val_acc] 0.5640 96 | [val_same_acc] 0.5580 97 | [val_mean_acc] 0.5610 98 | 99 | [iter] 14 100 | [loss] 726.9145 101 | [trn_acc] 0.8000 102 | [val_acc] 0.6180 103 | [val_same_acc] 0.5760 104 | [val_mean_acc] 0.5970 105 | 106 | [iter] 15 107 | [loss] 718.5194 108 | [trn_acc] 0.8300 109 | [val_acc] 0.5560 110 | [val_same_acc] 0.5740 111 | [val_mean_acc] 0.5650 112 | 113 | [iter] 16 114 | [loss] 705.0256 115 | [trn_acc] 0.7800 116 | [val_acc] 0.6520 117 | [val_same_acc] 0.5900 118 | [val_mean_acc] 0.6210 119 | 120 | [iter] 17 121 | [loss] 690.2497 122 | [trn_acc] 0.8700 123 | [val_acc] 0.5620 124 | [val_same_acc] 0.5760 125 | [val_mean_acc] 0.5690 126 | 127 | [iter] 18 128 | [loss] 679.5831 129 | [trn_acc] 0.8500 130 | [val_acc] 0.5840 131 | [val_same_acc] 0.5740 132 | [val_mean_acc] 0.5790 133 | 134 | [iter] 19 135 | [loss] 674.9844 136 | [trn_acc] 0.9100 137 | [val_acc] 0.5840 138 | [val_same_acc] 0.5900 139 | [val_mean_acc] 0.5870 140 | 141 | [iter] 20 142 | [loss] 664.1955 143 | [trn_acc] 0.9000 144 | [val_acc] 0.5880 145 | [val_same_acc] 0.5880 146 | [val_mean_acc] 0.5880 147 | 148 | [iter] 21 149 | [loss] 656.3805 150 | [trn_acc] 0.8800 151 | [val_acc] 0.6480 152 | [val_same_acc] 0.6160 153 | [val_mean_acc] 0.6320 154 | 155 | [iter] 22 156 | [loss] 642.0827 157 | [trn_acc] 0.8200 158 | [val_acc] 0.5600 159 | [val_same_acc] 0.5700 160 | [val_mean_acc] 0.5650 161 | 162 | [iter] 23 163 | [loss] 637.8745 164 | [trn_acc] 0.8000 165 | [val_acc] 0.6740 166 | [val_same_acc] 0.6300 167 | [val_mean_acc] 0.6520 168 | 169 | [iter] 24 170 | [loss] 629.8573 171 | [trn_acc] 0.9000 172 | [val_acc] 0.5860 173 | [val_same_acc] 0.5540 174 | [val_mean_acc] 0.5700 175 | 176 | [iter] 25 177 | [loss] 618.2125 178 | [trn_acc] 0.9000 179 | [val_acc] 0.6460 180 | [val_same_acc] 0.6160 181 | [val_mean_acc] 0.6310 182 | 183 | [iter] 26 184 | [loss] 612.3875 185 | [trn_acc] 0.8900 186 | [val_acc] 0.6560 187 | [val_same_acc] 0.6240 188 | [val_mean_acc] 0.6400 189 | 190 | [iter] 27 191 | [loss] 606.3396 192 | [trn_acc] 0.9300 193 | [val_acc] 0.6940 194 | [val_same_acc] 0.6300 195 | [val_mean_acc] 0.6620 196 | 197 | [FINAL val_acc] 0.6940 198 | [FINAL tst_acc] 0.6380 199 | [FINAL val_same_acc] 0.6300 200 | [FINAL tst_same_acc] 0.6360 201 | -------------------------------------------------------------------------------- /exp/cls_sim/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../cls.py \ 4 | --model=sim \ 5 | --learning_rate 0.001 \ 6 | --predict_hyp=true \ 7 | --infer_hyp=true \ 8 | --train \ 9 | --n_epochs=100 \ 10 | --test \ 11 | > train.out \ 12 | 2> train.err 13 | 14 | -------------------------------------------------------------------------------- /exp/cls_sim/train.err: -------------------------------------------------------------------------------- 1 | 2017-10-03 15:12:24.165888: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-10-03 15:12:24.165912: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-10-03 15:12:24.165915: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-10-03 15:12:24.165917: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-10-03 15:12:24.165920: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-10-03 15:12:26.169196: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-10-03 15:12:26.169624: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-10-03 15:12:26.169636: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-10-03 15:12:26.169640: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-10-03 15:12:26.169647: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | -------------------------------------------------------------------------------- /exp/cls_vis/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export PYTHONPATH=".:../../../shapeworld" 4 | 5 | python -u ../../cls.py \ 6 | --restore="../cls_hint/model.chk" \ 7 | --predict_hyp=true \ 8 | --infer_hyp=true \ 9 | --vis \ 10 | --n_sample_hyps=10 \ 11 | #> vis.out \ 12 | #2> vis.err 13 | 14 | -------------------------------------------------------------------------------- /exp/cls_vis/vis.err: -------------------------------------------------------------------------------- 1 | ./run.sh: 10: ./run.sh: --n_sample_hyps=10: not found 2 | -------------------------------------------------------------------------------- /exp/cls_vis/vis.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis.out -------------------------------------------------------------------------------- /exp/cls_vis/vis.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis.tar.gz -------------------------------------------------------------------------------- /exp/cls_vis/vis/0/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a red ellipse is to the right of an ellipse . 2 | pred desc: a red shape is to the right of a red semicircle . 3 | gold label: True 4 | pred label: True 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/0/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/0/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/0/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/0/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/0/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/0/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/0/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/0/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/0/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/0/input.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/1/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a shape is below a white ellipse . 2 | pred desc: a white shape is to the left of a yellow ellipse . 3 | gold label: False 4 | pred label: True 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/1/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/1/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/1/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/1/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/1/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/1/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/1/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/1/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/1/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/1/input.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/2/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a magenta triangle is to the left of a magenta pentagon . 2 | pred desc: a magenta triangle is to the left of a pentagon . 3 | gold label: True 4 | pred label: True 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/2/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/2/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/2/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/2/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/2/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/2/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/2/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/2/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/2/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/2/input.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/3/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a white shape is to the left of a square . 2 | pred desc: a cyan cross is to the right of a red shape . 3 | gold label: False 4 | pred label: False 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/3/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/3/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/3/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/3/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/3/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/3/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/3/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/3/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/3/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/3/input.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/4/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a cyan pentagon is to the right of a magenta shape . 2 | pred desc: a blue cross is above a pentagon . 3 | gold label: True 4 | pred label: False 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/4/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/4/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/4/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/4/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/4/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/4/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/4/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/4/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/4/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/4/input.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/5/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a cyan circle is to the left of a rectangle . 2 | pred desc: a circle is above a yellow circle . 3 | gold label: False 4 | pred label: False 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/5/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/5/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/5/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/5/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/5/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/5/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/5/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/5/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/5/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/5/input.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/6/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a red cross is below a square . 2 | pred desc: a square is above a red cross . 3 | gold label: True 4 | pred label: True 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/6/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/6/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/6/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/6/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/6/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/6/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/6/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/6/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/6/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/6/input.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/7/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a blue ellipse is to the right of a cyan ellipse . 2 | pred desc: a cyan ellipse is to the right of a white ellipse . 3 | gold label: True 4 | pred label: True 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/7/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/7/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/7/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/7/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/7/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/7/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/7/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/7/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/7/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/7/input.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/8/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a green pentagon is to the right of a yellow shape . 2 | pred desc: a green shape is to the right of a red semicircle . 3 | gold label: False 4 | pred label: False 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/8/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/8/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/8/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/8/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/8/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/8/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/8/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/8/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/8/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/8/input.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/9/desc.txt: -------------------------------------------------------------------------------- 1 | gold desc: a red circle is above a magenta semicircle . 2 | pred desc: a green triangle is above a red circle . 3 | gold label: False 4 | pred label: True 5 | -------------------------------------------------------------------------------- /exp/cls_vis/vis/9/ex_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/9/ex_0.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/9/ex_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/9/ex_1.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/9/ex_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/9/ex_2.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/9/ex_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/9/ex_3.png -------------------------------------------------------------------------------- /exp/cls_vis/vis/9/input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/exp/cls_vis/vis/9/input.png -------------------------------------------------------------------------------- /exp/pbd_ex/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../pbd.py \ 4 | --hint_type=none \ 5 | --predict_hyp=false \ 6 | --infer_hyp=false \ 7 | --learning_rate 0.001 \ 8 | --train \ 9 | --n_epochs=2000 \ 10 | --test \ 11 | > train.out \ 12 | 2> train.err 13 | 14 | -------------------------------------------------------------------------------- /exp/pbd_hint/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #export CUDA_VISIBLE_DEVICES="" 4 | 5 | python -u ../../pbd.py \ 6 | --hint_type=nl \ 7 | --predict_hyp=true \ 8 | --infer_hyp=true \ 9 | --n_sample_hyps=1 \ 10 | --learning_rate 0.001 \ 11 | --train \ 12 | --n_epochs=750 \ 13 | --test \ 14 | > train.out \ 15 | 2> train.err 16 | 17 | -------------------------------------------------------------------------------- /exp/pbd_hint_eval/eval.out: -------------------------------------------------------------------------------- 1 | [FINAL val_acc] 0.7960 2 | [FINAL tst_acc] 0.7640 3 | -------------------------------------------------------------------------------- /exp/pbd_hint_eval/eval_gold.err: -------------------------------------------------------------------------------- 1 | 2017-09-25 08:58:25.548205: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-09-25 08:58:25.548231: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-09-25 08:58:25.548235: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-09-25 08:58:25.548237: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-09-25 08:58:25.548240: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-09-25 08:58:27.553948: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-09-25 08:58:27.554373: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-09-25 08:58:27.554384: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-09-25 08:58:27.554388: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-09-25 08:58:27.554394: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | -------------------------------------------------------------------------------- /exp/pbd_hint_eval/eval_gold.out: -------------------------------------------------------------------------------- 1 | [FINAL val_acc] 0.7540 2 | [FINAL tst_acc] 0.1400 3 | -------------------------------------------------------------------------------- /exp/pbd_hint_eval/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../pbd.py \ 4 | --hint_type=nl \ 5 | --predict_hyp=true \ 6 | --infer_hyp=true \ 7 | --infer_by_likelihood=true \ 8 | --n_sample_hyps=100 \ 9 | --restore="../pbd_hint/model.chk" \ 10 | --test \ 11 | > eval.out \ 12 | 2> eval.err 13 | 14 | python -u ../../pbd.py \ 15 | --hint_type=nl \ 16 | --predict_hyp=true \ 17 | --infer_hyp=true \ 18 | --use_true_hyp=true \ 19 | --restore="../pbd_hint/model.chk" \ 20 | --test \ 21 | > eval_gold.out \ 22 | 2> eval_gold.err 23 | -------------------------------------------------------------------------------- /exp/pbd_joint/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../pbd.py \ 4 | --hint_type=nl \ 5 | --predict_hyp=true \ 6 | --infer_hyp=false \ 7 | --learning_rate 0.001 \ 8 | --train \ 9 | --n_epochs=720 \ 10 | --test \ 11 | > train.out \ 12 | 2> train.err 13 | 14 | -------------------------------------------------------------------------------- /exp/pbd_reval/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../pbd.py \ 4 | --hint_type=re \ 5 | --n_sample_hyps=10 \ 6 | --predict_hyp \ 7 | --infer_hyp \ 8 | --infer_by_likelihood \ 9 | --use_true_eval \ 10 | --learning_rate 0.001 \ 11 | --train \ 12 | --n_epochs=50 \ 13 | --test \ 14 | > train.out \ 15 | 2> train.err 16 | 17 | -------------------------------------------------------------------------------- /exp/pbd_reval/train.out: -------------------------------------------------------------------------------- 1 | [found_gold] 0.00 2 | [chose_gold] 0.00 3 | [found_exact] 0.00 4 | [loss] 5142.4078 5 | [trn_acc] 0.1700 6 | [found_gold] 0.00 7 | [chose_gold] 0.00 8 | [found_exact] 0.00 9 | [val_acc] 0.1880 10 | 11 | [found_gold] 0.00 12 | [chose_gold] 0.00 13 | [found_exact] 0.00 14 | [loss] 3820.1261 15 | [trn_acc] 0.2300 16 | [found_gold] 0.00 17 | [chose_gold] 0.00 18 | [found_exact] 0.00 19 | [val_acc] 0.1160 20 | 21 | [found_gold] 0.01 22 | [chose_gold] 0.01 23 | [found_exact] 0.01 24 | [loss] 3330.4243 25 | [trn_acc] 0.1400 26 | [found_gold] 0.00 27 | [chose_gold] 0.00 28 | [found_exact] 0.00 29 | [val_acc] 0.1160 30 | 31 | [found_gold] 0.00 32 | [chose_gold] 0.00 33 | [found_exact] 0.01 34 | [loss] 2887.1456 35 | [trn_acc] 0.1500 36 | [found_gold] 0.00 37 | [chose_gold] 0.00 38 | [found_exact] 0.00 39 | [val_acc] 0.1260 40 | 41 | [found_gold] 0.00 42 | [chose_gold] 0.00 43 | [found_exact] 0.00 44 | [loss] 2506.8392 45 | [trn_acc] 0.0900 46 | [found_gold] 0.00 47 | [chose_gold] 0.00 48 | [found_exact] 0.00 49 | [val_acc] 0.1080 50 | 51 | [found_gold] 0.02 52 | [chose_gold] 0.02 53 | [found_exact] 0.02 54 | [loss] 2191.8312 55 | [trn_acc] 0.1200 56 | [found_gold] 0.01 57 | [chose_gold] 0.01 58 | [found_exact] 0.01 59 | [val_acc] 0.1360 60 | 61 | [found_gold] 0.00 62 | [chose_gold] 0.00 63 | [found_exact] 0.01 64 | [loss] 1932.0898 65 | [trn_acc] 0.0700 66 | [found_gold] 0.01 67 | [chose_gold] 0.01 68 | [found_exact] 0.01 69 | [val_acc] 0.1380 70 | 71 | [found_gold] 0.01 72 | [chose_gold] 0.01 73 | [found_exact] 0.02 74 | [loss] 1735.1055 75 | [trn_acc] 0.1100 76 | [found_gold] 0.04 77 | [chose_gold] 0.04 78 | [found_exact] 0.05 79 | [val_acc] 0.1880 80 | 81 | [found_gold] 0.10 82 | [chose_gold] 0.10 83 | [found_exact] 0.12 84 | [loss] 1531.1536 85 | [trn_acc] 0.2000 86 | [found_gold] 0.07 87 | [chose_gold] 0.07 88 | [found_exact] 0.09 89 | [val_acc] 0.2380 90 | 91 | [found_gold] 0.21 92 | [chose_gold] 0.20 93 | [found_exact] 0.30 94 | [loss] 1312.2244 95 | [trn_acc] 0.4100 96 | [found_gold] 0.13 97 | [chose_gold] 0.12 98 | [found_exact] 0.19 99 | [val_acc] 0.3340 100 | 101 | [found_gold] 0.28 102 | [chose_gold] 0.26 103 | [found_exact] 0.36 104 | [loss] 1156.9384 105 | [trn_acc] 0.4600 106 | [found_gold] 0.22 107 | [chose_gold] 0.21 108 | [found_exact] 0.28 109 | [val_acc] 0.4320 110 | 111 | [found_gold] 0.43 112 | [chose_gold] 0.38 113 | [found_exact] 0.52 114 | [loss] 1053.6076 115 | [trn_acc] 0.6600 116 | [found_gold] 0.31 117 | [chose_gold] 0.27 118 | [found_exact] 0.37 119 | [val_acc] 0.5220 120 | 121 | [found_gold] 0.51 122 | [chose_gold] 0.44 123 | [found_exact] 0.58 124 | [loss] 940.1257 125 | [trn_acc] 0.6600 126 | [found_gold] 0.31 127 | [chose_gold] 0.26 128 | [found_exact] 0.38 129 | [val_acc] 0.5400 130 | 131 | [found_gold] 0.49 132 | [chose_gold] 0.45 133 | [found_exact] 0.52 134 | [loss] 868.9560 135 | [trn_acc] 0.7100 136 | [found_gold] 0.44 137 | [chose_gold] 0.36 138 | [found_exact] 0.50 139 | [val_acc] 0.6200 140 | 141 | [found_gold] 0.57 142 | [chose_gold] 0.49 143 | [found_exact] 0.65 144 | [loss] 795.5389 145 | [trn_acc] 0.6800 146 | [found_gold] 0.44 147 | [chose_gold] 0.37 148 | [found_exact] 0.53 149 | [val_acc] 0.6400 150 | 151 | [found_gold] 0.58 152 | [chose_gold] 0.53 153 | [found_exact] 0.66 154 | [loss] 756.1832 155 | [trn_acc] 0.6900 156 | [found_gold] 0.52 157 | [chose_gold] 0.42 158 | [found_exact] 0.60 159 | [val_acc] 0.7000 160 | 161 | [found_gold] 0.53 162 | [chose_gold] 0.47 163 | [found_exact] 0.61 164 | [loss] 712.2185 165 | [trn_acc] 0.7000 166 | [found_gold] 0.51 167 | [chose_gold] 0.44 168 | [found_exact] 0.57 169 | [val_acc] 0.6680 170 | 171 | [found_gold] 0.65 172 | [chose_gold] 0.55 173 | [found_exact] 0.76 174 | [loss] 668.4521 175 | [trn_acc] 0.8400 176 | [found_gold] 0.54 177 | [chose_gold] 0.47 178 | [found_exact] 0.63 179 | [val_acc] 0.7180 180 | 181 | [found_gold] 0.64 182 | [chose_gold] 0.55 183 | [found_exact] 0.74 184 | [loss] 626.0089 185 | [trn_acc] 0.7900 186 | [found_gold] 0.56 187 | [chose_gold] 0.48 188 | [found_exact] 0.62 189 | [val_acc] 0.7180 190 | 191 | [found_gold] 0.75 192 | [chose_gold] 0.64 193 | [found_exact] 0.80 194 | [loss] 582.6294 195 | [trn_acc] 0.8100 196 | [found_gold] 0.62 197 | [chose_gold] 0.52 198 | [found_exact] 0.69 199 | [val_acc] 0.7520 200 | 201 | [found_gold] 0.70 202 | [chose_gold] 0.59 203 | [found_exact] 0.80 204 | [loss] 561.4024 205 | [trn_acc] 0.8200 206 | [found_gold] 0.59 207 | [chose_gold] 0.49 208 | [found_exact] 0.67 209 | [val_acc] 0.7440 210 | 211 | [found_gold] 0.67 212 | [chose_gold] 0.55 213 | [found_exact] 0.75 214 | [loss] 533.5777 215 | [trn_acc] 0.8100 216 | [found_gold] 0.62 217 | [chose_gold] 0.53 218 | [found_exact] 0.70 219 | [val_acc] 0.7640 220 | 221 | [found_gold] 0.75 222 | [chose_gold] 0.64 223 | [found_exact] 0.80 224 | [loss] 495.2193 225 | [trn_acc] 0.8400 226 | [found_gold] 0.61 227 | [chose_gold] 0.51 228 | [found_exact] 0.68 229 | [val_acc] 0.7460 230 | 231 | [found_gold] 0.82 232 | [chose_gold] 0.69 233 | [found_exact] 0.88 234 | [loss] 476.5093 235 | [trn_acc] 0.9000 236 | [found_gold] 0.65 237 | [chose_gold] 0.55 238 | [found_exact] 0.71 239 | [val_acc] 0.7940 240 | 241 | [found_gold] 0.78 242 | [chose_gold] 0.72 243 | [found_exact] 0.83 244 | [loss] 444.8325 245 | [trn_acc] 0.8700 246 | [found_gold] 0.65 247 | [chose_gold] 0.53 248 | [found_exact] 0.72 249 | [val_acc] 0.7820 250 | 251 | [found_gold] 0.79 252 | [chose_gold] 0.71 253 | [found_exact] 0.88 254 | [loss] 426.7756 255 | [trn_acc] 0.9000 256 | [found_gold] 0.64 257 | [chose_gold] 0.55 258 | [found_exact] 0.70 259 | [val_acc] 0.7600 260 | 261 | [found_gold] 0.79 262 | [chose_gold] 0.74 263 | [found_exact] 0.79 264 | [loss] 410.9123 265 | [trn_acc] 0.8400 266 | [found_gold] 0.63 267 | [chose_gold] 0.52 268 | [found_exact] 0.70 269 | [val_acc] 0.7600 270 | 271 | [found_gold] 0.74 272 | [chose_gold] 0.65 273 | [found_exact] 0.81 274 | [loss] 394.1328 275 | [trn_acc] 0.8500 276 | [found_gold] 0.65 277 | [chose_gold] 0.54 278 | [found_exact] 0.70 279 | [val_acc] 0.7780 280 | 281 | [found_gold] 0.83 282 | [chose_gold] 0.75 283 | [found_exact] 0.85 284 | [loss] 392.4931 285 | [trn_acc] 0.8700 286 | [found_gold] 0.64 287 | [chose_gold] 0.55 288 | [found_exact] 0.72 289 | [val_acc] 0.7820 290 | 291 | [found_gold] 0.88 292 | [chose_gold] 0.82 293 | [found_exact] 0.90 294 | [loss] 362.4227 295 | [trn_acc] 0.9400 296 | [found_gold] 0.65 297 | [chose_gold] 0.56 298 | [found_exact] 0.72 299 | [val_acc] 0.7880 300 | 301 | [found_gold] 0.88 302 | [chose_gold] 0.77 303 | [found_exact] 0.93 304 | [loss] 344.4724 305 | [trn_acc] 0.9600 306 | [found_gold] 0.69 307 | [chose_gold] 0.56 308 | [found_exact] 0.75 309 | [val_acc] 0.8060 310 | 311 | [found_gold] 0.90 312 | [chose_gold] 0.73 313 | [found_exact] 0.93 314 | [loss] 332.6447 315 | [trn_acc] 0.9300 316 | [found_gold] 0.67 317 | [chose_gold] 0.55 318 | [found_exact] 0.73 319 | [val_acc] 0.8060 320 | 321 | [found_gold] 0.86 322 | [chose_gold] 0.73 323 | [found_exact] 0.90 324 | [loss] 314.9919 325 | [trn_acc] 0.9200 326 | [found_gold] 0.69 327 | [chose_gold] 0.58 328 | [found_exact] 0.75 329 | [val_acc] 0.8000 330 | 331 | [found_gold] 0.87 332 | [chose_gold] 0.81 333 | [found_exact] 0.92 334 | [loss] 309.5142 335 | [trn_acc] 0.9200 336 | [found_gold] 0.70 337 | [chose_gold] 0.60 338 | [found_exact] 0.76 339 | [val_acc] 0.8240 340 | 341 | [found_gold] 0.92 342 | [chose_gold] 0.82 343 | [found_exact] 0.94 344 | [loss] 289.6545 345 | [trn_acc] 0.9500 346 | [found_gold] 0.68 347 | [chose_gold] 0.56 348 | [found_exact] 0.75 349 | [val_acc] 0.8040 350 | 351 | [found_gold] 0.89 352 | [chose_gold] 0.83 353 | [found_exact] 0.91 354 | [loss] 284.1042 355 | [trn_acc] 0.9300 356 | [found_gold] 0.69 357 | [chose_gold] 0.58 358 | [found_exact] 0.77 359 | [val_acc] 0.8240 360 | 361 | [found_gold] 0.91 362 | [chose_gold] 0.85 363 | [found_exact] 0.93 364 | [loss] 279.9641 365 | [trn_acc] 0.9500 366 | [found_gold] 0.69 367 | [chose_gold] 0.58 368 | [found_exact] 0.77 369 | [val_acc] 0.8200 370 | 371 | [found_gold] 0.87 372 | [chose_gold] 0.77 373 | [found_exact] 0.88 374 | [loss] 270.2342 375 | [trn_acc] 0.9000 376 | [found_gold] 0.67 377 | [chose_gold] 0.59 378 | [found_exact] 0.76 379 | [val_acc] 0.8200 380 | 381 | [found_gold] 0.91 382 | [chose_gold] 0.84 383 | [found_exact] 0.93 384 | [loss] 247.9997 385 | [trn_acc] 0.9100 386 | [found_gold] 0.65 387 | [chose_gold] 0.54 388 | [found_exact] 0.75 389 | [val_acc] 0.8200 390 | 391 | [found_gold] 0.95 392 | [chose_gold] 0.88 393 | [found_exact] 0.97 394 | [loss] 238.7611 395 | [trn_acc] 0.9700 396 | [found_gold] 0.64 397 | [chose_gold] 0.55 398 | [found_exact] 0.75 399 | [val_acc] 0.7960 400 | 401 | [found_gold] 0.91 402 | [chose_gold] 0.81 403 | [found_exact] 0.94 404 | [loss] 237.4518 405 | [trn_acc] 0.9600 406 | [found_gold] 0.66 407 | [chose_gold] 0.55 408 | [found_exact] 0.75 409 | [val_acc] 0.7880 410 | 411 | [found_gold] 0.93 412 | [chose_gold] 0.87 413 | [found_exact] 0.96 414 | [loss] 222.4377 415 | [trn_acc] 0.9600 416 | [found_gold] 0.67 417 | [chose_gold] 0.54 418 | [found_exact] 0.75 419 | [val_acc] 0.8100 420 | 421 | [found_gold] 0.92 422 | [chose_gold] 0.85 423 | [found_exact] 0.94 424 | [loss] 219.1546 425 | [trn_acc] 0.9300 426 | [found_gold] 0.64 427 | [chose_gold] 0.54 428 | [found_exact] 0.74 429 | [val_acc] 0.8060 430 | 431 | [found_gold] 0.96 432 | [chose_gold] 0.88 433 | [found_exact] 0.98 434 | [loss] 212.8356 435 | [trn_acc] 0.9600 436 | [found_gold] 0.64 437 | [chose_gold] 0.55 438 | [found_exact] 0.74 439 | [val_acc] 0.8320 440 | 441 | [found_gold] 0.94 442 | [chose_gold] 0.88 443 | [found_exact] 0.95 444 | [loss] 201.1696 445 | [trn_acc] 0.9600 446 | [found_gold] 0.63 447 | [chose_gold] 0.54 448 | [found_exact] 0.72 449 | [val_acc] 0.7780 450 | 451 | [found_gold] 0.95 452 | [chose_gold] 0.84 453 | [found_exact] 0.97 454 | [loss] 196.8927 455 | [trn_acc] 0.9200 456 | [found_gold] 0.66 457 | [chose_gold] 0.55 458 | [found_exact] 0.75 459 | [val_acc] 0.8100 460 | 461 | [found_gold] 0.90 462 | [chose_gold] 0.83 463 | [found_exact] 0.95 464 | [loss] 194.0960 465 | [trn_acc] 0.9400 466 | [found_gold] 0.65 467 | [chose_gold] 0.55 468 | [found_exact] 0.75 469 | [val_acc] 0.8080 470 | 471 | [found_gold] 0.92 472 | [chose_gold] 0.85 473 | [found_exact] 0.95 474 | [loss] 189.1236 475 | [trn_acc] 0.9700 476 | [found_gold] 0.63 477 | [chose_gold] 0.54 478 | [found_exact] 0.75 479 | [val_acc] 0.8220 480 | 481 | [found_gold] 0.93 482 | [chose_gold] 0.88 483 | [found_exact] 0.97 484 | [loss] 182.4253 485 | [trn_acc] 0.9500 486 | [found_gold] 0.64 487 | [chose_gold] 0.55 488 | [found_exact] 0.73 489 | [val_acc] 0.8000 490 | 491 | [found_gold] 0.96 492 | [chose_gold] 0.92 493 | [found_exact] 0.98 494 | [loss] 176.5053 495 | [trn_acc] 0.9900 496 | [found_gold] 0.68 497 | [chose_gold] 0.57 498 | [found_exact] 0.76 499 | [val_acc] 0.8140 500 | 501 | [found_gold] 0.68 502 | [chose_gold] 0.57 503 | [found_exact] 0.76 504 | [found_gold] 0.00 505 | [chose_gold] 0.00 506 | [found_exact] 0.74 507 | [FINAL val_acc] 0.8140 508 | [FINAL tst_acc] 0.7860 509 | -------------------------------------------------------------------------------- /exp/pbd_reval_eval/eval.err: -------------------------------------------------------------------------------- 1 | 2017-10-06 15:41:36.344291: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-10-06 15:41:36.344317: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-10-06 15:41:36.344321: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-10-06 15:41:36.344323: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-10-06 15:41:36.344326: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-10-06 15:41:38.344271: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-10-06 15:41:38.344666: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-10-06 15:41:38.344676: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-10-06 15:41:38.344679: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-10-06 15:41:38.344686: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | /home/jda/concepts/models.py:640: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison 17 | if np.all(ex[:len(pred)] == pred): 18 | [5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 3.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 4.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 2.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 0.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 1.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 0.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0] 19 | < ( ) ( . C ) ( $ ) @ h \ 2 > 20 | < ( ) ( . o ) ( ) @ s > 21 | < ( ) ( r . ) ( ) @ f g > 22 | < ( ) ( C C ) ( $ ) @ h \ 2 > 23 | < ( ) ( . o ) ( ) @ s > 24 | < ( ) ( r V ) ( ) @ f g > 25 | 26 | [5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 0.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 3.0, 1.0, 5.0, 5.0, 5.0, 5.0, 2.0, 3.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 4.0, 1.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 0.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 2.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 0.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 0.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 1.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 0.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 5.0, 5.0, 5.0, 2.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 5.0] 27 | < ( ) ( C ) ( $ ) @ m > 28 | < ( ) ( . t ) ( ) @ \ 2 \ 2 > 29 | < ( ^ ) ( . C ) ( ) @ q f > 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /exp/pbd_reval_eval/eval.out: -------------------------------------------------------------------------------- 1 | [found_gold] 0.89 2 | [chose_gold] 0.68 3 | [found_exact] 0.93 4 | [found_gold] 0.00 5 | [chose_gold] 0.00 6 | [found_exact] 0.90 7 | [FINAL val_acc] 0.9220 8 | [FINAL tst_acc] 0.9040 9 | -------------------------------------------------------------------------------- /exp/pbd_reval_eval/eval_gold.err: -------------------------------------------------------------------------------- 1 | 2017-10-06 15:54:10.508889: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-10-06 15:54:10.508914: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-10-06 15:54:10.508917: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-10-06 15:54:10.508920: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-10-06 15:54:10.508922: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-10-06 15:54:12.516905: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-10-06 15:54:12.517315: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-10-06 15:54:12.517326: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-10-06 15:54:12.517330: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-10-06 15:54:12.517336: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | -------------------------------------------------------------------------------- /exp/pbd_reval_eval/eval_gold.out: -------------------------------------------------------------------------------- 1 | [FINAL val_acc] 1.0000 2 | [FINAL tst_acc] 0.0000 3 | -------------------------------------------------------------------------------- /exp/pbd_reval_eval/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../pbd.py \ 4 | --hint_type=re \ 5 | --predict_hyp \ 6 | --infer_hyp \ 7 | --infer_by_likelihood \ 8 | --n_sample_hyps=500 \ 9 | --use_true_eval \ 10 | --restore="../pbd_reval/model.chk" \ 11 | --test \ 12 | > eval.out \ 13 | 2> eval.err 14 | 15 | python -u ../../pbd.py \ 16 | --hint_type=re \ 17 | --predict_hyp \ 18 | --infer_hyp \ 19 | --use_true_eval \ 20 | --use_true_hyp \ 21 | --restore="../pbd_reval/model.chk" \ 22 | --test \ 23 | > eval_gold.out \ 24 | 2> eval_gold.err 25 | -------------------------------------------------------------------------------- /exp/pbd_rhint/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../pbd.py \ 4 | --hint_type=re \ 5 | --predict_hyp=true \ 6 | --infer_hyp=true \ 7 | --n_sample_hyps=1 \ 8 | --learning_rate 0.001 \ 9 | --train \ 10 | --n_epochs=750 \ 11 | --test \ 12 | > train.out \ 13 | 2> train.err 14 | 15 | -------------------------------------------------------------------------------- /exp/pbd_rhint_eval/eval.out: -------------------------------------------------------------------------------- 1 | [FINAL val_acc] 0.7600 2 | [FINAL tst_acc] 0.7420 3 | -------------------------------------------------------------------------------- /exp/pbd_rhint_eval/eval_gold.err: -------------------------------------------------------------------------------- 1 | 2017-09-25 14:45:21.888489: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-09-25 14:45:21.888514: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-09-25 14:45:21.888517: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-09-25 14:45:21.888520: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-09-25 14:45:21.888522: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-09-25 14:45:23.892373: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-09-25 14:45:23.892770: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-09-25 14:45:23.892781: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-09-25 14:45:23.892785: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-09-25 14:45:23.892791: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | -------------------------------------------------------------------------------- /exp/pbd_rhint_eval/eval_gold.out: -------------------------------------------------------------------------------- 1 | [FINAL val_acc] 0.8880 2 | [FINAL tst_acc] 0.0540 3 | -------------------------------------------------------------------------------- /exp/pbd_rhint_eval/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../pbd.py \ 4 | --hint_type=re \ 5 | --predict_hyp=true \ 6 | --infer_hyp=true \ 7 | --infer_by_likelihood=true \ 8 | --n_sample_hyps=100 \ 9 | --restore="../pbd_rhint/model.chk" \ 10 | --test \ 11 | > eval.out \ 12 | 2> eval.err 13 | 14 | python -u ../../pbd.py \ 15 | --hint_type=re \ 16 | --predict_hyp=true \ 17 | --infer_hyp=true \ 18 | --use_true_hyp=true \ 19 | --restore="../pbd_rhint/model.chk" \ 20 | --test \ 21 | > eval_gold.out \ 22 | 2> eval_gold.err 23 | -------------------------------------------------------------------------------- /exp/pbd_vis/eval.err: -------------------------------------------------------------------------------- 1 | 2017-10-07 14:30:31.353442: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2 | 2017-10-07 14:30:31.353468: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 3 | 2017-10-07 14:30:31.353472: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 4 | 2017-10-07 14:30:31.353474: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 5 | 2017-10-07 14:30:31.353477: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. 6 | 2017-10-07 14:30:33.371275: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:893] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 7 | 2017-10-07 14:30:33.371672: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: 8 | name: Tesla K40c 9 | major: 3 minor: 5 memoryClockRate (GHz) 0.745 10 | pciBusID 0000:04:00.0 11 | Total memory: 11.17GiB 12 | Free memory: 11.10GiB 13 | 2017-10-07 14:30:33.371683: I tensorflow/core/common_runtime/gpu/gpu_device.cc:961] DMA: 0 14 | 2017-10-07 14:30:33.371686: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0: Y 15 | 2017-10-07 14:30:33.371692: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 16 | 2017-10-07 14:30:45.302158: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:247] PoolAllocator: After 10757 get requests, put_count=10738 evicted_count=1000 eviction_rate=0.0931272 and unsatisfied allocation rate=0.104025 17 | 2017-10-07 14:30:45.302183: I tensorflow/core/common_runtime/gpu/pool_allocator.cc:259] Raising pool_size_limit_ from 100 to 110 18 | [-2.8418741, -10.415289, -1.8153647, -37.400433, -21.132236, -0.024474043, -0.014883434, -0.52999496, -17.485622, -2.2592385, -0.64741927, -0.74795115, -0.019725123, -0.015056101, -0.0015515734, -0.6068846, -0.0086151166, -0.026426716, -0.34573469, -2.8696146, -0.069931298, -0.016044764, -0.1935316, -0.18602964, -6.3033628, -0.022091087, -0.0076296721, -0.0057148263, -0.0029884495, -0.095146291, -0.4318262, -1.6047978, -4.3187361, -0.035147354, -0.012155383, -0.12328532, -3.8609509, -0.71912003, -3.7988715, -0.18248503, -21.600639, -1.0377922, -3.9351647, -0.013620982, -0.013080213, -0.0047476017, -2.3257563, -0.019664524, -1.7070702, -0.0066546747, -0.41170731, -0.20378669, -0.20080057, -1.1020348, -0.03018423, -0.033287577, -10.758209, -0.34818947, -4.9007716, -0.0073399157, -0.019036265, -0.0084778424, -0.16731827, -6.5987368, -0.34906626, -7.6476107, -0.28099829, -0.031510316, -1.4236754, -0.011347543, -1.5794218, -0.032781817, -0.78327918, -0.0036081569, -0.0053557456, -0.0021241601, -0.17976995, -3.0433514, -0.12091065, -15.040902, -0.12372475, -21.337322, -8.2939701, -6.3952408, -0.095896408, -21.094715, -20.423029, -0.16816451, -10.339314, -0.078383885, -0.021758404, -0.0050086984, -8.3452196, -0.12664831, -26.69014, -0.25361639, -0.011880727, -14.803966, -1.9743419, -0.014088811, -0.02379786, -1.5406592, -0.031128414, -0.0076816608, -1.2855771, -0.018933622, -0.68755841, -1.9323406, -1.4721496, -7.067379, -8.7665453, -50.543396, -1.0579095, -3.3328555, -0.27904949, -30.996059, -7.7046146, -0.93539351, -0.04762787, -0.32024789, -0.99287283, -0.032349315, -49.779182, -0.0047793705, -3.8083134, -37.896519, -0.020010892, -0.028158814, -12.446415, -0.027385866, -9.6486082, -0.44731617, -0.017218687, -12.905532, -0.058590874, -3.2873311, -1.7710928, -0.0051654372, -1.2783513, -27.631365, -0.079793155, -0.10808619, -0.0042615258, -0.010095217, -23.377186, -0.005440007, -0.15294519, -0.018499114, -7.6975574, -0.021672232, -2.4145091, -5.7915058, -0.0596359, -0.010783761, -0.94851285, -1.0830694, -0.28814965, -0.0080093564, -0.0268916, -5.2917585, -0.012111628, -0.0033842011, -0.043796666, -4.5538621, -2.3222699, -0.045045987, -0.60206783, -3.0691566, -0.021030342, -37.437283, -3.3381126, -0.15388019, -0.011883734, -0.0017434716, -0.0053925016, -1.0568188, -0.095033847, -0.52311265, -0.72434878, -0.010744181, -28.006584, -0.15340704, -0.30688804, -7.4301124, -0.005254006, -0.028426902, -0.099006042, -11.185539, -7.549099, -11.5994, -0.077904329, -10.884274, -1.7722626, -2.1424785, -1.1992733, -0.44382325, -4.5834818, -0.00042841752, -8.5407867, -0.56224102, -3.850219, -0.036012586, -0.0034982616, -28.123247, -0.37004817, -9.9713364, -0.00645896, -6.4276681, -0.00067279791, -0.066623762, -0.0086855432, -0.29446539, -8.5796614, -114.64086, -0.29868236, -6.1055474, -0.010587443, -1.650341, -40.215595, -0.06686651, -0.0035007368, -0.60829407, -0.67411953, -1.0480479, -0.001267886, -1.2326956, -0.12087401, -12.027596, -0.0049361293, -12.046106, -0.32844818, -0.079828814, -0.35547566, -0.014396608, -1.8199267, -0.020150121, -0.44320825, -0.024105376, -15.716759, -0.0062740119, -0.10492402, -9.9033804, -0.0022265806, -23.736916, -0.9079569, -0.49662304, -34.414265, -23.754633, -5.2808032, -0.3470889, -25.432058, -0.014024836, -0.0048666243, -0.12519728, -8.3822651, -6.1616178, -1.027508, -0.918019, -2.3606055, -0.052005984, -0.0082986234, -41.345112, -9.4128551, -0.0032180035, -0.027663244, -0.0042556655, -0.34166762, -51.351463, -0.012645154, -1.4192656, -0.019009713, -0.40070266, -2.5156558, -3.7184062, -0.11872041, -0.056744426, -0.016634949, -0.0052803396, -14.313699, -0.0050055599, -6.5396628, -8.9185066, -7.3333764, -1.29234, -7.501472, -0.064720027, -0.31088459, -0.0016540021, -0.0010543214, -0.54846829, -32.898956, -41.438873, -0.0055103758, -0.33043984, -0.28798658, -0.0069361301, -0.017368177, -2.3805425, -3.8717778, -0.071988389, -0.096132144, -0.01292406, -0.018176926, -28.990273, -3.8698165, -25.103878, -0.00080319319, -0.12433915, -0.16384318, -0.78798556, -0.74317658, -0.004915792, -0.016770959, -0.013136204, -0.0099831996, -0.06684798, -0.001335874, -0.0042620855, -0.14031133, -0.889319, -0.030651851, -1.0310504, -0.0040895031, -0.032199401, -0.0064863656, -11.931511, -0.0081919702, -11.968086, -0.54432833, -26.819201, -5.7653332, -0.013527393, -0.066303827, -37.973862, -0.014449019, -6.4659452, -18.000786, -0.4064607, -0.092162251, -1.271651, -0.0062184408, -41.230114, -4.4491801, -25.37995, -0.016697364, -0.17624655, -0.021330895, -0.10361408, -0.0093965931, -13.080112, -0.20255581, -0.018607685, -1.2793319, -72.091927, -1.814634, -0.012998911, -0.10723677, -1.1174715, -0.0093197944, -0.0019634573, -0.042241156, -0.630992, -0.013670417, -1.99945, -0.007171907, -0.033315048, -0.0034126826, -0.16807824, -0.016630223, -0.092305653, -0.037352305, -0.21644671, -0.043626059, -0.036922596, -0.67638242, -10.828901, -0.024141304, -10.170464, -2.1976671, -8.202713, -5.1859875, -0.046588875, -0.1589078, -0.0061685415, -0.0093892543, -7.1235585, -0.59074998, -0.14721429, -0.0040802499, -0.024768919, -16.239019, -0.020322993, -0.045273475, -2.1777947, -3.022203, -0.14958626, -0.039981626, -9.5209064, -0.18809065, -0.74538678, -16.530312, -0.0068132468, -0.046452291, -4.9996109, -4.5495396, -0.0699967, -0.026925372, -0.00096243795, -33.880215, -38.20726, -0.0053733615, -0.031131212, -24.472836, -36.425056, -0.0017416691, -10.18449, -0.0031860152, -6.9214811, -0.0076318174, -0.47742268, -0.0024576308, -12.888096, -2.2960739, -0.0039410819, -0.036676556, -0.76864946, -0.21065432, -0.21272409, -0.54856718, -1.0393218, -0.0055491282, -0.017578974, -0.0064324834, -0.004238843, -0.069340721, -13.090011, -21.638628, -5.1162877, -0.00042175315, -0.071379259, -5.3294768, -0.060271285, -0.0078984052, -0.1221441, -51.954128, -8.5827513, -8.6864328, -35.989727, -0.0098919123, -0.023540627, -27.979029, -0.0022928598, -0.1760906, -0.0011765582, -0.019556088, -20.252583, -91.061111, -0.0068458258, -0.0039710989, -0.74140835, -0.0058385623, -0.074035667, -4.1813874, -0.097396977, -17.504799, -0.010244532, -0.01075438, -0.11918137, -0.03947477, -1.2081227, -0.39178804, -5.8855643, -0.0018372631, -3.4539711, -1.643792, -3.492511, -2.1957316, -22.580029, -0.25277677, -5.307075, -0.011645172, -5.4911103, -1.4634254, -0.16232271, -0.97905338, -0.0029089784, -3.1141658, -0.010893187, -4.8460159, -0.49195191, -4.8713799, -0.026884874, -0.93078929, -0.0028447597, -5.4096131, -0.034786168, -1.2783766, -10.597557, -0.60241455, -0.0099944249] 19 | < insert a h as the third to last letter > 20 | < replace the alphabet o and the alphabet preceding it with s > 21 | < if the word has a r change the r and the following letter to a single letter f g > 22 | < if the last 2 letters do nt have a vowel then add h in front of them > 23 | < replace letter - o pairing with s > 24 | < replaced li with f g > 25 | 26 | -------------------------------------------------------------------------------- /exp/pbd_vis/eval.out: -------------------------------------------------------------------------------- 1 | [found_gold] 0.43 2 | [chose_gold] 0.03 3 | [found_exact] 0.67 4 | gerunds gerunhds 5 | reassess reassehss 6 | divorcing divorcihng 7 | mummers mummehrs 8 | marriageable marriageable 9 | 10 | gold: if the last 2 letters do nt have a vowel then add h in front of them 11 | pred: insert a h as the third to last letter 12 | 13 | 14 | gold: 15 | pred: 16 | === 17 | 18 | dissociation dissciatsn 19 | mandibles mandibles 20 | community smmunity 21 | buncombe bunsmbe 22 | enunciated enunciated 23 | 24 | gold: replace letter - o pairing with s 25 | pred: replace the alphabet o and the alphabet preceding it with s 26 | 27 | 28 | gold: 29 | pred: 30 | === 31 | 32 | anchorages anchofgges 33 | clamorous clamofgus 34 | itinerants itinefgnts 35 | daub daub 36 | batsman batsman 37 | 38 | gold: replaced li with f g 39 | pred: if the word has a r change the r and the following letter to a single letter f g 40 | 41 | 42 | gold: 43 | pred: 44 | === 45 | 46 | feigns fargns 47 | ploughshares plarghshares 48 | ambidextrously ambidextrarsly 49 | disrobes disrobes 50 | toilette tarlette 51 | 52 | gold: replace pairs of consecutive vowels with a a r 53 | pred: if there is a pair of two vowels next to each other , replace the vowels with a r 54 | 55 | 56 | gold: 57 | pred: 58 | === 59 | 60 | rankness rankness 61 | pilferers pilferers 62 | doublets donblets 63 | reimbursed ronmbursed 64 | supervening supervening 65 | 66 | gold: replace a series of 2 vowels with o n 67 | pred: if two vowels follow one another , replace them with n t 68 | 69 | 70 | gold: 71 | pred: 72 | === 73 | 74 | plummest plummesti 75 | bereaving bereavinti 76 | eddied eddieti 77 | struggles struggleti 78 | evils evilti 79 | 80 | gold: replace the last letter of the word with t i 81 | pred: change the last letter of the word into t i 82 | 83 | 84 | gold: 85 | pred: 86 | === 87 | 88 | canal ititititit 89 | unlabeled ititititititititit 90 | generalities itititititititititititit 91 | familiarly itititititititititit 92 | enured itititititit 93 | 94 | gold: replace every letter of the word with i t 95 | pred: repeat pattern i t for the length of the original word 96 | 97 | 98 | gold: 99 | pred: 100 | === 101 | 102 | mapper npnr 103 | concluding nncnnng 104 | excuse exnn 105 | effete efnn 106 | contracting nntncnng 107 | 108 | gold: replace pairs of letters consisting of a consonant followed by a vowel with an n 109 | pred: replace consonant - vowel pairings with n 110 | 111 | 112 | gold: 113 | pred: 114 | === 115 | 116 | supplicate supplicate 117 | finals finals 118 | incinerators intcinerators 119 | afterthought aftterthought 120 | theses theses 121 | 122 | gold: if the first letter is a vowel and the second letter is a consonant , add an t after them 123 | pred: add a t to the letter after an n in a word 124 | 125 | 126 | gold: 127 | pred: 128 | === 129 | 130 | artefacts ufrtuffufcts 131 | resourcefully rufsufufrcuffuflly 132 | regent rufgufnt 133 | taint tufufnt 134 | trimesters trufmufstufrs 135 | 136 | gold: replace every vowel with u f 137 | pred: change the each vowel to u f 138 | 139 | 140 | gold: 141 | pred: 142 | === 143 | 144 | -------------------------------------------------------------------------------- /exp/pbd_vis/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../pbd.py \ 4 | --hint_type=nl \ 5 | --predict_hyp=true \ 6 | --infer_hyp=true \ 7 | --infer_by_likelihood=true \ 8 | --n_sample_hyps=100 \ 9 | --restore="../pbd_hint/model.chk" \ 10 | --vis \ 11 | > eval.out \ 12 | 2> eval.err 13 | -------------------------------------------------------------------------------- /exp/rl_ex/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../rl.py \ 4 | --train \ 5 | --n_epochs=100 \ 6 | --predict_hyp=false \ 7 | --infer_hyp=false \ 8 | --use_expert=true \ 9 | > train.out \ 10 | 2> train.err 11 | 12 | -------------------------------------------------------------------------------- /exp/rl_ex/train.err: -------------------------------------------------------------------------------- 1 | 0%| | 0/2000 [00:00 (device: 0, name: Tesla K40c, pci bus id: 0000:04:00.0) 90 | -------------------------------------------------------------------------------- /exp/rl_ex_eval/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export CUDA_VISIBLE_DEVICES="" 4 | 5 | python -u ../../rl.py \ 6 | --test \ 7 | --n_epochs=20 \ 8 | --predict_hyp=false \ 9 | --infer_hyp=false \ 10 | --restore="../rl_ex" \ 11 | > eval.out \ 12 | 2> eval.err 13 | 14 | -------------------------------------------------------------------------------- /exp/rl_hint/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../rl.py \ 4 | --train \ 5 | --n_epochs=250 \ 6 | --predict_hyp=true \ 7 | --infer_hyp=true \ 8 | --use_expert=true \ 9 | > train.out \ 10 | 2> train.err 11 | 12 | -------------------------------------------------------------------------------- /exp/rl_hint_eval/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export CUDA_VISIBLE_DEVICES="" 4 | 5 | python -u ../../rl.py \ 6 | --test \ 7 | --n_epochs=20 \ 8 | --predict_hyp=true \ 9 | --infer_hyp=true \ 10 | --restore="../rl_hint" \ 11 | > eval.out \ 12 | 2> eval.err 13 | 14 | -------------------------------------------------------------------------------- /exp/rl_scratch/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python -u ../../rl.py \ 4 | --train \ 5 | --n_epochs=0 \ 6 | --predict_hyp=false \ 7 | --infer_hyp=false \ 8 | --use_expert=true \ 9 | > train.out \ 10 | 2> train.err 11 | 12 | -------------------------------------------------------------------------------- /exp/rl_scratch/train.out: -------------------------------------------------------------------------------- 1 | 2 | Loading local train environments with human annotations 3 | Found 1566 annotations 4 | 5 | Loading local test environments with human annotations 6 | Found 399 annotations 7 | 8 | Loading global train environments with human annotations 9 | Found 1071 annotations 10 | 11 | Loading global test environments with human annotations 12 | Found 272 annotations 13 | [n_train] 5000 14 | [n_test] 1000 test 15 | [task_ids] 5000 -> 5050 16 | [n_vocab] 74 17 | -------------------------------------------------------------------------------- /exp/rl_scratch_eval/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export CUDA_VISIBLE_DEVICES="" 4 | 5 | python -u ../../rl.py \ 6 | --test \ 7 | --n_epochs=20 \ 8 | --adapt_reprs=1 \ 9 | --adapt_samples=1 \ 10 | --predict_hyp=false \ 11 | --infer_hyp=false \ 12 | --restore="../rl_scratch" \ 13 | > eval.out \ 14 | 2> eval.err 15 | 16 | -------------------------------------------------------------------------------- /extras/featurize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | import tensorflow.contrib.slim as slim 6 | from tensorflow.contrib.slim.nets import inception, vgg 7 | from tensorflow.contrib import slim 8 | 9 | #image_size = inception.inception_v3.default_image_size 10 | image_size = vgg.vgg_16.default_image_size 11 | 12 | BATCH_SIZE = 100 13 | 14 | 15 | t_input = tf.placeholder(tf.float32, (None, 64, 64, 3)) 16 | 17 | t_proc = tf.image.resize_bilinear(t_input, [image_size, image_size], align_corners=False) 18 | t_proc = tf.subtract(t_proc, 0.5) 19 | t_proc = tf.multiply(t_proc, 2.0) 20 | 21 | logits, layers = vgg.vgg_16(t_proc, is_training=False) 22 | #t_output = layers["vgg_16/fc7"] 23 | #for name, tensor in layers.items(): 24 | # print name, tensor.shape 25 | #exit() 26 | 27 | #t_output_pre = layers["vgg_16/pool5"] 28 | #t_output = tf.reduce_mean(t_output_pre, axis=(1, 2)) 29 | #N_FEATS = 512 30 | 31 | t_output = layers["vgg_16/pool5"] 32 | t_output = slim.avg_pool2d(t_output, (3, 3), 2) 33 | t_output = slim.flatten(t_output) 34 | N_FEATS = t_output.shape[1] 35 | 36 | #t_output = layers["vgg_16/fc7"] 37 | #t_output = slim.flatten(t_output) 38 | #N_FEATS = t_output.shape[1] 39 | 40 | init_fn = slim.assign_from_checkpoint_fn( 41 | "/data/jda/concepts_features/vgg_16.ckpt", 42 | slim.get_model_variables()) 43 | 44 | with tf.Session() as session: 45 | init_fn(session) 46 | 47 | for fold in ("train", "val", "test", "val_same", "test_same"): 48 | print fold 49 | print "EXAMPLES" 50 | with open("%s/examples.npy" % fold) as ex_f: 51 | ex = np.load(ex_f) 52 | n_inp = ex.shape[0] 53 | n_ex = ex.shape[1] 54 | ex_feats = np.zeros((n_inp, n_ex, N_FEATS)) 55 | for i in range(0, n_inp, BATCH_SIZE/10): 56 | if i % 1000 == 0: 57 | print i 58 | batch = ex[i:i+BATCH_SIZE/10, ...] 59 | n_batch = batch.shape[0] 60 | batch = batch.reshape((n_batch * n_ex, batch.shape[2], 61 | batch.shape[3], batch.shape[4])) 62 | feats = session.run(t_output, {t_input: batch}) 63 | feats = feats.reshape((n_batch, n_ex, N_FEATS)) 64 | ex_feats[i:i+BATCH_SIZE/10, ...] = feats 65 | np.save("%s/examples.feats.npy" % fold, ex_feats) 66 | 67 | print "INPUTS" 68 | with open("%s/inputs.npy" % fold) as inp_f: 69 | inp = np.load(inp_f) 70 | n_inp = inp.shape[0] 71 | inp_feats = np.zeros((n_inp, N_FEATS)) 72 | for i in range(0, n_inp, BATCH_SIZE): 73 | if i % 1000 == 0: 74 | print i 75 | batch = inp[i:i+BATCH_SIZE, ...] 76 | feats = session.run(t_output, {t_input: batch}) 77 | feats = feats.reshape((-1, N_FEATS)) 78 | inp_feats[i:i+BATCH_SIZE, :] = feats 79 | np.save("%s/inputs.feats.npy" % fold, inp_feats) 80 | 81 | -------------------------------------------------------------------------------- /misc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/misc/__init__.py -------------------------------------------------------------------------------- /misc/array.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def pad_slice(array, slice_r, slice_c): 4 | assert len(array.shape) >= 2 5 | 6 | r1, r2 = slice_r 7 | c1, c2 = slice_c 8 | assert r2 > r1 9 | assert c2 > c1 10 | 11 | pr1 = max(r1, 0) 12 | pc1 = max(c1, 0) 13 | 14 | sl = array[pr1:r2, pc1:c2, :] 15 | slr, slc = sl.shape[:2] 16 | 17 | padded_sl = np.zeros((r2 - r1, c2 - c1) + array.shape[2:]) 18 | pad_fr_r = pr1 - r1 19 | pad_to_r = pad_fr_r + slr 20 | pad_fr_c = pc1 - c1 21 | pad_to_c = pad_fr_c + slc 22 | 23 | padded_sl[pad_fr_r:pad_to_r, pad_fr_c:pad_to_c, :] = sl 24 | 25 | return padded_sl 26 | -------------------------------------------------------------------------------- /misc/util.py: -------------------------------------------------------------------------------- 1 | import re 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | class Struct: 6 | def __init__(self, **entries): 7 | rec_entries = {} 8 | for k, v in entries.items(): 9 | if isinstance(v, dict): 10 | rv = Struct(**v) 11 | elif isinstance(v, list): 12 | rv = [] 13 | for item in v: 14 | if isinstance(item, dict): 15 | rv.append(Struct(**item)) 16 | else: 17 | rv.append(item) 18 | else: 19 | rv = v 20 | rec_entries[k] = rv 21 | self.__dict__.update(rec_entries) 22 | 23 | def __str_helper(self, depth): 24 | lines = [] 25 | for k, v in self.__dict__.items(): 26 | if isinstance(v, Struct): 27 | v_str = v.__str_helper(depth + 1) 28 | lines.append("%s:\n%s" % (k, v_str)) 29 | else: 30 | lines.append("%s: %r" % (k, v)) 31 | indented_lines = [" " * depth + l for l in lines] 32 | return "\n".join(indented_lines) 33 | 34 | def __str__(self): 35 | return "struct {\n%s\n}" % self.__str_helper(1) 36 | 37 | def __repr__(self): 38 | return "Struct(%r)" % self.__dict__ 39 | 40 | class Index: 41 | def __init__(self): 42 | self.contents = dict() 43 | self.ordered_contents = [] 44 | self.reverse_contents = dict() 45 | 46 | def __getitem__(self, item): 47 | if item not in self.contents: 48 | return None 49 | return self.contents[item] 50 | 51 | def index(self, item): 52 | if item not in self.contents: 53 | idx = len(self.contents) + 1 54 | self.ordered_contents.append(item) 55 | self.contents[item] = idx 56 | self.reverse_contents[idx] = item 57 | idx = self[item] 58 | assert idx != 0 59 | return idx 60 | 61 | def get(self, idx): 62 | if idx == 0: 63 | return "*invalid*" 64 | return self.reverse_contents[idx] 65 | 66 | def __len__(self): 67 | return len(self.contents) + 1 68 | 69 | def __iter__(self): 70 | return iter(self.ordered_contents) 71 | 72 | def flatten(lol): 73 | if isinstance(lol, tuple) or isinstance(lol, list): 74 | return sum([flatten(l) for l in lol], []) 75 | else: 76 | return [lol] 77 | 78 | def postorder(tree): 79 | if isinstance(tree, tuple): 80 | for subtree in tree[1:]: 81 | for node in postorder(subtree): 82 | yield node 83 | yield tree[0] 84 | else: 85 | yield tree 86 | 87 | def tree_map(function, tree): 88 | if isinstance(tree, tuple): 89 | head = function(tree) 90 | tail = tuple(tree_map(function, subtree) for subtree in tree[1:]) 91 | return (head,) + tail 92 | return function(tree) 93 | 94 | def tree_zip(*trees): 95 | if isinstance(trees[0], tuple): 96 | zipped_children = [[t[i] for t in trees] for i in range(len(trees[0]))] 97 | zipped_children_rec = [tree_zip(*z) for z in zipped_children] 98 | return tuple(zipped_children_rec) 99 | return trees 100 | 101 | FEXP_RE = re.compile(r"(.*)\[(.*)\]") 102 | def parse_fexp(fexp): 103 | m = FEXP_RE.match(fexp) 104 | return (m.group(1), m.group(2)) 105 | 106 | def pp_sexp(sexp): 107 | if not isinstance(sexp, tuple): 108 | return str(sexp) 109 | return "(" + " ".join([pp_sexp(s) for s in sexp]) + ")" 110 | 111 | random_counter = [0] 112 | def next_random(): 113 | random = np.random.RandomState(random_counter[0]) 114 | random_counter[0] += 1 115 | return random 116 | 117 | def batch_gather(params, indices): 118 | ids = tf.range(tf.shape(indices)[0]) 119 | full = tf.stack((ids, indices), axis=1) 120 | return tf.gather_nd(params, full) 121 | 122 | def vars_in_scope(scope): 123 | return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope.name) 124 | -------------------------------------------------------------------------------- /net.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | INIT_SCALE = 1.43 4 | 5 | def _linear(t_in, n_out): 6 | v_w = tf.get_variable( 7 | "w", 8 | shape=(t_in.get_shape()[-1], n_out), 9 | initializer=tf.uniform_unit_scaling_initializer( 10 | factor=INIT_SCALE)) 11 | v_b = tf.get_variable( 12 | "b", 13 | shape=n_out, 14 | initializer=tf.constant_initializer(0)) 15 | if len(t_in.get_shape()) == 2: 16 | return tf.einsum("ij,jk->ik", t_in, v_w) + v_b 17 | elif len(t_in.get_shape()) == 3: 18 | return tf.einsum("ijk,kl->ijl", t_in, v_w) + v_b 19 | else: 20 | assert False 21 | 22 | def _embed(t_in, n_embeddings, n_out): 23 | v = tf.get_variable( 24 | "embed", shape=(n_embeddings, n_out), 25 | initializer=tf.uniform_unit_scaling_initializer()) 26 | t_embed = tf.nn.embedding_lookup(v, t_in) 27 | return t_embed 28 | 29 | def _embed_dict(t_in, emb_dict): 30 | if isinstance(emb_dict, tf.Variable): 31 | v = emb_dict 32 | else: 33 | assert isinstance(emb_dict, np.array) 34 | v = tf.get_variable( 35 | "embed", shape=emb_dict.shape, 36 | initializer=tf.constant_initializer(emb_dict)) 37 | 38 | t_embed = tf.nn.embedding_lookup(v, t_in) 39 | return t_embed 40 | 41 | def _mlp(t_in, widths, activations): 42 | assert len(t_in.get_shape()) in (2, 3) 43 | assert len(widths) == len(activations) 44 | prev_width = t_in.get_shape()[1] 45 | prev_layer = t_in 46 | for i_layer, (width, act) in enumerate(zip(widths, activations)): 47 | with tf.variable_scope(str(i_layer)): 48 | layer = _linear(prev_layer, width) 49 | if act is not None: 50 | layer = act(layer) 51 | prev_layer = layer 52 | prev_width = width 53 | return prev_layer 54 | -------------------------------------------------------------------------------- /old/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | from models import Model3 4 | #from tasks import count, shapes, birds, regex, regex2 5 | from tasks import regex2 6 | import util 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | 11 | dataset = regex2 12 | 13 | model = Model3("model", dataset) 14 | print "built model" 15 | 16 | optimizer = tf.train.AdamOptimizer(0.001) 17 | o_train = optimizer.minimize(model.t_loss) 18 | 19 | session = tf.Session() 20 | session.run(tf.global_variables_initializer()) 21 | 22 | random = util.next_random() 23 | 24 | for i_epoch in range(1000): 25 | e_loss = 0. 26 | for i_batch in range(100): 27 | batch = dataset.sample_train() 28 | feed_dict = model.feed(batch) 29 | loss, _ = session.run( 30 | [model.t_loss, o_train], feed_dict) 31 | e_loss += loss 32 | 33 | batch = dataset.sample_train() 34 | e_acc = model.predict(batch, session) 35 | 36 | test_data = dataset.sample_test() 37 | t_acc = model.predict(test_data, session) 38 | 39 | #e_acc = np.mean([p == d.label for p, d in zip(preds, batch)]) 40 | #e_acc = 0. 41 | 42 | print "%01.3f %01.3f" % (e_loss, e_acc) 43 | print "%01.3f" % t_acc 44 | print 45 | 46 | #test_data = dataset.sample_test() 47 | #samp = model.sample(test_data, session) 48 | #idx = random.randint(len(test_data)) 49 | #print " ".join(dataset.nl_vocab.get(i) for i in samp[idx]) 50 | #print " " + " ".join(dataset.nl_vocab.get(i) for i in test_data[idx].hint) 51 | #preds = model.predict(test_data, session) 52 | #acc = np.mean([p == d.label for p, d in zip(preds, test_data)]) 53 | #print acc 54 | #print 55 | 56 | 57 | #test_exemplars = dataset.sample_test_exemplars() 58 | #model.fit(test_exemplars, session) 59 | # 60 | #test_data = dataset.sample_test_data() 61 | #preds = model.predict(test_data) 62 | #acc = np.mean([p == d.label for p, d in zip(preds, test_data)]) 63 | #print acc 64 | -------------------------------------------------------------------------------- /old/main.py.bak: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | from net import _mlp, _embed 4 | from tasks import count, shapes 5 | import util 6 | 7 | import numpy as np 8 | import tensorflow as tf 9 | 10 | HIDDEN = (256, 256) 11 | NONLIN = (tf.nn.relu, tf.nn.relu) 12 | 13 | dataset = shapes 14 | 15 | class Model(object): 16 | def __init__(self, name): 17 | with tf.variable_scope(name) as scope: 18 | t_image = tf.placeholder(tf.float32, (None,) + dataset.image_shape) 19 | t_concept = tf.placeholder(tf.int32, (None,)) 20 | t_hint = tf.placeholder(tf.int32, (None, None)) 21 | t_label = tf.placeholder(tf.int32, (None,)) 22 | 23 | t_image_rs = tf.reshape(t_image, (-1, reduce(lambda x, y: x*y, dataset.image_shape))) 24 | #t_concept_emb = _embed(t_concept, dataset.n_concepts+1, 64) 25 | #t_features = tf.concat((t_image_rs, t_concept_emb), axis=1) 26 | #t_hidden = _mlp(t_features, HIDDEN, NONLIN) 27 | n_weight = HIDDEN[-1] * 2 28 | t_hint_emb = _embed(t_hint, dataset.n_vocab, n_weight) 29 | t_hint_red = tf.reduce_sum(t_hint_emb, axis=1) 30 | t_weight = tf.reshape(t_hint_red, (-1, HIDDEN[-1], 2)) 31 | t_hidden = _mlp(t_image_rs, HIDDEN, NONLIN) 32 | t_pred = tf.einsum("ij,ijk->ik", t_hidden, t_weight) 33 | t_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 34 | labels=t_label, logits=t_pred)) 35 | 36 | params = util.vars_in_scope(scope) 37 | 38 | self.t_image = t_image 39 | self.t_concept = t_concept 40 | self.t_hint = t_hint 41 | self.t_label = t_label 42 | self.t_pred = t_pred 43 | self.t_loss = t_loss 44 | self.params = params 45 | 46 | model = Model("tr") 47 | test_model = Model("test") 48 | 49 | optimizer = tf.train.AdamOptimizer(0.001) 50 | o_train = optimizer.minimize(model.t_loss) 51 | o_train_test = optimizer.minimize(test_model.t_loss) 52 | o_assign = [tp.assign(p) for p, tp in zip(model.params, test_model.params)] 53 | 54 | session = tf.Session() 55 | session.run(tf.global_variables_initializer()) 56 | while True: 57 | c_loss = 0. 58 | c_acc = 0. 59 | for _ in range(100): 60 | batch = [dataset.sample() for _ in range(100)] 61 | images, concepts, labels, hints = zip(*batch) 62 | #concept_labels = [concept_index.index(c) for c in concepts] 63 | hint_data = np.zeros((len(hints), max(len(h) for h in hints))) 64 | for i, hint in enumerate(hints): 65 | hint_data[i, :len(hint)] = hints[i] 66 | loss, pred, _ = session.run( 67 | [model.t_loss, model.t_pred, o_train], 68 | {model.t_image: images, model.t_hint: hint_data, 69 | model.t_label: labels}) 70 | c_loss += loss 71 | c_acc += np.mean(np.argmax(pred, axis=1) == labels) 72 | 73 | print "%.3f %.3f" % (c_loss, c_acc) 74 | 75 | session.run([o_assign]) 76 | examples = [dataset.sample(test=True) for _ in range(100)] 77 | test_examples = [dataset.sample(test=True) for _ in range(500)] 78 | for _ in range(10): 79 | images, concepts, labels, hints = zip(*examples) 80 | hint_data = np.zeros((len(hints), max(len(h) for h in hints))) 81 | for i, hint in enumerate(hints): 82 | hint_data[i, :len(hint)] = hints[i] 83 | loss, pred, _ = session.run( 84 | [test_model.t_loss, test_model.t_pred, o_train_test], 85 | {test_model.t_image: images, test_model.t_hint: hint_data, 86 | test_model.t_label: labels}) 87 | acc = np.mean(np.argmax(pred, axis=1) == labels) 88 | 89 | images, concepts, labels, hints = zip(*test_examples) 90 | hint_data = np.zeros((len(hints), max(len(h) for h in hints))) 91 | for i, hint in enumerate(hints): 92 | hint_data[i, :len(hint)] = hints[i] 93 | pred, = session.run( 94 | [test_model.t_pred], 95 | {test_model.t_image: images, test_model.t_hint: hint_data, 96 | test_model.t_label: labels}) 97 | test_acc = np.mean(np.argmax(pred, axis=1) == labels) 98 | 99 | print " %.3f %.3f [%.3f]" % (loss, acc, test_acc) 100 | -------------------------------------------------------------------------------- /old/net.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | INIT_SCALE = 1.43 4 | 5 | def _linear(t_in, n_out): 6 | assert len(t_in.get_shape()) == 2 7 | v_w = tf.get_variable( 8 | "w", 9 | shape=(t_in.get_shape()[1], n_out), 10 | initializer=tf.uniform_unit_scaling_initializer( 11 | factor=INIT_SCALE)) 12 | v_b = tf.get_variable( 13 | "b", 14 | shape=n_out, 15 | initializer=tf.constant_initializer(0)) 16 | return tf.einsum("ij,jk->ik", t_in, v_w) + v_b 17 | 18 | def _embed(t_in, n_embeddings, n_out): 19 | v = tf.get_variable( 20 | "embed", shape=(n_embeddings, n_out), 21 | initializer=tf.uniform_unit_scaling_initializer()) 22 | t_embed = tf.nn.embedding_lookup(v, t_in) 23 | return t_embed 24 | 25 | def _embed_pretrained(t_in, emb_dict): 26 | v = tf.get_variable( 27 | "embed", shape=emb_dict.shape, 28 | initializer=tf.constant_initializer(emb_dict)) 29 | t_embed = tf.nn.embedding_lookup(v, t_in) 30 | return t_embed 31 | 32 | def _mlp(t_in, widths, activations): 33 | assert len(t_in.get_shape()) == 2 34 | assert len(widths) == len(activations) 35 | prev_width = t_in.get_shape()[1] 36 | prev_layer = t_in 37 | for i_layer, (width, act) in enumerate(zip(widths, activations)): 38 | with tf.variable_scope(str(i_layer)): 39 | layer = _linear(prev_layer, width) 40 | if act is not None: 41 | layer = act(layer) 42 | prev_layer = layer 43 | prev_width = width 44 | return prev_layer 45 | -------------------------------------------------------------------------------- /old/util.py: -------------------------------------------------------------------------------- 1 | import re 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | class Struct: 6 | def __init__(self, **entries): 7 | rec_entries = {} 8 | for k, v in entries.items(): 9 | if isinstance(v, dict): 10 | rv = Struct(**v) 11 | elif isinstance(v, list): 12 | rv = [] 13 | for item in v: 14 | if isinstance(item, dict): 15 | rv.append(Struct(**item)) 16 | else: 17 | rv.append(item) 18 | else: 19 | rv = v 20 | rec_entries[k] = rv 21 | self.__dict__.update(rec_entries) 22 | 23 | def __str_helper(self, depth): 24 | lines = [] 25 | for k, v in self.__dict__.items(): 26 | if isinstance(v, Struct): 27 | v_str = v.__str_helper(depth + 1) 28 | lines.append("%s:\n%s" % (k, v_str)) 29 | else: 30 | lines.append("%s: %r" % (k, v)) 31 | indented_lines = [" " * depth + l for l in lines] 32 | return "\n".join(indented_lines) 33 | 34 | def __str__(self): 35 | return "struct {\n%s\n}" % self.__str_helper(1) 36 | 37 | def __repr__(self): 38 | return "Struct(%r)" % self.__dict__ 39 | 40 | class Index: 41 | def __init__(self): 42 | self.contents = dict() 43 | self.ordered_contents = [] 44 | self.reverse_contents = dict() 45 | 46 | def __getitem__(self, item): 47 | if item not in self.contents: 48 | return None 49 | return self.contents[item] 50 | 51 | def index(self, item): 52 | if item not in self.contents: 53 | idx = len(self.contents) + 1 54 | self.ordered_contents.append(item) 55 | self.contents[item] = idx 56 | self.reverse_contents[idx] = item 57 | idx = self[item] 58 | assert idx != 0 59 | return idx 60 | 61 | def get(self, idx): 62 | if idx == 0: 63 | return "*invalid*" 64 | return self.reverse_contents[idx] 65 | 66 | def __len__(self): 67 | return len(self.contents) + 1 68 | 69 | def __iter__(self): 70 | return iter(self.ordered_contents) 71 | 72 | def flatten(lol): 73 | if isinstance(lol, tuple) or isinstance(lol, list): 74 | return sum([flatten(l) for l in lol], []) 75 | else: 76 | return [lol] 77 | 78 | def postorder(tree): 79 | if isinstance(tree, tuple): 80 | for subtree in tree[1:]: 81 | for node in postorder(subtree): 82 | yield node 83 | yield tree[0] 84 | else: 85 | yield tree 86 | 87 | def tree_map(function, tree): 88 | if isinstance(tree, tuple): 89 | head = function(tree) 90 | tail = tuple(tree_map(function, subtree) for subtree in tree[1:]) 91 | return (head,) + tail 92 | return function(tree) 93 | 94 | def tree_zip(*trees): 95 | if isinstance(trees[0], tuple): 96 | zipped_children = [[t[i] for t in trees] for i in range(len(trees[0]))] 97 | zipped_children_rec = [tree_zip(*z) for z in zipped_children] 98 | return tuple(zipped_children_rec) 99 | return trees 100 | 101 | FEXP_RE = re.compile(r"(.*)\[(.*)\]") 102 | def parse_fexp(fexp): 103 | m = FEXP_RE.match(fexp) 104 | return (m.group(1), m.group(2)) 105 | 106 | def pp_sexp(sexp): 107 | if not isinstance(sexp, tuple): 108 | return str(sexp) 109 | return "(" + " ".join([pp_sexp(s) for s in sexp]) + ")" 110 | 111 | random_counter = [0] 112 | def next_random(): 113 | random = np.random.RandomState(random_counter[0]) 114 | random_counter[0] += 1 115 | return random 116 | 117 | def batch_gather(params, indices): 118 | ids = tf.range(tf.shape(indices)[0]) 119 | full = tf.stack((ids, indices), axis=1) 120 | return tf.gather_nd(params, full) 121 | 122 | def vars_in_scope(scope): 123 | return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope.name) 124 | -------------------------------------------------------------------------------- /pbd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import models 4 | from models import TransducerModel 5 | from tasks import regex2 6 | 7 | import gflags 8 | import os 9 | import sys 10 | 11 | FLAGS = gflags.FLAGS 12 | gflags.DEFINE_boolean("train", False, "do a training run") 13 | gflags.DEFINE_boolean("test", False, "do a testing run") 14 | gflags.DEFINE_boolean("vis", False, "generate visualization output") 15 | gflags.DEFINE_integer("n_epochs", 0, "number of epochs to run for") 16 | gflags.DEFINE_integer("n_batch", 100, "batch size") 17 | models._set_flags() 18 | 19 | def main(): 20 | task = regex2.RegexTask() 21 | model = TransducerModel(task) 22 | 23 | if FLAGS.train: 24 | for i_epoch in range(FLAGS.n_epochs): 25 | e_loss = 0. 26 | for i_batch in range(100): 27 | batch = task.sample_train(FLAGS.n_batch) 28 | b_loss = model.train(batch) 29 | e_loss += b_loss 30 | 31 | batch = task.sample_train(FLAGS.n_batch) 32 | e_acc = model.predict(batch) 33 | print("[loss] %01.4f" % e_loss) 34 | print("[trn_acc] %01.4f" % e_acc) 35 | 36 | v_batch = task.sample_val() 37 | e_v_acc = model.predict(v_batch) 38 | print("[val_acc] %01.4f" % e_v_acc) 39 | print("") 40 | 41 | if i_epoch % 10 == 0: 42 | model.save() 43 | 44 | if FLAGS.test: 45 | v_batch = task.sample_val() 46 | e_v_acc = model.predict(v_batch) 47 | 48 | t_batch = task.sample_test() 49 | e_t_acc = model.predict(t_batch) 50 | 51 | print("[FINAL val_acc] %01.4f" % e_v_acc) 52 | print("[FINAL tst_acc] %01.4f" % e_t_acc) 53 | 54 | if FLAGS.vis: 55 | v_batch = task.sample_val() 56 | preds, labels, hyps = model.predict(v_batch, debug=True) 57 | for i in range(10): 58 | task.visualize(v_batch[i], hyps[i], preds[i][0]) 59 | 60 | if __name__ == "__main__": 61 | argv = FLAGS(sys.argv) 62 | main() 63 | -------------------------------------------------------------------------------- /rl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import rl_models 4 | from rl_models import Policy 5 | from tasks import minicraft2 6 | from tasks import nav 7 | from misc import util 8 | 9 | import sys 10 | import gflags 11 | 12 | FLAGS = gflags.FLAGS 13 | gflags.DEFINE_boolean("train", False, "do a training run") 14 | gflags.DEFINE_boolean("test", False, "do a testing run") 15 | gflags.DEFINE_integer("n_epochs", 0, "number of epochs to run for") 16 | gflags.DEFINE_integer("n_batch", 5000, "batch size") 17 | gflags.DEFINE_float("discount", 0.9, "discount factor") 18 | gflags.DEFINE_integer("max_steps", 50, "max rollout length") 19 | gflags.DEFINE_boolean("use_expert", False, "DAGGER training with expert feedback") 20 | rl_models._set_flags() 21 | 22 | N_PAR = 10 23 | 24 | random = util.next_random() 25 | 26 | def main(): 27 | task = nav.NavTask() 28 | policy = Policy(task) 29 | 30 | if FLAGS.train: 31 | if FLAGS.n_epochs == 0: 32 | policy.save() 33 | 34 | for i_epoch in range(FLAGS.n_epochs): 35 | total_rew = 0 36 | total_err = 0 37 | n_rollouts = 0 38 | for i in range(10): 39 | buf = [] 40 | while len(buf) < FLAGS.n_batch: 41 | states = [task.sample_train() for _ in range(N_PAR)] 42 | use_expert = FLAGS.use_expert and random.rand() < (0.95 ** i_epoch) 43 | rollouts, rews = do_rollout(task, policy, states, expert=use_expert) 44 | for rollout, rew in zip(rollouts, rews): 45 | buf.extend(rollout) 46 | if not use_expert: 47 | total_rew += rew 48 | n_rollouts += 1 49 | if FLAGS.use_expert: 50 | total_err += policy.train_dagger(buf) 51 | else: 52 | total_err += policy.train(buf) 53 | 54 | test_states = [task.sample_test(random.choice(task.test_ids)) for _ in range(100)] 55 | _, test_rews = do_rollout(task, policy, test_states, vis=False) 56 | total_test_rew = sum(test_rews) 57 | 58 | print("[iter] %d" % i_epoch) 59 | print("[loss] %01.4f" % (total_err / 10)) 60 | if n_rollouts > 0: 61 | print("[trn_rew] %01.4f" % (total_rew / n_rollouts)) 62 | print("[tst_rew] %01.4f" % (total_test_rew / len(test_rews))) 63 | hyps = policy.sample_hyps(3) 64 | for hyp in hyps: 65 | print " ".join(policy.task.vocab.get(w) for w in hyp) 66 | print 67 | 68 | if i_epoch % 10 == 0: 69 | policy.save() 70 | 71 | if FLAGS.test: 72 | for i_datum in task.test_ids: 73 | print "TEST DATUM", i_datum 74 | policy.restore(FLAGS.restore) 75 | policy.reset() 76 | state = task.sample_test(i_datum, erase_hint=False) 77 | print " ".join(task.vocab.get(w) for w in state.instruction) 78 | for i_epoch in range(FLAGS.n_epochs): 79 | total_rew = 0. 80 | n_rollouts = 0 81 | buf = [] 82 | episodes = [] 83 | while len(buf) < FLAGS.n_batch: 84 | states = [task.sample_test(i_datum, erase_hint=True) for _ in range(N_PAR)] 85 | states = policy.annotate(states) 86 | rollouts, rews = do_rollout(task, policy, states, expert=False) 87 | total_rew += sum(rews) 88 | n_rollouts += len(rews) 89 | for rollout, rew in zip(rollouts, rews): 90 | buf.extend(rollout) 91 | episodes.append((rollout, rew)) 92 | 93 | policy.adapt(buf, episodes) 94 | print("[FINAL tst_rew] %01.4f" % (total_rew / n_rollouts)) 95 | 96 | def do_rollout(task, policy, states, vis=False, expert=False): 97 | states = list(states) 98 | bufs = [[] for _ in states] 99 | done = [False for _ in states] 100 | for t in range(FLAGS.max_steps): 101 | actions = policy.act(states) 102 | for i_state in range(len(states)): 103 | if done[i_state]: 104 | continue 105 | state = states[i_state] 106 | if i_state == 0 and vis: 107 | print state.render() 108 | action = state.expert_a if expert else actions[i_state] 109 | state_, reward, stop = state.step(action) 110 | bufs[i_state].append((state, action, state_, reward)) 111 | states[i_state] = state_ 112 | if stop: 113 | done[i_state] = True 114 | if i_state == 0 and vis: 115 | print state_.render() 116 | if all(done): 117 | break 118 | 119 | discounted_bufs = [] 120 | total_rs = [] 121 | for buf in bufs: 122 | forward_r = 0. 123 | total_r = 0. 124 | discounted_buf = [] 125 | for s, a, s_, r in reversed(buf): 126 | forward_r *= FLAGS.discount 127 | r_ = r + forward_r 128 | discounted_buf.append((s, a, s_, r_)) 129 | forward_r += r 130 | total_r += r 131 | discounted_bufs.append(discounted_buf) 132 | total_rs.append(total_r) 133 | return discounted_bufs, total_rs 134 | 135 | if __name__ == "__main__": 136 | argv = FLAGS(sys.argv) 137 | main() 138 | -------------------------------------------------------------------------------- /rl_models.py: -------------------------------------------------------------------------------- 1 | from models import _encode, Decoder 2 | from models import N_HIDDEN as N_DEC_HIDDEN 3 | from net import _mlp, _linear, _embed_dict 4 | from misc import util 5 | 6 | from collections import defaultdict 7 | import gflags 8 | import numpy as np 9 | import tensorflow as tf 10 | 11 | FLAGS = gflags.FLAGS 12 | def _set_flags(): 13 | gflags.DEFINE_boolean("predict_hyp", False, "train to predict hypotheses") 14 | gflags.DEFINE_boolean("infer_hyp", False, "use hypotheses at test time") 15 | gflags.DEFINE_string("restore", None, "model to restore") 16 | gflags.DEFINE_float("concept_prior", None, "place a normal prior on concept representations") 17 | gflags.DEFINE_integer("adapt_reprs", 100, "number of representations to sample when doing adaptation") 18 | gflags.DEFINE_integer("adapt_samples", 1000, "number of episodes to spend evaluating sampled representations") 19 | 20 | N_EMBED = 64 21 | N_HIDDEN = 64 22 | 23 | random = util.next_random() 24 | 25 | class Policy(object): 26 | def __init__(self, task): 27 | self.task = task 28 | 29 | self.t_state = tf.placeholder(tf.float32, (None, task.n_features)) 30 | self.t_action = tf.placeholder(tf.int32, (None,)) 31 | self.t_reward = tf.placeholder(tf.float32, (None,)) 32 | self.t_hint = tf.placeholder(tf.int32, (None, None)) 33 | self.t_hint_len = tf.placeholder(tf.int32, (None,)) 34 | self.t_task = tf.placeholder(tf.int32, (None,)) 35 | 36 | self.t_last_hyp = tf.placeholder(tf.int32, (None,), "last_hyp") 37 | self.t_last_hyp_hidden = tf.placeholder(tf.float32, (None, N_DEC_HIDDEN), 38 | "last_hyp_hidden") 39 | t_hyp_init = tf.get_variable("hyp_init", shape=(1, N_DEC_HIDDEN), 40 | initializer=tf.uniform_unit_scaling_initializer()) 41 | self.t_n_batch = tf.shape(self.t_state)[0] 42 | #self.t_n_batch = tf.placeholder(tf.int32, ()) 43 | t_hyp_tile = tf.tile(t_hyp_init, (self.t_n_batch, 1)) 44 | 45 | t_hint_vecs = tf.get_variable( 46 | "hint_vec", (len(task.vocab), N_EMBED), 47 | initializer=tf.uniform_unit_scaling_initializer()) 48 | t_hint_repr = tf.reduce_mean(_embed_dict(self.t_hint, t_hint_vecs), axis=1) 49 | self.hyp_decoder = Decoder( 50 | "decode_hyp", t_hyp_tile, self.t_hint, self.t_last_hyp, 51 | self.t_last_hyp_hidden, t_hint_vecs) 52 | 53 | t_task_vecs = tf.get_variable( 54 | "task_vec", (task.n_tasks, N_EMBED), 55 | initializer=tf.uniform_unit_scaling_initializer()) 56 | t_task_repr = _embed_dict(self.t_task, t_task_vecs) 57 | 58 | if FLAGS.infer_hyp: 59 | self.t_concept = t_hint_repr 60 | else: 61 | self.t_concept = t_task_repr 62 | 63 | with tf.variable_scope("features"): 64 | t_features = _mlp(self.t_state, (N_HIDDEN, N_HIDDEN), (tf.nn.tanh, tf.nn.tanh)) 65 | with tf.variable_scope("param"): 66 | t_concept_param = _linear(self.t_concept, N_HIDDEN * task.n_actions) 67 | t_concept_mat = tf.reshape(t_concept_param, (-1, N_HIDDEN, task.n_actions)) 68 | self.t_score = tf.einsum("ij,ijk->ik", t_features, t_concept_mat) 69 | 70 | self.t_logprob = tf.nn.log_softmax(self.t_score) 71 | t_prob = tf.nn.softmax(self.t_score) 72 | t_entropy = -tf.reduce_mean(tf.reduce_sum(t_prob * self.t_logprob, axis=1)) 73 | 74 | with tf.variable_scope("baseline"): 75 | t_baseline = tf.squeeze(_linear(tf.stop_gradient(t_features), 1)) 76 | 77 | t_chosen_logprob = -tf.nn.sparse_softmax_cross_entropy_with_logits( 78 | logits=self.t_score, labels=self.t_action) 79 | t_loss_surrogate = -tf.reduce_mean( 80 | t_chosen_logprob * (self.t_reward - tf.stop_gradient(t_baseline))) 81 | t_baseline_err = tf.reduce_mean((t_baseline - self.t_reward) ** 2) 82 | 83 | self.t_rl_loss = t_loss_surrogate + t_baseline_err - 0.001 * t_entropy 84 | self.t_dagger_loss = -tf.reduce_mean(t_chosen_logprob) 85 | 86 | if FLAGS.concept_prior is not None: 87 | def normal(x): 88 | return tf.reduce_mean(tf.reduce_sum(tf.square(x), axis=1)) 89 | self.t_rl_loss += normal(self.t_concept) / FLAGS.concept_prior 90 | self.t_dagger_loss += normal(self.t_concept) / FLAGS.concept_prior 91 | 92 | if FLAGS.predict_hyp: 93 | self.t_loss = self.t_rl_loss + self.hyp_decoder.t_loss 94 | self.t_dagger_loss = self.t_dagger_loss + self.hyp_decoder.t_loss 95 | else: 96 | self.t_loss = self.t_rl_loss 97 | 98 | optimizer = tf.train.AdamOptimizer(0.001) 99 | self.o_train = optimizer.minimize(self.t_loss) 100 | self.o_rl_train = optimizer.minimize(self.t_rl_loss) 101 | self.o_dagger_train = optimizer.minimize(self.t_dagger_loss) 102 | 103 | self.session = tf.Session() 104 | self.session.run(tf.global_variables_initializer()) 105 | self.saver = tf.train.Saver() 106 | if FLAGS.restore is not None: 107 | self.restore(FLAGS.restore) 108 | 109 | def load_hint(self, states): 110 | max_len = max(len(s.instruction) for s in states) 111 | hint = np.zeros((len(states), max_len)) 112 | hint_len = np.zeros((len(states),)) 113 | for i, state in enumerate(states): 114 | hint[i, :len(state.instruction)] = state.instruction 115 | hint_len[i] = len(state.instruction) 116 | return hint, hint_len 117 | 118 | def act(self, states): 119 | hint, hint_len = self.load_hint(states) 120 | feed_dict = { 121 | self.t_state: [s.features for s in states], 122 | self.t_hint: hint, 123 | self.t_hint_len: hint_len, 124 | self.t_task: [s.task_id for s in states] 125 | } 126 | 127 | logprobs, = self.session.run([self.t_logprob], feed_dict) 128 | probs = np.exp(logprobs) 129 | actions = [] 130 | for i in range(len(states)): 131 | action = random.choice(self.task.n_actions, p=probs[i, :]) 132 | actions.append(action) 133 | return actions 134 | 135 | def train(self, transitions, ignore_hyp=False): 136 | states, actions, _, rewards = zip(*transitions) 137 | features = [s.features for s in states] 138 | hint, hint_len = self.load_hint(states) 139 | feed_dict = { 140 | self.t_state: features, 141 | self.t_action: actions, 142 | self.t_reward: rewards, 143 | self.t_hint: hint, 144 | self.t_hint_len: hint_len, 145 | self.t_task: [s.task_id for s in states] 146 | } 147 | t_loss, o_train = (self.t_rl_loss, self.o_rl_train) if ignore_hyp else (self.t_loss, self.o_train) 148 | loss, _ = self.session.run([t_loss, o_train], feed_dict) 149 | return loss 150 | 151 | def train_dagger(self, transitions): 152 | states = [t[0] for t in transitions] 153 | actions = [s.expert_a for s in states] 154 | features = [s.features for s in states] 155 | hint, hint_len = self.load_hint(states) 156 | feed_dict = { 157 | self.t_state: features, 158 | self.t_action: actions, 159 | self.t_hint: hint, 160 | self.t_hint_len: hint_len, 161 | self.t_task: [s.task_id for s in states] 162 | } 163 | 164 | loss, _ = self.session.run([self.t_dagger_loss, self.o_dagger_train], feed_dict) 165 | return loss 166 | 167 | def sample_hyps(self, n): 168 | init = [self.task.vocab[self.task.START] for _ in range(n)] 169 | hyps = self.hyp_decoder.decode( 170 | init, 171 | self.task.vocab[self.task.STOP], 172 | {self.t_n_batch: len(init)}, 173 | self.session, 174 | temp=1) 175 | return hyps 176 | 177 | def reset(self): 178 | self._adapt_scores = defaultdict(lambda: 0.) 179 | self._adapt_counts = defaultdict(lambda: 0) 180 | self._adapt_all = defaultdict(list) 181 | self._adapt_total = 0 182 | 183 | if FLAGS.infer_hyp: 184 | adapt_reprs = self.sample_hyps(FLAGS.adapt_reprs) 185 | self._adapt_reprs = list(set(tuple(r) for r in adapt_reprs)) 186 | self._n_ad = len(self._adapt_reprs) 187 | self._ann_counter = 0 188 | else: 189 | self._adapt_reprs = random.randint(self.task.n_tasks, 190 | size=FLAGS.adapt_reprs) 191 | self._n_ad = FLAGS.adapt_reprs 192 | self._ann_counter = 0 193 | 194 | self.ready = False 195 | 196 | def adapt(self, transitions, episodes): 197 | if self._adapt_total < FLAGS.adapt_samples: 198 | for ss, r in episodes: 199 | s = ss[0][0] 200 | # we might have written over the real task id 201 | self._adapt_scores[s.datum.task_id, s.meta] += r 202 | self._adapt_counts[s.datum.task_id, s.meta] += 1 203 | self._adapt_all[s.datum.task_id, s.meta].append(r) 204 | self._adapt_total += 1 205 | else: 206 | self.train(transitions) 207 | 208 | def annotate(self, states): 209 | if self._adapt_total < FLAGS.adapt_samples: 210 | n_ad = self._n_ad 211 | repr_ids = [random.randint(n_ad) for state in states] 212 | reprs = [self._adapt_reprs[i] for i in repr_ids] 213 | new_states = [] 214 | for i, state in enumerate(states): 215 | if FLAGS.infer_hyp: 216 | new_states.append(state.annotate_instruction(reprs[i], repr_ids[i])) 217 | else: 218 | new_states.append(state.annotate_task(reprs[i], repr_ids[i])) 219 | return new_states 220 | else: 221 | # TODO outside 222 | adapt_means = {} 223 | for k in self._adapt_scores: 224 | adapt_means[k] = self._adapt_scores[k] / self._adapt_counts[k] 225 | 226 | if not self.ready: 227 | print "TRANSITION" 228 | self.ready = True 229 | 230 | for k, v in sorted(self._adapt_counts.items(), 231 | key=lambda x: adapt_means[x[0]]): 232 | if FLAGS.infer_hyp: 233 | print ( 234 | k, 235 | v, 236 | " ".join(self.task.vocab.get(w) for w in self._adapt_reprs[k[1]]), 237 | adapt_means[k] 238 | ) 239 | else: 240 | print (k, v, adapt_means[k]) 241 | 242 | new_states = [] 243 | for state in states: 244 | # we might have written over the real task id 245 | valid_keys = [k for k in adapt_means if k[0] == state.datum.task_id] 246 | best_key = max(valid_keys, key=lambda x: adapt_means[x]) 247 | choose_repr = self._adapt_reprs[best_key[1]] 248 | if FLAGS.infer_hyp: 249 | new_states.append(state.annotate_instruction(choose_repr, best_key[1])) 250 | else: 251 | new_states.append(state.annotate_task(choose_repr, best_key[1])) 252 | return new_states 253 | 254 | def save(self): 255 | self.saver.save(self.session, "model.chk") 256 | 257 | def restore(self, path): 258 | self.saver.restore(self.session, tf.train.latest_checkpoint(path)) 259 | -------------------------------------------------------------------------------- /tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobandreas/l3/e1748ef42f37d3eff090223e413465440b2daa12/tasks/__init__.py -------------------------------------------------------------------------------- /tasks/birds.py: -------------------------------------------------------------------------------- 1 | from misc import util 2 | 3 | from collections import namedtuple 4 | import csv 5 | import numpy as np 6 | import os 7 | import pickle 8 | import sys 9 | 10 | N_EX = 4 11 | 12 | Datum = namedtuple("Datum", ["hint", "ex_inputs", "input", "label"]) 13 | 14 | START = "" 15 | STOP = "" 16 | 17 | random = util.next_random() 18 | 19 | birds_path = os.path.join(sys.path[0], "data/birds") 20 | 21 | def choose_except(options, reject, random=random): 22 | choice = None 23 | while choice is None: 24 | choice = random.choice(options) 25 | if choice in reject: 26 | choice = None 27 | return choice 28 | 29 | class BirdsTask(): 30 | def __init__(self): 31 | self.hint_vocab = util.Index() 32 | self.START = START 33 | self.STOP = STOP 34 | 35 | with open(os.path.join(birds_path, "hendricks_data", "CUB_feature_dict.pkl")) as feat_f: 36 | self.features = pickle.load(feat_f) 37 | #file_to_full = {k.split("/")[1]: k for k in self.features} 38 | 39 | #self.captions = {} 40 | #for fname in os.listdir(os.path.join(birds_path, "captions")): 41 | # name = file_to_full[fname[:-4] + ".jpg"] 42 | # inst_capts = [] 43 | # with open(os.path.join(birds_path, "captions", fname)) as capt_f: 44 | # for line in capt_f: 45 | # line = line.strip().replace(".", " .").replace(",", " ,") 46 | # toks = [START] + line.split() + [STOP] 47 | # toks = [self.hint_vocab.index(w) for w in toks] 48 | # inst_capts.append(tuple(toks)) 49 | # self.captions[name] = tuple(inst_capts) 50 | self.captions = {} 51 | with open(os.path.join(birds_path, "hendricks_data", "captions.tsv")) as capt_f: 52 | reader = csv.DictReader(capt_f, delimiter="\t") 53 | for row in reader: 54 | caption = row["Description"].lower().replace(".", " .").replace(",", " ,") 55 | toks = [START] + caption.split() + [STOP] 56 | toks = [self.hint_vocab.index(w) for w in toks] 57 | url = row["Input.image_url"] 58 | inst = "/".join(url.split("/")[-2:]) 59 | if inst not in self.captions: 60 | self.captions[inst] = [] 61 | self.captions[inst].append(toks) 62 | 63 | classes = sorted(list(set(k.split("/")[0] for k in self.captions))) 64 | classes.remove("cub_missing") 65 | shuf_random = np.random.RandomState(999) 66 | shuf_random.shuffle(classes) 67 | assert len(classes) == 200 68 | data_classes = { 69 | "train": classes[:100], 70 | "val": classes[100:110], 71 | "test": classes[100:200] 72 | } 73 | 74 | data_insts = {} 75 | for fold in ("train", "val", "test"): 76 | classes = data_classes[fold] 77 | data_classes[fold] = classes 78 | 79 | instances = {cls: [] for cls in classes} 80 | for key in self.features.keys(): 81 | cls, inst = key.split("/") 82 | if cls in instances: 83 | instances[cls].append(key) 84 | data_insts[fold] = instances 85 | 86 | # print fold 87 | # for cls in classes: 88 | # print cls, len(instances[cls]) 89 | # print 90 | #exit() 91 | 92 | self.train_classes = data_classes["train"] 93 | self.val_classes = data_classes["val"] 94 | self.test_classes = data_classes["test"] 95 | 96 | self.train_insts = data_insts["train"] 97 | self.val_insts = data_insts["val"] 98 | self.test_insts = data_insts["test"] 99 | 100 | self.n_features = self.features[self.features.keys()[0]].size 101 | 102 | def sample_train(self, n_batch, augment): 103 | assert not augment 104 | 105 | batch = [] 106 | for _ in range(n_batch): 107 | cls = random.choice(self.train_classes) 108 | insts = [random.choice(self.train_insts[cls]) for _ in range(N_EX)] 109 | captions = self.captions[insts[0]] 110 | caption = captions[random.choice(len(captions))] 111 | feats = np.asarray([self.features[inst] for inst in insts]) 112 | 113 | label = random.randint(2) 114 | if label == 0: 115 | other_cls = choose_except(self.train_classes, [cls]) 116 | other_inst = random.choice(self.train_insts[other_cls]) 117 | else: 118 | other_inst = choose_except(self.train_insts[cls], insts) 119 | 120 | other_feats = self.features[other_inst] 121 | datum = Datum(caption, feats, other_feats, label) 122 | batch.append(datum) 123 | return batch 124 | 125 | def sample_heldout(self, classes, insts): 126 | batch = [] 127 | local_random = np.random.RandomState(0) 128 | for i, cls in enumerate(classes): 129 | datum_insts = insts[cls][:N_EX] 130 | caption = self.captions[datum_insts[0]][0] 131 | feats = np.asarray([self.features[inst] for inst in datum_insts]) 132 | label = i % 2 133 | if label == 0: 134 | other_cls = choose_except(classes, [cls], local_random) 135 | other_inst = insts[other_cls][N_EX] 136 | else: 137 | other_inst = insts[cls][N_EX] 138 | 139 | other_feats = self.features[other_inst] 140 | datum = Datum(caption, feats, other_feats, label) 141 | batch.append(datum) 142 | return batch 143 | 144 | def sample_val(self, same=False): 145 | return self.sample_heldout(self.val_classes, self.val_insts) 146 | 147 | def sample_test(self, same=False): 148 | return self.sample_heldout(self.test_classes, self.test_insts) 149 | -------------------------------------------------------------------------------- /tasks/nav/library.py: -------------------------------------------------------------------------------- 1 | spritepath = 'sprites/' 2 | 3 | objects = { 4 | 'grass': { 5 | 'index': 0, 6 | 'value': 0, 7 | 'sprite': 'environment/sprites/white.png', # 'sprites/white.png', 8 | 'background': True, 9 | 'unique': False, 10 | }, 11 | 'puddle': { 12 | 'index': 1, 13 | 'value': -1, 14 | 'sprite': 'environment/sprites/white.png', 15 | 'background': True, 16 | 'unique': False, 17 | }, 18 | ## unique 19 | 'star': { 20 | 'index': 2, 21 | 'value': 0, 22 | 'sprite': 'environment/sprites/white_alpha.png', ## white_alpha.png 23 | 'background': False, 24 | 'unique': True, 25 | }, 26 | 'circle': { 27 | 'index': 3, 28 | 'value': 0, 29 | 'sprite': 'environment/sprites/white_alpha.png', 30 | 'background': False, 31 | 'unique': True, 32 | }, 33 | 'triangle': { 34 | 'index': 4, 35 | 'value': 0, 36 | 'sprite': 'environment/sprites/white_alpha.png', 37 | 'background': False, 38 | 'unique': True, 39 | }, 40 | 'heart': { 41 | 'index': 5, 42 | 'value': 0, 43 | 'sprite': 'environment/sprites/white_alpha.png', 44 | 'background': False, 45 | 'unique': True, 46 | }, 47 | 'spade': { 48 | 'index': 6, 49 | 'value': 0, 50 | 'sprite': 'environment/sprites/white_alpha.png', 51 | 'background': False, 52 | 'unique': True, 53 | }, 54 | 'diamond': { 55 | 'index': 7, 56 | 'value': 0, 57 | 'sprite': 'environment/sprites/white_alpha.png', 58 | 'background': False, 59 | 'unique': True, 60 | }, 61 | ## non-unique 62 | 'rock': { 63 | 'index': 8, 64 | 'value': 0, 65 | 'sprite': 'environment/sprites/rock_hires.png', 66 | 'background': False, 67 | 'unique': False, 68 | }, 69 | 'tree': { 70 | 'index': 9, 71 | 'value': 0, 72 | 'sprite': 'environment/sprites/tree_hires.png', 73 | 'background': False, 74 | 'unique': False, 75 | }, 76 | 'house': { 77 | 'index': 10, 78 | 'value': 0, 79 | 'sprite': 'environment/sprites/house_hires.png', 80 | 'background': False, 81 | 'unique': False, 82 | }, 83 | 'horse': { 84 | 'index': 11, 85 | 'value': 0, 86 | 'sprite': 'environment/sprites/horse_hires.png', 87 | 'background': False, 88 | 'unique': False, 89 | }, 90 | } 91 | 92 | unique_instructions = { 93 | ## original 94 | 'to top left of': (-1, -1), 95 | 'on top of': (-1, 0), 96 | 'to top right of': (-1, 1), 97 | 'to left of': (0, -1), 98 | 'with': (0, 0), 99 | 'to right of': (0, 1), 100 | 'to bottom left of': (1, -1), 101 | 'on bottom of': (1, 0), 102 | 'to bottom right of': (1, 1), 103 | ## two steps away 104 | 'two to the left and two above': (-2, -2), 105 | 'one to the left and two above': (-2, -1), 106 | 'two above': (-2, 0), 107 | 'one to the right and two above': (-2, 1), 108 | 'two to the right and two above': (-2, 2), 109 | 'two to the right and one above': (-1, 2), 110 | 'two to the right of': (0, 2), 111 | 'two to the right and one below': (1, 2), 112 | 'two to the right and two below': (2, 2), 113 | 'one to the right and two below': (2, 1), 114 | 'two below': (2, 0), 115 | 'one to the left and two below': (2, -1), 116 | 'two to the left and two below': (2, -2), 117 | 'two to the left and one below': (1, -2), 118 | 'two to the left': (0, -2), 119 | 'two to the left and one above': (-1, -2) 120 | 121 | } 122 | 123 | background = 'environment/sprites/white.png' 124 | 125 | # print objects -------------------------------------------------------------------------------- /tasks/regex2.py: -------------------------------------------------------------------------------- 1 | from misc import util 2 | 3 | from collections import namedtuple 4 | import gflags 5 | import json 6 | import re 7 | import sys 8 | import os 9 | 10 | FLAGS = gflags.FLAGS 11 | gflags.DEFINE_string("hint_type", None, "hint format") 12 | 13 | SEP = "@" 14 | START = "<" 15 | STOP = ">" 16 | N_EX = 5 17 | 18 | FullDatum = namedtuple("FullDatum", ["hints", "pairs"]) 19 | Datum = namedtuple("Datum", ["hint", "ex_inputs", "ex_outputs", "input", "output"]) 20 | 21 | random = util.next_random() 22 | 23 | class RegexTask(): 24 | def __init__(self): 25 | assert FLAGS.hint_type in ("re", "nl", "none") 26 | 27 | with open(os.path.join(sys.path[0], "data/re2/corpus.json")) as corpus_f: 28 | corpus = json.load(corpus_f) 29 | 30 | self.hint_vocab = util.Index() 31 | self.str_vocab = util.Index() 32 | self.str_vocab.index(SEP) 33 | self.SEP = SEP 34 | self.START = START 35 | self.STOP = STOP 36 | 37 | data = {} 38 | for fold in ["train", "val", "test"]: 39 | data[fold] = [] 40 | for example in corpus[fold]: 41 | if FLAGS.hint_type == "re": 42 | hint = example["re"] 43 | hint = [self.hint_vocab.index(c) for c in hint] 44 | hints = [hint] 45 | elif FLAGS.hint_type == "nl": 46 | hints = [] 47 | for hint in example["hints_aug"]: 48 | hint = [self.hint_vocab.index(w) for w in hint] 49 | hints.append(hint) 50 | elif FLAGS.hint_type == "none": 51 | hints = [[]] 52 | 53 | pairs = [] 54 | for inp, out in example["examples"]: 55 | inp = [self.str_vocab.index(c) for c in inp] 56 | out = [self.str_vocab.index(c) for c in out] 57 | pairs.append((inp, out)) 58 | 59 | datum = FullDatum(hints, pairs) 60 | data[fold].append(datum) 61 | 62 | self.train_data = data["train"] 63 | self.val_data = data["val"] 64 | self.test_data = data["test"] 65 | 66 | def sample_train(self, n_batch): 67 | batch = [] 68 | n_train = len(self.train_data) 69 | for _ in range(n_batch): 70 | full_datum = self.train_data[random.randint(n_train)] 71 | pairs = list(full_datum.pairs) 72 | random.shuffle(pairs) 73 | hint = full_datum.hints[random.randint(len(full_datum.hints))] 74 | inp, out = pairs.pop() 75 | pairs = pairs[:N_EX] 76 | ex_inputs, ex_outputs = zip(*pairs) 77 | assert len(ex_inputs) == N_EX 78 | datum = Datum(hint, ex_inputs, ex_outputs, inp, out) 79 | batch.append(datum) 80 | return batch 81 | 82 | def sample_val(self): 83 | batch = [] 84 | for full_datum in self.val_data: 85 | pairs = list(full_datum.pairs) 86 | inp, out = pairs.pop() 87 | ex_inputs, ex_outputs = zip(*pairs) 88 | assert len(ex_inputs) == N_EX 89 | if FLAGS.use_true_hyp or FLAGS.use_true_eval or FLAGS.vis: 90 | hint = full_datum.hints[random.randint(len(full_datum.hints))] 91 | else: 92 | hint = [] 93 | batch.append(Datum(hint, ex_inputs, ex_outputs, inp, out)) 94 | return batch 95 | 96 | def sample_test(self): 97 | batch = [] 98 | for full_datum in self.test_data: 99 | pairs = list(full_datum.pairs) 100 | inp, out = pairs.pop() 101 | ex_inputs, ex_outputs = zip(*pairs) 102 | assert len(ex_inputs) == N_EX 103 | batch.append(Datum([], ex_inputs, ex_outputs, inp, out)) 104 | return batch 105 | 106 | def execute(self, hint, inps, outs): 107 | if 0 in hint: 108 | return [-1 for _ in inps], ["" for _ in inps] 109 | hint = "".join(self.hint_vocab.get(t) for t in hint) 110 | hint = hint[1:-1] 111 | if hint.count("@") != 1: 112 | return [-1 for _ in inps], ["" for _ in inps] 113 | before, after = hint.split("@") 114 | before = before.replace("C", "[^aeiou]").replace("V", "[aeiou]") 115 | 116 | inps = [inp[1:-1] for inp in inps] 117 | inps = ["".join(self.str_vocab.get(w) for w in inp) for inp in inps] 118 | outs = [out[1:-1] for out in outs] 119 | outs = ["".join(self.str_vocab.get(w) for w in out) for out in outs] 120 | 121 | try: 122 | predicted_out_strs = [re.sub(before, after, inp) for inp in inps] 123 | except Exception as e: 124 | return [-1 for _ in inps], ["" for _ in inps] 125 | score = [float(p == o) for p, o in zip(outs, predicted_out_strs)] 126 | 127 | predicted_outs = [] 128 | for s in predicted_out_strs: 129 | predicted_outs.append( 130 | [self.str_vocab[START]] 131 | + [self.str_vocab[w] for w in s] 132 | + [self.str_vocab[STOP]]) 133 | 134 | return score, predicted_outs 135 | 136 | def visualize(self, datum, hyp, pred): 137 | for i in range(N_EX): 138 | print "".join(self.str_vocab.get(w) for w in datum.ex_inputs[i][1:-1]), 139 | print "".join(self.str_vocab.get(w) for w in datum.ex_outputs[i][1:-1]) 140 | print 141 | print "gold:", " ".join(self.hint_vocab.get(w) for w in datum.hint[1:-1]) 142 | print "pred:", " ".join(self.hint_vocab.get(w) for w in hyp[1:-1]) 143 | print 144 | print "".join(self.str_vocab.get(w) for w in datum.input) 145 | print "gold:", "".join(self.str_vocab.get(w) for w in datum.output) 146 | print "pred:", "".join(self.str_vocab.get(w) for w in pred) 147 | print "===" 148 | print 149 | -------------------------------------------------------------------------------- /tasks/shapes.py: -------------------------------------------------------------------------------- 1 | from misc import util 2 | 3 | from collections import namedtuple 4 | import numpy as np 5 | import os 6 | from PIL import Image 7 | import sys 8 | import json 9 | import scipy 10 | import gflags 11 | 12 | FLAGS = gflags.FLAGS 13 | 14 | USE_IMAGES = False 15 | #N_EX = 6 16 | N_EX = 4 17 | 18 | sw_path = os.path.join(sys.path[0], "data/shapeworld") 19 | 20 | Fold = namedtuple("Fold", ["hints", "examples", "inputs", "labels"]) 21 | Datum = namedtuple("Datum", ["hint", "ex_inputs", "input", "label"]) 22 | VisDatum = namedtuple("VisDatum", ["hint", "ex_inputs", "input", "label", "vis_ex_inputs", "vis_input"]) 23 | 24 | START = "" 25 | STOP = "" 26 | 27 | random = util.next_random() 28 | 29 | class ShapeworldTask(): 30 | def __init__(self): 31 | self.hint_vocab = util.Index() 32 | self.feature_index = util.Index() 33 | self.START = START 34 | self.STOP = STOP 35 | 36 | #with open(os.path.join(sw_path, "train", "examples.struct.json")) as feature_f: 37 | # feature_data = json.load(feature_f) 38 | # for datum in feature_data: 39 | # for example in datum: 40 | # for feature in example: 41 | # self.feature_index.index(tuple(feature)) 42 | 43 | data = {} 44 | for fold in ("train", "val", "test", "val_same", "test_same"): 45 | examples = np.load(os.path.join(sw_path, fold, "examples.npy")) 46 | inputs = np.load(os.path.join(sw_path, fold, "inputs.npy")) 47 | labels = np.load(os.path.join(sw_path, fold, "labels.npy")) 48 | 49 | with open(os.path.join(sw_path, fold, "hints.json")) as hint_f: 50 | hints = json.load(hint_f) 51 | 52 | #new_hints = [] 53 | #for hint in hints: 54 | # hint = hint.split() 55 | # new_hint = [] 56 | # for i in range(len(hint) - 1): 57 | # new_hint.append(hint[i] + "/" + hint[i+1]) 58 | # new_hints.append(" ".join(new_hint)) 59 | #hints = new_hints 60 | 61 | indexed_hints = [] 62 | for hint in hints: 63 | hint = [START] + hint.split() + [STOP] 64 | indexed_hint = [self.hint_vocab.index(w) for w in hint] 65 | indexed_hints.append(indexed_hint) 66 | hints = indexed_hints 67 | 68 | #ex_features = np.zeros((examples.shape[0], examples.shape[1], len(self.feature_index))) 69 | #inp_features = np.zeros((examples.shape[0], len(self.feature_index))) 70 | #with open(os.path.join(sw_path, fold, "examples.struct.json")) as ex_struct_f: 71 | # examples_struct = json.load(ex_struct_f) 72 | # for i_datum, expls in enumerate(examples_struct): 73 | # for i_ex, example in enumerate(expls): 74 | # for feature in example: 75 | # i_feat = self.feature_index[tuple(feature)] 76 | # if i_feat: 77 | # ex_features[i_datum, i_ex, i_feat] = 1 78 | #with open(os.path.join(sw_path, fold, "inputs.struct.json")) as in_struct_f: 79 | # inputs_struct = json.load(in_struct_f) 80 | # for i_datum, example in enumerate(inputs_struct): 81 | # for feature in example: 82 | # i_feat = self.feature_index[tuple(feature)] 83 | # if i_feat is not None: 84 | # inp_features[i_datum, i_feat] = 1 85 | ex_features = np.load(os.path.join(sw_path, fold, "examples.feats.npy")) 86 | inp_features = np.load(os.path.join(sw_path, fold, "inputs.feats.npy")) 87 | 88 | fold_data = [] 89 | 90 | for i in range(len(hints)): 91 | if USE_IMAGES: 92 | fold_data.append(Datum( 93 | hints[i], examples[i, ...], inputs[i, ...], labels[i])) 94 | else: 95 | fold_data.append(Datum( 96 | hints[i], ex_features[i, ...], inp_features[i, ...], labels[i])) 97 | if FLAGS.vis: 98 | # TODO this is so dirty! 99 | datum = fold_data[-1] 100 | fold_data[-1] = VisDatum( 101 | datum.hint, datum.ex_inputs, datum.input, 102 | datum.label, examples[i, ...], inputs[i, ...]) 103 | data[fold] = fold_data 104 | 105 | self.train_data = data["train"] 106 | self.val_data = data["val"] 107 | self.test_data = data["test"] 108 | self.val_same_data = data["val_same"] 109 | self.test_same_data = data["test_same"] 110 | 111 | #self.train_data = data["train"][:8000] 112 | #self.val_data = data["train"][8000:8500] 113 | #self.test_data = data["train"][8500:9000] 114 | 115 | if USE_IMAGES: 116 | self.width, self.height, self.channels = self.train_data[0].input.shape 117 | else: 118 | #self.n_features = len(self.feature_index) 119 | self.n_features = inp_features.shape[1] 120 | 121 | def sample_train(self, n_batch, augment): 122 | n_train = len(self.train_data) 123 | batch = [] 124 | 125 | #for _ in range(n_batch): 126 | # datum = self.train_data[random.randint(n_train)] 127 | # batch.append(datum) 128 | 129 | for _ in range(n_batch): 130 | datum = self.train_data[random.randint(n_train)] 131 | if not augment: 132 | batch.append(datum) 133 | continue 134 | 135 | label = random.randint(2) 136 | if label == 0: 137 | alt_datum = self.train_data[random.randint(n_train)] 138 | swap = random.randint(N_EX + 1) 139 | if swap == N_EX: 140 | feats = alt_datum.input 141 | else: 142 | feats = alt_datum.ex_inputs[swap, ...] 143 | datum = datum._replace(input=feats, label=0) 144 | 145 | elif label == 1: 146 | swap = random.randint((N_EX + 1 if datum.label == 1 else N_EX)) 147 | if swap != N_EX: 148 | examples = datum.ex_inputs.copy() 149 | feats = examples[swap, ...] 150 | if datum.label == 1: 151 | examples[swap, ...] = datum.input 152 | else: 153 | examples[swap, ...] = examples[random.randint(N_EX), ...] 154 | datum = datum._replace(input=feats, ex_inputs=examples, label=1) 155 | 156 | batch.append(datum) 157 | 158 | #if datum.label == 0: 159 | # batch.append(datum) 160 | # continue 161 | #swap = random.randint(N_EX + 1) 162 | #if swap == N_EX: 163 | # batch.append(datum) 164 | # continue 165 | #examples = datum.ex_inputs.copy() 166 | #tmp = examples[swap, ...] 167 | #examples[swap, ...] = datum.input 168 | #datum = datum._replace(ex_inputs=examples, input=tmp) 169 | #batch.append(datum) 170 | 171 | #for _ in range(n_batch): 172 | # datum = self.train_data[random.randint(n_train)] 173 | # in_examples = datum.ex_inputs 174 | # out_examples = [] 175 | # #for i_ex in range(N_EX): 176 | # # out_examples.append( 177 | # # in_examples[random.randint(in_examples.shape[0]), ...]) 178 | # indices = list(range(in_examples.shape[0])) 179 | # random.shuffle(indices) 180 | # indices = indices[:N_EX] 181 | # out_examples = [in_examples[i, ...] for i in indices] 182 | # #out_examples = in_examples[:N_EX, ...] 183 | # datum = datum._replace(ex_inputs=np.asarray(out_examples)) 184 | # batch.append(datum) 185 | 186 | return batch 187 | 188 | def sample_val(self, same=False): 189 | if same: 190 | return self.val_same_data 191 | else: 192 | return self.val_data 193 | 194 | def sample_test(self, same=False): 195 | if same: 196 | return self.test_same_data 197 | else: 198 | return self.test_data 199 | 200 | def visualize(self, datum, hyp, pred, dest): 201 | hint = " ".join(self.hint_vocab.get(w) for w in datum.hint[1:-1]) 202 | hyp = " ".join(self.hint_vocab.get(w) for w in hyp[1:-1]) 203 | os.mkdir(dest) 204 | with open(os.path.join(dest, "desc.txt"), "w") as desc_f: 205 | print >>desc_f, "gold desc:", hint 206 | print >>desc_f, "pred desc:", hyp 207 | print >>desc_f, "gold label:", bool(datum.label) 208 | print >>desc_f, "pred label:", bool(pred) 209 | for i in range(datum.ex_inputs.shape[0]): 210 | scipy.misc.imsave( 211 | os.path.join(dest, "ex_%d.png" % i), 212 | datum.vis_ex_inputs[i, ...]) 213 | scipy.misc.imsave( 214 | os.path.join(dest, "input.png"), 215 | datum.vis_input) 216 | --------------------------------------------------------------------------------