├── .gitignore ├── README.md ├── build_vocab.py ├── caption_gan_encoder_decoder_model.py ├── caption_gan_model.py ├── config.yaml ├── convert_birds_to_skipthought_hd5_script.py ├── convert_cub_to_hd5_script.py ├── convert_flowers_skipthought_hd5_script.py ├── convert_flowers_to_hd5_script.py ├── cycleresults ├── a curly red flower with many dark spots on it's petals..jpg ├── a flower with long and wide petals that are yellow..jpg ├── a flower with tiny yellow petals and very few yellow pistils..jpg ├── a golden flower with a spiky look and a white center..jpg ├── a white flower with a golden and purple center..jpg ├── flower is enclosed in a bud like leaf with yellow petals on the outside and long purple style..jpg ├── the flower has big yellow upright petals attached to a thick vine.jpg ├── the flower is red in color with a long pedicel with leaves on it.jpg ├── the flower is yellow with petals that are smooth and has pointy tip and they are in the formation around the stamen like a disc.jpg ├── the flower pedals looks like colored leaves that are yellow and purple..jpg ├── the petal of the flower is white in color with a large yellow pistil in the center..jpg ├── the petals of the flower resemble the heightened tail of a peacock, and have a bold yellow color..jpg ├── the petals of this flower are purple and white with a long stigma.jpg ├── the petals of this flower are purple with a short stigma.jpg ├── the petals on this flower are yellow with an a few brown stripes..jpg ├── the petals on this flower are yellow with dark lines..jpg ├── the petals range in color from orange to yellow to purple with hints of white. the petals are different shapes and close as the flower goes up..jpg ├── the pink petals of the flower is bended and the stamen is not seen..jpg ├── the spiky, thin yellow petals of this flower radiate outward from its bushy stamen..jpg ├── there are many pink petals below red anthers and filaments..jpg ├── this flower consist of a bunch of purple pedals climbing down the stock with black stamen..jpg ├── this flower features broad yellow petals surrounding the ovary in a ball like pattern..jpg ├── this flower has a light green sepal and strange looking blue pistil..jpg ├── this flower has a spiky brown center and down-turned pink petals with tapered tips..jpg ├── this flower has a star-like configuration of bright blue petals with darker veins..jpg ├── this flower has bright yellow petals curved into a bell shape and a simple long green stem..jpg ├── this flower has oddly shaped petals that vary in shades of purple and yellow..jpg ├── this flower has orange spiky petals and purple spiky petals..jpg ├── this flower has pale pink petals with a large rounded bright red anther in the center..jpg ├── this flower has petals that are blue and are very thin.jpg ├── this flower has petals that are orange with black dotd.jpg ├── this flower has petals that are pink and ruffled at the edges.jpg ├── this flower has petals that are purple and are very thin.jpg ├── this flower has petals that are purple with purple stigma.jpg ├── this flower has petals that are white and has patches of pink.jpg ├── this flower has petals that are white on the outer petal and red on the center with red anthers...jpg ├── this flower has petals that are yellow and are folded together.jpg ├── this flower has petals that are yellow and are very thin.jpg ├── this flower has tall and spiky yellow petals with no visible outer stamen.jpg ├── this flower has yellow rounded petals with stripe brown markings..jpg ├── this flower is made up of flowy white petals that have some pink highlights toward the tips..jpg ├── this flower is orange in color, with petals that are layered..jpg ├── this flower is pink and white in color, and has petals that are striped..jpg ├── this flower is pink and white in color, with petals that are wavy and curled..jpg ├── this flower is pink in color, with petals that have veins..jpg ├── this flower is purple and white in color, and has petals that are very skinny..jpg ├── this flower is purple in color, and has petals that are bell shaped..jpg ├── this flower is purple in color, with petals that are oddly shaped..jpg ├── this flower is rose shaped with bright yellow overlapping petals..jpg ├── this flower is white and green in color, and has petals that are pointed..jpg ├── this flower is white and pink in color, and has petals that are oval shaped..jpg ├── this flower is white and yellow in color, with petals that are wavy and curled..jpg ├── this flower is yellow and purple in color, with petals that are spotted..jpg ├── this flower is yellow in color, with petals that are ruffled..jpg ├── this flower is yellow in color, with petals that are very skinny..jpg ├── this has a spiny type pedicel with a pink-lavendar hue flower that has many pollen and stigma leaves..jpg ├── this is a blue flower with short and rounded petals that turns white toward the receptacle..jpg ├── this is a soft petaled rose colored flower with large flappy petals and a firm pedicel.jpg ├── this orange and blue flower has pointed petals and a green pedicel..jpg ├── this orange and brown flower has rounded petals and bright orange stamen..jpg ├── this petals on this flower are purple with no visible stamen..jpg ├── this pink flower has the string like petals arranged on the horny pedicel.jpg ├── this pink flower of asteraceae family has outer florets forming a pink corolla and the central florets are red with yellow pistils..jpg └── this yellow, pink, and white flower has pointed petals with a pink pedicel..jpg ├── cycleresults220 ├── a blue flower with petals and the yellow stigma and anther and the leaves are green and thin.jpg ├── a flower with white petals with brown stripes..jpg ├── an odd shaped flower that has dark red spots on the salmon colored petals.jpg ├── purple bell shaped flower with straight white stamen..jpg ├── the flower has long yellow petals that are thin and a yellow stamen..jpg ├── the flower has needle like stigma sticking out of the downward facing petals that are surrounding it.jpg ├── the flower has pointed pale pink petals and several white anthers..jpg ├── the flower has pointed petals and purple stamen and pistil..jpg ├── the flower has white and purple petals as well as purple stamen.jpg ├── the flower has white and yellow petals that pleum out..jpg ├── the flower has yellow petals with a yellow stigma and green pedicel..jpg ├── the flower shown has red petals with a green pedicel.jpg ├── the flower shown here has blue petals which surround the white stamen.jpg ├── the large round center of this flower is covered with whitish pink anthers and the petals are white closest to the center and end in a red point..jpg ├── the petal is closed and it appears white and the stamen is orange..jpg ├── the petals of the flower are orange in color and have green leaves..jpg ├── the petals of this flower are red with a long stigma.jpg ├── the petals of this flower are white with dark purple tips and the stamen are green with purple tips..jpg ├── the petals on this flower are round and purple shaped in threes..jpg ├── the pistil of this flower is of radial symmetry petals brightly yellow colored. there are stamen in the center which are of the same color of the stamen. no stigma visible..jpg ├── the pretty flower has white petals with purple at their tips..jpg ├── there are flat, white petals and white petals that curl upwards around yellow and pink stamens..jpg ├── there are many long and narrow floppy pink petals surrounding many red stamen and a green stigma on this flower..jpg ├── there is a simple fat white bell shaped bloom with round petals and visible yellow stamen..jpg ├── this flower has a large, white cup shaped petal with a thick bright yellow stigma..jpg ├── this flower has a lot of pointed red petals in a ray-like shape around the many stamen..jpg ├── this flower has a thick yellow stamen wrapped in a thick pointed white petal..jpg ├── this flower has blue petals that are folded together in a bulb.jpg ├── this flower has long petals that are orange with spots and a very long pistil..jpg ├── this flower has many bright orange petals with a brown center..jpg ├── this flower has petals that are pink and folded together.jpg ├── this flower has petals that are purple and very thin.jpg ├── this flower has petals that are red with patches of yellow.jpg ├── this flower has petals that are white with many ruffles.jpg ├── this flower has petals that are white with pink shading.jpg ├── this flower has petals that are yellow and are folded together.jpg ├── this flower has petals that are yellow and are very thin.jpg ├── this flower has petals that are yellow and folded together.jpg ├── this flower has petals that are yellow and has red dots.jpg ├── this flower has petals that are yellow and has yellow stamen.jpg ├── this flower has petals that are yellow with dark lines.jpg ├── this flower has pink petals that have long yellow stamen.jpg ├── this flower has purple petals with spikes at the bottom.jpg ├── this flower has red and purple petals with tiny pointy receptacles around it..jpg ├── this flower has several large white and pink petals with slight accents of yellow and brown..jpg ├── this flower has two rows of long, rounded petals that are multi-tonal (orange to yellow) in color..jpg ├── this flower has white petals that have a small yellow patch in the center.jpg ├── this flower is characterized by its spiky purple petals on top, and its spiky green petals directly under the purple petals..jpg ├── this flower is pink and white in color, with petals that are pointed on the ends..jpg ├── this flower is pink and yellow in color, with petals that are multi colored..jpg ├── this flower is purple in color, and has petals that are thin and wavy..jpg ├── this flower is purple in color, and has petals that are very skinny..jpg ├── this flower is red white and yellow in color, with petals that are curled and layered vertically..jpg ├── this flower is white, pink, and yellow in color, and has petals that are multi colored..jpg ├── this flower is yellow and purple in color, with petals that are striped near the center..jpg ├── this flower is yellow in color, and has petals that are rounded and curled around the center..jpg ├── this is a flower with round purple upward facing petals..jpg ├── this is a group of white flowers, with spider shaped centers..jpg ├── this large yellow blossom has numerous yellow stamen and hundreds of very thin yellow petals..jpg ├── this orange and blue flower has pointed petals and a green pedicel..jpg ├── this spikey flower has a green pointed barrier around the bottom with pink stamen protruding from the top..jpg ├── this white flower has rounded petals and a yellow orange stamen..jpg └── this white flower is delicate and a beautiful white color with strong green leaves that surround it..jpg ├── cycleresults250 ├── a blue flower with petals and the yellow stigma and anther and the leaves are green and thin.jpg ├── a flower with white petals with brown stripes..jpg ├── an odd shaped flower that has dark red spots on the salmon colored petals.jpg ├── purple bell shaped flower with straight white stamen..jpg ├── the flower has long yellow petals that are thin and a yellow stamen..jpg ├── the flower has needle like stigma sticking out of the downward facing petals that are surrounding it.jpg ├── the flower has pointed pale pink petals and several white anthers..jpg ├── the flower has pointed petals and purple stamen and pistil..jpg ├── the flower has white and purple petals as well as purple stamen.jpg ├── the flower has white and yellow petals that pleum out..jpg ├── the flower has yellow petals with a yellow stigma and green pedicel..jpg ├── the flower shown has red petals with a green pedicel.jpg ├── the flower shown here has blue petals which surround the white stamen.jpg ├── the large round center of this flower is covered with whitish pink anthers and the petals are white closest to the center and end in a red point..jpg ├── the petal is closed and it appears white and the stamen is orange..jpg ├── the petals of the flower are orange in color and have green leaves..jpg ├── the petals of this flower are red with a long stigma.jpg ├── the petals of this flower are white with dark purple tips and the stamen are green with purple tips..jpg ├── the petals on this flower are round and purple shaped in threes..jpg ├── the pistil of this flower is of radial symmetry petals brightly yellow colored. there are stamen in the center which are of the same color of the stamen. no stigma visible..jpg ├── the pretty flower has white petals with purple at their tips..jpg ├── there are flat, white petals and white petals that curl upwards around yellow and pink stamens..jpg ├── there are many long and narrow floppy pink petals surrounding many red stamen and a green stigma on this flower..jpg ├── there is a simple fat white bell shaped bloom with round petals and visible yellow stamen..jpg ├── this flower has a large, white cup shaped petal with a thick bright yellow stigma..jpg ├── this flower has a lot of pointed red petals in a ray-like shape around the many stamen..jpg ├── this flower has a thick yellow stamen wrapped in a thick pointed white petal..jpg ├── this flower has blue petals that are folded together in a bulb.jpg ├── this flower has long petals that are orange with spots and a very long pistil..jpg ├── this flower has many bright orange petals with a brown center..jpg ├── this flower has petals that are pink and folded together.jpg ├── this flower has petals that are purple and very thin.jpg ├── this flower has petals that are red with patches of yellow.jpg ├── this flower has petals that are white with many ruffles.jpg ├── this flower has petals that are white with pink shading.jpg ├── this flower has petals that are yellow and are folded together.jpg ├── this flower has petals that are yellow and are very thin.jpg ├── this flower has petals that are yellow and folded together.jpg ├── this flower has petals that are yellow and has red dots.jpg ├── this flower has petals that are yellow and has yellow stamen.jpg ├── this flower has petals that are yellow with dark lines.jpg ├── this flower has pink petals that have long yellow stamen.jpg ├── this flower has purple petals with spikes at the bottom.jpg ├── this flower has red and purple petals with tiny pointy receptacles around it..jpg ├── this flower has several large white and pink petals with slight accents of yellow and brown..jpg ├── this flower has two rows of long, rounded petals that are multi-tonal (orange to yellow) in color..jpg ├── this flower has white petals that have a small yellow patch in the center.jpg ├── this flower is characterized by its spiky purple petals on top, and its spiky green petals directly under the purple petals..jpg ├── this flower is pink and white in color, with petals that are pointed on the ends..jpg ├── this flower is pink and yellow in color, with petals that are multi colored..jpg ├── this flower is purple in color, and has petals that are thin and wavy..jpg ├── this flower is purple in color, and has petals that are very skinny..jpg ├── this flower is red white and yellow in color, with petals that are curled and layered vertically..jpg ├── this flower is white, pink, and yellow in color, and has petals that are multi colored..jpg ├── this flower is yellow and purple in color, with petals that are striped near the center..jpg ├── this flower is yellow in color, and has petals that are rounded and curled around the center..jpg ├── this is a flower with round purple upward facing petals..jpg ├── this is a group of white flowers, with spider shaped centers..jpg ├── this large yellow blossom has numerous yellow stamen and hundreds of very thin yellow petals..jpg ├── this orange and blue flower has pointed petals and a green pedicel..jpg ├── this spikey flower has a green pointed barrier around the bottom with pink stamen protruding from the top..jpg ├── this white flower has rounded petals and a yellow orange stamen..jpg └── this white flower is delicate and a beautiful white color with strong green leaves that surround it..jpg ├── eval_dataset.py ├── figures ├── cycle_a_losses.png ├── pretraining_caption_disc_losses.png ├── pretraining_gen_losses.png ├── stage_1_disc_losses.png └── stage_1_gen_losses.png ├── inception_score.py ├── models ├── gan.py ├── gan_cls.py └── gan_factory.py ├── nocycle_results ├── a curly red flower with many dark spots on it's petals..jpg ├── a flower with long and wide petals that are yellow..jpg ├── a flower with tiny yellow petals and very few yellow pistils..jpg ├── a golden flower with a spiky look and a white center..jpg ├── a white flower with a golden and purple center..jpg ├── flower is enclosed in a bud like leaf with yellow petals on the outside and long purple style..jpg ├── the flower has big yellow upright petals attached to a thick vine.jpg ├── the flower is red in color with a long pedicel with leaves on it.jpg ├── the flower is yellow with petals that are smooth and has pointy tip and they are in the formation around the stamen like a disc.jpg ├── the flower pedals looks like colored leaves that are yellow and purple..jpg ├── the petal of the flower is white in color with a large yellow pistil in the center..jpg ├── the petals of the flower resemble the heightened tail of a peacock, and have a bold yellow color..jpg ├── the petals of this flower are purple and white with a long stigma.jpg ├── the petals of this flower are purple with a short stigma.jpg ├── the petals on this flower are yellow with an a few brown stripes..jpg ├── the petals on this flower are yellow with dark lines..jpg ├── the petals range in color from orange to yellow to purple with hints of white. the petals are different shapes and close as the flower goes up..jpg ├── the pink petals of the flower is bended and the stamen is not seen..jpg ├── the spiky, thin yellow petals of this flower radiate outward from its bushy stamen..jpg ├── there are many pink petals below red anthers and filaments..jpg ├── this flower consist of a bunch of purple pedals climbing down the stock with black stamen..jpg ├── this flower features broad yellow petals surrounding the ovary in a ball like pattern..jpg ├── this flower has a light green sepal and strange looking blue pistil..jpg ├── this flower has a spiky brown center and down-turned pink petals with tapered tips..jpg ├── this flower has a star-like configuration of bright blue petals with darker veins..jpg ├── this flower has bright yellow petals curved into a bell shape and a simple long green stem..jpg ├── this flower has oddly shaped petals that vary in shades of purple and yellow..jpg ├── this flower has orange spiky petals and purple spiky petals..jpg ├── this flower has pale pink petals with a large rounded bright red anther in the center..jpg ├── this flower has petals that are blue and are very thin.jpg ├── this flower has petals that are orange with black dotd.jpg ├── this flower has petals that are pink and ruffled at the edges.jpg ├── this flower has petals that are purple and are very thin.jpg ├── this flower has petals that are purple with purple stigma.jpg ├── this flower has petals that are white and has patches of pink.jpg ├── this flower has petals that are white on the outer petal and red on the center with red anthers...jpg ├── this flower has petals that are yellow and are folded together.jpg ├── this flower has petals that are yellow and are very thin.jpg ├── this flower has tall and spiky yellow petals with no visible outer stamen.jpg ├── this flower has yellow rounded petals with stripe brown markings..jpg ├── this flower is made up of flowy white petals that have some pink highlights toward the tips..jpg ├── this flower is orange in color, with petals that are layered..jpg ├── this flower is pink and white in color, and has petals that are striped..jpg ├── this flower is pink and white in color, with petals that are wavy and curled..jpg ├── this flower is pink in color, with petals that have veins..jpg ├── this flower is purple and white in color, and has petals that are very skinny..jpg ├── this flower is purple in color, and has petals that are bell shaped..jpg ├── this flower is purple in color, with petals that are oddly shaped..jpg ├── this flower is rose shaped with bright yellow overlapping petals..jpg ├── this flower is white and green in color, and has petals that are pointed..jpg ├── this flower is white and pink in color, and has petals that are oval shaped..jpg ├── this flower is white and yellow in color, with petals that are wavy and curled..jpg ├── this flower is yellow and purple in color, with petals that are spotted..jpg ├── this flower is yellow in color, with petals that are ruffled..jpg ├── this flower is yellow in color, with petals that are very skinny..jpg ├── this has a spiny type pedicel with a pink-lavendar hue flower that has many pollen and stigma leaves..jpg ├── this is a blue flower with short and rounded petals that turns white toward the receptacle..jpg ├── this is a soft petaled rose colored flower with large flappy petals and a firm pedicel.jpg ├── this orange and blue flower has pointed petals and a green pedicel..jpg ├── this orange and brown flower has rounded petals and bright orange stamen..jpg ├── this petals on this flower are purple with no visible stamen..jpg ├── this pink flower has the string like petals arranged on the horny pedicel.jpg ├── this pink flower of asteraceae family has outer florets forming a pink corolla and the central florets are red with yellow pistils..jpg └── this yellow, pink, and white flower has pointed petals with a pink pedicel..jpg ├── run_all_caption_inference_cycle.sh ├── run_inference_cycle.sh ├── run_inference_pretrained.sh ├── runtime.py ├── sample.py ├── scripts ├── start_continue_train.sh ├── start_continue_train_stackgan.sh ├── start_cycle_inference.sh ├── start_cycle_stackgan_inference_flowers.sh ├── start_cycle_train.sh ├── start_inference.sh ├── start_pretraining.sh ├── start_stackgan_inference.sh ├── start_stackgan_inference_flowers.sh ├── start_train.sh ├── start_train_inverse.sh ├── start_train_stackgan.sh └── train_cycle.sh ├── trainer.py ├── txt2image_dataset.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | pre-trained models/* 2 | dataset/* 3 | data/* 4 | *.pyc 5 | .idea/ 6 | checkpoints/* 7 | results/* 8 | *.pkl 9 | bounding_boxes.txt 10 | images.txt 11 | # Byte-compiled / optimized / DLL files 12 | __pycache__/ 13 | *.py[cod] 14 | *$py.class 15 | *.swp 16 | fitures/* 17 | *.zip 18 | *.swp 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | build/ 26 | develop-eggs/ 27 | dist/ 28 | downloads/ 29 | eggs/ 30 | .eggs/ 31 | lib/ 32 | lib64/ 33 | parts/ 34 | sdist/ 35 | var/ 36 | wheels/ 37 | *.egg-info/ 38 | .installed.cfg 39 | *.egg 40 | MANIFEST 41 | 42 | # PyInstaller 43 | # Usually these files are written by a python script from a template 44 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 45 | *.manifest 46 | *.spec 47 | 48 | # Installer logs 49 | pip-log.txt 50 | pip-delete-this-directory.txt 51 | 52 | # Unit test / coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .coverage 56 | .coverage.* 57 | .cache 58 | nosetests.xml 59 | coverage.xml 60 | *.cover 61 | .hypothesis/ 62 | .pytest_cache/ 63 | 64 | # Translations 65 | *.mo 66 | *.pot 67 | 68 | # Django stuff: 69 | *.log 70 | local_settings.py 71 | db.sqlite3 72 | 73 | # Flask stuff: 74 | instance/ 75 | .webassets-cache 76 | 77 | # Scrapy stuff: 78 | .scrapy 79 | 80 | # Sphinx documentation 81 | docs/_build/ 82 | 83 | # PyBuilder 84 | target/ 85 | 86 | # Jupyter Notebook 87 | .ipynb_checkpoints 88 | 89 | # pyenv 90 | .python-version 91 | 92 | # celery beat schedule file 93 | celerybeat-schedule 94 | 95 | # SageMath parsed files 96 | *.sage.py 97 | 98 | # Environments 99 | .env 100 | .venv 101 | env/ 102 | venv/ 103 | ENV/ 104 | env.bak/ 105 | venv.bak/ 106 | 107 | # Spyder project settings 108 | .spyderproject 109 | .spyproject 110 | 111 | # Rope project settings 112 | .ropeproject 113 | 114 | # mkdocs documentation 115 | /site 116 | 117 | # mypy 118 | .mypy_cache/ 119 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Text to Image to Text Translation using Cycle Consistent Adversarial networks. 2 | 3 | This project tries to improve text to image synthesis using Cycle Gan. You can read our paper here. https://arxiv.org/abs/1808.04538. 4 | 5 | -------------------------------------------------------------------------------- /build_vocab.py: -------------------------------------------------------------------------------- 1 | import nltk 2 | import pickle 3 | import argparse 4 | from collections import Counter 5 | import os 6 | import pdb 7 | from tqdm import tqdm 8 | import re 9 | import string 10 | 11 | class Vocabulary(object): 12 | """Simple vocabulary wrapper.""" 13 | def __init__(self): 14 | self.word2idx = {} 15 | self.idx2word = {} 16 | self.idx = 0 17 | 18 | def add_word(self, word): 19 | if not word in self.word2idx: 20 | self.word2idx[word] = self.idx 21 | self.idx2word[self.idx] = word 22 | self.idx += 1 23 | 24 | def __call__(self, word): 25 | if not word in self.word2idx: 26 | return self.word2idx[''] 27 | return self.word2idx[word] 28 | 29 | def __len__(self): 30 | return len(self.word2idx) 31 | 32 | def build_vocab(filepath, threshold): 33 | """Build a simple vocabulary wrapper.""" 34 | # go through all files 35 | counter = Counter() 36 | for subdir, dirs, files in os.walk(filepath): 37 | for caption_file in tqdm(files): 38 | if caption_file[-3:] != 'txt': 39 | continue 40 | with open(os.path.join(subdir, caption_file), 'r') as f: 41 | captions = f.readlines() 42 | for caption in captions: 43 | caption = caption.strip() 44 | caption = caption.encode('ascii', 'ignore') 45 | caption = caption.decode('ascii') 46 | exclude = set(string.punctuation) 47 | preproc_caption = ''.join(ch for ch in caption if ch not in exclude) 48 | tokens = nltk.tokenize.word_tokenize(preproc_caption.lower()) 49 | counter.update(tokens) 50 | 51 | # If the word frequency is less than 'threshold', then the word is discarded. 52 | words = [word for word, cnt in counter.items() if cnt >= threshold] 53 | 54 | # Creates a vocab wrapper and add some special tokens. 55 | vocab = Vocabulary() 56 | vocab.add_word('') 57 | vocab.add_word('') 58 | vocab.add_word('') 59 | vocab.add_word('') 60 | 61 | # Adds the words to the vocabulary. 62 | for i, word in enumerate(words): 63 | vocab.add_word(word) 64 | return vocab 65 | 66 | def main(args): 67 | vocab = build_vocab(filepath=args.caption_path, 68 | threshold=args.threshold) 69 | vocab_path = args.vocab_path 70 | with open(vocab_path, 'wb') as f: 71 | pickle.dump(vocab, f) 72 | print("Total vocabulary size: %d" %len(vocab)) 73 | print("Saved the vocabulary wrapper to '%s'" %vocab_path) 74 | 75 | 76 | if __name__ == '__main__': 77 | parser = argparse.ArgumentParser() 78 | parser.add_argument('--caption_path', type=str, 79 | #default='/usr/share/mscoco/annotations/captions_train2014.json', 80 | # default='./data/annotations/captions_train2014.json', 81 | # default='./data/birds_captions/', 82 | default='./data/text_c10/', 83 | help='path for train annotation file') 84 | parser.add_argument('--vocab_path', type=str, default='./data/flowers_vocab.pkl', 85 | help='path for saving vocabulary wrapper') 86 | parser.add_argument('--threshold', type=int, default=4, 87 | help='minimum word count threshold') 88 | args = parser.parse_args() 89 | main(args) 90 | -------------------------------------------------------------------------------- /caption_gan_encoder_decoder_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.models as models 4 | from torch.nn.utils.rnn import * 5 | from torch.autograd import Variable 6 | import pdb 7 | from tqdm import tqdm 8 | 9 | class EncoderCNN(nn.Module): 10 | def __init__(self, embed_size): 11 | """Load the pretrained ResNet-152 and replace top fc layer.""" 12 | super(EncoderCNN, self).__init__() 13 | # resnet = models.resnet152(pretrained=True) 14 | # modules = list(resnet.children())[:-1] # delete the last fc layer. 15 | 16 | # self.resnet = nn.Sequential(*modules) 17 | # self.linear = nn.Linear(resnet.fc.in_features, embed_size) 18 | 19 | alexnet = models.alexnet(pretrained=True) 20 | self.alexnet_features = alexnet.features 21 | # modules = list(alexnet.classifier.children())[:-1] # delete the last fc layer. 22 | 23 | # self.alexnet_classifier = nn.Sequential(*modules) 24 | # self.linear = nn.Linear(list(alexnet.children())[-1][-1].in_features, embed_size) 25 | self.linear = nn.Linear(2304, embed_size) 26 | self.bn = nn.BatchNorm1d(embed_size, momentum=0.01) 27 | self.init_weights() 28 | 29 | def init_weights(self): 30 | """Initialize the weights.""" 31 | self.linear.weight.data.normal_(0.0, 0.02) 32 | self.linear.bias.data.fill_(0) 33 | 34 | def forward(self, images): 35 | """Extract the image feature vectors.""" 36 | #features = self.resnet(images) 37 | features = self.alexnet_features(images) 38 | features = Variable(features.data) 39 | features = features.view(features.size(0), -1) 40 | # features = self.alexnet_classifier(features) 41 | # features = features.view(features.size(0), -1) 42 | features = self.bn(self.linear(features)) 43 | return features 44 | 45 | 46 | class DecoderRNN(nn.Module): 47 | def __init__(self, embed_size, hidden_size, vocab_size, num_layers): 48 | """Set the hyper-parameters and build the layers.""" 49 | super(DecoderRNN, self).__init__() 50 | self.embed = nn.Embedding(vocab_size, embed_size) 51 | self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True) 52 | self.linear = nn.Linear(hidden_size, vocab_size) 53 | self.softmax = nn.Softmax(dim=1) 54 | self.init_weights() 55 | 56 | def init_weights(self): 57 | """Initialize weights.""" 58 | self.embed.weight.data.uniform_(-0.1, 0.1) 59 | self.linear.weight.data.uniform_(-0.1, 0.1) 60 | self.linear.bias.data.fill_(0) 61 | 62 | def forward(self, features, captions, lengths, noise=False): 63 | """Decode image feature vectors and generates captions.""" 64 | # TODO: should not use all teacher forcing 65 | # return: outputs (s, V), lengths list(Tmax) 66 | embeddings = self.embed(captions) 67 | if not noise: 68 | embeddings = torch.cat((features.unsqueeze(1), embeddings), 1) 69 | else: 70 | max_embedding = torch.max(features[0,:]).data.numpy()[0] 71 | min_embedding = torch.min(features[0,:]).data.numpy()[0] 72 | concat_noise = (max_embedding - min_embedding) * torch.rand((embeddings.size(0), 1, embeddings.size(2))) + torch.FloatTensor([float(min_embedding)]).unsqueeze(0).unsqueeze(1) 73 | features = torch.cat((features.unsqueeze(1), Variable(concat_noise, requires_grad=False)), 1) 74 | embeddings = torch.cat((features, embeddings), 1) 75 | packed = pack_padded_sequence(embeddings, lengths, batch_first=True) 76 | hiddens, _ = self.lstm(packed) 77 | outputs = self.linear(hiddens[0]) 78 | return outputs, hiddens[1] 79 | 80 | def sample(self, features, states=None): 81 | """Samples captions for given image features (Greedy search).""" 82 | sampled_ids = [] 83 | inputs = features.unsqueeze(1) 84 | for i in range(20): # maximum sampling length 85 | hiddens, states = self.lstm(inputs, states) # (batch_size, 1, hidden_size), 86 | outputs = self.linear(hiddens.squeeze(1)) # (batch_size, vocab_size) 87 | predicted = outputs.max(1)[1] 88 | # pdb.set_trace() 89 | # outputs = self.softmax(outputs) 90 | # predicted_index = outputs.multinomial(1) 91 | # predicted = outputs[predicted_index] 92 | sampled_ids.append(predicted) 93 | inputs = self.embed(predicted) 94 | inputs = inputs.unsqueeze(1) # (batch_size, 1, embed_size) 95 | #sampled_ids = torch.cat(sampled_ids, 1) # (batch_size, 20) 96 | sampled_ids = torch.cat(sampled_ids, 0) # (batch_size, 20) 97 | sampled_ids = sampled_ids.view(-1, 20) 98 | # return sampled_ids.squeeze() 99 | # pdb.set_trace() 100 | return sampled_ids 101 | 102 | 103 | def pre_compute(self, features, gen_samples, eval_t, states=None): 104 | 105 | best_sample_nums = 5 # number of vocabs to sample 106 | 107 | inputs = features.unsqueeze(1) # (b, 1, e) 108 | if torch.cuda.is_available(): 109 | gen_samples = gen_samples.type(torch.cuda.LongTensor) 110 | else: 111 | gen_samples = gen_samples.type(torch.LongTensor) 112 | forced_inputs = gen_samples[:,:eval_t] 113 | 114 | for i in range(eval_t): 115 | hiddens, states = self.lstm(inputs, states) # hiddens = (b, 1, h) 116 | inputs = self.embed(forced_inputs[:,i]) 117 | inputs = inputs.unsqueeze(1) # (batch_size, 1, embed_size) 118 | 119 | outputs = self.linear(hiddens.squeeze(1)) 120 | outputs = self.softmax(outputs) 121 | predicted_indices = outputs.multinomial(best_sample_nums) 122 | 123 | return predicted_indices, states 124 | 125 | 126 | def rollout(self, features, gen_samples, t, Tmax, states=None): 127 | """ 128 | sample caption from a specific time t 129 | 130 | features = (b, e) 131 | t = scalar 132 | Tmax = scalar 133 | states = cell states = tuple 134 | """ 135 | sampled_ids = [] 136 | # inputs = features.unsqueeze(1) # (b, 1, e) 137 | if torch.cuda.is_available(): 138 | gen_samples = gen_samples.type(torch.cuda.LongTensor) 139 | else: 140 | gen_samples = gen_samples.type(torch.LongTensor) 141 | # forced_inputs = gen_samples[:,:t+1] 142 | # for i in range(t): 143 | # hiddens, states = self.lstm(inputs, states) # hiddens = (b, 1, h) 144 | # inputs = self.embed(forced_inputs[:,i]) 145 | # inputs = inputs.unsqueeze(1) # (batch_size, 1, embed_size) 146 | 147 | inputs = self.embed(gen_samples[:,t]).unsqueeze(1) 148 | for i in range(t, Tmax): # maximum sampling length 149 | # pdb.set_trace() 150 | hiddens, states = self.lstm(inputs, states) # hiddens = (b, 1, h) 151 | outputs = self.linear(hiddens.squeeze(1)) # outputs = (b, V) 152 | predicted = outputs.max(1)[1] 153 | 154 | # pdb.set_trace() 155 | # TODO maybe need to sample? 156 | # outputs = self.softmax(outputs) 157 | # predicted_index = outputs.multinomial(1) 158 | # predicted = outputs[predicted_index] 159 | 160 | sampled_ids.append(predicted) 161 | inputs = self.embed(predicted) 162 | inputs = inputs.unsqueeze(1) # (batch_size, 1, embed_size) 163 | 164 | sampled_ids = torch.cat(sampled_ids, 0) # (batch_size, 20) 165 | # pdb.set_trace() 166 | sampled_ids = sampled_ids.view(-1, Tmax-t) 167 | return sampled_ids 168 | -------------------------------------------------------------------------------- /caption_gan_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | from torch.nn.utils.rnn import * 5 | from caption_gan_encoder_decoder_model import EncoderCNN 6 | from caption_gan_encoder_decoder_model import DecoderRNN 7 | import pdb 8 | 9 | class CaptionDiscriminator(nn.Module): 10 | def __init__(self, embed_size, hidden_size, vocab_size, num_layers): 11 | super(CaptionDiscriminator, self).__init__() 12 | self.embed = nn.Embedding(vocab_size, embed_size) 13 | self.image_feature_encoder = EncoderCNN(embed_size) 14 | self.sentence_feature_encoder = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True) 15 | self.hidden_fine_tune_linear = nn.Linear(hidden_size, embed_size) 16 | 17 | def forward(self, images, captions, lengths): 18 | """Calculate reward score: r = logistic(dot_prod(f, h))""" 19 | # print(captions) 20 | features = self.image_feature_encoder(images) #(batch_size=128, embed_size=256) 21 | 22 | if torch.cuda.is_available(): 23 | embeddings = self.embed(captions.type(torch.cuda.LongTensor)) # (batch_size, embed_size) 24 | else: 25 | embeddings = self.embed(captions.type(torch.LongTensor)) # (batch_size, embed_size) 26 | packed = pack_padded_sequence(embeddings, lengths, batch_first=True) 27 | hiddens, _ = self.sentence_feature_encoder(packed) 28 | 29 | padded = pad_packed_sequence(hiddens, batch_first=True) 30 | # padded[0] # (batch_size, T_max, hidden_size) 31 | last_padded_indices = [index-1 for index in padded[1]] 32 | hidden_outputs = padded[0][range(captions.size(0)), last_padded_indices, :] 33 | hidden_outputs = self.hidden_fine_tune_linear(hidden_outputs) 34 | 35 | dot_prod = torch.bmm(features.unsqueeze(1), hidden_outputs.unsqueeze(1).transpose(2,1)).squeeze() 36 | return nn.Sigmoid()(dot_prod) 37 | 38 | 39 | class CaptionGenerator(nn.Module): 40 | def __init__(self, embed_size, hidden_size, vocab_size, num_layers): 41 | super(CaptionGenerator, self).__init__() 42 | self.encoder = EncoderCNN(embed_size) 43 | self.decoder = DecoderRNN(embed_size, hidden_size, vocab_size, num_layers) 44 | self.features = None 45 | 46 | def forward(self, images, captions, lengths): 47 | """Getting captions""" 48 | self.features = self.encoder(images) 49 | outputs, packed_lengths = self.decoder(self.features, captions, lengths, noise=False) # TODO (packed_size, vocab_size) 50 | # outputs = self.decoder(self.features, captions, lengths, noise=False) # TODO (packed_size, vocab_size) 51 | # outputs = PackedSequence(outputs, packed_lengths) 52 | # outputs = pad_packed_sequence(outputs, batch_first=True) # (b, T, V) 53 | return outputs, packed_lengths 54 | 55 | 56 | def pre_compute(self, gen_samples, t): 57 | """ 58 | pre compute the most likely vocabs and their states 59 | """ 60 | if self.features is None: 61 | print('must do forward before calling this function') 62 | return None 63 | 64 | predicted_ids, saved_states = self.decoder.pre_compute(self.features, gen_samples, t) 65 | return predicted_ids, saved_states 66 | 67 | def rollout(self, gen_samples, t, saved_states): 68 | """ inputs: 69 | * gen_samples: (b, Tmax) 70 | * t: scalar 71 | 72 | outputs: 73 | * gen_rollouts: (b, Tmax - t) 74 | * lengths_rollouts: list (b) 75 | """ 76 | if self.features is None: 77 | print('must do forward before calling this function') 78 | return None 79 | 80 | Tmax = gen_samples.size(1) 81 | sampled_ids = self.decoder.rollout(self.features, gen_samples, t, Tmax, states=saved_states) 82 | # pdb.set_trace() 83 | return sampled_ids 84 | 85 | def sample(self, images, states=None): 86 | features = self.encoder(images) 87 | return self.decoder.sample(features, states) 88 | 89 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | # Birds CUB dataset 2 | birds_images_path: '/home/krishna/Documents/UofT/courses/mlcv/project/datasets/cvpr2016_cub/images/' 3 | birds_embedding_path: '/home/krishna/Documents/UofT/courses/mlcv/project/datasets/cub_icml/' 4 | birds_text_path: '/home/krishna/Documents/UofT/courses/mlcv/project/datasets/cvpr2016_cub/text_c10/' 5 | birds_dataset_path: '/home/jeremy/Downloads/birds.hdf5' 6 | 7 | val_split_path: '/home/krishna/Documents/UofT/courses/mlcv/project/datasets/cub_icml/valclasses.txt' 8 | train_split_path: '/home/krishna/Documents/UofT/courses/mlcv/project/datasets/cub_icml/trainclasses.txt' 9 | test_split_path: '/home/krishna/Documents/UofT/courses/mlcv/project/datasets/cub_icml/testclasses.txt' 10 | 11 | 12 | # flowers dataset 13 | flowers_images_path: '/home/jeremy/Downloads/flowers_images' 14 | flowers_embedding_path: '/home/jeremy/Downloads/flowers_icml' 15 | flowers_text_path: '/home/jeremy/Downloads/cvpr2016_flowers/text_c10' 16 | flowers_dataset_path: '/home/jeremy/Downloads/flowers_v2.hdf5' 17 | 18 | flowers_val_split_path: '/home/jeremy/Downloads/flowers_icml/valclasses.txt' 19 | flowers_train_split_path: '/home/jeremy/Downloads/flowers_icml/trainclasses.txt' 20 | flowers_test_split_path: '/home/jeremy/Downloads/flowers_icml/testclasses.txt' 21 | -------------------------------------------------------------------------------- /convert_birds_to_skipthought_hd5_script.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os.path import join, isfile 3 | import numpy as np 4 | import h5py 5 | from glob import glob 6 | from torch.utils.serialization import load_lua 7 | from PIL import Image 8 | import yaml 9 | import io 10 | import pdb 11 | import nltk 12 | import re 13 | from collections import Counter 14 | import torch 15 | from torch.autograd import Variable 16 | import sys 17 | sys.path.append('skip-thoughts.torch/pytorch') 18 | from skipthoughts import UniSkip 19 | dir_st = 'data/skip-thoughts' 20 | import tqdm 21 | import pickle 22 | 23 | 24 | 25 | with open('config.yaml', 'r') as f: 26 | config = yaml.load(f) 27 | 28 | images_path = config['birds_images_path'] 29 | embedding_path = config['birds_embedding_path'] 30 | text_path = config['birds_text_path'] 31 | datasetDir = config['birds_dataset_path'] 32 | 33 | val_classes = open(config['val_split_path']).read().splitlines() 34 | train_classes = open(config['train_split_path']).read().splitlines() 35 | test_classes = open(config['test_split_path']).read().splitlines() 36 | 37 | f = h5py.File(datasetDir, 'w') 38 | train = f.create_group('train') 39 | valid = f.create_group('valid') 40 | test = f.create_group('test') 41 | 42 | class Vocabulary(object): 43 | """Simple vocabulary wrapper.""" 44 | def __init__(self): 45 | self.word2idx = {} 46 | self.idx2word = {} 47 | self.idx = 0 48 | 49 | def add_word(self, word): 50 | if not word in self.word2idx: 51 | self.word2idx[word] = self.idx 52 | self.idx2word[self.idx] = word 53 | self.idx += 1 54 | 55 | def __call__(self, word): 56 | if not word in self.word2idx: 57 | return self.word2idx[''] 58 | return self.word2idx[word] 59 | 60 | def __len__(self): 61 | return len(self.word2idx) 62 | 63 | def sanitize_string(caption): 64 | caption = caption.strip() 65 | caption = caption.encode('ascii', 'ignore') 66 | caption = caption.decode('ascii') 67 | caption = re.sub(r'[!\"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~]+', '', caption) 68 | tokens = nltk.tokenize.word_tokenize(caption.lower()) 69 | return tokens 70 | 71 | def build_vocab(): 72 | 73 | threshold = 4 74 | """Build a simple vocabulary wrapper.""" 75 | # go through all files 76 | counter = Counter() 77 | for _class in sorted(os.listdir(embedding_path)): 78 | txt_path = os.path.join(text_path, _class) 79 | for txt_file in sorted(glob(txt_path + "/*.txt")): 80 | lines = open(txt_file, "r").readlines() 81 | for caption in lines: 82 | tokens = sanitize_string(caption) 83 | counter.update(tokens) 84 | 85 | # If the word frequency is less than 'threshold', then the word is discarded. 86 | words = [word for word, cnt in counter.items() if cnt >= threshold] 87 | 88 | # Creates a vocab wrapper and add some special tokens. 89 | vocab = Vocabulary() 90 | vocab.add_word('') 91 | vocab.add_word('') 92 | vocab.add_word('') 93 | vocab.add_word('') 94 | 95 | # Adds the words to the vocabulary. 96 | for i, word in enumerate(words): 97 | vocab.add_word(word) 98 | return vocab 99 | 100 | # vocab = build_vocab() 101 | # with open('data/birds_vocab.pkl', 'wb') as f: 102 | # pickle.dump(vocab, f) 103 | 104 | with open('data/birds_vocab.pkl', 'rb') as f: 105 | vocab = pickle.load(f) 106 | 107 | 108 | all_words_in_vocab = vocab.word2idx.keys() 109 | uniskip = UniSkip(dir_st, all_words_in_vocab) 110 | 111 | 112 | def get_ids(tokens, vocab): 113 | ids = [] 114 | # appending start and eos at the beginning and the end respectively for every sequence 115 | ids.append(vocab('')) 116 | for word in tokens: 117 | ids.append(vocab(word)) 118 | ids.append(vocab('')) 119 | return ids 120 | 121 | for _class in sorted(os.listdir(embedding_path)): 122 | split = '' 123 | if _class in train_classes: 124 | split = train 125 | elif _class in val_classes: 126 | split = valid 127 | elif _class in test_classes: 128 | split = test 129 | 130 | data_path = os.path.join(embedding_path, _class) 131 | txt_path = os.path.join(text_path, _class) 132 | for example, txt_file in zip(sorted(glob(data_path + "/*.t7")), sorted(glob(txt_path + "/*.txt"))): 133 | example_data = load_lua(example) 134 | img_path = example_data['img'] 135 | embeddings = example_data['txt'].numpy() 136 | example_name = img_path.split('/')[-1][:-4] 137 | 138 | f = open(txt_file, "r") 139 | txt = f.readlines() 140 | f.close() 141 | 142 | img_path = os.path.join(images_path, img_path) 143 | img = open(img_path, 'rb').read() 144 | 145 | txt_choice = np.random.choice(range(10), 5) 146 | 147 | embeddings = embeddings[txt_choice] 148 | 149 | 150 | txt = np.array(txt) 151 | txt = txt[txt_choice] 152 | dt = h5py.special_dtype(vlen=str) 153 | 154 | 155 | batch_txt_ids = [] 156 | max_len = 0 157 | id_lens = [] 158 | for t in txt: 159 | skip_thought_txt = sanitize_string(t) 160 | txt_ids = get_ids(skip_thought_txt, vocab) 161 | max_len = len(txt_ids) if max_len < len(txt_ids) else max_len 162 | batch_txt_ids.append(txt_ids) 163 | id_lens.append(len(txt_ids)) 164 | 165 | # padding with eos 166 | for arr in batch_txt_ids: 167 | n = len(arr) 168 | rem = max_len - n 169 | concat_arr = [vocab('')]*rem 170 | arr+=concat_arr 171 | 172 | 173 | input = Variable(torch.LongTensor(batch_txt_ids)) 174 | output_seq2vec = uniskip(input, lengths=id_lens).data.numpy() 175 | 176 | # for c, e in enumerate(embeddings): 177 | for c, e in enumerate(output_seq2vec): 178 | 179 | ex = split.create_group(example_name + '_' + str(c)) 180 | ex.create_dataset('name', data=example_name) 181 | ex.create_dataset('img', data=np.void(img)) 182 | ex.create_dataset('embeddings', data=e) 183 | ex.create_dataset('class', data=_class) 184 | ex.create_dataset('txt', data=txt[c].astype(object), dtype=dt) 185 | 186 | print(example_name) 187 | 188 | 189 | -------------------------------------------------------------------------------- /convert_cub_to_hd5_script.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os.path import join, isfile 3 | import numpy as np 4 | import h5py 5 | from glob import glob 6 | from torch.utils.serialization import load_lua 7 | from PIL import Image 8 | import yaml 9 | import io 10 | import pdb 11 | 12 | with open('config.yaml', 'r') as f: 13 | config = yaml.load(f) 14 | 15 | images_path = config['birds_images_path'] 16 | embedding_path = config['birds_embedding_path'] 17 | text_path = config['birds_text_path'] 18 | datasetDir = config['birds_dataset_path'] 19 | 20 | val_classes = open(config['val_split_path']).read().splitlines() 21 | train_classes = open(config['train_split_path']).read().splitlines() 22 | test_classes = open(config['test_split_path']).read().splitlines() 23 | 24 | f = h5py.File(datasetDir, 'w') 25 | train = f.create_group('train') 26 | valid = f.create_group('valid') 27 | test = f.create_group('test') 28 | 29 | for _class in sorted(os.listdir(embedding_path)): 30 | split = '' 31 | if _class in train_classes: 32 | split = train 33 | elif _class in val_classes: 34 | split = valid 35 | elif _class in test_classes: 36 | split = test 37 | 38 | data_path = os.path.join(embedding_path, _class) 39 | txt_path = os.path.join(text_path, _class) 40 | for example, txt_file in zip(sorted(glob(data_path + "/*.t7")), sorted(glob(txt_path + "/*.txt"))): 41 | example_data = load_lua(example) 42 | img_path = example_data['img'] 43 | embeddings = example_data['txt'].numpy() 44 | example_name = img_path.split('/')[-1][:-4] 45 | 46 | f = open(txt_file, "r") 47 | txt = f.readlines() 48 | f.close() 49 | 50 | img_path = os.path.join(images_path, img_path) 51 | img = open(img_path, 'rb').read() 52 | 53 | txt_choice = np.random.choice(range(10), 5) 54 | 55 | embeddings = embeddings[txt_choice] 56 | txt = np.array(txt) 57 | txt = txt[txt_choice] 58 | dt = h5py.special_dtype(vlen=str) 59 | 60 | for c, e in enumerate(embeddings): 61 | ex = split.create_group(example_name + '_' + str(c)) 62 | ex.create_dataset('name', data=example_name) 63 | ex.create_dataset('img', data=np.void(img)) 64 | ex.create_dataset('embeddings', data=e) 65 | ex.create_dataset('class', data=_class) 66 | ex.create_dataset('txt', data=txt[c].astype(object), dtype=dt) 67 | 68 | print(example_name) 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /convert_flowers_skipthought_hd5_script.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os.path import join, isfile 3 | import numpy as np 4 | import h5py 5 | from glob import glob 6 | from torch.utils.serialization import load_lua 7 | from PIL import Image 8 | import yaml 9 | import io 10 | import pdb 11 | from collections import Counter 12 | import torch 13 | from torch.autograd import Variable 14 | import sys 15 | sys.path.append('skip-thoughts.torch/pytorch') 16 | from skipthoughts import UniSkip 17 | dir_st = 'data/skip-thoughts' 18 | from tqdm import tqdm 19 | import pickle 20 | import re 21 | import nltk 22 | import string 23 | 24 | with open('config.yaml', 'r') as f: 25 | config = yaml.load(f) 26 | 27 | images_path = config['flowers_images_path'] 28 | embedding_path = config['flowers_embedding_path'] 29 | text_path = config['flowers_text_path'] 30 | datasetDir = config['flowers_dataset_path'] 31 | 32 | val_classes = open(config['flowers_val_split_path']).read().splitlines() 33 | train_classes = open(config['flowers_train_split_path']).read().splitlines() 34 | test_classes = open(config['flowers_test_split_path']).read().splitlines() 35 | 36 | f = h5py.File(datasetDir, 'w') 37 | train = f.create_group('train') 38 | valid = f.create_group('valid') 39 | test = f.create_group('test') 40 | 41 | class Vocabulary(object): 42 | """Simple vocabulary wrapper.""" 43 | def __init__(self): 44 | self.word2idx = {} 45 | self.idx2word = {} 46 | self.idx = 0 47 | 48 | def add_word(self, word): 49 | if not word in self.word2idx: 50 | self.word2idx[word] = self.idx 51 | self.idx2word[self.idx] = word 52 | self.idx += 1 53 | 54 | def __call__(self, word): 55 | if not word in self.word2idx: 56 | return self.word2idx[''] 57 | return self.word2idx[word] 58 | 59 | def __len__(self): 60 | return len(self.word2idx) 61 | 62 | def sanitize_string(caption): 63 | caption = caption.strip() 64 | caption = caption.encode('ascii', 'ignore') 65 | caption = caption.decode('ascii') 66 | exclude = set(string.punctuation) 67 | preproc_caption = ''.join(ch for ch in caption if ch not in exclude) 68 | tokens = nltk.tokenize.word_tokenize(preproc_caption.lower()) 69 | return tokens 70 | 71 | def build_vocab(): 72 | 73 | threshold = 4 74 | """Build a simple vocabulary wrapper.""" 75 | # go through all files 76 | counter = Counter() 77 | for _class in sorted(os.listdir(embedding_path)): 78 | txt_path = os.path.join(text_path, _class) 79 | for txt_file in sorted(glob(txt_path + "/*.txt")): 80 | lines = open(txt_file, "r").readlines() 81 | for caption in lines: 82 | tokens = sanitize_string(caption) 83 | counter.update(tokens) 84 | 85 | # If the word frequency is less than 'threshold', then the word is discarded. 86 | words = [word for word, cnt in counter.items() if cnt >= threshold] 87 | 88 | # Creates a vocab wrapper and add some special tokens. 89 | vocab = Vocabulary() 90 | vocab.add_word('') 91 | vocab.add_word('') 92 | vocab.add_word('') 93 | vocab.add_word('') 94 | 95 | # Adds the words to the vocabulary. 96 | for i, word in enumerate(words): 97 | vocab.add_word(word) 98 | return vocab 99 | 100 | # vocab = build_vocab() 101 | # with open('data/flowers_vocab.pkl', 'wb') as f: 102 | # pickle.dump(vocab, f) 103 | 104 | with open('data/flowers_vocab.pkl', 'rb') as f: 105 | vocab = pickle.load(f) 106 | 107 | all_words_in_vocab = vocab.word2idx.keys() 108 | uniskip = UniSkip(dir_st, all_words_in_vocab) 109 | 110 | def get_ids(tokens, vocab): 111 | ids = [] 112 | # appending start and eos at the beginning and the end respectively for every sequence 113 | ids.append(vocab('')) 114 | for word in tokens: 115 | ids.append(vocab(word)) 116 | ids.append(vocab('')) 117 | return ids 118 | 119 | 120 | for _class in tqdm(sorted(os.listdir(embedding_path))): 121 | split = '' 122 | if _class in train_classes: 123 | split = train 124 | elif _class in val_classes: 125 | split = valid 126 | elif _class in test_classes: 127 | split = test 128 | 129 | data_path = os.path.join(embedding_path, _class) 130 | txt_path = os.path.join(text_path, _class) 131 | for example, txt_file in zip(sorted(glob(data_path + "/*.t7")), sorted(glob(txt_path + "/*.txt"))): 132 | example_data = load_lua(example) 133 | img_path = example_data['img'] 134 | embeddings = example_data['txt'].numpy() 135 | example_name = img_path.split('/')[-1][:-4] 136 | 137 | f = open(txt_file, "r") 138 | txt = f.readlines() 139 | f.close() 140 | 141 | img_path = os.path.join(images_path, img_path) 142 | img = open(img_path, 'rb').read() 143 | 144 | txt_choice = np.random.choice(range(10), 5) 145 | 146 | embeddings = embeddings[txt_choice] 147 | txt = np.array(txt) 148 | txt = txt[txt_choice] 149 | dt = h5py.special_dtype(vlen=str) 150 | 151 | batch_txt_ids = [] 152 | max_len = 0 153 | id_lens = [] 154 | for t in txt: 155 | skip_thought_txt = sanitize_string(t) 156 | txt_ids = get_ids(skip_thought_txt, vocab) 157 | max_len = len(txt_ids) if max_len < len(txt_ids) else max_len 158 | batch_txt_ids.append(txt_ids) 159 | id_lens.append(len(txt_ids)) 160 | 161 | # padding with eos 162 | for arr in batch_txt_ids: 163 | n = len(arr) 164 | rem = max_len - n 165 | concat_arr = [vocab('')]*rem 166 | arr+=concat_arr 167 | 168 | 169 | input = Variable(torch.LongTensor(batch_txt_ids)) 170 | output_seq2vec = uniskip(input, lengths=id_lens).data.numpy() 171 | 172 | # for c, e in enumerate(embeddings): 173 | for c, e in enumerate(output_seq2vec): 174 | 175 | ex = split.create_group(example_name + '_' + str(c)) 176 | ex.create_dataset('name', data=example_name) 177 | ex.create_dataset('img', data=np.void(img)) 178 | ex.create_dataset('embeddings', data=e) 179 | ex.create_dataset('class', data=_class) 180 | ex.create_dataset('txt', data=txt[c].astype(object), dtype=dt) 181 | 182 | #print(example_name, txt[1], _class) 183 | -------------------------------------------------------------------------------- /convert_flowers_to_hd5_script.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os.path import join, isfile 3 | import numpy as np 4 | import h5py 5 | from glob import glob 6 | from torch.utils.serialization import load_lua 7 | from PIL import Image 8 | import yaml 9 | import io 10 | import pdb 11 | 12 | with open('config.yaml', 'r') as f: 13 | config = yaml.load(f) 14 | 15 | images_path = config['flowers_images_path'] 16 | embedding_path = config['flowers_embedding_path'] 17 | text_path = config['flowers_text_path'] 18 | datasetDir = config['flowers_dataset_path'] 19 | 20 | val_classes = open(config['flowers_val_split_path']).read().splitlines() 21 | train_classes = open(config['flowers_train_split_path']).read().splitlines() 22 | test_classes = open(config['flowers_test_split_path']).read().splitlines() 23 | 24 | f = h5py.File(datasetDir, 'w') 25 | train = f.create_group('train') 26 | valid = f.create_group('valid') 27 | test = f.create_group('test') 28 | 29 | for _class in sorted(os.listdir(embedding_path)): 30 | split = '' 31 | if _class in train_classes: 32 | split = train 33 | elif _class in val_classes: 34 | split = valid 35 | elif _class in test_classes: 36 | split = test 37 | 38 | data_path = os.path.join(embedding_path, _class) 39 | txt_path = os.path.join(text_path, _class) 40 | for example, txt_file in zip(sorted(glob(data_path + "/*.t7")), sorted(glob(txt_path + "/*.txt"))): 41 | example_data = load_lua(example) 42 | img_path = example_data['img'] 43 | embeddings = example_data['txt'].numpy() 44 | example_name = img_path.split('/')[-1][:-4] 45 | 46 | f = open(txt_file, "r") 47 | txt = f.readlines() 48 | f.close() 49 | 50 | img_path = os.path.join(images_path, img_path) 51 | img = open(img_path, 'rb').read() 52 | 53 | txt_choice = np.random.choice(range(10), 5) 54 | 55 | embeddings = embeddings[txt_choice] 56 | txt = np.array(txt) 57 | txt = txt[txt_choice] 58 | dt = h5py.special_dtype(vlen=str) 59 | 60 | for c, e in enumerate(embeddings): 61 | ex = split.create_group(example_name + '_' + str(c)) 62 | ex.create_dataset('name', data=example_name) 63 | ex.create_dataset('img', data=np.void(img)) 64 | ex.create_dataset('embeddings', data=e) 65 | ex.create_dataset('class', data=_class) 66 | ex.create_dataset('txt', data=txt[c].astype(object), dtype=dt) 67 | 68 | print(example_name, txt[1], _class) 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /cycleresults/a curly red flower with many dark spots on it's petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/a curly red flower with many dark spots on it's petals..jpg -------------------------------------------------------------------------------- /cycleresults/a flower with long and wide petals that are yellow..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/a flower with long and wide petals that are yellow..jpg -------------------------------------------------------------------------------- /cycleresults/a flower with tiny yellow petals and very few yellow pistils..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/a flower with tiny yellow petals and very few yellow pistils..jpg -------------------------------------------------------------------------------- /cycleresults/a golden flower with a spiky look and a white center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/a golden flower with a spiky look and a white center..jpg -------------------------------------------------------------------------------- /cycleresults/a white flower with a golden and purple center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/a white flower with a golden and purple center..jpg -------------------------------------------------------------------------------- /cycleresults/flower is enclosed in a bud like leaf with yellow petals on the outside and long purple style..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/flower is enclosed in a bud like leaf with yellow petals on the outside and long purple style..jpg -------------------------------------------------------------------------------- /cycleresults/the flower has big yellow upright petals attached to a thick vine.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the flower has big yellow upright petals attached to a thick vine.jpg -------------------------------------------------------------------------------- /cycleresults/the flower is red in color with a long pedicel with leaves on it.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the flower is red in color with a long pedicel with leaves on it.jpg -------------------------------------------------------------------------------- /cycleresults/the flower is yellow with petals that are smooth and has pointy tip and they are in the formation around the stamen like a disc.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the flower is yellow with petals that are smooth and has pointy tip and they are in the formation around the stamen like a disc.jpg -------------------------------------------------------------------------------- /cycleresults/the flower pedals looks like colored leaves that are yellow and purple..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the flower pedals looks like colored leaves that are yellow and purple..jpg -------------------------------------------------------------------------------- /cycleresults/the petal of the flower is white in color with a large yellow pistil in the center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the petal of the flower is white in color with a large yellow pistil in the center..jpg -------------------------------------------------------------------------------- /cycleresults/the petals of the flower resemble the heightened tail of a peacock, and have a bold yellow color..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the petals of the flower resemble the heightened tail of a peacock, and have a bold yellow color..jpg -------------------------------------------------------------------------------- /cycleresults/the petals of this flower are purple and white with a long stigma.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the petals of this flower are purple and white with a long stigma.jpg -------------------------------------------------------------------------------- /cycleresults/the petals of this flower are purple with a short stigma.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the petals of this flower are purple with a short stigma.jpg -------------------------------------------------------------------------------- /cycleresults/the petals on this flower are yellow with an a few brown stripes..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the petals on this flower are yellow with an a few brown stripes..jpg -------------------------------------------------------------------------------- /cycleresults/the petals on this flower are yellow with dark lines..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the petals on this flower are yellow with dark lines..jpg -------------------------------------------------------------------------------- /cycleresults/the petals range in color from orange to yellow to purple with hints of white. the petals are different shapes and close as the flower goes up..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the petals range in color from orange to yellow to purple with hints of white. the petals are different shapes and close as the flower goes up..jpg -------------------------------------------------------------------------------- /cycleresults/the pink petals of the flower is bended and the stamen is not seen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the pink petals of the flower is bended and the stamen is not seen..jpg -------------------------------------------------------------------------------- /cycleresults/the spiky, thin yellow petals of this flower radiate outward from its bushy stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/the spiky, thin yellow petals of this flower radiate outward from its bushy stamen..jpg -------------------------------------------------------------------------------- /cycleresults/there are many pink petals below red anthers and filaments..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/there are many pink petals below red anthers and filaments..jpg -------------------------------------------------------------------------------- /cycleresults/this flower consist of a bunch of purple pedals climbing down the stock with black stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower consist of a bunch of purple pedals climbing down the stock with black stamen..jpg -------------------------------------------------------------------------------- /cycleresults/this flower features broad yellow petals surrounding the ovary in a ball like pattern..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower features broad yellow petals surrounding the ovary in a ball like pattern..jpg -------------------------------------------------------------------------------- /cycleresults/this flower has a light green sepal and strange looking blue pistil..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has a light green sepal and strange looking blue pistil..jpg -------------------------------------------------------------------------------- /cycleresults/this flower has a spiky brown center and down-turned pink petals with tapered tips..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has a spiky brown center and down-turned pink petals with tapered tips..jpg -------------------------------------------------------------------------------- /cycleresults/this flower has a star-like configuration of bright blue petals with darker veins..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has a star-like configuration of bright blue petals with darker veins..jpg -------------------------------------------------------------------------------- /cycleresults/this flower has bright yellow petals curved into a bell shape and a simple long green stem..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has bright yellow petals curved into a bell shape and a simple long green stem..jpg -------------------------------------------------------------------------------- /cycleresults/this flower has oddly shaped petals that vary in shades of purple and yellow..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has oddly shaped petals that vary in shades of purple and yellow..jpg -------------------------------------------------------------------------------- /cycleresults/this flower has orange spiky petals and purple spiky petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has orange spiky petals and purple spiky petals..jpg -------------------------------------------------------------------------------- /cycleresults/this flower has pale pink petals with a large rounded bright red anther in the center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has pale pink petals with a large rounded bright red anther in the center..jpg -------------------------------------------------------------------------------- /cycleresults/this flower has petals that are blue and are very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has petals that are blue and are very thin.jpg -------------------------------------------------------------------------------- /cycleresults/this flower has petals that are orange with black dotd.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has petals that are orange with black dotd.jpg -------------------------------------------------------------------------------- /cycleresults/this flower has petals that are pink and ruffled at the edges.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has petals that are pink and ruffled at the edges.jpg -------------------------------------------------------------------------------- /cycleresults/this flower has petals that are purple and are very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has petals that are purple and are very thin.jpg -------------------------------------------------------------------------------- /cycleresults/this flower has petals that are purple with purple stigma.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has petals that are purple with purple stigma.jpg -------------------------------------------------------------------------------- /cycleresults/this flower has petals that are white and has patches of pink.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has petals that are white and has patches of pink.jpg -------------------------------------------------------------------------------- /cycleresults/this flower has petals that are white on the outer petal and red on the center with red anthers...jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has petals that are white on the outer petal and red on the center with red anthers...jpg -------------------------------------------------------------------------------- /cycleresults/this flower has petals that are yellow and are folded together.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has petals that are yellow and are folded together.jpg -------------------------------------------------------------------------------- /cycleresults/this flower has petals that are yellow and are very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has petals that are yellow and are very thin.jpg -------------------------------------------------------------------------------- /cycleresults/this flower has tall and spiky yellow petals with no visible outer stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has tall and spiky yellow petals with no visible outer stamen.jpg -------------------------------------------------------------------------------- /cycleresults/this flower has yellow rounded petals with stripe brown markings..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower has yellow rounded petals with stripe brown markings..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is made up of flowy white petals that have some pink highlights toward the tips..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is made up of flowy white petals that have some pink highlights toward the tips..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is orange in color, with petals that are layered..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is orange in color, with petals that are layered..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is pink and white in color, and has petals that are striped..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is pink and white in color, and has petals that are striped..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is pink and white in color, with petals that are wavy and curled..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is pink and white in color, with petals that are wavy and curled..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is pink in color, with petals that have veins..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is pink in color, with petals that have veins..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is purple and white in color, and has petals that are very skinny..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is purple and white in color, and has petals that are very skinny..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is purple in color, and has petals that are bell shaped..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is purple in color, and has petals that are bell shaped..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is purple in color, with petals that are oddly shaped..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is purple in color, with petals that are oddly shaped..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is rose shaped with bright yellow overlapping petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is rose shaped with bright yellow overlapping petals..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is white and green in color, and has petals that are pointed..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is white and green in color, and has petals that are pointed..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is white and pink in color, and has petals that are oval shaped..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is white and pink in color, and has petals that are oval shaped..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is white and yellow in color, with petals that are wavy and curled..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is white and yellow in color, with petals that are wavy and curled..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is yellow and purple in color, with petals that are spotted..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is yellow and purple in color, with petals that are spotted..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is yellow in color, with petals that are ruffled..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is yellow in color, with petals that are ruffled..jpg -------------------------------------------------------------------------------- /cycleresults/this flower is yellow in color, with petals that are very skinny..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this flower is yellow in color, with petals that are very skinny..jpg -------------------------------------------------------------------------------- /cycleresults/this has a spiny type pedicel with a pink-lavendar hue flower that has many pollen and stigma leaves..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this has a spiny type pedicel with a pink-lavendar hue flower that has many pollen and stigma leaves..jpg -------------------------------------------------------------------------------- /cycleresults/this is a blue flower with short and rounded petals that turns white toward the receptacle..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this is a blue flower with short and rounded petals that turns white toward the receptacle..jpg -------------------------------------------------------------------------------- /cycleresults/this is a soft petaled rose colored flower with large flappy petals and a firm pedicel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this is a soft petaled rose colored flower with large flappy petals and a firm pedicel.jpg -------------------------------------------------------------------------------- /cycleresults/this orange and blue flower has pointed petals and a green pedicel..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this orange and blue flower has pointed petals and a green pedicel..jpg -------------------------------------------------------------------------------- /cycleresults/this orange and brown flower has rounded petals and bright orange stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this orange and brown flower has rounded petals and bright orange stamen..jpg -------------------------------------------------------------------------------- /cycleresults/this petals on this flower are purple with no visible stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this petals on this flower are purple with no visible stamen..jpg -------------------------------------------------------------------------------- /cycleresults/this pink flower has the string like petals arranged on the horny pedicel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this pink flower has the string like petals arranged on the horny pedicel.jpg -------------------------------------------------------------------------------- /cycleresults/this pink flower of asteraceae family has outer florets forming a pink corolla and the central florets are red with yellow pistils..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this pink flower of asteraceae family has outer florets forming a pink corolla and the central florets are red with yellow pistils..jpg -------------------------------------------------------------------------------- /cycleresults/this yellow, pink, and white flower has pointed petals with a pink pedicel..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults/this yellow, pink, and white flower has pointed petals with a pink pedicel..jpg -------------------------------------------------------------------------------- /cycleresults220/a blue flower with petals and the yellow stigma and anther and the leaves are green and thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/a blue flower with petals and the yellow stigma and anther and the leaves are green and thin.jpg -------------------------------------------------------------------------------- /cycleresults220/a flower with white petals with brown stripes..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/a flower with white petals with brown stripes..jpg -------------------------------------------------------------------------------- /cycleresults220/an odd shaped flower that has dark red spots on the salmon colored petals.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/an odd shaped flower that has dark red spots on the salmon colored petals.jpg -------------------------------------------------------------------------------- /cycleresults220/purple bell shaped flower with straight white stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/purple bell shaped flower with straight white stamen..jpg -------------------------------------------------------------------------------- /cycleresults220/the flower has long yellow petals that are thin and a yellow stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the flower has long yellow petals that are thin and a yellow stamen..jpg -------------------------------------------------------------------------------- /cycleresults220/the flower has needle like stigma sticking out of the downward facing petals that are surrounding it.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the flower has needle like stigma sticking out of the downward facing petals that are surrounding it.jpg -------------------------------------------------------------------------------- /cycleresults220/the flower has pointed pale pink petals and several white anthers..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the flower has pointed pale pink petals and several white anthers..jpg -------------------------------------------------------------------------------- /cycleresults220/the flower has pointed petals and purple stamen and pistil..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the flower has pointed petals and purple stamen and pistil..jpg -------------------------------------------------------------------------------- /cycleresults220/the flower has white and purple petals as well as purple stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the flower has white and purple petals as well as purple stamen.jpg -------------------------------------------------------------------------------- /cycleresults220/the flower has white and yellow petals that pleum out..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the flower has white and yellow petals that pleum out..jpg -------------------------------------------------------------------------------- /cycleresults220/the flower has yellow petals with a yellow stigma and green pedicel..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the flower has yellow petals with a yellow stigma and green pedicel..jpg -------------------------------------------------------------------------------- /cycleresults220/the flower shown has red petals with a green pedicel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the flower shown has red petals with a green pedicel.jpg -------------------------------------------------------------------------------- /cycleresults220/the flower shown here has blue petals which surround the white stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the flower shown here has blue petals which surround the white stamen.jpg -------------------------------------------------------------------------------- /cycleresults220/the large round center of this flower is covered with whitish pink anthers and the petals are white closest to the center and end in a red point..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the large round center of this flower is covered with whitish pink anthers and the petals are white closest to the center and end in a red point..jpg -------------------------------------------------------------------------------- /cycleresults220/the petal is closed and it appears white and the stamen is orange..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the petal is closed and it appears white and the stamen is orange..jpg -------------------------------------------------------------------------------- /cycleresults220/the petals of the flower are orange in color and have green leaves..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the petals of the flower are orange in color and have green leaves..jpg -------------------------------------------------------------------------------- /cycleresults220/the petals of this flower are red with a long stigma.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the petals of this flower are red with a long stigma.jpg -------------------------------------------------------------------------------- /cycleresults220/the petals of this flower are white with dark purple tips and the stamen are green with purple tips..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the petals of this flower are white with dark purple tips and the stamen are green with purple tips..jpg -------------------------------------------------------------------------------- /cycleresults220/the petals on this flower are round and purple shaped in threes..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the petals on this flower are round and purple shaped in threes..jpg -------------------------------------------------------------------------------- /cycleresults220/the pistil of this flower is of radial symmetry petals brightly yellow colored. there are stamen in the center which are of the same color of the stamen. no stigma visible..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the pistil of this flower is of radial symmetry petals brightly yellow colored. there are stamen in the center which are of the same color of the stamen. no stigma visible..jpg -------------------------------------------------------------------------------- /cycleresults220/the pretty flower has white petals with purple at their tips..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/the pretty flower has white petals with purple at their tips..jpg -------------------------------------------------------------------------------- /cycleresults220/there are flat, white petals and white petals that curl upwards around yellow and pink stamens..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/there are flat, white petals and white petals that curl upwards around yellow and pink stamens..jpg -------------------------------------------------------------------------------- /cycleresults220/there are many long and narrow floppy pink petals surrounding many red stamen and a green stigma on this flower..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/there are many long and narrow floppy pink petals surrounding many red stamen and a green stigma on this flower..jpg -------------------------------------------------------------------------------- /cycleresults220/there is a simple fat white bell shaped bloom with round petals and visible yellow stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/there is a simple fat white bell shaped bloom with round petals and visible yellow stamen..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has a large, white cup shaped petal with a thick bright yellow stigma..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has a large, white cup shaped petal with a thick bright yellow stigma..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has a lot of pointed red petals in a ray-like shape around the many stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has a lot of pointed red petals in a ray-like shape around the many stamen..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has a thick yellow stamen wrapped in a thick pointed white petal..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has a thick yellow stamen wrapped in a thick pointed white petal..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has blue petals that are folded together in a bulb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has blue petals that are folded together in a bulb.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has long petals that are orange with spots and a very long pistil..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has long petals that are orange with spots and a very long pistil..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has many bright orange petals with a brown center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has many bright orange petals with a brown center..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are pink and folded together.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are pink and folded together.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are purple and very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are purple and very thin.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are red with patches of yellow.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are red with patches of yellow.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are white with many ruffles.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are white with many ruffles.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are white with pink shading.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are white with pink shading.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are yellow and are folded together.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are yellow and are folded together.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are yellow and are very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are yellow and are very thin.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are yellow and folded together.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are yellow and folded together.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are yellow and has red dots.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are yellow and has red dots.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are yellow and has yellow stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are yellow and has yellow stamen.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has petals that are yellow with dark lines.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has petals that are yellow with dark lines.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has pink petals that have long yellow stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has pink petals that have long yellow stamen.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has purple petals with spikes at the bottom.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has purple petals with spikes at the bottom.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has red and purple petals with tiny pointy receptacles around it..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has red and purple petals with tiny pointy receptacles around it..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has several large white and pink petals with slight accents of yellow and brown..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has several large white and pink petals with slight accents of yellow and brown..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has two rows of long, rounded petals that are multi-tonal (orange to yellow) in color..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has two rows of long, rounded petals that are multi-tonal (orange to yellow) in color..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower has white petals that have a small yellow patch in the center.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower has white petals that have a small yellow patch in the center.jpg -------------------------------------------------------------------------------- /cycleresults220/this flower is characterized by its spiky purple petals on top, and its spiky green petals directly under the purple petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower is characterized by its spiky purple petals on top, and its spiky green petals directly under the purple petals..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower is pink and white in color, with petals that are pointed on the ends..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower is pink and white in color, with petals that are pointed on the ends..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower is pink and yellow in color, with petals that are multi colored..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower is pink and yellow in color, with petals that are multi colored..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower is purple in color, and has petals that are thin and wavy..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower is purple in color, and has petals that are thin and wavy..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower is purple in color, and has petals that are very skinny..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower is purple in color, and has petals that are very skinny..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower is red white and yellow in color, with petals that are curled and layered vertically..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower is red white and yellow in color, with petals that are curled and layered vertically..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower is white, pink, and yellow in color, and has petals that are multi colored..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower is white, pink, and yellow in color, and has petals that are multi colored..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower is yellow and purple in color, with petals that are striped near the center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower is yellow and purple in color, with petals that are striped near the center..jpg -------------------------------------------------------------------------------- /cycleresults220/this flower is yellow in color, and has petals that are rounded and curled around the center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this flower is yellow in color, and has petals that are rounded and curled around the center..jpg -------------------------------------------------------------------------------- /cycleresults220/this is a flower with round purple upward facing petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this is a flower with round purple upward facing petals..jpg -------------------------------------------------------------------------------- /cycleresults220/this is a group of white flowers, with spider shaped centers..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this is a group of white flowers, with spider shaped centers..jpg -------------------------------------------------------------------------------- /cycleresults220/this large yellow blossom has numerous yellow stamen and hundreds of very thin yellow petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this large yellow blossom has numerous yellow stamen and hundreds of very thin yellow petals..jpg -------------------------------------------------------------------------------- /cycleresults220/this orange and blue flower has pointed petals and a green pedicel..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this orange and blue flower has pointed petals and a green pedicel..jpg -------------------------------------------------------------------------------- /cycleresults220/this spikey flower has a green pointed barrier around the bottom with pink stamen protruding from the top..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this spikey flower has a green pointed barrier around the bottom with pink stamen protruding from the top..jpg -------------------------------------------------------------------------------- /cycleresults220/this white flower has rounded petals and a yellow orange stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this white flower has rounded petals and a yellow orange stamen..jpg -------------------------------------------------------------------------------- /cycleresults220/this white flower is delicate and a beautiful white color with strong green leaves that surround it..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults220/this white flower is delicate and a beautiful white color with strong green leaves that surround it..jpg -------------------------------------------------------------------------------- /cycleresults250/a blue flower with petals and the yellow stigma and anther and the leaves are green and thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/a blue flower with petals and the yellow stigma and anther and the leaves are green and thin.jpg -------------------------------------------------------------------------------- /cycleresults250/a flower with white petals with brown stripes..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/a flower with white petals with brown stripes..jpg -------------------------------------------------------------------------------- /cycleresults250/an odd shaped flower that has dark red spots on the salmon colored petals.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/an odd shaped flower that has dark red spots on the salmon colored petals.jpg -------------------------------------------------------------------------------- /cycleresults250/purple bell shaped flower with straight white stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/purple bell shaped flower with straight white stamen..jpg -------------------------------------------------------------------------------- /cycleresults250/the flower has long yellow petals that are thin and a yellow stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the flower has long yellow petals that are thin and a yellow stamen..jpg -------------------------------------------------------------------------------- /cycleresults250/the flower has needle like stigma sticking out of the downward facing petals that are surrounding it.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the flower has needle like stigma sticking out of the downward facing petals that are surrounding it.jpg -------------------------------------------------------------------------------- /cycleresults250/the flower has pointed pale pink petals and several white anthers..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the flower has pointed pale pink petals and several white anthers..jpg -------------------------------------------------------------------------------- /cycleresults250/the flower has pointed petals and purple stamen and pistil..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the flower has pointed petals and purple stamen and pistil..jpg -------------------------------------------------------------------------------- /cycleresults250/the flower has white and purple petals as well as purple stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the flower has white and purple petals as well as purple stamen.jpg -------------------------------------------------------------------------------- /cycleresults250/the flower has white and yellow petals that pleum out..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the flower has white and yellow petals that pleum out..jpg -------------------------------------------------------------------------------- /cycleresults250/the flower has yellow petals with a yellow stigma and green pedicel..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the flower has yellow petals with a yellow stigma and green pedicel..jpg -------------------------------------------------------------------------------- /cycleresults250/the flower shown has red petals with a green pedicel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the flower shown has red petals with a green pedicel.jpg -------------------------------------------------------------------------------- /cycleresults250/the flower shown here has blue petals which surround the white stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the flower shown here has blue petals which surround the white stamen.jpg -------------------------------------------------------------------------------- /cycleresults250/the large round center of this flower is covered with whitish pink anthers and the petals are white closest to the center and end in a red point..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the large round center of this flower is covered with whitish pink anthers and the petals are white closest to the center and end in a red point..jpg -------------------------------------------------------------------------------- /cycleresults250/the petal is closed and it appears white and the stamen is orange..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the petal is closed and it appears white and the stamen is orange..jpg -------------------------------------------------------------------------------- /cycleresults250/the petals of the flower are orange in color and have green leaves..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the petals of the flower are orange in color and have green leaves..jpg -------------------------------------------------------------------------------- /cycleresults250/the petals of this flower are red with a long stigma.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the petals of this flower are red with a long stigma.jpg -------------------------------------------------------------------------------- /cycleresults250/the petals of this flower are white with dark purple tips and the stamen are green with purple tips..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the petals of this flower are white with dark purple tips and the stamen are green with purple tips..jpg -------------------------------------------------------------------------------- /cycleresults250/the petals on this flower are round and purple shaped in threes..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the petals on this flower are round and purple shaped in threes..jpg -------------------------------------------------------------------------------- /cycleresults250/the pistil of this flower is of radial symmetry petals brightly yellow colored. there are stamen in the center which are of the same color of the stamen. no stigma visible..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the pistil of this flower is of radial symmetry petals brightly yellow colored. there are stamen in the center which are of the same color of the stamen. no stigma visible..jpg -------------------------------------------------------------------------------- /cycleresults250/the pretty flower has white petals with purple at their tips..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/the pretty flower has white petals with purple at their tips..jpg -------------------------------------------------------------------------------- /cycleresults250/there are flat, white petals and white petals that curl upwards around yellow and pink stamens..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/there are flat, white petals and white petals that curl upwards around yellow and pink stamens..jpg -------------------------------------------------------------------------------- /cycleresults250/there are many long and narrow floppy pink petals surrounding many red stamen and a green stigma on this flower..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/there are many long and narrow floppy pink petals surrounding many red stamen and a green stigma on this flower..jpg -------------------------------------------------------------------------------- /cycleresults250/there is a simple fat white bell shaped bloom with round petals and visible yellow stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/there is a simple fat white bell shaped bloom with round petals and visible yellow stamen..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has a large, white cup shaped petal with a thick bright yellow stigma..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has a large, white cup shaped petal with a thick bright yellow stigma..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has a lot of pointed red petals in a ray-like shape around the many stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has a lot of pointed red petals in a ray-like shape around the many stamen..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has a thick yellow stamen wrapped in a thick pointed white petal..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has a thick yellow stamen wrapped in a thick pointed white petal..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has blue petals that are folded together in a bulb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has blue petals that are folded together in a bulb.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has long petals that are orange with spots and a very long pistil..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has long petals that are orange with spots and a very long pistil..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has many bright orange petals with a brown center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has many bright orange petals with a brown center..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are pink and folded together.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are pink and folded together.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are purple and very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are purple and very thin.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are red with patches of yellow.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are red with patches of yellow.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are white with many ruffles.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are white with many ruffles.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are white with pink shading.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are white with pink shading.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are yellow and are folded together.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are yellow and are folded together.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are yellow and are very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are yellow and are very thin.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are yellow and folded together.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are yellow and folded together.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are yellow and has red dots.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are yellow and has red dots.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are yellow and has yellow stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are yellow and has yellow stamen.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has petals that are yellow with dark lines.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has petals that are yellow with dark lines.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has pink petals that have long yellow stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has pink petals that have long yellow stamen.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has purple petals with spikes at the bottom.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has purple petals with spikes at the bottom.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has red and purple petals with tiny pointy receptacles around it..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has red and purple petals with tiny pointy receptacles around it..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has several large white and pink petals with slight accents of yellow and brown..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has several large white and pink petals with slight accents of yellow and brown..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has two rows of long, rounded petals that are multi-tonal (orange to yellow) in color..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has two rows of long, rounded petals that are multi-tonal (orange to yellow) in color..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower has white petals that have a small yellow patch in the center.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower has white petals that have a small yellow patch in the center.jpg -------------------------------------------------------------------------------- /cycleresults250/this flower is characterized by its spiky purple petals on top, and its spiky green petals directly under the purple petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower is characterized by its spiky purple petals on top, and its spiky green petals directly under the purple petals..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower is pink and white in color, with petals that are pointed on the ends..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower is pink and white in color, with petals that are pointed on the ends..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower is pink and yellow in color, with petals that are multi colored..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower is pink and yellow in color, with petals that are multi colored..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower is purple in color, and has petals that are thin and wavy..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower is purple in color, and has petals that are thin and wavy..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower is purple in color, and has petals that are very skinny..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower is purple in color, and has petals that are very skinny..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower is red white and yellow in color, with petals that are curled and layered vertically..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower is red white and yellow in color, with petals that are curled and layered vertically..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower is white, pink, and yellow in color, and has petals that are multi colored..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower is white, pink, and yellow in color, and has petals that are multi colored..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower is yellow and purple in color, with petals that are striped near the center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower is yellow and purple in color, with petals that are striped near the center..jpg -------------------------------------------------------------------------------- /cycleresults250/this flower is yellow in color, and has petals that are rounded and curled around the center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this flower is yellow in color, and has petals that are rounded and curled around the center..jpg -------------------------------------------------------------------------------- /cycleresults250/this is a flower with round purple upward facing petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this is a flower with round purple upward facing petals..jpg -------------------------------------------------------------------------------- /cycleresults250/this is a group of white flowers, with spider shaped centers..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this is a group of white flowers, with spider shaped centers..jpg -------------------------------------------------------------------------------- /cycleresults250/this large yellow blossom has numerous yellow stamen and hundreds of very thin yellow petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this large yellow blossom has numerous yellow stamen and hundreds of very thin yellow petals..jpg -------------------------------------------------------------------------------- /cycleresults250/this orange and blue flower has pointed petals and a green pedicel..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this orange and blue flower has pointed petals and a green pedicel..jpg -------------------------------------------------------------------------------- /cycleresults250/this spikey flower has a green pointed barrier around the bottom with pink stamen protruding from the top..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this spikey flower has a green pointed barrier around the bottom with pink stamen protruding from the top..jpg -------------------------------------------------------------------------------- /cycleresults250/this white flower has rounded petals and a yellow orange stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this white flower has rounded petals and a yellow orange stamen..jpg -------------------------------------------------------------------------------- /cycleresults250/this white flower is delicate and a beautiful white color with strong green leaves that surround it..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/cycleresults250/this white flower is delicate and a beautiful white color with strong green leaves that surround it..jpg -------------------------------------------------------------------------------- /eval_dataset.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.transforms as transforms 3 | import torch.utils.data as data 4 | from PIL import Image 5 | import os 6 | from tqdm import tqdm 7 | import numpy as np 8 | 9 | class EvalDataset(data.Dataset): 10 | def __init__(self, data_dir, transform=None): 11 | self.transform = transform 12 | self.root = data_dir 13 | self.dataset = [] 14 | 15 | # load images into memory 16 | for subdir, dirs, files in os.walk(self.root): 17 | for img_path in tqdm(files): 18 | image = Image.open(os.path.join(self.root, img_path)) 19 | if self.transform is not None: 20 | image = self.transform(image) 21 | self.dataset.append(image) 22 | 23 | def __len__(self): 24 | length = len(self.dataset) 25 | return length 26 | 27 | def __getitem__(self, idx): 28 | image = self.dataset[idx] 29 | image = np.array(image, dtype=float) 30 | image = torch.FloatTensor(image) 31 | return image 32 | 33 | -------------------------------------------------------------------------------- /figures/cycle_a_losses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/figures/cycle_a_losses.png -------------------------------------------------------------------------------- /figures/pretraining_caption_disc_losses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/figures/pretraining_caption_disc_losses.png -------------------------------------------------------------------------------- /figures/pretraining_gen_losses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/figures/pretraining_gen_losses.png -------------------------------------------------------------------------------- /figures/stage_1_disc_losses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/figures/stage_1_disc_losses.png -------------------------------------------------------------------------------- /figures/stage_1_gen_losses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/figures/stage_1_gen_losses.png -------------------------------------------------------------------------------- /inception_score.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch.autograd import Variable 4 | from torch.nn import functional as F 5 | import torch.utils.data 6 | 7 | from torchvision.models.inception import inception_v3 8 | 9 | import numpy as np 10 | from scipy.stats import entropy 11 | 12 | from tqdm import tqdm 13 | import pdb 14 | 15 | from txt2image_dataset import Text2ImageDataset 16 | from eval_dataset import EvalDataset 17 | 18 | def inception_score(imgs, cuda=True, batch_size=32, resize=False, splits=1): 19 | """Computes the inception score of the generated images imgs 20 | imgs -- Torch dataset of (3xHxW) numpy images normalized in the range [-1, 1] 21 | cuda -- whether or not to run on GPU 22 | batch_size -- batch size for feeding into Inception v3 23 | splits -- number of splits 24 | """ 25 | N = len(imgs) 26 | 27 | assert batch_size > 0 28 | assert N > batch_size 29 | 30 | # Set up dtype 31 | if cuda: 32 | dtype = torch.cuda.FloatTensor 33 | else: 34 | if torch.cuda.is_available(): 35 | print("WARNING: You have a CUDA device, so you should probably set cuda=True") 36 | dtype = torch.FloatTensor 37 | 38 | # Set up dataloader 39 | dataloader = torch.utils.data.DataLoader(imgs, batch_size=batch_size) 40 | 41 | # Load inception model 42 | inception_model = inception_v3(pretrained=True, transform_input=False).type(dtype) 43 | inception_model.eval(); 44 | up = nn.Upsample(size=(299, 299), mode='bilinear').type(dtype) 45 | def get_pred(x): 46 | if resize: 47 | x = up(x) 48 | x = inception_model(x) 49 | return F.softmax(x).data.cpu().numpy() 50 | 51 | # Get predictions 52 | preds = np.zeros((N, 1000)) 53 | 54 | for i, batch in tqdm(enumerate(dataloader, 0)): 55 | batch = batch.type(dtype) 56 | batchv = Variable(batch) 57 | batch_size_i = batch.size()[0] 58 | 59 | preds[i*batch_size:i*batch_size + batch_size_i] = get_pred(batchv) 60 | 61 | # Now compute the mean kl-div 62 | split_scores = [] 63 | 64 | for k in tqdm(range(splits)): 65 | part = preds[k * (N // splits): (k+1) * (N // splits), :] # N/split, 1000 66 | py = np.mean(part, axis=0) 67 | scores = [] 68 | for i in range(part.shape[0]): 69 | pyx = part[i, :] 70 | scores.append(entropy(pyx, py)) 71 | split_scores.append(np.exp(np.mean(scores))) 72 | 73 | return np.mean(split_scores), np.std(split_scores) 74 | 75 | if __name__ == '__main__': 76 | class IgnoreLabelDataset(torch.utils.data.Dataset): 77 | def __init__(self, orig): 78 | self.orig = orig 79 | 80 | def __getitem__(self, index): 81 | return self.orig[index][0] 82 | 83 | def __len__(self): 84 | return len(self.orig) 85 | 86 | import torchvision.datasets as dset 87 | import torchvision.transforms as transforms 88 | 89 | # cifar = dset.CIFAR10(root='data/', download=True, 90 | # transform=transforms.Compose([ 91 | # transforms.Scale(32), 92 | # transforms.ToTensor(), 93 | # transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) 94 | # ]) 95 | # ) 96 | 97 | # IgnoreLabelDataset(cifar) 98 | 99 | # dataset = Text2ImageDataset(config['flowers_dataset_path'], dataset_type='flowers', vocab=self.vocab, split=split) 100 | dataset = EvalDataset('./results/', transform=transforms.Compose([ 101 | transforms.Scale(32), 102 | transforms.ToTensor(), 103 | transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) 104 | ]) 105 | ) 106 | print ("Calculating Inception Score...") 107 | 108 | print (inception_score(dataset, cuda=True, batch_size=32, resize=True, splits=10)) 109 | 110 | -------------------------------------------------------------------------------- /models/gan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | import numpy as np 5 | 6 | class generator(nn.Module): 7 | def __init__(self): 8 | super(generator, self).__init__() 9 | self.image_size = 64 10 | self.num_channels = 3 11 | self.noise_dim = 100 12 | self.ngf = 64 13 | 14 | # based on: https://github.com/pytorch/examples/blob/master/dcgan/main.py 15 | self.netG = nn.Sequential( 16 | nn.ConvTranspose2d(self.noise_dim, self.ngf * 8, 4, 1, 0, bias=False), 17 | nn.BatchNorm2d(self.ngf * 8), 18 | nn.ReLU(True), 19 | # state size. (ngf*8) x 4 x 4 20 | nn.ConvTranspose2d(self.ngf * 8, self.ngf * 4, 4, 2, 1, bias=False), 21 | nn.BatchNorm2d(self.ngf * 4), 22 | nn.ReLU(True), 23 | # state size. (ngf*4) x 8 x 8 24 | nn.ConvTranspose2d(self.ngf * 4, self.ngf * 2, 4, 2, 1, bias=False), 25 | nn.BatchNorm2d(self.ngf * 2), 26 | nn.ReLU(True), 27 | # state size. (ngf*2) x 16 x 16 28 | nn.ConvTranspose2d(self.ngf * 2,self.ngf, 4, 2, 1, bias=False), 29 | nn.BatchNorm2d(self.ngf), 30 | nn.ReLU(True), 31 | # state size. (ngf) x 32 x 32 32 | nn.ConvTranspose2d(self.ngf, self.num_channels, 4, 2, 1, bias=False), 33 | nn.Tanh() 34 | # state size. (num_channels) x 64 x 64 35 | ) 36 | 37 | def forward(self, z): 38 | return self.netG(z) 39 | 40 | 41 | class discriminator(nn.Module): 42 | def __init__(self): 43 | super(discriminator, self).__init__() 44 | self.image_size = 64 45 | self.num_channels = 3 46 | self.ndf = 64 47 | 48 | self.B_dim = 128 49 | self.C_dim = 16 50 | 51 | self.netD_1 = nn.Sequential( 52 | # input is (nc) x 64 x 64 53 | nn.Conv2d(self.num_channels, self.ndf, 4, 2, 1, bias=False), 54 | nn.LeakyReLU(0.2, inplace=True), 55 | # state size. (ndf) x 32 x 32 56 | nn.Conv2d(self.ndf, self.ndf * 2, 4, 2, 1, bias=False), 57 | nn.BatchNorm2d(self.ndf * 2), 58 | nn.LeakyReLU(0.2, inplace=True), 59 | # state size. (ndf*2) x 16 x 16 60 | nn.Conv2d(self.ndf * 2, self.ndf * 4, 4, 2, 1, bias=False), 61 | nn.BatchNorm2d(self.ndf * 4), 62 | nn.LeakyReLU(0.2, inplace=True), 63 | # state size. (ndf*4) x 8 x 8 64 | nn.Conv2d(self.ndf * 4, self.ndf * 8, 4, 2, 1, bias=False), 65 | nn.BatchNorm2d(self.ndf * 8), 66 | nn.LeakyReLU(0.2, inplace=True) 67 | ) 68 | 69 | self.netD_2 = nn.Sequential( 70 | # state size. (ndf*8) x 4 x 4 71 | nn.Conv2d(self.ndf * 8, 1, 4, 1, 0, bias=False), 72 | nn.Sigmoid() 73 | ) 74 | 75 | def forward(self, inp): 76 | x_intermediate = self.netD_1(inp) 77 | output = self.netD_2(x_intermediate) 78 | return output.view(-1, 1).squeeze(1), x_intermediate 79 | -------------------------------------------------------------------------------- /models/gan_cls.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | import numpy as np 5 | from utils import Concat_embed 6 | import pdb 7 | 8 | def conv3x3(in_planes, out_planes, stride=1): 9 | "3x3 convolution with padding" 10 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 11 | padding=1, bias=True) 12 | 13 | 14 | # Upsale the spatial size by a factor of 2 15 | def upBlock(in_planes, out_planes): 16 | block = nn.Sequential( 17 | nn.Upsample(scale_factor=2, mode='nearest'), 18 | conv3x3(in_planes, out_planes), 19 | nn.BatchNorm2d(out_planes), 20 | nn.ReLU(True)) 21 | return block 22 | 23 | 24 | class ResBlock(nn.Module): 25 | def __init__(self, channel_num): 26 | super(ResBlock, self).__init__() 27 | self.block = nn.Sequential( 28 | conv3x3(channel_num, channel_num), 29 | nn.BatchNorm2d(channel_num), 30 | nn.ReLU(True), 31 | conv3x3(channel_num, channel_num), 32 | nn.BatchNorm2d(channel_num)) 33 | self.relu = nn.ReLU(inplace=True) 34 | 35 | def forward(self, x): 36 | residual = x 37 | out = self.block(x) 38 | out += residual 39 | out = self.relu(out) 40 | return out 41 | 42 | class generator(nn.Module): 43 | def __init__(self): 44 | super(generator, self).__init__() 45 | self.image_size = 64 46 | self.num_channels = 3 47 | self.noise_dim = 100 48 | self.embed_dim = 2400 # compatible with skip thought (1024) 49 | self.projected_embed_dim = 128 50 | self.latent_dim = self.noise_dim + self.projected_embed_dim 51 | self.ngf = 64 52 | 53 | self.projection = nn.Sequential( 54 | nn.Linear(in_features=self.embed_dim, out_features=self.projected_embed_dim), 55 | nn.BatchNorm1d(num_features=self.projected_embed_dim), 56 | nn.LeakyReLU(negative_slope=0.2, inplace=True) 57 | ) 58 | 59 | # based on: https://github.com/pytorch/examples/blob/master/dcgan/main.py 60 | self.netG = nn.Sequential( 61 | nn.ConvTranspose2d(self.latent_dim, self.ngf * 8, 4, 1, 0, bias=True), 62 | nn.BatchNorm2d(self.ngf * 8), 63 | nn.ReLU(True), 64 | 65 | # adding extra convs will give output (ngf*8) x 4 x 4 66 | nn.Conv2d(self.ngf*8, self.ngf*2, 1, 1, 0), 67 | nn.BatchNorm2d(self.ngf*2), 68 | nn.ReLU(True), 69 | 70 | nn.Conv2d(self.ngf*2, self.ngf*2, 3, 1, 1), 71 | nn.BatchNorm2d(self.ngf*2), 72 | nn.ReLU(True), 73 | 74 | nn.Conv2d(self.ngf*2, self.ngf*8, 3, 1, 1), 75 | nn.BatchNorm2d(self.ngf*8), 76 | nn.ReLU(True), 77 | 78 | # state size. (ngf*8) x 4 x 4 79 | nn.ConvTranspose2d(self.ngf * 8, self.ngf * 4, 4, 2, 1, bias=True), 80 | nn.BatchNorm2d(self.ngf * 4), 81 | nn.ReLU(True), 82 | 83 | # adding extra convs will give output (ngf*4) x 4 x 4 84 | nn.Conv2d(self.ngf*4, self.ngf, 1, 1, 0), 85 | nn.BatchNorm2d(self.ngf), 86 | nn.ReLU(True), 87 | 88 | nn.Conv2d(self.ngf, self.ngf, 3, 1, 1), 89 | nn.BatchNorm2d(self.ngf), 90 | nn.ReLU(True), 91 | 92 | nn.Conv2d(self.ngf, self.ngf*4, 3, 1, 1), 93 | nn.BatchNorm2d(self.ngf*4), 94 | nn.ReLU(True), 95 | 96 | 97 | # state size. (ngf*4) x 8 x 8 98 | nn.ConvTranspose2d(self.ngf * 4, self.ngf * 2, 4, 2, 1, bias=True), 99 | nn.BatchNorm2d(self.ngf * 2), 100 | nn.ReLU(True), 101 | # state size. (ngf*2) x 16 x 16 102 | nn.ConvTranspose2d(self.ngf * 2,self.ngf, 4, 2, 1, bias=True), 103 | nn.BatchNorm2d(self.ngf), 104 | nn.ReLU(True), 105 | # state size. (ngf) x 32 x 32 106 | nn.ConvTranspose2d(self.ngf, self.num_channels, 4, 2, 1, bias=True), 107 | nn.Tanh() 108 | # state size. (num_channels) x 64 x 64 109 | ) 110 | 111 | 112 | def forward(self, embed_vector, z): 113 | 114 | # embed_vector = 64 by 1024 115 | # projected_embed = 64 by 128 by 1 by 1 116 | # z = 64 by 100 by 1 by 1 117 | 118 | projected_embed = self.projection(embed_vector).unsqueeze(2).unsqueeze(3) 119 | latent_vector = torch.cat([projected_embed, z], 1) 120 | output = self.netG(latent_vector) 121 | 122 | return output 123 | 124 | class discriminator(nn.Module): 125 | def __init__(self): 126 | super(discriminator, self).__init__() 127 | self.image_size = 64 128 | self.num_channels = 3 129 | self.embed_dim = 2400 # compatible with skip thought (1024) 130 | self.projected_embed_dim = 128 131 | self.ndf = 64 132 | self.B_dim = 128 133 | self.C_dim = 16 134 | 135 | self.netD_1 = nn.Sequential( 136 | # input is (nc) x 64 x 64 137 | nn.Conv2d(self.num_channels, self.ndf, 4, 2, 1, bias=True), 138 | nn.LeakyReLU(0.2, inplace=True), 139 | # state size. (ndf) x 32 x 32 140 | nn.Conv2d(self.ndf, self.ndf * 2, 4, 2, 1, bias=True), 141 | nn.BatchNorm2d(self.ndf * 2), 142 | nn.LeakyReLU(0.2, inplace=True), 143 | # state size. (ndf*2) x 16 x 16 144 | nn.Conv2d(self.ndf * 2, self.ndf * 4, 4, 2, 1, bias=True), 145 | nn.BatchNorm2d(self.ndf * 4), 146 | nn.LeakyReLU(0.2, inplace=True), 147 | # state size. (ndf*4) x 8 x 8 148 | nn.Conv2d(self.ndf * 4, self.ndf * 8, 4, 2, 1, bias=True), 149 | nn.BatchNorm2d(self.ndf * 8), 150 | nn.LeakyReLU(0.2, inplace=True), 151 | 152 | nn.Conv2d(self.ndf*8, self.ndf*2, 1, 1, 0), 153 | nn.BatchNorm2d(self.ndf*2), 154 | nn.LeakyReLU(0.2, True), 155 | 156 | nn.Conv2d(self.ndf*2, self.ndf*2, 3, 1, 1), 157 | nn.BatchNorm2d(self.ndf*2), 158 | nn.LeakyReLU(0.2, True), 159 | 160 | nn.Conv2d(self.ndf*2, self.ndf*8, 3, 1, 1), 161 | nn.BatchNorm2d(self.ndf*8), 162 | nn.LeakyReLU(0.2, True) 163 | 164 | #output size (ndf*8) x 8 x 8 165 | ) 166 | 167 | self.projector = Concat_embed(self.embed_dim, self.projected_embed_dim) 168 | 169 | self.netD_2 = nn.Sequential( 170 | # state size. (ndf*8) x 4 x 4 171 | nn.Conv2d(self.ndf * 8 + self.projected_embed_dim, self.ndf*8, 1), 172 | nn.BatchNorm2d(self.ndf*8), 173 | nn.LeakyReLU(0.2, True), 174 | 175 | nn.Conv2d(self.ndf * 8, 1, 4, 1, 0, bias=True), 176 | nn.Sigmoid() 177 | ) 178 | 179 | def forward(self, inp, embed): 180 | x_intermediate = self.netD_1(inp) 181 | x = self.projector(x_intermediate, embed) 182 | x = self.netD_2(x) 183 | 184 | return x.view(-1, 1).squeeze(1) , x_intermediate 185 | 186 | 187 | class generator2(nn.Module): 188 | def __init__(self): 189 | super(generator2, self).__init__() 190 | self.image_size = 128 191 | self.num_channels = 3 192 | self.noise_dim = 100 193 | self.embed_dim = 2400 # compatible with skip thought (1024) 194 | self.projected_embed_dim = 128 195 | self.latent_dim = self.noise_dim + self.projected_embed_dim 196 | self.ngf = 64 197 | 198 | self.projection = nn.Sequential( 199 | nn.Linear(in_features=self.embed_dim, out_features=self.projected_embed_dim), 200 | nn.BatchNorm1d(num_features=self.projected_embed_dim), 201 | nn.LeakyReLU(negative_slope=0.2, inplace=True) 202 | ) 203 | 204 | 205 | # downsample 206 | self.netG_1 = nn.Sequential( 207 | # input is (nc) x 64 x 64 208 | nn.Conv2d(self.num_channels, self.ngf, 3, 1, 1), 209 | nn.ReLU(True), 210 | # state size. (ngf) x 64 x 64 211 | nn.Conv2d(self.ngf, self.ngf * 2, 4, 2, 1, bias=True), 212 | nn.BatchNorm2d(self.ngf * 2), 213 | nn.ReLU(True), 214 | # state size. (ngf*2) x 32 x 32 215 | nn.Conv2d(self.ngf * 2, self.ngf * 4, 4, 2, 1, bias=True), 216 | nn.BatchNorm2d(self.ngf * 4), 217 | nn.ReLU(True) 218 | # state size. (ngf*4) x 16 x 16 219 | ) 220 | 221 | self.join_embed = nn.Sequential( 222 | 223 | nn.Conv2d(self.projected_embed_dim + self.ngf*4 , self.ngf * 4, 3, 1, 1), 224 | nn.BatchNorm2d(self.ngf * 4), 225 | nn.ReLU(True) 226 | ) 227 | 228 | self.residual = self._make_layer(ResBlock, self.ngf * 4, 4) 229 | 230 | # --> 2ngf x 32 x 32 231 | self.upsample1 = upBlock(self.ngf * 4, self.ngf * 2) 232 | # --> ngf x 64 x 64 233 | self.upsample2 = upBlock(self.ngf * 2, self.ngf) 234 | # --> ngf // 2 x 128 x 128 235 | self.upsample3 = upBlock(self.ngf, self.ngf // 2) 236 | # --> ngf // 4 x 128 x 128 237 | self.flatten = nn.Conv2d(self.ngf // 2 , self.ngf //4, 3, 1, 1) 238 | # --> 3 x 128 x 128 239 | self.img = nn.Sequential( 240 | nn.Conv2d(self.ngf // 4 , 3, 3, 1, 1), 241 | nn.Tanh() 242 | ) 243 | 244 | def _make_layer(self, block, channel_num, r_num): 245 | layers = [] 246 | for i in range(r_num): 247 | layers.append(block(channel_num)) 248 | return nn.Sequential(*layers) 249 | 250 | 251 | def forward(self, inp, embed): 252 | 253 | # embed_vector = 64 by 1024 254 | # projected_embed = 64 by 128 by 1 by 1 255 | # z = 64 by 100 by 1 by 1 256 | g1_output = self.netG_1(inp) # shape is (ngf*4) x 16 x 16 257 | projected_embed = self.projection(embed) 258 | replicated_embed = projected_embed.repeat(16, 16, 1, 1).permute(2, 3, 0, 1) 259 | hidden_concat = torch.cat([g1_output, replicated_embed], 1) 260 | x = self.join_embed(hidden_concat) 261 | x = self.residual(x) 262 | x = self.upsample1(x) 263 | x = self.upsample2(x) 264 | x = self.upsample3(x) 265 | x = self.flatten(x) 266 | output = self.img(x) 267 | 268 | return output 269 | 270 | class discriminator2(nn.Module): 271 | def __init__(self): 272 | super(discriminator2, self).__init__() 273 | self.image_size = 64 274 | self.num_channels = 3 275 | self.embed_dim = 2400 # compatible with skip thought (1024) 276 | self.projected_embed_dim = 128 277 | self.ndf = 64 278 | self.B_dim = 128 279 | self.C_dim = 16 280 | 281 | self.encode_img = nn.Sequential( 282 | # state size = 3 x 128 x 128 283 | nn.Conv2d(3, self.ndf, 4, 2, 1, bias=True), # 128 * 128 * ndf 284 | nn.LeakyReLU(0.2, inplace=True), 285 | # state size = ndf x 64 x 64 286 | nn.Conv2d(self.ndf , self.ndf * 2, 3, 1, 1, bias=True), 287 | nn.BatchNorm2d(self.ndf * 2), 288 | nn.LeakyReLU(0.2, inplace=True), 289 | nn.Conv2d(self.ndf * 2, self.ndf * 4, 4, 2, 1, bias=True), 290 | nn.BatchNorm2d(self.ndf * 4), 291 | nn.LeakyReLU(0.2, inplace=True), # 32 * 32 * ndf * 4 292 | nn.Conv2d(self.ndf * 4, self.ndf * 8, 4, 2, 1, bias=True), 293 | nn.BatchNorm2d(self.ndf * 8), 294 | nn.LeakyReLU(0.2, inplace=True), # 16 * 16 * ndf * 8 295 | nn.Conv2d(self.ndf * 8, self.ndf * 8, 3, 1, 1), 296 | nn.BatchNorm2d(self.ndf * 8), 297 | nn.LeakyReLU(0.2, inplace=True), # 8 * 8 * ndf * 8 298 | nn.Conv2d(self.ndf * 8, self.ndf * 2, 4, 2, 1), 299 | nn.BatchNorm2d(self.ndf * 2), 300 | nn.LeakyReLU(0.2, inplace=True), # 4 * 4 * ndf * 2 301 | nn.Conv2d(self.ndf * 2, self.ndf * 8, 3, 1, 1), 302 | nn.BatchNorm2d(self.ndf * 8), 303 | nn.LeakyReLU(0.2, inplace=True), 304 | nn.Conv2d(self.ndf * 8, self.ndf * 8, 4, 2, 1), 305 | nn.BatchNorm2d(self.ndf * 8), 306 | nn.LeakyReLU(0.2, inplace=True) 307 | # nn.Conv2d(self.ndf * 8, self.ndf * 16, 4, 2, 1, bias=True), 308 | # nn.BatchNorm2d(self.ndf * 16), 309 | # nn.LeakyReLU(0.2, inplace=True), # 8 * 8 * ndf * 16 310 | # nn.Conv2d(self.ndf * 16, self.ndf * 32, 4, 2, 1, bias=True), 311 | # nn.BatchNorm2d(self.ndf * 32), 312 | # nn.LeakyReLU(0.2, inplace=True), # 4 * 4 * ndf * 32 313 | # conv3x3(self.ndf * 32, self.ndf * 16), 314 | # nn.BatchNorm2d(self.ndf * 16), 315 | # nn.LeakyReLU(0.2, inplace=True), # 4 * 4 * ndf * 16 316 | # nn.Conv2d(self.ndf * 16, self.ndf * 8, 3, 1, 1, bias=True), 317 | # nn.BatchNorm2d(self.ndf * 8), 318 | # nn.LeakyReLU(0.2, inplace=True) # 4 * 4 * ndf * 8 319 | ) 320 | 321 | self.outlogitscond = nn.Sequential( 322 | conv3x3(self.ndf * 8 + self.projected_embed_dim, self.ndf * 8), 323 | nn.BatchNorm2d(self.ndf * 8), 324 | nn.LeakyReLU(0.2, inplace=True), 325 | nn.Conv2d(self.ndf * 8, 1, kernel_size=4, stride=4), 326 | nn.Sigmoid() 327 | ) 328 | 329 | # self.outlogits = nn.Sequential( 330 | # nn.Conv2d(self.ndf * 8, 1, kernel_size=4, stride=4), 331 | # nn.Sigmoid() 332 | # ) 333 | 334 | 335 | self.projector = Concat_embed(self.embed_dim, self.projected_embed_dim) 336 | 337 | def forward(self, inp, embed): 338 | x_intermediate = self.encode_img(inp) 339 | x = self.projector(x_intermediate, embed) 340 | x_cond = self.outlogitscond(x) 341 | # x_uncond = self.outlogits(x_intermediate) 342 | 343 | return x_cond.view(-1, 1).squeeze(1) 344 | -------------------------------------------------------------------------------- /models/gan_factory.py: -------------------------------------------------------------------------------- 1 | from models import gan_cls 2 | 3 | class gan_factory(object): 4 | 5 | @staticmethod 6 | def generator_factory(type): 7 | if type == 'gan': 8 | return gan_cls.generator() 9 | elif type == 'stage2_gan': 10 | return gan_cls.generator2() 11 | 12 | @staticmethod 13 | def discriminator_factory(type): 14 | if type == 'gan': 15 | return gan_cls.discriminator() 16 | elif type == 'stage2_gan': 17 | return gan_cls.discriminator2() 18 | 19 | 20 | -------------------------------------------------------------------------------- /nocycle_results/a curly red flower with many dark spots on it's petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/a curly red flower with many dark spots on it's petals..jpg -------------------------------------------------------------------------------- /nocycle_results/a flower with long and wide petals that are yellow..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/a flower with long and wide petals that are yellow..jpg -------------------------------------------------------------------------------- /nocycle_results/a flower with tiny yellow petals and very few yellow pistils..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/a flower with tiny yellow petals and very few yellow pistils..jpg -------------------------------------------------------------------------------- /nocycle_results/a golden flower with a spiky look and a white center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/a golden flower with a spiky look and a white center..jpg -------------------------------------------------------------------------------- /nocycle_results/a white flower with a golden and purple center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/a white flower with a golden and purple center..jpg -------------------------------------------------------------------------------- /nocycle_results/flower is enclosed in a bud like leaf with yellow petals on the outside and long purple style..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/flower is enclosed in a bud like leaf with yellow petals on the outside and long purple style..jpg -------------------------------------------------------------------------------- /nocycle_results/the flower has big yellow upright petals attached to a thick vine.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the flower has big yellow upright petals attached to a thick vine.jpg -------------------------------------------------------------------------------- /nocycle_results/the flower is red in color with a long pedicel with leaves on it.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the flower is red in color with a long pedicel with leaves on it.jpg -------------------------------------------------------------------------------- /nocycle_results/the flower is yellow with petals that are smooth and has pointy tip and they are in the formation around the stamen like a disc.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the flower is yellow with petals that are smooth and has pointy tip and they are in the formation around the stamen like a disc.jpg -------------------------------------------------------------------------------- /nocycle_results/the flower pedals looks like colored leaves that are yellow and purple..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the flower pedals looks like colored leaves that are yellow and purple..jpg -------------------------------------------------------------------------------- /nocycle_results/the petal of the flower is white in color with a large yellow pistil in the center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the petal of the flower is white in color with a large yellow pistil in the center..jpg -------------------------------------------------------------------------------- /nocycle_results/the petals of the flower resemble the heightened tail of a peacock, and have a bold yellow color..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the petals of the flower resemble the heightened tail of a peacock, and have a bold yellow color..jpg -------------------------------------------------------------------------------- /nocycle_results/the petals of this flower are purple and white with a long stigma.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the petals of this flower are purple and white with a long stigma.jpg -------------------------------------------------------------------------------- /nocycle_results/the petals of this flower are purple with a short stigma.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the petals of this flower are purple with a short stigma.jpg -------------------------------------------------------------------------------- /nocycle_results/the petals on this flower are yellow with an a few brown stripes..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the petals on this flower are yellow with an a few brown stripes..jpg -------------------------------------------------------------------------------- /nocycle_results/the petals on this flower are yellow with dark lines..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the petals on this flower are yellow with dark lines..jpg -------------------------------------------------------------------------------- /nocycle_results/the petals range in color from orange to yellow to purple with hints of white. the petals are different shapes and close as the flower goes up..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the petals range in color from orange to yellow to purple with hints of white. the petals are different shapes and close as the flower goes up..jpg -------------------------------------------------------------------------------- /nocycle_results/the pink petals of the flower is bended and the stamen is not seen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the pink petals of the flower is bended and the stamen is not seen..jpg -------------------------------------------------------------------------------- /nocycle_results/the spiky, thin yellow petals of this flower radiate outward from its bushy stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/the spiky, thin yellow petals of this flower radiate outward from its bushy stamen..jpg -------------------------------------------------------------------------------- /nocycle_results/there are many pink petals below red anthers and filaments..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/there are many pink petals below red anthers and filaments..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower consist of a bunch of purple pedals climbing down the stock with black stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower consist of a bunch of purple pedals climbing down the stock with black stamen..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower features broad yellow petals surrounding the ovary in a ball like pattern..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower features broad yellow petals surrounding the ovary in a ball like pattern..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has a light green sepal and strange looking blue pistil..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has a light green sepal and strange looking blue pistil..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has a spiky brown center and down-turned pink petals with tapered tips..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has a spiky brown center and down-turned pink petals with tapered tips..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has a star-like configuration of bright blue petals with darker veins..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has a star-like configuration of bright blue petals with darker veins..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has bright yellow petals curved into a bell shape and a simple long green stem..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has bright yellow petals curved into a bell shape and a simple long green stem..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has oddly shaped petals that vary in shades of purple and yellow..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has oddly shaped petals that vary in shades of purple and yellow..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has orange spiky petals and purple spiky petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has orange spiky petals and purple spiky petals..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has pale pink petals with a large rounded bright red anther in the center..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has pale pink petals with a large rounded bright red anther in the center..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has petals that are blue and are very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has petals that are blue and are very thin.jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has petals that are orange with black dotd.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has petals that are orange with black dotd.jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has petals that are pink and ruffled at the edges.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has petals that are pink and ruffled at the edges.jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has petals that are purple and are very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has petals that are purple and are very thin.jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has petals that are purple with purple stigma.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has petals that are purple with purple stigma.jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has petals that are white and has patches of pink.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has petals that are white and has patches of pink.jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has petals that are white on the outer petal and red on the center with red anthers...jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has petals that are white on the outer petal and red on the center with red anthers...jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has petals that are yellow and are folded together.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has petals that are yellow and are folded together.jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has petals that are yellow and are very thin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has petals that are yellow and are very thin.jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has tall and spiky yellow petals with no visible outer stamen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has tall and spiky yellow petals with no visible outer stamen.jpg -------------------------------------------------------------------------------- /nocycle_results/this flower has yellow rounded petals with stripe brown markings..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower has yellow rounded petals with stripe brown markings..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is made up of flowy white petals that have some pink highlights toward the tips..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is made up of flowy white petals that have some pink highlights toward the tips..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is orange in color, with petals that are layered..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is orange in color, with petals that are layered..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is pink and white in color, and has petals that are striped..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is pink and white in color, and has petals that are striped..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is pink and white in color, with petals that are wavy and curled..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is pink and white in color, with petals that are wavy and curled..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is pink in color, with petals that have veins..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is pink in color, with petals that have veins..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is purple and white in color, and has petals that are very skinny..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is purple and white in color, and has petals that are very skinny..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is purple in color, and has petals that are bell shaped..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is purple in color, and has petals that are bell shaped..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is purple in color, with petals that are oddly shaped..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is purple in color, with petals that are oddly shaped..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is rose shaped with bright yellow overlapping petals..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is rose shaped with bright yellow overlapping petals..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is white and green in color, and has petals that are pointed..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is white and green in color, and has petals that are pointed..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is white and pink in color, and has petals that are oval shaped..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is white and pink in color, and has petals that are oval shaped..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is white and yellow in color, with petals that are wavy and curled..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is white and yellow in color, with petals that are wavy and curled..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is yellow and purple in color, with petals that are spotted..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is yellow and purple in color, with petals that are spotted..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is yellow in color, with petals that are ruffled..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is yellow in color, with petals that are ruffled..jpg -------------------------------------------------------------------------------- /nocycle_results/this flower is yellow in color, with petals that are very skinny..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this flower is yellow in color, with petals that are very skinny..jpg -------------------------------------------------------------------------------- /nocycle_results/this has a spiny type pedicel with a pink-lavendar hue flower that has many pollen and stigma leaves..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this has a spiny type pedicel with a pink-lavendar hue flower that has many pollen and stigma leaves..jpg -------------------------------------------------------------------------------- /nocycle_results/this is a blue flower with short and rounded petals that turns white toward the receptacle..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this is a blue flower with short and rounded petals that turns white toward the receptacle..jpg -------------------------------------------------------------------------------- /nocycle_results/this is a soft petaled rose colored flower with large flappy petals and a firm pedicel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this is a soft petaled rose colored flower with large flappy petals and a firm pedicel.jpg -------------------------------------------------------------------------------- /nocycle_results/this orange and blue flower has pointed petals and a green pedicel..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this orange and blue flower has pointed petals and a green pedicel..jpg -------------------------------------------------------------------------------- /nocycle_results/this orange and brown flower has rounded petals and bright orange stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this orange and brown flower has rounded petals and bright orange stamen..jpg -------------------------------------------------------------------------------- /nocycle_results/this petals on this flower are purple with no visible stamen..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this petals on this flower are purple with no visible stamen..jpg -------------------------------------------------------------------------------- /nocycle_results/this pink flower has the string like petals arranged on the horny pedicel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this pink flower has the string like petals arranged on the horny pedicel.jpg -------------------------------------------------------------------------------- /nocycle_results/this pink flower of asteraceae family has outer florets forming a pink corolla and the central florets are red with yellow pistils..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this pink flower of asteraceae family has outer florets forming a pink corolla and the central florets are red with yellow pistils..jpg -------------------------------------------------------------------------------- /nocycle_results/this yellow, pink, and white flower has pointed petals with a pink pedicel..jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CSC2548/text2image2textGAN/5b756d866feaab961c24e7014685afb059d612dd/nocycle_results/this yellow, pink, and white flower has pointed petals with a pink pedicel..jpg -------------------------------------------------------------------------------- /run_all_caption_inference_cycle.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | for filename in results/*.jpg; do 3 | python3 sample.py --gen_path=./checkpoints/cycle_caption_gen-40.pkl --image="$filename" >> cyclecaptions140.txt 4 | printf ", $filename \n" >> cyclecaptions140.txt 5 | done 6 | 7 | 8 | -------------------------------------------------------------------------------- /run_inference_cycle.sh: -------------------------------------------------------------------------------- 1 | python3 sample.py --image=./results/this\ blue\ and\ orange\ flower\ has\ pointed\ petals\ and\ a\ red\ and\ green\ pedicel..jpg --gen_path=./checkpoints/cycle_caption_gen-10.pkl 2 | 3 | # --encoder_path=./birds_gan_models/encoder-10-47.pkl --decoder_path=./birds_gan_models/decoder-10-47.pkl --vocab_path=./data/birds_vocab.pkl 4 | -------------------------------------------------------------------------------- /run_inference_pretrained.sh: -------------------------------------------------------------------------------- 1 | python3 sample.py --image=./results/this\ blue\ and\ orange\ flower\ has\ pointed\ petals\ and\ a\ red\ and\ green\ pedicel..jpg --gen_path=./checkpoints/pretrained-generator-20.pkl 2 | 3 | # --encoder_path=./birds_gan_models/encoder-10-47.pkl --decoder_path=./birds_gan_models/decoder-10-47.pkl --vocab_path=./data/birds_vocab.pkl 4 | -------------------------------------------------------------------------------- /runtime.py: -------------------------------------------------------------------------------- 1 | from trainer import Trainer 2 | import argparse 3 | from PIL import Image 4 | import os 5 | from build_vocab import * 6 | 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument("--type", default='gan') 9 | parser.add_argument("--lr", default=0.0002, type=float) 10 | parser.add_argument("--l1_coef", default=50, type=float) 11 | parser.add_argument("--l2_coef", default=100, type=float) 12 | parser.add_argument("--diter", default=5, type=int) 13 | parser.add_argument("--cls", default=False, action='store_true') 14 | parser.add_argument("--interp", default=False, action='store_true') 15 | parser.add_argument("--vis_screen", default='gan') 16 | parser.add_argument("--save_path", default='') 17 | parser.add_argument("--inference", default=False, action='store_true') 18 | parser.add_argument('--pre_trained_disc_A', default=None) 19 | parser.add_argument('--pre_trained_gen_A', default=None) 20 | parser.add_argument('--pre_trained_disc_B', default=None) 21 | parser.add_argument('--pre_trained_gen_B', default=None) 22 | parser.add_argument('--dataset', default='flowers') 23 | parser.add_argument('--split', default=0, type=int) 24 | parser.add_argument('--batch_size', default=64, type=int) 25 | parser.add_argument('--num_workers', default=16, type=int) 26 | parser.add_argument('--epochs', default=55, type=int) 27 | args = parser.parse_args() 28 | 29 | trainer = Trainer(type=args.type, 30 | dataset=args.dataset, 31 | split=args.split, 32 | lr=args.lr, 33 | diter=args.diter, 34 | vis_screen=args.vis_screen, 35 | save_path=args.save_path, 36 | l1_coef=args.l1_coef, 37 | l2_coef=args.l2_coef, 38 | pre_trained_disc=args.pre_trained_disc_A, 39 | pre_trained_gen=args.pre_trained_gen_A, 40 | batch_size=args.batch_size, 41 | num_workers=args.num_workers, 42 | epochs=args.epochs, 43 | pre_trained_disc_B=args.pre_trained_disc_B, 44 | pre_trained_gen_B=args.pre_trained_gen_B 45 | ) 46 | 47 | print(args.inference) 48 | print(args.type) 49 | 50 | if not args.inference and args.type!='cycle_gan': 51 | print("gan") 52 | trainer.train(args.cls, args.interp) 53 | elif not args.inference and args.type=='cycle_gan': 54 | print('cycle gan') 55 | cycle_trainer.train(args.cls) 56 | elif args.inference and args.type=='cycle_gan': 57 | # trainer.predict() 58 | print('cycle gan prediction') 59 | cycle_trainer.predict() 60 | elif args.inference and args.type=='gan': 61 | print('gan prediction') 62 | trainer.predict() 63 | elif args.inference and args.type=='stackgan': 64 | trainer.predict(args.type) 65 | else: 66 | print('wrong input...') 67 | 68 | -------------------------------------------------------------------------------- /sample.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import argparse 5 | import pickle 6 | import os 7 | from torch.autograd import Variable 8 | from torchvision import transforms 9 | from build_vocab import Vocabulary 10 | # from gan_encoder_decoder_model import EncoderCNN, DecoderRNN 11 | from caption_gan_model import CaptionGenerator 12 | from PIL import Image 13 | import pdb 14 | 15 | def to_var(x, volatile=False): 16 | if torch.cuda.is_available(): 17 | x = x.cuda() 18 | return Variable(x, volatile=volatile) 19 | 20 | def load_image(image_path, transform=None): 21 | image = Image.open(image_path) 22 | # image = image.resize([224, 224], Image.LANCZOS) 23 | 24 | if transform is not None: 25 | image = transform(image).unsqueeze(0) 26 | 27 | return image 28 | 29 | def main(args): 30 | # Image preprocessing 31 | transform = transforms.Compose([ 32 | transforms.ToTensor(), 33 | transforms.Normalize((0.485, 0.456, 0.406), 34 | (0.229, 0.224, 0.225))]) 35 | 36 | # Load vocabulary wrapper 37 | with open(args.vocab_path, 'rb') as f: 38 | vocab = pickle.load(f) 39 | 40 | # Build Models 41 | # encoder = EncoderCNN(args.embed_size) 42 | # encoder.eval() # evaluation mode (BN uses moving mean/variance) 43 | # decoder = DecoderRNN(args.embed_size, args.hidden_size, 44 | # len(vocab), args.num_layers) 45 | generator = CaptionGenerator(args.embed_size, args.hidden_size, len(vocab), args.num_layers) 46 | generator.encoder.eval() 47 | # Load the trained model parameters 48 | # encoder.load_state_dict(torch.load(args.encoder_path)) 49 | # decoder.load_state_dict(torch.load(args.decoder_path)) 50 | generator.load_state_dict(torch.load(args.gen_path)) 51 | 52 | # Prepare Image 53 | image = load_image(args.image, transform) 54 | image_tensor = to_var(image, volatile=True) 55 | 56 | # If use gpu 57 | if torch.cuda.is_available(): 58 | # encoder.cuda() 59 | # decoder.cuda() 60 | generator.cuda() 61 | 62 | # Generate caption from image 63 | # feature = encoder(image_tensor) 64 | 65 | # sampled_ids = decoder.sample(feature) 66 | sampled_ids = generator.sample(image_tensor) 67 | sampled_ids = sampled_ids.cpu().data.numpy()[0] 68 | # Decode word_ids to words 69 | sampled_caption = [] 70 | for word_id in sampled_ids: 71 | word = vocab.idx2word[word_id] 72 | sampled_caption.append(word) 73 | if word == '': 74 | break 75 | sentence = ' '.join(sampled_caption) 76 | 77 | # Print out image and generated caption. 78 | print (sentence) 79 | # image = Image.open(args.image) 80 | # plt.imshow(np.asarray(image)) 81 | # pdb.set_trace() 82 | 83 | if __name__ == '__main__': 84 | parser = argparse.ArgumentParser() 85 | parser.add_argument('--image', type=str, required=True, 86 | help='input image for generating caption') 87 | parser.add_argument('--encoder_path', type=str, default='./models/encoder-5-3000.pkl', 88 | help='path for trained encoder') 89 | parser.add_argument('--decoder_path', type=str, default='./models/decoder-5-3000.pkl', 90 | help='path for trained decoder') 91 | parser.add_argument('--vocab_path', type=str, default='./data/flowers_vocab.pkl', 92 | help='path for vocabulary wrapper') 93 | 94 | # Model parameters (should be same as paramters in train.py) 95 | parser.add_argument('--embed_size', type=int , default=256, 96 | help='dimension of word embedding vectors') 97 | parser.add_argument('--hidden_size', type=int , default=512, 98 | help='dimension of lstm hidden states') 99 | parser.add_argument('--num_layers', type=int , default=1 , 100 | help='number of layers in lstm') 101 | parser.add_argument('--gen_path', type=str, default='./birds_gan_models/generator-gan-10-47.pkl') 102 | args = parser.parse_args() 103 | main(args) 104 | -------------------------------------------------------------------------------- /scripts/start_continue_train.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --cls --interp --type=gan --epochs=600 --dataset=birds --pre_trained_disc_A=./checkpoints/disc_150.pth --pre_trained_gen_A=./checkpoints/gen_150.pth 2 | -------------------------------------------------------------------------------- /scripts/start_continue_train_stackgan.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --cls --type=stackgan --epochs=103 --dataset=birds --pre_trained_disc_A=./checkpoints/disc_stage_1_epoch_100.pth --pre_trained_gen_A=./checkpoints/gen_stage_1_epoch_100.pth 2 | 3 | # --pre_trained_disc_B=./checkpoints/disc_stage_2_epoch_50.pth --pre_trained_gen_B=./checkpoints/gen_stage_2_epoch_50.pth 4 | -------------------------------------------------------------------------------- /scripts/start_cycle_inference.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --pre_trained_disc_A=./checkpoints/disc_99.pth --pre_trained_gen_A=./checkpoints/gen_99.pth --pre_trained_disc_B=./checkpoints/inv_disc_99.pth --pre_trained_gen_B=./checkpoints/inv_gen_99.pth --type=cycle_gan --inference --split=2 --dataset=birds 2 | -------------------------------------------------------------------------------- /scripts/start_cycle_stackgan_inference_flowers.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --pre_trained_disc_A=./checkpoints/cycle_disc_stage_1_epoch_100.pth --pre_trained_gen_A=./checkpoints/cycle_gen_stage_1_epoch_100.pth --pre_trained_disc_B=./checkpoints/cycle_disc_stage_2_epoch_100.pth --pre_trained_gen_B=./checkpoints/cycle_gen_stage_2_epoch_100.pth --type=stackgan --inference --split=2 --dataset=flowers 2 | -------------------------------------------------------------------------------- /scripts/start_cycle_train.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --cls --type=cycle_gan --epochs=100 --dataset=birds 2 | -------------------------------------------------------------------------------- /scripts/start_inference.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --pre_trained_disc_A=./checkpoints/disc_200.pth --pre_trained_gen_A=./checkpoints/gen_200.pth --type=gan --inference --split=1 --dataset=birds 2 | -------------------------------------------------------------------------------- /scripts/start_pretraining.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --type=pretrain_caption --dataset=flowers 2 | -------------------------------------------------------------------------------- /scripts/start_stackgan_inference.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --pre_trained_disc_A=./checkpoints/disc_stage_1_epoch_2.pth --pre_trained_gen_A=./checkpoints/gen_stage_1_epoch_2.pth --pre_trained_disc_B=./checkpoints/disc_stage_2_epoch_2.pth --pre_trained_gen_B=./checkpoints/gen_stage_2_epoch_2.pth --type=stackgan --inference --split=2 --dataset=birds 2 | -------------------------------------------------------------------------------- /scripts/start_stackgan_inference_flowers.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --pre_trained_disc_A=./checkpoints/disc_stage_1_epoch_20.pth --pre_trained_gen_A=./checkpoints/gen_stage_1_epoch_20.pth --pre_trained_disc_B=./checkpoints/disc_stage_2_epoch_20.pth --pre_trained_gen_B=./checkpoints/gen_stage_2_epoch_20.pth --type=stackgan --inference --split=2 --dataset=flowers 2 | -------------------------------------------------------------------------------- /scripts/start_train.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --interp --type=wgan --cls --epochs=202 --dataset=birds 2 | -------------------------------------------------------------------------------- /scripts/start_train_inverse.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --type=inverse_gan --epochs=200 -------------------------------------------------------------------------------- /scripts/start_train_stackgan.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --type=stackgan --cls --epochs=102 --dataset=flowers 2 | -------------------------------------------------------------------------------- /scripts/train_cycle.sh: -------------------------------------------------------------------------------- 1 | python3 runtime.py --cls --type=stackgan --epochs=103 --dataset=flowers --pre_trained_disc_A=./checkpoints/cycle_disc_stage_1_epoch_40.pth --pre_trained_gen_A=./checkpoints/cycle_gen_stage_1_epoch_40.pth --pre_trained_disc_B=./checkpoints/cycle_disc_stage_2_epoch_40.pth --pre_trained_gen_B=./checkpoints/cycle_gen_stage_2_epoch_40.pth 2 | 3 | -------------------------------------------------------------------------------- /trainer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import yaml 4 | from torch import nn 5 | from torch.autograd import Variable 6 | from torch.utils.data import DataLoader 7 | 8 | from txt2image_dataset import Text2ImageDataset, collate_fn 9 | from models.gan_factory import gan_factory 10 | from utils import Utils 11 | from PIL import Image 12 | import os 13 | from tqdm import tqdm 14 | import matplotlib 15 | matplotlib.use('Agg') 16 | 17 | import matplotlib.pyplot as plt 18 | 19 | import pdb 20 | import pickle 21 | from build_vocab import Vocabulary 22 | from caption_gan_model import CaptionDiscriminator, CaptionGenerator 23 | from torch.nn.utils.rnn import * 24 | 25 | is_cuda = torch.cuda.is_available() 26 | 27 | def to_var(x, volatile=False): 28 | if torch.cuda.is_available(): 29 | x = x.cuda() 30 | return Variable(x, volatile=volatile) 31 | 32 | class Trainer(object): 33 | def __init__(self, type, dataset, split, lr, diter, vis_screen, save_path, l1_coef, l2_coef, pre_trained_gen, pre_trained_disc, batch_size, num_workers, epochs, pre_trained_disc_B, pre_trained_gen_B): 34 | with open('config.yaml', 'r') as f: 35 | config = yaml.load(f) 36 | 37 | # forward gan 38 | if is_cuda: 39 | self.generator = torch.nn.DataParallel(gan_factory.generator_factory('gan').cuda()) 40 | self.discriminator = torch.nn.DataParallel(gan_factory.discriminator_factory('gan').cuda()) 41 | self.generator2 = torch.nn.DataParallel(gan_factory.generator_factory('stage2_gan').cuda()) 42 | self.discriminator2 = torch.nn.DataParallel(gan_factory.discriminator_factory('stage2_gan').cuda()) 43 | else: 44 | self.generator = torch.nn.DataParallel(gan_factory.generator_factory('gan')) 45 | self.discriminator = torch.nn.DataParallel(gan_factory.discriminator_factory('gan')) 46 | self.generator2 = torch.nn.DataParallel(gan_factory.generator_factory('stage2_gan')) 47 | self.discriminator2 = torch.nn.DataParallel(gan_factory.discriminator_factory('stage2_gan')) 48 | 49 | if pre_trained_disc: 50 | self.discriminator.load_state_dict(torch.load(pre_trained_disc)) 51 | else: 52 | self.discriminator.apply(Utils.weights_init) 53 | 54 | if pre_trained_gen: 55 | self.generator.load_state_dict(torch.load(pre_trained_gen)) 56 | else: 57 | self.generator.apply(Utils.weights_init) 58 | 59 | if pre_trained_disc_B: 60 | self.discriminator2.load_state_dict(torch.load(pre_trained_disc_B)) 61 | else: 62 | self.discriminator2.apply(Utils.weights_init) 63 | 64 | if pre_trained_gen_B: 65 | self.generator2.load_state_dict(torch.load(pre_trained_gen_B)) 66 | else: 67 | self.generator2.apply(Utils.weights_init) 68 | 69 | if dataset == 'birds': 70 | with open('./data/birds_vocab.pkl', 'rb') as f: 71 | self.vocab = pickle.load(f) 72 | self.dataset = Text2ImageDataset(config['birds_dataset_path'], dataset_type='birds', vocab=self.vocab, split=split) 73 | elif dataset == 'flowers': 74 | with open('./data/flowers_vocab.pkl', 'rb') as f: 75 | self.vocab = pickle.load(f) 76 | self.dataset = Text2ImageDataset(config['flowers_dataset_path'], dataset_type='flowers', vocab=self.vocab, split=split) 77 | else: 78 | print('Dataset not supported, please select either birds or flowers.') 79 | exit() 80 | 81 | self.noise_dim = 100 82 | self.batch_size = batch_size 83 | self.lr = lr 84 | self.beta1 = 0.5 85 | self.num_epochs = epochs 86 | self.DITER = diter 87 | self.num_workers = num_workers 88 | 89 | self.l1_coef = l1_coef 90 | self.l2_coef = l2_coef 91 | 92 | self.data_loader = DataLoader(self.dataset, batch_size=self.batch_size, shuffle=True, 93 | num_workers=self.num_workers, collate_fn=collate_fn) 94 | 95 | self.optimD = torch.optim.Adam(self.discriminator.parameters(), lr=self.lr, betas=(self.beta1, 0.999)) 96 | self.optimG = torch.optim.Adam(self.generator.parameters(), lr=self.lr, betas=(self.beta1, 0.999)) 97 | 98 | self.optimD2 = torch.optim.Adam(self.discriminator2.parameters(), lr=self.lr, betas=(self.beta1, 0.999)) 99 | self.optimG2 = torch.optim.Adam(self.generator2.parameters(), lr=self.lr, betas=(self.beta1, 0.999)) 100 | 101 | self.checkpoints_path = './checkpoints/' 102 | self.save_path = save_path 103 | self.type = type 104 | 105 | # TODO: put these as runtime.py params 106 | self.embed_size = 256 107 | self.hidden_size = 512 108 | self.num_layers = 1 109 | 110 | self.gen_pretrain_num_epochs = 100 111 | self.disc_pretrain_num_epochs = 20 112 | 113 | self.figure_path = './figures/' 114 | 115 | self.caption_generator = CaptionGenerator(self.embed_size, self.hidden_size, len(self.vocab), self.num_layers).cuda() 116 | self.caption_discriminator = CaptionDiscriminator(self.embed_size, self.hidden_size, len(self.vocab), self.num_layers).cuda() 117 | 118 | pretrained_caption_gen = './checkpoints/pretrained-generator-20.pkl' 119 | pretrained_caption_disc = './checkpoints/pretrained-discriminator-5.pkl' 120 | 121 | if os.path.exists(pretrained_caption_gen): 122 | print('loaded pretrained caption generator') 123 | self.caption_generator.load_state_dict(torch.load(pretrained_caption_gen)) 124 | 125 | if os.path.exists(pretrained_caption_disc): 126 | print('loaded pretrained caption discriminator') 127 | self.caption_discriminator.load_state_dict(torch.load(pretrained_caption_disc)) 128 | 129 | self.optim_captionG = torch.optim.Adam(list(self.caption_generator.parameters())) 130 | self.optim_captionD = torch.optim.Adam(list(self.caption_discriminator.parameters())) 131 | 132 | 133 | def train(self, cls=False, interp=False): 134 | 135 | if self.type == 'gan': 136 | self._train_gan(cls, interp) 137 | elif self.type == 'stackgan': 138 | self._train_stack_gan(cls, interp) 139 | elif self.type == 'pretrain_caption': 140 | self._pretrain_caption() 141 | 142 | def _pretrain_caption(self): 143 | 144 | # Create model directory 145 | if not os.path.exists(self.checkpoints_path): 146 | os.makedirs(self.checkpoints_path) 147 | 148 | if not os.path.exists(self.figure_path): 149 | os.makedirs(self.figure_path) 150 | 151 | # Build the models (Gen) 152 | generator = CaptionGenerator(self.embed_size, self.hidden_size, len(self.vocab), self.num_layers) 153 | 154 | # Build the models (Disc) 155 | discriminator = CaptionDiscriminator(self.embed_size, self.hidden_size, len(self.vocab), self.num_layers) 156 | 157 | if torch.cuda.is_available(): 158 | generator.cuda() 159 | discriminator.cuda() 160 | 161 | # Loss and Optimizer (Gen) 162 | mle_criterion = nn.CrossEntropyLoss() 163 | params_gen = list(generator.parameters()) 164 | optimizer_gen = torch.optim.Adam(params_gen) 165 | 166 | # Loss and Optimizer (Disc) 167 | params_disc = list(discriminator.parameters()) 168 | optimizer_disc = torch.optim.Adam(params_disc) 169 | 170 | disc_losses = [] 171 | gen_losses = [] 172 | for epoch in tqdm(range(max([int(self.gen_pretrain_num_epochs), int(self.disc_pretrain_num_epochs)]))): 173 | for sample in self.data_loader: 174 | images = sample['right_images128'] # 64x3x128x128 175 | captions = sample['captions'] 176 | lengths = sample['lengths'] 177 | wrong_captions = sample['wrong_captions'] 178 | wrong_lengths = sample['wrong_lengths'] 179 | 180 | images = to_var(images, volatile=True) 181 | captions = to_var(captions) 182 | wrong_captions = to_var(wrong_captions) 183 | 184 | targets = pack_padded_sequence(captions, lengths, batch_first=True)[0] 185 | 186 | if epoch < int(self.gen_pretrain_num_epochs): 187 | generator.zero_grad() 188 | outputs, _ = generator(images, captions, lengths) 189 | loss_gen = mle_criterion(outputs, targets) 190 | gen_losses.append(loss_gen.cpu().data.numpy()[0]) 191 | loss_gen.backward() 192 | optimizer_gen.step() 193 | 194 | if epoch < int(self.disc_pretrain_num_epochs): 195 | discriminator.zero_grad() 196 | rewards_real = discriminator(images, captions, lengths) 197 | # rewards_fake = discriminator(images, sampled_captions, sampled_lengths) 198 | rewards_wrong = discriminator(images, wrong_captions, wrong_lengths) 199 | real_loss = -torch.mean(torch.log(rewards_real)) 200 | # fake_loss = -torch.mean(torch.clamp(torch.log(1 - rewards_fake), min=-1000)) 201 | wrong_loss = -torch.mean(torch.clamp(torch.log(1 - rewards_wrong), min=-1000)) 202 | loss_disc = real_loss + wrong_loss # + fake_loss, no fake_loss because this is pretraining 203 | 204 | disc_losses.append(loss_disc.cpu().data.numpy()[0]) 205 | loss_disc.backward() 206 | optimizer_disc.step() 207 | 208 | 209 | # Save pretrained models 210 | torch.save(discriminator.state_dict(), os.path.join(self.checkpoints_path, 'pretrained-discriminator-%d.pkl' %int(self.disc_pretrain_num_epochs))) 211 | torch.save(generator.state_dict(), os.path.join(self.checkpoints_path, 'pretrained-generator-%d.pkl' %int(self.gen_pretrain_num_epochs))) 212 | 213 | # Plot pretraining figures 214 | plt.plot(disc_losses, label='pretraining_caption_disc_loss') 215 | plt.savefig(self.figure_path + 'pretraining_caption_disc_losses.png') 216 | plt.clf() 217 | 218 | plt.plot(gen_losses, label='pretraining_gen_loss') 219 | plt.savefig(self.figure_path + 'pretraining_gen_losses.png') 220 | plt.clf() 221 | 222 | 223 | 224 | def _train_gan(self, cls, interp): 225 | criterion = nn.BCELoss() 226 | l2_loss = nn.MSELoss() 227 | l1_loss = nn.L1Loss() 228 | iteration = 0 229 | 230 | gen_losses = [] 231 | disc_losses = [] 232 | for epoch in tqdm(range(self.num_epochs)): 233 | for sample in tqdm(self.data_loader): 234 | # pdb.set_trace() 235 | iteration += 1 236 | # sample.keys() = dict_keys(['right_images', 'wrong_images', 'inter_embed', 'right_embed', 'txt']) 237 | right_images = sample['right_images'] # 64x3x64x64 238 | right_embed = sample['right_embed'] # 64x1024 239 | wrong_images = sample['wrong_images'] # 64x3x64x64 240 | 241 | if is_cuda: 242 | right_images = Variable(right_images.float()).cuda() 243 | right_embed = Variable(right_embed.float()).cuda() 244 | wrong_images = Variable(wrong_images.float()).cuda() 245 | else: 246 | right_images = Variable(right_images.float()) 247 | right_embed = Variable(right_embed.float()) 248 | wrong_images = Variable(wrong_images.float()) 249 | 250 | real_labels = torch.ones(right_images.size(0)) 251 | fake_labels = torch.zeros(right_images.size(0)) 252 | 253 | # ======== One sided label smoothing ========== 254 | # Helps preventing the discriminator from overpowering the 255 | # generator adding penalty when the discriminator is too confident 256 | # ============================================= 257 | smoothed_real_labels = torch.FloatTensor(Utils.smooth_label(real_labels.numpy(), -0.1)) 258 | 259 | if is_cuda: 260 | real_labels = Variable(real_labels).cuda() 261 | smoothed_real_labels = Variable(smoothed_real_labels).cuda() 262 | fake_labels = Variable(fake_labels).cuda() 263 | else: 264 | real_labels = Variable(real_labels) 265 | smoothed_real_labels = Variable(smoothed_real_labels) 266 | fake_labels = Variable(fake_labels) 267 | 268 | # Train the discriminator 269 | self.discriminator.zero_grad() 270 | outputs, activation_real = self.discriminator(right_images, right_embed) 271 | real_loss = criterion(outputs, smoothed_real_labels) 272 | real_score = outputs 273 | 274 | if cls: 275 | outputs, _ = self.discriminator(wrong_images, right_embed) 276 | wrong_loss = criterion(outputs, fake_labels) 277 | wrong_score = outputs 278 | 279 | if is_cuda: 280 | noise = Variable(torch.randn(right_images.size(0), 100)).cuda() 281 | else: 282 | noise = Variable(torch.randn(right_images.size(0), 100)) 283 | noise = noise.view(noise.size(0), 100, 1, 1) 284 | fake_images = self.generator(right_embed, noise) 285 | outputs, _ = self.discriminator(fake_images, right_embed) 286 | fake_loss = criterion(outputs, fake_labels) 287 | fake_score = outputs 288 | 289 | if cls: 290 | d_loss = real_loss + 0.5 * wrong_loss + 0.5 * fake_loss 291 | else: 292 | d_loss = real_loss + fake_loss 293 | 294 | d_loss.backward() 295 | self.optimD.step() 296 | 297 | # Train the generator 298 | self.generator.zero_grad() 299 | if is_cuda: 300 | noise = Variable(torch.randn(right_images.size(0), 100)).cuda() 301 | else: 302 | noise = Variable(torch.randn(right_images.size(0), 100)) 303 | 304 | noise = noise.view(noise.size(0), 100, 1, 1) 305 | fake_images = self.generator(right_embed, noise) 306 | outputs, activation_fake = self.discriminator(fake_images, right_embed) 307 | _, activation_real = self.discriminator(right_images, right_embed) 308 | 309 | activation_fake = torch.mean(activation_fake, 0) 310 | activation_real = torch.mean(activation_real, 0) 311 | 312 | 313 | 314 | #======= Generator Loss function============ 315 | # This is a customized loss function, the first term is the regular cross entropy loss 316 | # The second term is feature matching loss, this measure the distance between the real and generated 317 | # images statistics by comparing intermediate layers activations 318 | # The third term is L1 distance between the generated and real images, this is helpful for the conditional case 319 | # because it links the embedding feature vector directly to certain pixel values. 320 | #=========================================== 321 | g_loss = criterion(outputs, real_labels) 322 | # \ 323 | # + self.l2_coef * l2_loss(activation_fake, activation_real.detach()) \ 324 | # + self.l1_coef * l1_loss(fake_images, right_images) 325 | 326 | if (interp): 327 | """ GAN INT loss""" 328 | # pdb.set_trace() 329 | # print('iter {}, size {}, right {}'.format(iteration, self.batch_size, right_embed.size()))i 330 | available_batch_size = int(right_embed.size(0)) 331 | first_part = right_embed[:int(available_batch_size/2),:] 332 | second_part = right_embed[int(available_batch_size/2):,:] 333 | interp_embed = (first_part + second_part)*0.5 334 | 335 | if is_cuda: 336 | noise = Variable(torch.randn(int(available_batch_size/2), 100)).cuda() 337 | else: 338 | noise = Variable(torch.randn(int(available_batch_size), 100)) 339 | 340 | interp_real_labels = torch.ones(int(available_batch_size/2)) 341 | if is_cuda: 342 | interp_real_labels = Variable(interp_real_labels).cuda() 343 | else: 344 | interp_real_labels = Variable(interp_real_labels) 345 | 346 | fake_images = self.generator(interp_embed, noise) 347 | outputs, activation_fake = self.discriminator(fake_images, interp_embed) 348 | g_int_loss = criterion(outputs, interp_real_labels) 349 | g_loss = g_loss + 0.2 * g_int_loss 350 | 351 | g_loss.backward() 352 | self.optimG.step() 353 | 354 | gen_losses.append(g_loss.data[0]) 355 | disc_losses.append(d_loss.data[0]) 356 | 357 | with open('gen.pkl', 'wb') as f_gen, open('disc.pkl', 'wb') as f_disc: 358 | pickle.dump(gen_losses, f_gen) 359 | pickle.dump(disc_losses, f_disc) 360 | 361 | 362 | x = list(range(len(gen_losses))) 363 | plt.plot(x, gen_losses, 'g-', label='gen loss') 364 | plt.plot(x, disc_losses, 'b-', label='disc loss') 365 | plt.legend() 366 | plt.savefig('gen_vs_disc_.png') 367 | plt.clf() 368 | 369 | # if (epoch) % 10 == 0: 370 | if (epoch) % 50 == 0: 371 | Utils.save_checkpoint(self.discriminator, self.generator, self.checkpoints_path, self.save_path, epoch) 372 | 373 | 374 | def _train_stack_gan(self, cls, interp): 375 | 376 | criterion = nn.BCELoss() 377 | l2_loss = nn.MSELoss() 378 | l1_loss = nn.L1Loss() 379 | iteration = 0 380 | 381 | # cycle gan params 382 | lambda_a = 2 383 | lambda_b = 2 384 | mle_criterion = nn.CrossEntropyLoss() 385 | 386 | gen_losses = [] 387 | disc_losses = [] 388 | cycle_a_losses = [] 389 | for epoch in tqdm(range(self.num_epochs)): 390 | for sample in tqdm(self.data_loader): 391 | # pdb.set_trace() 392 | iteration += 1 393 | # sample.keys() = dict_keys(['right_images', 'wrong_images', 'inter_embed', 'right_embed', 'txt']) 394 | right_images = sample['right_images'] # 64x3x64x64 395 | right_embed = sample['right_embed'] # 64x1024 396 | wrong_images = sample['wrong_images'] # 64x3x64x64 397 | right_images128 = sample['right_images128'] # 64x3x128x128 398 | wrong_images128 = sample['wrong_images128'] # 64x3x128x128 399 | right_captions = sample['captions'] 400 | right_lengths = sample['lengths'] 401 | 402 | if is_cuda: 403 | right_images = Variable(right_images.float()).cuda() 404 | right_embed = Variable(right_embed.float()).cuda() 405 | wrong_images = Variable(wrong_images.float()).cuda() 406 | right_images128 = Variable(right_images128.float()).cuda() 407 | wrong_images128 = Variable(wrong_images128.float()).cuda() 408 | right_captions = Variable(right_captions.long()).cuda() 409 | else: 410 | right_images = Variable(right_images.float()) 411 | right_embed = Variable(right_embed.float()) 412 | wrong_images = Variable(wrong_images.float()) 413 | right_images128 = Variable(right_images128.float()) 414 | wrong_images128 = Variable(wrong_images128.float()) 415 | right_captions = Variable(right_captions.long()) 416 | 417 | real_labels = torch.ones(right_images.size(0)) 418 | fake_labels = torch.zeros(right_images.size(0)) 419 | 420 | # ======== One sided label smoothing ========== 421 | # Helps preventing the discriminator from overpowering the 422 | # generator adding penalty when the discriminator is too confident 423 | # ============================================= 424 | smoothed_real_labels = torch.FloatTensor(Utils.smooth_label(real_labels.numpy(), -0.1)) 425 | 426 | if is_cuda: 427 | real_labels = Variable(real_labels).cuda() 428 | smoothed_real_labels = Variable(smoothed_real_labels).cuda() 429 | fake_labels = Variable(fake_labels).cuda() 430 | else: 431 | real_labels = Variable(real_labels) 432 | smoothed_real_labels = Variable(smoothed_real_labels) 433 | fake_labels = Variable(fake_labels) 434 | 435 | # Train the discriminator 436 | self.discriminator.zero_grad() 437 | 438 | # ------------------- Training D stage 1 ------------------------------- 439 | outputs, activation_real = self.discriminator(right_images, right_embed) 440 | real_loss = criterion(outputs, smoothed_real_labels) 441 | real_score = outputs 442 | 443 | if cls: 444 | outputs, _ = self.discriminator(wrong_images, right_embed) 445 | wrong_loss = criterion(outputs, fake_labels) 446 | wrong_score = outputs 447 | 448 | if is_cuda: 449 | noise = Variable(torch.randn(right_images.size(0), 100)).cuda() 450 | else: 451 | noise = Variable(torch.randn(right_images.size(0), 100)) 452 | noise = noise.view(noise.size(0), 100, 1, 1) 453 | fake_images = self.generator(right_embed, noise) 454 | outputs, _ = self.discriminator(fake_images, right_embed) 455 | fake_loss = criterion(outputs, fake_labels) 456 | fake_score = outputs 457 | 458 | if cls: 459 | d_loss = real_loss + 0.5 * wrong_loss + 0.5 * fake_loss 460 | else: 461 | d_loss = real_loss + fake_loss 462 | 463 | d_loss.backward() 464 | self.optimD.step() 465 | 466 | # -------------------- Training G stage 1 ------------------------------- 467 | self.generator.zero_grad() 468 | self.discriminator.zero_grad() 469 | if is_cuda: 470 | noise = Variable(torch.randn(right_images.size(0), 100)).cuda() 471 | else: 472 | noise = Variable(torch.randn(right_images.size(0), 100)) 473 | 474 | noise = noise.view(noise.size(0), 100, 1, 1) 475 | fake_images = self.generator(right_embed, noise) 476 | outputs, activation_fake = self.discriminator(fake_images, right_embed) 477 | 478 | g_loss = criterion(outputs, real_labels) 479 | 480 | if (interp): 481 | """ GAN INT loss""" 482 | available_batch_size = int(right_embed.size(0)) 483 | first_part = right_embed[:int(available_batch_size/2),:] 484 | second_part = right_embed[int(available_batch_size/2):,:] 485 | interp_embed = (first_part + second_part)*0.5 486 | 487 | if is_cuda: 488 | noise = Variable(torch.randn(int(available_batch_size/2), 100)).cuda() 489 | else: 490 | noise = Variable(torch.randn(int(available_batch_size), 100)) 491 | 492 | noise = noise.view(noise.size(0), 100, 1, 1) 493 | 494 | interp_real_labels = torch.ones(int(available_batch_size/2)) 495 | if is_cuda: 496 | interp_real_labels = Variable(interp_real_labels).cuda() 497 | else: 498 | interp_real_labels = Variable(interp_real_labels) 499 | 500 | fake_images = self.generator(interp_embed, noise) 501 | outputs, activation_fake = self.discriminator(fake_images, interp_embed) 502 | g_int_loss = criterion(outputs, interp_real_labels) 503 | g_loss = g_loss + 0.2 * g_int_loss 504 | 505 | g_loss.backward() 506 | self.optimG.step() 507 | 508 | # -------------------- Training D stage 2 ------------------------------- 509 | self.discriminator2.zero_grad() 510 | outputs = self.discriminator2(right_images128, right_embed) 511 | real_loss = criterion(outputs, smoothed_real_labels) 512 | real_score = outputs 513 | 514 | if cls: 515 | outputs = self.discriminator2(wrong_images128, right_embed) 516 | wrong_loss = criterion(outputs, fake_labels) 517 | wrong_score = outputs 518 | 519 | if is_cuda: 520 | noise = Variable(torch.randn(right_images.size(0), 100)).cuda() 521 | else: 522 | noise = Variable(torch.randn(right_images.size(0), 100)) 523 | 524 | noise = noise.view(noise.size(0), 100, 1, 1) 525 | fake_images_v1 = self.generator(right_embed, noise) 526 | fake_images_v1 = fake_images_v1.detach() 527 | fake_images = self.generator2(fake_images_v1, right_embed) 528 | fake_images = fake_images.detach() 529 | outputs = self.discriminator2(fake_images, right_embed) 530 | fake_loss = criterion(outputs, fake_labels) 531 | fake_score = outputs 532 | 533 | if cls: 534 | d_loss2 = real_loss + 0.5 * wrong_loss + 0.5 * fake_loss 535 | else: 536 | d_loss2 = real_loss + fake_loss 537 | 538 | d_loss2.backward() 539 | self.optimD2.step() 540 | 541 | 542 | # -------------------- Training G stage 2 ------------------------------- 543 | self.generator2.zero_grad() 544 | self.discriminator2.zero_grad() 545 | if is_cuda: 546 | noise = Variable(torch.randn(right_images.size(0), 100)).cuda() 547 | else: 548 | noise = Variable(torch.randn(right_images.size(0), 100)) 549 | 550 | noise = noise.view(noise.size(0), 100, 1, 1) 551 | fake_images_v1 = self.generator(right_embed, noise) 552 | fake_images_v1 = fake_images_v1.detach() 553 | fake_images = self.generator2(fake_images_v1, right_embed) 554 | outputs = self.discriminator2(fake_images, right_embed) 555 | 556 | g_loss2 = criterion(outputs, real_labels) 557 | g_loss2.backward() 558 | self.optimG2.step() 559 | 560 | gen_losses.append(g_loss2.data[0]) 561 | disc_losses.append(d_loss2.data[0]) 562 | 563 | # Generate caption with caption GAN (inverse GAN) 564 | # fake_images.requires_grad = False # freeze the caption generator 565 | self.caption_generator.zero_grad() 566 | sampled_captions, _ = self.caption_generator.forward(fake_images, right_captions, right_lengths) 567 | targets = pack_padded_sequence(right_captions, right_lengths, batch_first=True)[0] 568 | loss_cycle_A = mle_criterion(sampled_captions, targets)* lambda_a 569 | loss_cycle_A.backward() 570 | self.optimG2.step() 571 | self.optim_captionG.step() 572 | cycle_a_losses.append(loss_cycle_A.data[0]) 573 | 574 | with open('gen.pkl', 'wb') as f_gen, open('disc.pkl', 'wb') as f_disc: 575 | pickle.dump(gen_losses, f_gen) 576 | pickle.dump(disc_losses, f_disc) 577 | 578 | if (epoch+1) % 10 == 0: 579 | # if (epoch+1) % 5 == 0: 580 | Utils.save_checkpoint(self.discriminator, self.generator, self.checkpoints_path, self.save_path, epoch+1) 581 | Utils.save_checkpoint(self.discriminator2, self.generator2, self.checkpoints_path, self.save_path, epoch+1, False, 2) 582 | torch.save(self.caption_discriminator.state_dict(), os.path.join(self.checkpoints_path, 'cycle_caption_disc-%d.pkl' % (epoch + 1))) 583 | torch.save(self.caption_generator.state_dict(), os.path.join(self.checkpoints_path, 'cycle_caption_gen-%d.pkl' % (epoch + 1) )) 584 | 585 | # Plot pretraining figures 586 | plt.plot(disc_losses, label='stage 1 disc losses') 587 | plt.savefig(self.figure_path + 'stage_1_disc_losses.png') 588 | plt.clf() 589 | 590 | plt.plot(gen_losses, label='stage_1_gen_loss') 591 | plt.savefig(self.figure_path + 'stage_1_gen_losses.png') 592 | plt.clf() 593 | 594 | plt.plot(disc_losses, label='cycle_a_losses') 595 | plt.savefig(self.figure_path + 'cycle_a_losses.png') 596 | plt.clf() 597 | 598 | 599 | def predict(self, gan_type='gan'): 600 | torch.manual_seed(7) 601 | count = 0 602 | for sample in self.data_loader: 603 | right_images = sample['right_images'] 604 | right_embed = sample['right_embed'] 605 | txt = sample['txt'] 606 | 607 | if not os.path.exists('results/{0}'.format(self.save_path)): 608 | os.makedirs('results/{0}'.format(self.save_path)) 609 | 610 | if is_cuda: 611 | right_images = Variable(right_images.float()).cuda() 612 | right_embed = Variable(right_embed.float()).cuda() 613 | else: 614 | right_images = Variable(right_images.float()) 615 | right_embed = Variable(right_embed.float()) 616 | 617 | # Train the generator 618 | if is_cuda: 619 | noise = Variable(torch.randn(right_images.size(0), 100)).cuda() 620 | else: 621 | noise = Variable(torch.randn(right_images.size(0), 100)) 622 | noise = noise.view(noise.size(0), 100, 1, 1) 623 | fake_images = self.generator(right_embed, noise) 624 | 625 | if(gan_type=='stackgan'): 626 | fake_images = self.generator2(fake_images, right_embed) 627 | 628 | for image, t in zip(fake_images, txt): 629 | im = Image.fromarray(image.data.mul_(127.5).add_(127.5).byte().permute(1, 2, 0).cpu().numpy()) 630 | im.save('results/{0}/{1}.jpg'.format(self.save_path, t.replace("/", "")[:200])) 631 | print(t) 632 | count += 1 633 | if count == 1: 634 | break 635 | 636 | 637 | 638 | 639 | 640 | 641 | -------------------------------------------------------------------------------- /txt2image_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import io 3 | from torch.utils.data import Dataset, DataLoader 4 | import h5py 5 | import numpy as np 6 | import pdb 7 | from PIL import Image 8 | import torch 9 | from torch.autograd import Variable 10 | import pdb 11 | import torch.nn.functional as F 12 | import pandas as pd 13 | import torchvision.transforms as transforms 14 | from build_vocab import * 15 | import nltk 16 | 17 | class Text2ImageDataset(Dataset): 18 | 19 | def __init__(self, datasetFile, dataset_type, vocab, transform=None, split=0): 20 | self.datasetFile = datasetFile 21 | self.dataset_type = dataset_type 22 | self.vocab = vocab 23 | self.transform = transform 24 | self.dataset = None 25 | self.dataset_keys = None 26 | self.split = 'train' if split == 0 else 'valid' if split == 1 else 'test' 27 | self.bboxes_df = pd.read_table('data/bounding_boxes.txt', sep=' ', header=None) 28 | self.image_paths_df = pd.read_table('data/images.txt', sep='\s+|\/+', header=None) 29 | self.birds_caption_path_root = './data/birds_captions/' 30 | self.h5py2int = lambda x: int(np.array(x)) 31 | 32 | def __len__(self): 33 | f = h5py.File(self.datasetFile, 'r') 34 | self.dataset_keys = [str(k) for k in f[self.split].keys()] 35 | length = len(f[self.split]) 36 | f.close() 37 | 38 | return length 39 | 40 | def crop_image(self, img, bbox): 41 | width, height = img.size 42 | r = int(np.maximum(bbox[2], bbox[3]) * 0.75) 43 | center_x = int((2 * bbox[0] + bbox[2]) / 2) 44 | center_y = int((2 * bbox[1] + bbox[3]) / 2) 45 | y1 = np.maximum(0, center_y - r) 46 | y2 = np.minimum(height, center_y + r) 47 | x1 = np.maximum(0, center_x - r) 48 | x2 = np.minimum(width, center_x + r) 49 | img = img.crop([x1, y1, x2, y2]) 50 | return img 51 | 52 | 53 | def __getitem__(self, idx): 54 | if self.dataset is None: 55 | self.dataset = h5py.File(self.datasetFile, mode='r') 56 | self.dataset_keys = [str(k) for k in self.dataset[self.split].keys()] 57 | 58 | example_name = self.dataset_keys[idx] 59 | # example_name = self.dataset_keys[0] 60 | example = self.dataset[self.split][example_name] 61 | 62 | right_image = bytes(np.array(example['img'])) 63 | right_embed = np.array(example['embeddings'], dtype=float) 64 | 65 | wrong_name, wrong_example, wrong_txt = self.find_wrong_image(example['class']) 66 | wrong_image = bytes(np.array(wrong_example)) 67 | if self.dataset_type=='birds': 68 | wrong_index_found = self.image_paths_df.index[self.image_paths_df[2]==(example_name[:-2]+'.jpg')].values[0] 69 | if wrong_index_found == None: 70 | print('ERROR: cannot find image index') 71 | wrong_txt_sub_path = self.image_paths_df.iloc[[wrong_index_found]][1].values[0] + '/' + self.image_paths_df.iloc[[wrong_index_found]][2].values[0][:-3] + 'txt' 72 | wrong_txt_path = self.birds_caption_path_root + wrong_txt_sub_path 73 | wrong_txt = open(wrong_txt_path, 'r').readlines()[0] 74 | 75 | index_found = self.image_paths_df.index[self.image_paths_df[2]==(example_name[:-2]+'.jpg')].values[0] 76 | if index_found == None: 77 | print('ERROR: cannot find image index') 78 | 79 | # find right image bbox 80 | df_bbox = self.bboxes_df.iloc[[index_found]] 81 | bbox_x = df_bbox[1].values[0] 82 | bbox_y = df_bbox[2].values[0] 83 | bbox_w = df_bbox[3].values[0] 84 | bbox_h = df_bbox[4].values[0] 85 | 86 | # find wrong image bbox 87 | index_found_wrong = self.image_paths_df.index[self.image_paths_df[2]==(wrong_name[:-2]+'.jpg')].values[0] 88 | if index_found_wrong == None: 89 | print('ERROR: cannot find wrong image') 90 | 91 | df_bbox_wrong = self.bboxes_df.iloc[[index_found_wrong]] 92 | wrong_bbox_x = df_bbox_wrong[1].values[0] 93 | wrong_bbox_y = df_bbox_wrong[2].values[0] 94 | wrong_bbox_w = df_bbox_wrong[3].values[0] 95 | wrong_bbox_h = df_bbox_wrong[4].values[0] 96 | 97 | byte_right_image = io.BytesIO(right_image) 98 | byte_wrong_image = io.BytesIO(wrong_image) 99 | 100 | right_image = Image.open(byte_right_image) 101 | wrong_image = Image.open(byte_wrong_image) 102 | 103 | if self.dataset_type == 'birds': 104 | right_image = self.crop_image(right_image, bbox=[bbox_x, bbox_y, bbox_w, bbox_h]) 105 | wrong_image = self.crop_image(wrong_image, bbox=[wrong_bbox_x, wrong_bbox_y, wrong_bbox_w, wrong_bbox_h]) 106 | 107 | # right_image = Image.open(byte_right_image).resize((64, 64)) 108 | # wrong_image = Image.open(byte_wrong_image).resize((64, 64)) 109 | #right_image128 = Image.open(byte_right_image).resize((128, 128)) 110 | #wrong_image128 = Image.open(byte_wrong_image).resize((128, 128)) 111 | 112 | right_image = transforms.Resize((64,64))(right_image) 113 | wrong_image = transforms.Resize((64,64))(wrong_image) 114 | 115 | right_image128 = transforms.Resize((128,128))(right_image) 116 | wrong_image128 = transforms.Resize((128,128))(wrong_image) 117 | 118 | # print(right_image.size, wrong_image.size, right_image128.size, wrong_image128.size) 119 | 120 | right_image = self.validate_image(right_image) 121 | wrong_image = self.validate_image(wrong_image) 122 | 123 | right_image128 = self.validate_image128(right_image128) 124 | wrong_image128 = self.validate_image128(wrong_image128) 125 | 126 | txt = np.array(example['txt']).astype(str) 127 | 128 | # preprocess txt and wrong_txt 129 | txt = str(txt) 130 | txt = txt.strip() 131 | txt = txt.encode('ascii', 'ignore') 132 | txt = txt.decode('ascii') 133 | exclude = set(string.punctuation) 134 | preproc_txt = ''.join(ch for ch in txt if ch not in exclude) 135 | tokens = nltk.tokenize.word_tokenize(preproc_txt.lower()) 136 | caption = [] 137 | caption.append(self.vocab('')) 138 | caption.extend([self.vocab(token) for token in tokens]) 139 | caption.append(self.vocab('')) 140 | caption = torch.Tensor(caption) 141 | 142 | 143 | wrong_txt = wrong_txt.strip() 144 | wrong_txt = wrong_txt.encode('ascii', 'ignore') 145 | wrong_txt = wrong_txt.decode('ascii') 146 | exclude = set(string.punctuation) 147 | preproc_wrong_txt = ''.join(ch for ch in wrong_txt if ch not in exclude) 148 | wrong_tokens = nltk.tokenize.word_tokenize(preproc_wrong_txt.lower()) 149 | wrong_caption = [] 150 | wrong_caption.append(self.vocab('')) 151 | wrong_caption.extend([self.vocab(token) for token in wrong_tokens]) 152 | wrong_caption.append(self.vocab('')) 153 | wrong_caption = torch.Tensor(wrong_caption) 154 | 155 | sample = { 156 | 'right_images': torch.FloatTensor(right_image), 157 | 'right_embed': torch.FloatTensor(right_embed), 158 | 'wrong_images': torch.FloatTensor(wrong_image), 159 | 'caption': caption, 160 | 'txt': txt, 161 | 'right_images128': torch.FloatTensor(right_image128), 162 | 'wrong_images128': torch.FloatTensor(wrong_image128), 163 | 'wrong_caption': wrong_caption 164 | } 165 | 166 | sample['right_images'] = sample['right_images'].sub_(127.5).div_(127.5) 167 | sample['wrong_images'] =sample['wrong_images'].sub_(127.5).div_(127.5) 168 | 169 | sample['right_images128'] = sample['right_images128'].sub_(127.5).div_(127.5) 170 | sample['wrong_images128'] =sample['wrong_images128'].sub_(127.5).div_(127.5) 171 | 172 | return sample 173 | 174 | def find_wrong_image(self, category): 175 | idx = np.random.randint(len(self.dataset_keys)) 176 | example_name = self.dataset_keys[idx] 177 | example = self.dataset[self.split][example_name] 178 | _category = example['class'] 179 | 180 | if _category != category: 181 | return example_name, example['img'], str(np.array(example['txt'])) 182 | 183 | return self.find_wrong_image(category) 184 | 185 | def find_inter_embed(self): 186 | idx = np.random.randint(len(self.dataset_keys)) 187 | example_name = self.dataset_keys[idx] 188 | example = self.dataset[self.split][example_name] 189 | return example['embeddings'] 190 | 191 | 192 | def validate_image(self, img): 193 | img = np.array(img, dtype=float) 194 | if len(img.shape) < 3: 195 | rgb = np.empty((64, 64, 3), dtype=np.float32) 196 | rgb[:, :, 0] = img 197 | rgb[:, :, 1] = img 198 | rgb[:, :, 2] = img 199 | img = rgb 200 | 201 | return img.transpose(2, 0, 1) 202 | 203 | def validate_image128(self, img): 204 | img = np.array(img, dtype=float) 205 | if len(img.shape) < 3: 206 | rgb = np.empty((128, 128, 3), dtype=np.float32) 207 | rgb[:, :, 0] = img 208 | rgb[:, :, 1] = img 209 | rgb[:, :, 2] = img 210 | img = rgb 211 | 212 | return img.transpose(2, 0, 1) 213 | 214 | 215 | def collate_fn(data): 216 | """Creates mini-batch tensors from the list of tuples (image, caption). 217 | 218 | We should build custom collate_fn rather than using default collate_fn, 219 | because merging caption (including padding) is not supported in default. 220 | 221 | Args: 222 | data: dictionary with keys (right_images, right_embed, wrong_images, caption, 223 | right_images128, wrong_images128, wrong_caption). 224 | 225 | Returns: 226 | sample: dictionary with keys sorted by len(caption) for right images/embed etc, 227 | and by len(wrong_caption) for wrong images, wrong_caption. 228 | """ 229 | 230 | collate_data = {} 231 | 232 | # Sort a data list by caption length (descending order). 233 | data.sort(key=lambda x: x['caption'].size(0), reverse=True) 234 | 235 | captions = [] 236 | right_images = [] 237 | right_embeds = [] 238 | wrong_images = [] 239 | right_images128 = [] 240 | wrong_images128 = [] 241 | collate_data['txt'] = [] 242 | for i in range(len(data)): 243 | collate_data['txt'].append(data[i]['txt']) 244 | captions.append(data[i]['caption']) 245 | right_images.append(data[i]['right_images']) 246 | right_embeds.append(data[i]['right_embed']) 247 | wrong_images.append(data[i]['wrong_images']) 248 | right_images128.append(data[i]['right_images128']) 249 | wrong_images128.append(data[i]['wrong_images128']) 250 | 251 | 252 | # sort and get captions, lengths, images, embeds etc 253 | lengths = [len(cap) for cap in captions] 254 | collate_data['lengths'] = lengths 255 | collate_data['captions'] = torch.zeros(len(captions), max(lengths)).long() 256 | for i, cap in enumerate(captions): 257 | end = lengths[i] 258 | collate_data['captions'][i, :end] = cap[:end] 259 | 260 | collate_data['right_images'] = torch.stack(right_images, 0) 261 | collate_data['right_embed'] = torch.stack(right_embeds, 0) 262 | collate_data['wrong_images'] = torch.stack(wrong_images, 0) 263 | collate_data['right_images128'] = torch.stack(right_images128, 0) 264 | collate_data['wrong_images128'] = torch.stack(wrong_images128, 0) 265 | 266 | # sort and get wrong_captions, wrong_lengths 267 | wrong_captions = [] 268 | wrong_lengths = [] 269 | for i in range(len(data)): 270 | wrong_captions.append(data[i]['wrong_caption']) 271 | wrong_captions.sort(key=lambda x: len(x), reverse=True) 272 | wrong_lengths = [len(cap) for cap in wrong_captions] 273 | collate_data['wrong_lengths'] = wrong_lengths 274 | collate_data['wrong_captions'] = torch.zeros(len(wrong_captions), max(wrong_lengths)).long() 275 | for i, cap in enumerate(wrong_captions): 276 | end = wrong_lengths[i] 277 | collate_data['wrong_captions'][i, :end] = cap[:end] 278 | 279 | 280 | return collate_data 281 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from torch import nn 3 | from torch import autograd 4 | import torch 5 | # from visualize import VisdomPlotter 6 | import os 7 | import pdb 8 | 9 | class Concat_embed(nn.Module): 10 | 11 | def __init__(self, embed_dim, projected_embed_dim): 12 | super(Concat_embed, self).__init__() 13 | self.projection = nn.Sequential( 14 | nn.Linear(in_features=embed_dim, out_features=projected_embed_dim), 15 | nn.BatchNorm1d(num_features=projected_embed_dim), 16 | nn.LeakyReLU(negative_slope=0.2, inplace=True) 17 | ) 18 | 19 | def forward(self, inp, embed): 20 | projected_embed = self.projection(embed) 21 | replicated_embed = projected_embed.repeat(4, 4, 1, 1).permute(2, 3, 0, 1) 22 | hidden_concat = torch.cat([inp, replicated_embed], 1) 23 | 24 | return hidden_concat 25 | 26 | 27 | class minibatch_discriminator(nn.Module): 28 | def __init__(self, num_channels, B_dim, C_dim): 29 | super(minibatch_discriminator, self).__init__() 30 | self.B_dim = B_dim 31 | self.C_dim =C_dim 32 | self.num_channels = num_channels 33 | T_init = torch.randn(num_channels * 4 * 4, B_dim * C_dim) * 0.1 34 | self.T_tensor = nn.Parameter(T_init, requires_grad=True) 35 | 36 | def forward(self, inp): 37 | inp = inp.view(-1, self.num_channels * 4 * 4) 38 | M = inp.mm(self.T_tensor) 39 | M = M.view(-1, self.B_dim, self.C_dim) 40 | 41 | op1 = M.unsqueeze(3) 42 | op2 = M.permute(1, 2, 0).unsqueeze(0) 43 | 44 | output = torch.sum(torch.abs(op1 - op2), 2) 45 | output = torch.sum(torch.exp(-output), 2) 46 | output = output.view(M.size(0), -1) 47 | 48 | output = torch.cat((inp, output), 1) 49 | 50 | return output 51 | 52 | 53 | class Utils(object): 54 | 55 | @staticmethod 56 | def smooth_label(tensor, offset): 57 | return tensor + offset 58 | 59 | @staticmethod 60 | 61 | # based on: https://github.com/caogang/wgan-gp/blob/master/gan_cifar10.py 62 | def compute_GP(netD, real_data, real_embed, fake_data, LAMBDA): 63 | BATCH_SIZE = real_data.size(0) 64 | alpha = torch.rand(BATCH_SIZE, 1) 65 | alpha = alpha.expand(BATCH_SIZE, int(real_data.nelement() / BATCH_SIZE)).contiguous().view(BATCH_SIZE, 3, 64, 64) 66 | alpha = alpha.cuda() 67 | 68 | interpolates = alpha * real_data + ((1 - alpha) * fake_data) 69 | 70 | interpolates = interpolates.cuda() 71 | 72 | interpolates = autograd.Variable(interpolates, requires_grad=True) 73 | 74 | disc_interpolates, _ = netD(interpolates, real_embed) 75 | 76 | gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates, 77 | grad_outputs=torch.ones(disc_interpolates.size()).cuda(), 78 | create_graph=True, retain_graph=True, only_inputs=True)[0] 79 | 80 | gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * LAMBDA 81 | 82 | return gradient_penalty 83 | 84 | @staticmethod 85 | def save_checkpoint(netD, netG, dir_path, subdir_path, epoch, inverse=False, stage = 1): 86 | path = os.path.join(dir_path, subdir_path) 87 | if not os.path.exists(path): 88 | os.makedirs(path) 89 | if not inverse: 90 | torch.save(netD.state_dict(), '{0}/cycle_disc_stage_{2}_epoch_{1}.pth'.format(path, epoch, stage)) 91 | torch.save(netG.state_dict(), '{0}/cycle_gen_stage_{2}_epoch_{1}.pth'.format(path, epoch, stage)) 92 | else: 93 | torch.save(netD.state_dict(), '{0}/inv_disc_{1}.pth'.format(path, epoch)) 94 | torch.save(netG.state_dict(), '{0}/inv_gen_{1}.pth'.format(path, epoch)) 95 | 96 | @staticmethod 97 | def weights_init(m): 98 | classname = m.__class__.__name__ 99 | if classname.find('Conv') != -1: 100 | m.weight.data.normal_(0.0, 0.02) 101 | m.bias.data.fill_(0) 102 | elif classname.find('BatchNorm') != -1: 103 | m.weight.data.normal_(1.0, 0.02) 104 | m.bias.data.fill_(0) 105 | 106 | --------------------------------------------------------------------------------