├── .gitignore ├── README.md ├── demo.py ├── examples ├── example.png ├── gts.json └── res.json └── pycocoevalcap ├── bleu ├── LICENSE ├── __init__.py ├── bleu.py └── bleu_scorer.py ├── cider ├── __init__.py ├── cider.py └── cider_scorer.py ├── eval.py ├── meteor ├── __init__.py ├── data │ └── paraphrase-en.gz ├── meteor-1.5.jar └── meteor.py ├── rouge ├── __init__.py └── rouge.py ├── spice ├── __init__.py ├── cache │ ├── data.mdb │ └── lock.mdb ├── lib │ ├── Meteor-1.5.jar │ ├── SceneGraphParser-1.0.jar │ ├── ejml-0.23.jar │ ├── fst-2.47.jar │ ├── guava-19.0.jar │ ├── hamcrest-core-1.3.jar │ ├── jackson-core-2.5.3.jar │ ├── javassist-3.19.0-GA.jar │ ├── json-simple-1.1.1.jar │ ├── junit-4.12.jar │ ├── lmdbjni-0.4.6.jar │ ├── lmdbjni-linux64-0.4.6.jar │ ├── lmdbjni-osx64-0.4.6.jar │ ├── lmdbjni-win64-0.4.6.jar │ ├── objenesis-2.4.jar │ ├── slf4j-api-1.7.12.jar │ └── slf4j-simple-1.7.21.jar ├── spice-1.0.jar ├── spice.py └── tmp │ ├── tmpia7f3mb5 │ └── tmpy5tu8h2v ├── tokenizer ├── __init__.py ├── ptbtokenizer.py └── stanford-corenlp-3.4.1.jar └── wmd ├── data └── stopwords.txt └── wmd.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | GoogleNews-vectors-negative300.bin 3 | stanford-corenlp-3.6.0.jar 4 | stanford-corenlp-3.6.0-models.jar -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # image-caption-metrics 2 | a py3 lib for NLP & image-caption metrics : BLEU METEOR CIDEr ROUGE SPICE WMD 3 | 4 | Features below: 5 | - python3 support 6 | - add new metric `WMD` 7 | 8 | 9 | ## Requirements 10 | - java 1.8+ 11 | - python 3 (Python2 has not been tested) 12 | - gensim 13 | - Stanford CoreNLP 3.6.0[(download)](http://nlp.stanford.edu/software/stanford-corenlp-full-2015-12-09.zip) 14 | - add stanford-corenlp-3.6.0.jar to `pycocoevalcap/spice/lib/` 15 | - add stanford-corenlp-3.6.0-models.jar to `pycocoevalcap/spice/lib/` 16 | 17 | - `google_word2vec_model` for WMD[(download)](https://docs.google.com/uc?export=download&id=0B7XkCwpI5KDYNlNUTTlSS21pQmM) 18 | - unzip it and add GoogleNews-vectors-negative300.bin to `pycocoevalcap/wmd/data` 19 | 20 | ## Usage 21 | See in `demo.py` 22 | - Note that the input format must be the same as the file in `examples/gts.json` and `examples/res.json` 23 | - It seems can't run in windows(error about java), run it on Linux 24 | ``` 25 | import pycocoevalcap.eval as E 26 | 27 | with open('examples/gts.json', 'r') as f: 28 | gts = json.load(f) 29 | with open('examples/res.json', 'r') as f: 30 | res = json.load(f) 31 | 32 | ans = E.eval(gts,tes) 33 | print(ans) 34 | bleu = E.get_bleu(gts,res) 35 | print(bleu) 36 | cider = E.get_cider(gts,res) 37 | print(cider) 38 | ``` 39 | ## References 40 | - WMD metric from [https://github.com/mtanti/coco-caption](https://github.com/mtanti/coco-caption) 41 | - main code from [https://github.com/wangleihitcs/CaptionMetrics](https://github.com/wangleihitcs/CaptionMetrics) 42 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | from pycocoevalcap.eval import eval 2 | import json 3 | 4 | with open('examples/gts.json', 'r') as f: 5 | gts = json.load(f) 6 | with open('examples/res.json', 'r') as f: 7 | res = json.load(f) 8 | 9 | if __name__ == '__main__': 10 | mp = eval(gts,res) 11 | print(mp) -------------------------------------------------------------------------------- /examples/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/examples/example.png -------------------------------------------------------------------------------- /examples/res.json: -------------------------------------------------------------------------------- 1 | {"184321": ["train traveling down a track in front of a road"], "81922": ["plane is flying through the sky"], "577539": ["birthday cake sitting on top of a white plate"], "380932": ["group of people are on the side of a snowy field"], "204805": ["black and white photo of people that are parked in front of a building"], "339974": ["traffic light on a city street at night"], "153607": ["vase that is sitting on top of a table"], "440329": ["group of people that are parked in front of a road"], "501762": ["man in the water holding a frisbee in the ocean"], "419856": ["bunch of bananas on a table next to each other"], "1369": ["man riding a wave in the water holding a surfboard in the ocean"], "234500": ["man eating a hot dog on a plate with a sandwich on a table"], "241691": ["black and white photo of a man standing next to each other"], "444444": ["dog that is sitting on the back of a road"], "280607": ["close up of a piece of pizza sitting on a table"], "514083": ["man riding a skate board doing a trick on his skateboard"], "393258": ["black and white cat sitting on top of a bed"], "57387": ["group of people standing on a tennis court holding a tennis racket at a ball"], "260141": ["boat that is parked in the water"], "301102": ["man sitting on top of a tennis court"], "432176": ["black and white cat sitting in front of a window"], "122934": ["man riding a horse drawn carriage on the back of a motorcycle"], "153656": ["white and white cat is sitting on top of a table"], "325690": ["stop sign on the side of a street"], "58915": ["red and white photo of a building with a clock tower in front of a city street"], "162827": ["black and white teddy bear sitting on a table"], "260166": ["group of people standing on a dirt road"], "3084": ["clock tower with a clock on top of a building"], "356427": ["man is riding a motorcycle down the road"], "452684": ["close up of a bowl of oranges sitting on a table"], "235597": ["group of people sitting in a living room with two laptops are playing a video game"], "579664": ["bunch of oranges sitting on a table"], "506552": ["group of people are playing a video game"], "493652": ["living room with a christmas tree in front of a living room"], "28758": ["giraffe standing on top of a lush green field"], "540763": ["bathroom with a white toilet sitting on top of a table"], "381021": ["man standing in front of a refrigerator"], "213086": ["laptop computer sitting on top of a kitchen counter"], "346207": ["black and white cat laying on top of a desk in front of a computer desk"], "469088": ["brown bear standing next to each other"], "323682": ["group of people standing in front of a building"], "276580": ["group of people standing in front of a baseball game"], "237669": ["baseball player swinging a bat at a game"], "73830": ["giraffe is standing next to a fence"], "120935": ["group of zebras are standing next to each other"], "299116": ["man sitting at a table with a plate of food"], "98416": ["group of sheep standing next to a herd of sheep grazing on a lush green field"], "16497": ["man standing on top of a skateboard"], "284350": ["man in a suit standing on a tennis court holding a tennis racket"], "65267": ["living room with a television sitting on top of a table"], "317560": ["young boy sitting on a table holding a banana"], "327807": ["group of people standing in front of a train station"], "125059": ["bathroom with a white toilet sitting on top of a table"], "350341": ["close up of a pizza sitting on top of a table"], "129159": ["city street with a clock tower in front of a building"], "321674": ["man sitting in front of a window"], "32907": ["black and white fire hydrant sitting on top of a wooden bench"], "262284": ["bathroom with a sink and a mirror"], "395405": ["living room with a couch next to a window in front of a wooden table"], "26767": ["couple of a bench in front of a street sign"], "173350": ["black and white dog laying on a bed next to each other"], "473237": ["man is holding a slice of pizza"], "209604": ["group of people standing in front of a city street"], "317595": ["group of people sitting on a table"], "398362": ["black and white photo of a living room"], "12448": ["little girl sitting on top of a teddy bear"], "383137": ["man riding a skateboard on a skate board"], "104612": ["bowl of food on top of a plate"], "524456": ["cat sitting on top of a dog"], "266409": ["man riding skis down a snow covered slope"], "450247": ["black and white cat laying on top of a computer desk"], "37038": ["group of people standing on a beach"], "247984": ["black and white cat sitting on top of a computer desk"], "522418": ["black and white photo of a bathroom with a mirror"], "143541": ["man riding a wave on the beach holding a surfboard in the ocean"], "366141": ["living room with a flat screen tv"], "413044": ["little girl sitting on a bench in front of a fire hydrant"], "288955": ["black and white cat laying on a table next to a bowl of food"], "16574": ["group of people riding skis down the side of a snow covered slope"], "426175": ["man sitting on a bench in front of a building"], "2240": ["close up of a teddy bear sitting on a table"], "303298": ["man standing on a tennis court holding a tennis racket at a tennis ball"], "129739": ["man sitting in front of a living room with a laptop computer"], "217285": ["baseball player is holding a bat in a baseball game"], "57545": ["man sitting on a park bench in front of a tree"], "10442": ["group of baseball players are standing in a baseball game"], "340175": ["living room with a lot of furniture"], "207056": ["group of elephants are standing in the water"], "315601": ["group of people walking down a busy city street"], "241876": ["black and white cat laying on top of a bed"], "360661": ["group of people riding horses on a horse"], "427135": ["group of people standing on a tennis court holding a baseball bat"], "516316": ["herd of zebra standing in a grassy field"], "180447": ["group of zebras grazing on a lush green field"], "32992": ["black and white cat sitting on top of a wooden table"], "8418": ["close up of a laptop computer sitting on a table"], "316795": ["man riding a horse on a sunny day"], "499940": ["young boy standing in a field holding a kite in the air"], "155877": ["group of sheep standing in front of a fence"], "336102": ["close up of a pizza sitting on top of a plate"], "579815": ["man standing in front of a city street"], "28377": ["large clock tower in front of a building"], "538859": ["man sitting at a table with plates of food"], "354540": ["close up of a laptop computer sitting on top of a table"], "155885": ["black and white boat is sitting on top of a beach"], "542959": ["kitchen with a microwave oven"], "520433": ["train is parked in front of a train station"], "372979": ["black and white cat sitting in front of a window"], "436470": ["living room with a chair in front of a living room"], "353807": ["man riding a motorcycle down a street"], "250108": ["laptop computer sitting on top of a desk next to each other"], "57597": ["group of men playing a game of soccer on a field with a baseball bat"], "440575": ["large airplane flying through the sky"], "311553": ["plate of food on a table"], "461063": ["woman standing on a tennis court holding a tennis racket on a tennis court"], "46859": ["group of people playing a game of frisbee in a field"], "139530": ["group of people standing next to each other"], "229643": ["black and white photo of a bathroom with a large window"], "268556": ["man riding a motorcycle parked in front of a city street"], "430125": ["group of people standing in front of a store"], "286994": ["herd of elephants are standing in the water"], "328757": ["young boy sitting at a table with a plate of food"], "102446": ["living room with a television and a window"], "69911": ["stop sign on a pole next to a traffic light"], "125211": ["zebra standing on top of a lush green field"], "67868": ["black and white dog sitting on top of a bench"], "577821": ["large clock tower with a clock on top of a building"], "98590": ["group of people standing on top of a snow covered slope"], "446751": ["herd of sheep standing next to each other in a field"], "387362": ["man sitting on top of a cell phone"], "184613": ["group of cows are standing in front of a lush green field"], "53542": ["red and white photo of a building with a clock tower in front of a city street"], "172327": ["man riding a wave on a surfboard in the ocean"], "233771": ["man standing in front of a city street"], "493868": ["bowl of vegetables sitting on top of a plate"], "190767": ["herd of cows that are standing in a grassy field"], "86320": ["bathroom with a sink and a mirror"], "252213": ["cat laying on top of a white plate"], "336182": ["woman sitting at a table in a living room"], "299319": ["man holding a surfboard on the beach"], "69946": ["group of boats are docked in the water"], "252219": ["group of people standing in front of a building"], "168330": ["city bus driving down the street at night"], "26942": ["close up of a cat laying on top of a table"], "158015": ["man standing on a tennis court holding a tennis racket in front of a tennis ball"], "577858": ["group of people sitting at a table with plates of food"], "360772": ["bathroom that is sitting on top of a toilet"], "434230": ["group of people sitting on top of a table with a clock on it"], "440646": ["man sitting on a bed"], "410337": ["man holding a cell phone"], "502090": ["blue airplane flying a kite in the air"], "16716": ["group of zebras standing next to each other in a grassy field"], "485709": ["baseball player holding a baseball bat on a baseball field"], "504142": ["cat laying on top of a bed"], "180560": ["group of people standing next to each other"], "368978": ["blue and white airplane flying through the sky"], "409944": ["man sitting on top of a laptop computer"], "500062": ["tennis player is playing tennis on a tennis court holding a tennis racket at a tennis ball"], "18149": ["man standing next to a building"], "481635": ["man sitting at a table with a plate of food"], "356708": ["group of people riding skis down a snowy slope"], "520727": ["group of zebra standing on top of a lush green field"], "387431": ["black and white photo of a city street in front of a building"], "403817": ["man standing in front of a tv"], "358763": ["laptop computer sitting on top of a desk next to each other"], "410002": ["pizza sitting on top of a table"], "307569": ["group of people riding horses on the beach"], "517822": ["man holding a frisbee in a field with a baseball bat"], "502136": ["clock tower sitting on top of a fire hydrant in front of a building"], "319865": ["cat laying on top of a black and white dog sitting on top of a couch"], "219514": ["traffic light with a stop sign on the side of a street"], "35197": ["man riding a skateboard on top of a skate board"], "94590": ["cup of donuts are sitting on top of a plate"], "297343": ["close up of a stop sign with a pair of scissors"], "270721": ["bathroom with a white toilet and a sink and a mirror"], "420229": ["close up of a bowl of scissors sitting on a table"], "86408": ["kitchen with wood cabinets and stainless steel appliances"], "235914": ["group of zebras are standing in a dirt field"], "49559": ["group of people standing in the air on the beach flying a kite in the ocean"], "119181": ["white plate that is sitting on top of a table"], "178592": ["herd of elephants standing on top of a lush green field"], "15085": ["couple of zebras are standing in the water"], "400": ["black and white dog sitting on top of a wall"], "174482": ["man riding a motorcycle down the side of a road"], "289173": ["plane that is parked on top of a runway"], "338327": ["close up of a clock sitting on top of a table"], "383384": ["baseball player swinging a bat in front of a baseball game"], "143769": ["baby elephant standing next to a fence"], "240028": ["young boy standing on a tennis court holding a frisbee in a field"], "331162": ["laptop computer sitting on top of a table in front of a living room"], "207264": ["herd of elephants are standing in a dirt field"], "319908": ["group of people standing in front of a baseball game"], "551334": ["cat sitting on top of a computer desk"], "569768": ["young man holding a frisbee in front of a baseball bat"], "541010": ["group of people standing in a field with a frisbee"], "534957": ["living room with a white toilet sitting on top of a table"], "383406": ["man sitting on top of a bed"], "248242": ["man riding a snowboard down the side of a snow covered slope"], "166323": ["bathroom with a sink and mirror"], "246199": ["bunch of fruits and vegetables on a table"], "219578": ["close up of a dog sitting on a table"], "102843": ["large boat that is on the beach"], "313789": ["group of fruit in front of an outdoor market"], "572517": ["black and white polar bear standing in the water"], "292301": ["white plate that is sitting on a table next to a cup of coffee"], "223648": ["wooden bench sitting on top of a table"], "438723": ["man sitting on top of a motorcycle"], "340420": ["pizza that is sitting on top of a table"], "10694": ["black and white cat sitting on top of a table"], "580041": ["black and white photo of a street sign with a traffic light"], "514508": ["group of people sitting on a table"], "109005": ["herd of cows are grazing on a lush green field"], "143824": ["man sitting in front of a dog"], "364521": ["close up of a banana sitting on top of a table"], "258516": ["man is standing in front of a window"], "383445": ["close up of a close up of a donut sitting on a table"], "184791": ["vase that is sitting on top of a table next to a glass of wine"], "8665": ["bird sitting on top of a branch of water"], "242139": ["city street with a traffic light on the side of a building"], "360926": ["man holding a tennis racket on a tennis court holding a tennis racquet on a baseball field"], "399839": ["man holding a teddy bear sitting on a table"], "49633": ["yellow and white train is parked in front of a lush green grass covered field"], "553442": ["man sitting on top of a bed"], "111076": ["bathroom with a white toilet and a sink and a mirror"], "428454": ["man riding a surf board on the beach flying a kite in the snow"], "350694": ["group of people sitting on a boat in the water"], "418281": ["giraffe standing in front of a lush green field"], "269394": ["close up of a bus that is parked in a parking lot"], "383470": ["stop sign on the side of a snow covered slope"], "569839": ["man sitting at a table with a plate of food"], "504304": ["young girl holding a tennis racquet on a tennis court holding a tennis racket"], "334321": ["group of people sitting on a horse"], "127474": ["group of people standing next to a woman holding a baseball bat"], "557556": ["man holding a tennis racquet on a tennis court holding a tennis racket"], "412151": ["group of motorcycles parked in front of a street"], "216482": ["giraffe standing next to a tree in the background"], "182784": ["group of people standing next to each other in the background"], "245453": ["man flying a kite in the sky"], "576939": ["pair of scissors hanging on the side of a stop sign"], "76292": ["man holding a frisbee in the air"], "176649": ["black and white photo of a street sign sitting on top of a table"], "147980": ["group of people playing a game of frisbee in a field"], "545293": ["park bench sitting on top of a lush green field"], "10766": ["young boy holding a cell phone"], "565778": ["train traveling down a train station"], "419281": ["group of boats that are sitting on a boat in the water"], "14869": ["giraffe standing next to a park bench in front of a lush green field"], "549399": ["black and white motorcycle parked in front of a building"], "522778": ["group of people standing on top of a skateboard"], "426523": ["group of people standing on a park bench holding a frisbee in a field"], "188958": ["group of people standing on a bench in front of a street"], "78371": ["man riding a skateboard on top of a baseball field"], "125476": ["black and white cat sitting on a park bench in front of a lush green field"], "565797": ["black and white cat sitting on top of a car"], "227879": ["baseball player swinging a bat at a baseball game"], "129576": ["living room with a flat screen tv"], "377385": ["baseball player is holding a tennis racket"], "544519": ["young boy brushing his teeth with a toothbrush in her hand"], "559665": ["man riding a motorcycle down a street"], "371250": ["black and white cat sitting on top of a bed"], "291380": ["man sitting on top of a bench in front of a building"], "391735": ["yellow fire hydrant in front of a city street"], "143931": ["close up of a bus stop sign on a city street"], "180800": ["bowl of fruit and vegetables on a table"], "133698": ["pizza sitting on top of a pan on a table"], "375363": ["bench sitting in front of a tree"], "334405": ["double decker bus driving down the street with a traffic light"], "283210": ["man riding a horse in the middle of an elephant"], "476258": ["man riding a skateboard down the side of a street"], "41550": ["man sitting on top of a computer desk"], "340559": ["bathroom with a sink in front of a window"], "426578": ["man riding a wave on a surfboard in the ocean"], "300814": ["group of people riding horses down a street"], "6005": ["group of giraffes standing next to each other"], "154202": ["laptop computer sitting on top of a desk"], "74331": ["bunch of bananas on display in front of a street"], "309852": ["cat sitting on top of a teddy bear"], "518586": ["black and white photo of a train traveling down the tracks in front of a train station"], "387173": ["boat that is sitting on top of a body of water"], "490081": ["group of cows that are standing in a dirt field"], "383419": ["young boy sitting on a table holding a hot dog"], "399462": ["young boy holding a frisbee in a field with a kite in a park"], "143975": ["living room with a coffee table"], "311913": ["brown horse standing in front of a lush green field"], "285291": ["man is skiing down the side of a snow covered slope"], "62060": ["group of people standing in a field with a frisbee"], "336493": ["man riding a bike on a baseball field"], "459374": ["yellow fire hydrant sitting in front of a building"], "289393": ["black and white cat sitting on top of a table"], "25202": ["plate that is sitting on a table"], "244339": ["man sitting on top of a table"], "39540": ["pizza that is sitting on top of a plate"], "191096": ["baseball player swinging a bat at a baseball game"], "2684": ["group of zebras standing next to each other in a field"], "74369": ["young boy playing a game of frisbee in a field"], "560323": ["man standing on a tennis court holding a tennis racket at a tennis ball"], "475779": ["baby elephant standing in front of a tree"], "166532": ["bathroom with a sink and a mirror"], "170629": ["double decker bus is parked in front of a city street"], "197254": ["large truck parked in front of a building with a clock on top of a building"], "53015": ["woman sitting at a table holding a pizza on a table"], "295564": ["bunch of vases sitting on display in the background"], "516750": ["black and white cat is sitting on top of a wall"], "289423": ["group of young men playing a game of frisbee in the air"], "35474": ["baby elephant standing in front of a tree"], "572051": ["close up of a plate of food on a table"], "248468": ["black and white photo of a man riding a horse drawn carriage in the background"], "273045": ["close up of a cat sitting on top of a table"], "236182": ["close up of a street sign sitting on top of a tree"], "256668": ["white toilet sitting next to a small bathroom"], "305821": ["giraffe standing on top of a lush green field"], "117407": ["black and white cat sitting on top of a bed"], "370677": ["man standing in front of a store"], "6818": ["bathroom with a toilet and sink"], "129699": ["group of people standing in front of a train station"], "289444": ["group of young children playing a game of frisbee in a field"], "295589": ["baseball player holding a baseball bat on a baseball field"], "344860": ["zebra standing on top of a lush green field"], "324266": ["bathroom with a toilet and a sink"], "76460": ["group of elephants standing in a grassy field"], "175831": ["group of motorcycles parked in front of a building in front of a city street"], "527023": ["man swinging a tennis racket on a tennis court holding a tennis racquet on a baseball field"], "527025": ["group of people sitting on a table"], "322226": ["black and white cat is sitting on top of a wooden bench"], "420532": ["group of men standing next to a man and a woman wearing a suit and tie"], "144053": ["street sign in front of a building"], "213687": ["group of people walking down the street in front of a city street"], "271032": ["small boat in a body of water"], "555705": ["cat sitting on top of a computer desk"], "78522": ["herd of sheep grazing on a lush green grass covered field"], "80671": ["person riding a snowboard down the side of a snow covered slope"], "438862": ["group of young men playing a game of frisbee in a field"], "570045": ["street sign on a pole next to a building"], "527040": ["man sitting on a bed with a cake on a table"], "310177": ["bowl of hot dogs are sitting on a table"], "553667": ["man holding a hot dog in his mouth"], "299716": ["close up of a teddy bear sitting on top of it"], "62151": ["herd of cows are grazing on a lush green field"], "303818": ["group of people riding a motorcycle on a city street"], "336587": ["stop sign sitting on a pole in front of a building"], "185036": ["group of people standing on top of a snow covered slope"], "397773": ["close up of a plate of food on a table"], "348881": ["white toilet sitting on top of a bathroom"], "557780": ["black and white dog holding a frisbee in the background"], "41687": ["man riding a snowboard down the side of a snow covered slope"], "27353": ["close up of a plate of food on a table"], "369370": ["close up of a plate of food on a table"], "318171": ["giraffe standing in front of a lush green field"], "93725": ["man swinging a tennis racket on a tennis court holding a tennis racquet on a baseball field"], "437370": ["black and white airplane sitting on top of a lush green field"], "307936": ["baby elephant standing in a dirt field"], "375521": ["group of people standing on top of a snow covered slope"], "559842": ["group of people playing a game of soccer on a field"], "328421": ["herd of elephants walking through a field"], "53990": ["man eating a hot dog sitting on a plate with a slice of cake"], "140007": ["man standing on top of a tennis court"], "574184": ["black and white cat sitting on top of a computer desk"], "365289": ["little girl sitting on a bed with a teddy bear"], "156397": ["herd of sheep grazing on a lush green field"], "453757": ["man standing on a tennis court holding a baseball bat at a ball"], "27235": ["man flying a kite in the sky"], "578292": ["stop sign in front of a street"], "193271": ["kitchen with white cabinets and stainless steel appliances"], "82680": ["dog is standing in the grass"], "385786": ["living room with a couch sitting on a bed"], "376959": ["young boy brushing his teeth in front of a bed"], "535292": ["herd of sheep standing on top of a lush green field"], "477949": ["vase filled with flowers sitting on a table"], "431573": ["red fire hydrant sitting on a park bench in front of a road"], "156416": ["man riding a wave on a surfboard in the ocean"], "168706": ["cow standing on top of a lush green field"], "301867": ["group of people standing in the rain holding umbrellas"], "550426": ["vase of flowers sitting on top of a table"], "72833": ["man riding a wave on top of a surfboard"], "279305": ["close up of a close up of a pair of scissors sitting on a table"], "318219": ["black and white cat sitting on top of a computer desk"], "142092": ["close up of a pizza sitting on top of a table"], "146193": ["man riding a wave on top of a beach"], "404243": ["bunch of donuts sitting on top of a grill in front of a display case"], "522713": ["boat that is sitting on top of a beach"], "209692": ["red fire truck parked in front of a building"], "324383": ["black and white cat laying on top of a bed"], "269089": ["hot dog sitting on top of a plate next to a cup of food on a table"], "510755": ["couple of a motorcycle parked in front of a lush green field"], "568101": ["group of people flying kites in the sky"], "191270": ["group of people sitting on a table"], "421681": ["laptop computer sitting on top of a wooden table"], "45864": ["group of kids playing a game of soccer on a lush green field"], "314154": ["man standing in a living room holding a wii game controller"], "37675": ["group of cows are standing in a grassy field"], "95022": ["close up of a bird perched on top of a tree branch"], "228144": ["living room with a bed next to a window"], "269105": ["white toilet sitting on top of a bathroom sink"], "415746": ["display of donuts sitting on top of a table"], "365366": ["man is holding a tennis racket"], "90935": ["living room with a bed next to a window"], "553162": ["black bear standing next to a tree in the woods"], "308026": ["man riding a skateboard in front of a building"], "455483": ["bathroom with a sink and a mirror"], "226111": ["stop sign in front of a street sign"], "242499": ["group of people standing next to a boat in the water"], "573206": ["pizza that is sitting on top of a table"], "410437": ["red fire hydrant sitting on top of a building"], "232262": ["group of people walking down a busy city street"], "54088": ["red motorcycle parked in front of a city street"], "271177": ["man riding a motorcycle parked in front of a road"], "262986": ["black bear standing in the water"], "480076": ["black and white cat sitting on top of a bed"], "351053": ["laptop computer sitting on top of a table"], "511117": ["young girl holding a baseball bat in a field"], "39760": ["man flying a kite in the sky"], "287570": ["man riding skis down a snow covered slope"], "197461": ["group of sheep standing on top of a lush green field"], "310103": ["pile of luggage sitting on a road"], "508730": ["woman holding a stuffed teddy bear sitting on a table"], "334686": ["man eating a hot dog on a plate"], "138079": ["bathroom with a sink and a mirror"], "414560": ["black and white cat sitting on top of a lush green field"], "486203": ["herd of horses standing in a grassy field"], "476005": ["double decker bus is parked in front of a building"], "20965": ["woman swinging a tennis ball on a tennis court holding a tennis racket on a tennis court"], "66412": ["man standing in front of a bathroom"], "421010": ["group of people sitting on a bench in front of a park"], "332654": ["yellow fire hydrant in front of a parking meter on the side of a street"], "281455": ["person riding a wave on top of a beach"], "353136": ["man sitting in front of a dog"], "430961": ["baseball player getting ready to hit a ball on a tennis court"], "463730": ["group of people walking down a city street"], "78707": ["baseball player swinging a bat at home plate during a game"], "181666": ["group of cows are standing next to each other in the background"], "232309": ["man riding a kite on a surfboard in the ocean"], "66423": ["group of people sitting on the beach"], "39743": ["woman holding a stuffed teddy bear sitting on a table"], "236412": ["slice of pizza sitting on top of a plate"], "199551": ["man sitting on top of a bed"], "291712": ["blue and white photo of a plane flying through the water"], "293554": ["man is sitting on top of it"], "406404": ["man riding a skateboard at a skate park bench in front of a snow covered slope"], "148358": ["group of giraffes standing next to each other in a grassy field"], "336777": ["stop sign in front of a city street"], "150410": ["man sitting on top of a field"], "451468": ["cat sitting on top of a table in front of a living room"], "326541": ["man holding a cell phone"], "527248": ["group of people standing in front of a building"], "37777": ["black and white photo of a living room"], "191381": ["bathroom with a white toilet sitting on top of a toilet"], "129942": ["small bird sitting on top of a tree branch"], "154971": ["group of people playing a game of frisbee in a field with a soccer ball"], "557981": ["young woman holding a cell phone"], "19358": ["close up of a pizza sitting on top of a plate"], "385861": ["man sitting at a table eating a birthday cake with a hot dog"], "41888": ["zebra standing on top of a lush green field"], "132001": ["black and white cat sitting on top of a computer keyboard"], "185250": ["young boy holding a frisbee in a field"], "410533": ["black and white cat sitting on top of a bed"], "443303": ["black and white cat sitting on top of a bed"], "258985": ["white toilet sitting on top of a table"], "293802": ["man on a skateboard doing a trick on a tennis court holding a tennis racket"], "474028": ["group of young children playing a game of frisbee in a field"], "144200": ["close up of a plate of food on a table"], "199602": ["young woman holding a kite on the beach"], "242611": ["bathroom with a sink and a mirror"], "113588": ["group of people sitting at a desk in front of a laptop computer"], "207797": ["giraffe standing in the water"], "314294": ["herd of elephants standing next to each other in the background"], "500663": ["man flying a kite on top of a lush green field"], "355257": ["laptop computer sitting on top of a desk"], "87199": ["man riding a horse in a field"], "281533": ["woman sitting on top of a living room"], "232383": ["man sitting on top of a laptop"], "511136": ["giraffe standing in front of a building"], "578498": ["man sitting on a bed in front of a living room"], "578500": ["living room filled with lots of chairs in a living room"], "261061": ["young girl holding a frisbee in a field"], "5064": ["baseball player swinging a bat at a ball during a game"], "562121": ["giraffe standing on top of a lush green field"], "472621": ["bathroom with a sink and a mirror"], "209868": ["man sitting on a cell phone"], "50125": ["double decker bus driving down a parking lot"], "502737": ["woman holding a piece of cake on a table"], "207826": ["woman holding a laptop on her cell phone"], "310227": ["group of people sitting on a bed"], "256091": ["herd of zebra standing on a dirt road"], "414679": ["brown and white cow standing in a field"], "25560": ["black and white cat sitting on top of a tv"], "41945": ["red and white photo of a boat that is parked in front of a building"], "162952": ["woman standing in a living room holding a laptop computer"], "66523": ["bathroom with a white bath tub"], "463836": ["man riding a skateboard down a street"], "263136": ["man standing on top of a field"], "437218": ["living room with a television sitting on top of a desk"], "183803": ["living room with a couch next to a wooden table"], "562150": ["woman sitting on top of a plate"], "117125": ["black and white photo of a train traveling down the tracks"], "48636": ["baseball player is holding a bat in a baseball game"], "146411": ["cat laying on top of a bed"], "560108": ["young boy riding a skateboard down a baseball bat"], "404464": ["black and white photo of a man standing in front of a building"], "84982": ["stop sign on a pole next to a traffic light"], "201723": ["cat sitting on top of a table with books"], "87038": ["group of people riding a skateboard down a street in front of a city street"], "394240": ["man riding a motorcycle on a city street"], "102912": ["group of elephants standing next to a fire hydrant"], "31747": ["boat that is on a body of boats in the water"], "338948": ["group of people standing in front of a building"], "54277": ["man standing on top of a wall"], "138246": ["man standing in the middle of a grassy field"], "410632": ["group of people standing on a tennis court holding a soccer ball"], "373591": ["black and white bird sitting on top of a tree branch"], "384012": ["baseball player swinging a bat in front of a baseball game"], "191501": ["red and white photo of a street sign sitting on top of a building"], "551952": ["truck is parked in front of a train station with a parking meter on a city street"], "93201": ["black and white cat laying on top of a bed"], "37907": ["black and white cat sitting on top of a tree"], "39956": ["living room with a chair in the living room"], "246809": ["boat that is sitting on top of a body of water"], "457754": ["group of men standing in front of wine"], "193565": ["man sitting at a table eating a hot dog"], "310302": ["pizza sitting on top of a table"], "375840": ["close up of a bowl of bananas sitting on a table"], "347170": ["young boy riding a skateboard on a tennis court holding a baseball bat"], "486438": ["close up of donuts that are sitting on a table"], "400139": ["plate of food on a table"], "7211": ["train is parked in front of a train station"], "381106": ["man flying a kite on the beach playing a game of water"], "341041": ["young man riding a skateboard on a tennis court holding a tennis racket at a skate park"], "310325": ["black and white cat sitting on top of a toilet"], "171062": ["dog is sitting in a living room with a computer screen"], "244575": ["brown bear standing next to a horse"], "461885": ["man standing on a lush green field"], "398076": ["man is holding a cell phone"], "78915": ["group of people standing on a tennis court holding a tennis racket"], "13383": ["woman that is sitting on top of a plate on a table"], "203849": ["kitchen with white cabinets and stainless steel appliances"], "461898": ["woman is holding a cell phone"], "234572": ["man doing a trick on a skateboard in the air on top of a ramp"], "511058": ["close up of giraffe standing next to each other in the background"], "330835": ["man sitting on top of a skateboard"], "42069": ["giraffe standing in front of a building"], "240727": ["woman sitting on a bed"], "16228": ["man riding a horse drawn carriage in front of a street"], "91227": ["giraffe standing on top of a lush green field"], "322654": ["street sign on a pole next to a street sign"], "222304": ["vase filled with flowers sitting on a table"], "570465": ["plate of food sitting on top of a table"], "386146": ["giraffe standing on top of a lush green field"], "369763": ["group of people standing in front of a street"], "312421": ["black and white cat sitting on top of a street"], "207151": ["pizza that is sitting on top of a plate"], "300138": ["vase of flowers sitting on top of a table"], "289899": ["group of people standing on top of a body of water"], "189550": ["black and white photo of a person standing next to each other"], "400573": ["man eating a hot dog on a cell phone"], "386164": ["close up of a piece of scissors sitting on a table"], "310391": ["man riding a motorcycle parked on the side of a truck"], "476280": ["man holding a surfboard on the beach"], "348654": ["close up of a laptop computer sitting on top of a cell phone"], "1146": ["man wearing a suit and tie holding a cell phone"], "453756": ["close up of a table that is sitting on top of a table"], "326781": ["bathroom with a sink and a mirror"], "402559": ["giraffe standing in front of a tree"], "496768": ["man flying a kite in a body of water"], "461953": ["close up of a hot dog is sitting on a table"], "554114": ["man standing in front of a street"], "418949": ["baseball player holding a bat on a field playing a game"], "459912": ["group of motorcycles are parked in front of a city street"], "31255": ["black and white dog is standing in front of an elephant"], "38029": ["group of people sitting on a city street"], "38034": ["laptop computer sitting on top of a desk next to a computer monitor"], "150675": ["cat is sitting on top of a wall"], "400538": ["group of people sitting at a table with food in a living room"], "103579": ["group of people standing on a lush green field holding a frisbee in the park"], "463633": ["stop sign with a stop sign on the street"], "519706": ["man standing on top of a tennis court"], "32965": ["man in the water holding a kite on the beach"], "156832": ["bathroom with a toilet and sink and a mirror"], "369826": ["laptop computer sitting on top of a street"], "216228": ["man standing in front of a store"], "165029": ["man sitting on top of a bed"], "40102": ["group of giraffes standing next to each other in a grassy field"], "545959": ["fire hydrant sitting on top of a lush green field"], "85160": ["group of people sitting in a living room"], "308394": ["cat sitting on top of a bed next to a window"], "365426": ["group of people playing tennis on a tennis court holding tennis rackets on a tennis court"], "152751": ["laptop computer sitting in front of a living room"], "304305": ["group of elephants standing next to each other in a grassy field"], "455859": ["group of people playing a game of frisbee"], "472246": ["close up of a bowl of oranges sitting on a table"], "210103": ["plate of food on a table next to a birthday cake"], "521400": ["woman on a tennis court holding a tennis racket in front of a tennis ball"], "507065": ["man sitting on top of a bench"], "453819": ["boat that is sitting on the water"], "372938": ["man riding a motorcycle down the road"], "114549": ["bunch of bananas are standing in front of a tree"], "217005": ["bird perched on top of a tree branch"], "146627": ["group of people sitting on a table"], "201925": ["close up of a pizza sitting on top of a table"], "64710": ["man on a beach flying a kite in the snow"], "443591": ["zebra standing on top of a lush green field"], "181449": ["black and white cat sitting on top of a cell phone"], "206027": ["sandwich sitting on top of a table"], "163020": ["flock of birds flying in the water"], "60623": ["man sitting at a table with a plate of food"], "216273": ["teddy bear sitting on top of a bed"], "9426": ["large plane flying through the sky"], "570579": ["group of people sitting at a table with a plate of food"], "467522": ["group of zebras grazing on a lush green field"], "384213": ["man sitting on top of a table"], "515289": ["group of people riding a bike down a road in front of a street"], "17627": ["group of a bus that is driving down a street with a traffic light"], "503005": ["man standing on top of a lush green field"], "511204": ["man standing in front of an elephant"], "388325": ["fire hydrant in front of a street sign in front of a building"], "109798": ["hot dog sitting on top of a plate of food"], "121041": ["man riding a wave on a surfboard in the ocean"], "158952": ["stop sign on the side of a street sign"], "320039": ["little girl sitting at a table with a plate of food"], "15113": ["close up of a bird perched on top of a tree"], "138477": ["group of people standing in a field flying a kite in a grassy field"], "66800": ["herd of elephants standing in front of a lush green field"], "412914": ["person riding a wave in the water"], "195829": ["group of people playing a game of frisbee in a field with a soccer ball"], "533750": ["close up of a donut sitting on a table"], "349737": ["desk with a laptop computer sitting on top of a table"], "191738": ["black and white cat laying on top of a lush green field"], "109819": ["group of people walking down a street"], "95427": ["close up of a pile of oranges sitting on a table"], "41369": ["black and white cat sitting on top of a laptop"], "309120": ["man in a field holding a frisbee in the air"], "240903": ["group of sheep standing on top of a lush green field"], "505099": ["baseball player swinging a bat at the ball during a baseball game"], "58636": ["street sign on a pole in front of a building"], "165133": ["man standing in front of a street"], "391895": ["man holding a red umbrella in the rain"], "60687": ["herd of sheep standing on top of a lush green field"], "71171": ["close up of a pizza sitting on top of a plate"], "204049": ["man riding a snowboard down a hill in front of a snow covered slope"], "238866": ["black and white photo of a teddy bear sitting on top of a table"], "281878": ["cat is sitting on top of a bed in front of a living room"], "29913": ["red fire hydrant sitting on top of a motorcycle"], "155743": ["zebra standing on top of a lush green field"], "318671": ["man standing on top of a tennis court"], "371999": ["clock tower with a clock on the side of a building"], "451872": ["group of elephants are standing in a dirt field"], "466211": ["baseball player is holding a bat at a baseball game"], "453926": ["white plate topped with a piece of pizza sitting on top of a table"], "75051": ["white toilet sitting next to a small bathroom"], "196311": ["living room with lots of luggage sitting on top of a table"], "568623": ["group of people standing in front of a luggage bag"], "322864": ["black and white cat sitting on top of a computer desk"], "574769": ["group of people standing in a living room"], "146738": ["living room with flowers in front of a wooden table"], "267571": ["close up of giraffe standing next to each other"], "433460": ["group of zebras standing next to each other in a field"], "136501": ["baseball player is holding a bat in front of a field"], "64822": ["red and white cat sitting on top of a street sign"], "175417": ["couple of a bench sitting on top of a lush green field"], "77123": ["plane that is sitting on top of a runway"], "458052": ["black and white cat laying on top of a bed"], "458054": ["bird that is sitting on top of a wooden table"], "173383": ["group of people sitting on a table"], "324937": ["cat laying on a bed with a teddy bear sitting on top of a table"], "15690": ["herd of elephants walking through the water"], "75083": ["street sign in front of a city street"], "85329": ["man wearing a hat and tie"], "564563": ["black and white photo of a plane flying through the water"], "546702": ["black and white cat laying on top of a bed"], "60760": ["bathroom with a sink and white toilet and a mirror"], "511321": ["large body of boats docked in the water"], "23899": ["man sitting on top of a bed"], "17756": ["man sitting on a park bench in the water"], "187743": ["little girl brushing her teeth"], "118113": ["cat sitting in front of a living room"], "436795": ["man holding a wii video game controller"], "222564": ["man standing in front of a kitchen"], "179558": ["giraffe standing in front of a fence"], "121745": ["close up of a bowl of oranges sitting on a table"], "437609": ["black and white photo of a herd of cows are standing next to each other"], "294119": ["close up of a stop sign on a cell phone"], "554348": ["man standing on top of a bench in front of a street"], "497006": ["bunch of bananas in front of an outdoor market"], "296303": ["cat is sitting in front of a window"], "337264": ["man standing in front of a kitchen"], "419048": ["woman holding a frisbee on a lush green grass covered field"], "183666": ["train traveling down the tracks in front of a train station"], "251252": ["man riding a motorcycle parked in front of a street"], "515445": ["group of people standing on the beach"], "308599": ["stop sign sitting on the side of a building"], "370043": ["baby elephant standing next to each other in the background"], "77184": ["couple of boats docked in the water"], "208589": ["close up of a pizza sitting on top of a table"], "77187": ["brown bear standing in front of a tree"], "34180": ["young boy holding a banana"], "262873": ["black and white cat sitting on top of a table"], "205378": ["black and white cat sitting on top of a bed"], "531854": ["group of birds swimming in a body of water"], "109092": ["man in a suit and tie"], "341393": ["dog laying on top of a lush green field"], "406932": ["dog is standing in front of a car mirror"], "189845": ["young man holding a tennis racquet on a tennis court"], "99734": ["boat that is sitting on top of a body of water"], "75162": ["zebra standing on top of a lush green field"], "377715": ["baseball player holding a baseball bat at a game"], "72944": ["black and white cat is sitting in front of a window"], "521634": ["street sign on a pole in front of a traffic light"], "227227": ["man standing next to a dog"], "52644": ["parking meter sitting on top of a beach"], "185768": ["close up of a teddy bear sitting on top of it"], "458153": ["baby elephant standing on top of a lush green field"], "490923": ["man brushing his teeth on her cell phone"], "80172": ["young boy holding a video game controller"], "427438": ["group of zebras standing next to each other in a field"], "295837": ["man sitting at a table in front of a pizza"], "409009": ["passenger train is parked in front of a train station"], "564659": ["close up of a plate of food on a table"], "476597": ["group of people standing in a living room playing a video game"], "562614": ["group of people playing a game of a soccer ball on a lush green field"], "579003": ["black and white photo of a bathroom with a mirror"], "558524": ["white and white vase that is sitting on top of a table"], "449981": ["group of people standing next to each other in the background"], "232894": ["young boy sitting on a table holding a cell phone"], "542145": ["bathroom with a sink in front of a mirror"], "351683": ["baseball player swinging a bat at a home plate"], "386500": ["cat laying on a bed with a teddy bear"], "101622": ["man holding a frisbee in the air"], "140743": ["woman holding a teddy bear sitting on a cell phone"], "183757": ["group of people standing next to each other"], "484816": ["yellow airplane flying through the sky"], "245201": ["black and white cat sitting on top of a wall"], "181714": ["woman holding a red umbrella"], "15827": ["group of motorcycles parked in front of a parking lot"], "231339": ["woman sitting at a table with food in a kitchen"], "112085": ["herd of horses grazing on a lush green field"], "480726": ["group of people standing on a tennis court with a tennis ball during a baseball game"], "113914": ["woman sitting in front of a car"], "224736": ["bathroom that is sitting on top of a kitchen counter"], "62790": ["man sitting on top of a tv"], "420775": ["man riding a snowboard down the side of a snow covered slope"], "171500": ["truck driving down a road in front of a lush green field"], "560623": ["plane that is parked on a runway"], "579056": ["man standing on top of a tennis court"], "247285": ["woman sitting at a table with a cell phone"], "97434": ["group of motorcycles parked in front of a city street"], "105975": ["baby elephant standing in the grass"], "515579": ["young boy standing on a tennis court holding a baseball bat"], "523772": ["plate of donuts sitting on top of a plate"], "529917": ["group of people standing next to each other in the background"], "235006": ["truck parked in front of a fire hydrant in front of a lush green field"], "161470": ["bird sitting on top of a lush green field"], "142592": ["double decker bus parked in front of a building"], "7682": ["man wearing a suit and tie holding a cell phone"], "495107": ["man eating a hot dog on a cell phone"], "462341": ["clock tower with a clock on the side of a building"], "173574": ["herd of sheep are standing in the grass field"], "132615": ["baseball player is holding a bat in a baseball game"], "564745": ["man holding a tennis racket on a tennis court holding a tennis racquet on a skateboard"], "247306": ["group of boats docked in the water"], "54796": ["young girl playing tennis on a tennis court holding a tennis racket on a tennis court"], "331807": ["bowl of bananas sitting on top of a wooden table"], "540174": ["young boy eating a hot dog in his mouth"], "503311": ["man flying a kite in the water holding a kite in the air"], "79380": ["group of people sitting on a park bench in front of a lush green field"], "122390": ["black and white dog laying on a bed"], "52759": ["plane that is parked on a runway"], "540186": ["bathroom with a sink in front of a kitchen"], "146973": ["white plate that is sitting on a table"], "354533": ["motorcycle parked in front of a lush green field"], "353830": ["pizza that is sitting on top of a table"], "384553": ["man riding a dog in the water"], "95786": ["white toilet sitting on top of a table"], "224757": ["group of people standing on a park bench"], "550444": ["group of people playing a game of soccer on a field"], "347693": ["living room with chairs sitting on a bed in front of a window"], "30255": ["large black and white photo of people sitting on top of a boat"], "321079": ["chocolate cake sitting on top of a white plate"], "460347": ["double decker bus driving down the road next to a car"], "284220": ["woman holding a tennis racket on a tennis court holding a tennis racquet on a baseball field"], "180490": ["black and white photo of a living room with a refrigerator"], "108094": ["man riding a bike parked in front of a building with a clock on a city street"], "489023": ["group of people playing a game of frisbee in a field"], "84235": ["hot dog sitting on a bun next to a plate of food"], "89668": ["herd of elephants standing on top of a grass field"], "403013": ["bathroom with a sink in the kitchen"], "328289": ["group of people standing on top of a lush green field"], "556616": ["man standing next to a parking meter"], "403020": ["zebra is standing next to a giraffe"], "132686": ["man flying a kite in the background"], "321107": ["man riding a skateboard down a street"], "157269": ["group of people standing in front of a bus"], "556462": ["close up of a pizza sitting on top of a plate"], "331352": ["bathroom with a toilet and a sink"], "104025": ["black and white horse is riding a wave on the beach"], "199951": ["man riding a skateboard on a boat in the water"], "224861": ["black and white photo of a man holding a cell phone"], "129637": ["group of horses standing next to each other"], "505440": ["group of zebras standing on top of a lush green field"], "563964": ["young boy holding a frisbee in the air"], "421478": ["man in a suit and tie holding a cell phone"], "190056": ["group of zebras are standing next to each other in the background"], "329323": ["group of people standing next to each other in the background"], "405778": ["group of zebras grazing on top of a lush green field"], "335472": ["young man riding skis down a snow covered slope"], "540264": ["woman holding a cell phone in front of a man"], "110196": ["group of people standing in front of a city street"], "484982": ["living room with a clock tower in front of a window"], "385633": ["bathroom that is sitting on top of a kitchen"], "403065": ["truck is parked in front of a road"], "134778": ["group of birds flying in the water"], "468604": ["baby elephant standing in front of a lush green field"], "261757": ["pizza that is sitting on top of a wooden table"], "452224": ["small bird perched on top of a tree branch"], "550529": ["red motorcycle parked in front of a building"], "173704": ["man sitting at a table holding a cell phone"], "108169": ["black and white photo of a snow covered in the middle of water"], "430359": ["baseball player holding a bat in front of a field"], "394892": ["young boy riding a skateboard on a baseball field"], "513681": ["large white airplane that is sitting on top of a runway"], "288403": ["group of people standing in front of a soccer ball"], "278166": ["black and white cat sitting on top of a toilet"], "46743": ["young boy sitting on a bench in front of a fire hydrant"], "81561": ["traffic light with a clock tower in the air"], "382088": ["horse that is standing next to a fence"], "562261": ["man holding a frisbee in a field"], "24223": ["pizza sitting on top of a plate"], "497312": ["man riding a horse in a field"], "79472": ["laptop computer sitting on top of a desk next to a wooden table"], "409251": ["group of people standing next to each other"], "349860": ["man doing a trick on a skateboard in the air on top of a ramp"], "282225": ["young man playing tennis on a tennis court holding a tennis racquet on a field"], "5802": ["group of people sitting at a table with food in front of a kitchen"], "165547": ["living room with a large window"], "335532": ["man sitting on a piece of a plate of food"], "353968": ["man sitting on top of a bed"], "63154": ["man riding a wave on a surfboard in the ocean"], "43635": ["group of people standing in front of a building"], "562870": ["person sitting on top of a cell phone"], "212663": ["close up of a bus stop sign on a city street"], "321214": ["little boy sitting at a table eating a slice of pizza"], "515779": ["group of people sitting on a cell phone"], "493174": ["baseball player holding a baseball bat at a game"], "280819": ["group of people sitting at a table with plates of food"], "296649": ["group of people riding bikes down a city street"], "181962": ["group of people riding horses on the beach"], "65227": ["man in a field holding a frisbee in a park"], "577826": ["group of people riding a skateboard down a street"], "551215": ["man standing on a tennis court holding a frisbee on a tennis court holding a tennis racket"], "146723": ["man playing a game of a soccer ball"], "384204": ["black and white cat laying on top of a couch next to a window"], "124629": ["group of young men playing a game of frisbee on a tennis court"], "450263": ["group of people sitting on a bench in front of a city street"], "554625": ["man sitting on top of a laptop computer"], "480985": ["group of people standing next to each other"], "335578": ["young boy holding a tennis racquet on a tennis court holding a baseball bat at a ball"], "472795": ["cow standing next to a horse"], "65244": ["man sitting on top of a wall"], "5754": ["close up of donuts that are sitting on a table"], "138975": ["group of people standing on top of a snow covered slope"], "212704": ["group of people sitting on a couch playing a video game"], "550627": ["plate of food that are sitting on a table"], "376549": ["man riding a snowboard down a snow covered slope"], "196053": ["woman standing on a tennis court holding a tennis racket on a tennis court"], "272110": ["man on a skateboard doing a trick on a bench in front of a building"], "521967": ["stuffed teddy bear sitting on top of a bed"], "429580": ["group of people standing in a field holding a frisbee in the grass"], "239347": ["man sitting on top of a bed"], "379977": ["black and white horse is standing on top of a lush green field"], "370423": ["black and white cat sitting on top of a bed"], "231163": ["train is parked in front of a train station"], "61181": ["group of cars are driving down a busy city street"], "153343": ["teddy bear sitting on top of a fire hydrant"], "535668": ["baseball player swinging a bat at home plate holding a bat in a baseball game"], "17707": ["black and white cat sitting on top of a beach"], "190081": ["woman is playing a game in a living room with a tennis racket"], "216841": ["man sitting on top of a table"], "290570": ["herd of sheep are standing on top of a lush green field"], "74711": ["group of birds are standing in the water"], "529636": ["large clock tower in front of a building"], "448269": ["group of people standing on a baseball field"], "329486": ["group of people standing in the middle of water"], "69392": ["man sitting on top of a tennis court"], "368402": ["man standing in front of a kitchen"], "177941": ["close up of a pizza that is sitting on a table"], "472854": ["group of people standing in front of a street"], "208663": ["close up of a plate of food on a table"], "308441": ["man standing on a tennis court holding a tennis racket at a baseball game"], "205103": ["black and white cat sitting on top of a bed"], "309022": ["kitchen counter in front of a table"], "281221": ["group of giraffes standing in a grassy field"], "557360": ["close up of a close up of a close up of a hand holding a tree"], "457691": ["street sign with a clock tower in front of a building"], "530212": ["red fire hydrant sitting on top of a table"], "235302": ["group of people sitting on a table"], "296098": ["large clock tower on top of a building"], "71466": ["bed is sitting on top of a wall"], "137003": ["young girl holding a kite in a field"], "241453": ["couple of cows are standing next to each other"], "542510": ["truck parked in front of a lush green field"], "476975": ["group of giraffes standing next to each other"], "198448": ["woman sitting on a bench in front of a cell phone"], "159537": ["large clock tower with a clock on top of a building"], "227125": ["street sign sitting in front of a traffic light"], "413321": ["man riding a wave on a surfboard in the ocean"], "536831": ["man riding skis down a snow covered slope"], "266041": ["man wearing a tie standing in front of a cell phone"], "127451": ["man riding a snowboard on top of a snow covered slope"], "59202": ["group of people doing a trick on a skateboard"], "313155": ["hot dog sitting on top of a plate on a table"], "471009": ["man flying a kite on the beach flying kites in the air"], "407368": ["polar bear in a body of water"], "169802": ["little girl sitting in front of a birthday cake"], "143671": ["close up of a close up of a plate of food on top of a plate"], "397133": ["group of people sitting at a table with a glass of wine"], "14941": ["cat laying on a bed next to each other"], "479057": ["herd of sheep grazing on a lush green grass covered field"], "477010": ["baseball player swinging a bat on a tennis court holding a tennis racket"], "462565": ["man riding a bike down a street"], "14990": ["group of elephants walking down a dirt road"], "466774": ["baby elephant standing in front of a tree"], "289610": ["group of people standing in front of a stuffed teddy bear sitting on top of a table"], "239448": ["clock tower with a clock on the side of a building"], "16356": ["man flying a kite in the air"], "456496": ["man riding a skateboard on top of a bench"], "386912": ["group of people standing in front of a living room with a laptop computer"], "82327": ["clock tower sitting in front of a building"], "466787": ["bowl of fruits and vegetables on it"], "374628": ["living room with a kitchen with a wooden table"], "226278": ["group of people standing on a city street"], "573291": ["giraffe standing on top of a lush green grass covered field"], "448365": ["man riding a skate board doing a trick on a skateboard in the air"], "24430": ["boat that is sitting on top of a city street"], "12543": ["man flying a kite in front of a field"], "446322": ["group of donuts sitting on a table"], "261779": ["man holding a skateboard on top of a tennis court"], "434494": ["train is parked in front of a city street"], "153299": ["giraffe is standing in front of a building"], "129001": ["bathroom with a toilet and a sink"], "249720": ["man riding a skateboard down a street"], "122745": ["stop sign on the side of a street sign"], "509822": ["black and white bike parked in front of a car"], "144941": ["bathroom with a toilet sitting on top of a bathroom"], "362368": ["man sitting at a table with a plate of food"], "362373": ["laptop computer sitting on top of a table in front of a living room"], "272262": ["man holding a tennis racket on a tennis court holding a tennis racquet on a baseball field"], "65415": ["young boy riding skis down a snow covered slope"], "467477": ["man riding a wave on a surfboard in the ocean"], "345998": ["man riding a horse in a dirt field"], "96493": ["young girl laying on top of a couch holding a baby sitting on a bed"], "303590": ["woman holding a hot dog standing in front of a hot dog"], "122203": ["black and white photo of a herd of sheep grazing on a lush green field"], "503707": ["man riding a motorcycle down a road next to a car"], "571034": ["group of people sitting on a lush green field"], "10142": ["man riding skis down a snow covered slope"], "137658": ["man sitting on a cell phone"], "169891": ["black and white cat is standing in front of a kitchen"], "270244": ["zebra standing on top of a lush green field"], "573349": ["group of people walking down a street with an umbrella"], "180135": ["man is holding a slice of cake"], "128939": ["traffic light on the side of a car"], "331692": ["pizza that is sitting on top of a plate"], "436141": ["bathroom with a toilet and a sink"], "167854": ["woman holding a tennis racket on the beach"], "236189": ["man in a red shirt holding a frisbee in a field"], "294832": ["bathroom with a sink and a mirror"], "40881": ["close up of a plate of food on a white plate"], "192440": ["living room is sitting on top of a wooden table"], "403385": ["bathroom with a toilet sitting next to a bathroom"], "368117": ["traffic light on the side of a red stop sign"], "352194": ["man riding a motorcycle down a busy city street"], "142667": ["group of people standing in front of a street"], "450500": ["group of people walking down a street with an umbrella"], "358342": ["black and white photo of a building with a clock tower in front of a city street"], "439969": ["black and white fire hydrant sitting on top of a boat"], "356298": ["train is parked in front of a lush green field"], "83915": ["large clock tower with a clock tower in front of a building"], "561100": ["plane that is parked on top of a lush green field"], "578210": ["teddy bear sitting on a table next to a birthday cake"], "356302": ["black and white cat sitting on top of a desk"], "335833": ["city street with a clock tower on the side of a building"], "153563": ["man sitting on a cell phone in front of a laptop computer"], "503772": ["kitchen with white cabinets and stainless steel appliances"], "125997": ["stop sign in front of a street sign"], "79841": ["man riding a bike down a street in front of a city street"], "153570": ["man riding a wave on a surfboard in the ocean"], "122851": ["group of people standing next to each other in the background"], "75748": ["train traveling down the tracks in the water"], "182245": ["group of people sitting at a table with plates of food in front of a birthday cake"], "491497": ["bathroom that is sitting on top of a bed"], "278506": ["group of people standing on a beach flying kites"], "483108": ["double decker bus is parked in front of a building"], "82258": ["clock sitting on top of books"], "28655": ["street sign sitting on top of a pole in front of a building"], "231408": ["black and white bird sitting on top of a lush green field"], "94194": ["man sitting in front of a kitchen"], "399012": ["close up of a donut sitting on top of a plate"], "329717": ["woman sitting on top of a bed"], "342006": ["large clock tower on a cloudy day"], "51191": ["bathroom with a sink and a mirror"], "495612": ["street sign sitting on top of a building"], "239274": ["couple of boats docked in the water"], "483234": ["man standing on top of a snow covered slope"]} -------------------------------------------------------------------------------- /pycocoevalcap/bleu/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Xinlei Chen, Hao Fang, Tsung-Yi Lin, and Ramakrishna Vedantam 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /pycocoevalcap/bleu/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /pycocoevalcap/bleu/bleu.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # File Name : bleu.py 4 | # 5 | # Description : Wrapper for BLEU scorer. 6 | # 7 | # Creation Date : 06-01-2015 8 | # Last Modified : Thu 19 Mar 2015 09:13:28 PM PDT 9 | # Authors : Hao Fang and Tsung-Yi Lin 10 | 11 | from .bleu_scorer import BleuScorer 12 | 13 | 14 | class Bleu: 15 | def __init__(self, n=4): 16 | # default compute Blue score up to 4 17 | self._n = n 18 | self._hypo_for_image = {} 19 | self.ref_for_image = {} 20 | 21 | def compute_score(self, gts, res): 22 | 23 | assert(list(gts.keys()) == list(res.keys())) 24 | imgIds = list(gts.keys()) 25 | 26 | bleu_scorer = BleuScorer(n=self._n) 27 | for id in imgIds: 28 | hypo = res[id] 29 | ref = gts[id] 30 | 31 | # Sanity check. 32 | assert(type(hypo) is list) 33 | assert(len(hypo) == 1) 34 | assert(type(ref) is list) 35 | assert(len(ref) >= 1) 36 | 37 | bleu_scorer += (hypo[0], ref) 38 | 39 | #score, scores = bleu_scorer.compute_score(option='shortest') 40 | score, scores = bleu_scorer.compute_score(option='closest', verbose=1) 41 | #score, scores = bleu_scorer.compute_score(option='average', verbose=1) 42 | 43 | # return (bleu, bleu_info) 44 | return score, scores 45 | 46 | def method(self): 47 | return "Bleu" 48 | -------------------------------------------------------------------------------- /pycocoevalcap/bleu/bleu_scorer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # bleu_scorer.py 4 | # David Chiang 5 | 6 | # Copyright (c) 2004-2006 University of Maryland. All rights 7 | # reserved. Do not redistribute without permission from the 8 | # author. Not for commercial use. 9 | 10 | # Modified by: 11 | # Hao Fang 12 | # Tsung-Yi Lin 13 | 14 | '''Provides: 15 | cook_refs(refs, n=4): Transform a list of reference sentences as strings into a form usable by cook_test(). 16 | cook_test(test, refs, n=4): Transform a test sentence as a string (together with the cooked reference sentences) into a form usable by score_cooked(). 17 | ''' 18 | 19 | import copy 20 | import sys, math, re 21 | from collections import defaultdict 22 | 23 | def precook(s, n=4, out=False): 24 | """Takes a string as input and returns an object that can be given to 25 | either cook_refs or cook_test. This is optional: cook_refs and cook_test 26 | can take string arguments as well.""" 27 | words = s.split() 28 | counts = defaultdict(int) 29 | for k in range(1,n+1): 30 | for i in range(len(words)-k+1): 31 | ngram = tuple(words[i:i+k]) 32 | counts[ngram] += 1 33 | return (len(words), counts) 34 | 35 | def cook_refs(refs, eff=None, n=4): ## lhuang: oracle will call with "average" 36 | '''Takes a list of reference sentences for a single segment 37 | and returns an object that encapsulates everything that BLEU 38 | needs to know about them.''' 39 | 40 | reflen = [] 41 | maxcounts = {} 42 | for ref in refs: 43 | rl, counts = precook(ref, n) 44 | reflen.append(rl) 45 | for (ngram,count) in counts.items(): 46 | maxcounts[ngram] = max(maxcounts.get(ngram,0), count) 47 | 48 | # Calculate effective reference sentence length. 49 | if eff == "shortest": 50 | reflen = min(reflen) 51 | elif eff == "average": 52 | reflen = float(sum(reflen))/len(reflen) 53 | 54 | ## lhuang: N.B.: leave reflen computaiton to the very end!! 55 | 56 | ## lhuang: N.B.: in case of "closest", keep a list of reflens!! (bad design) 57 | 58 | return (reflen, maxcounts) 59 | 60 | def cook_test(test, xxx_todo_changeme, eff=None, n=4): 61 | '''Takes a test sentence and returns an object that 62 | encapsulates everything that BLEU needs to know about it.''' 63 | (reflen, refmaxcounts) = xxx_todo_changeme 64 | testlen, counts = precook(test, n, True) 65 | 66 | result = {} 67 | 68 | # Calculate effective reference sentence length. 69 | 70 | if eff == "closest": 71 | result["reflen"] = min((abs(l-testlen), l) for l in reflen)[1] 72 | else: ## i.e., "average" or "shortest" or None 73 | result["reflen"] = reflen 74 | 75 | result["testlen"] = testlen 76 | 77 | result["guess"] = [max(0,testlen-k+1) for k in range(1,n+1)] 78 | 79 | result['correct'] = [0]*n 80 | for (ngram, count) in counts.items(): 81 | result["correct"][len(ngram)-1] += min(refmaxcounts.get(ngram,0), count) 82 | 83 | return result 84 | 85 | class BleuScorer(object): 86 | """Bleu scorer. 87 | """ 88 | 89 | __slots__ = "n", "crefs", "ctest", "_score", "_ratio", "_testlen", "_reflen", "special_reflen" 90 | # special_reflen is used in oracle (proportional effective ref len for a node). 91 | 92 | def copy(self): 93 | ''' copy the refs.''' 94 | new = BleuScorer(n=self.n) 95 | new.ctest = copy.copy(self.ctest) 96 | new.crefs = copy.copy(self.crefs) 97 | new._score = None 98 | return new 99 | 100 | def __init__(self, test=None, refs=None, n=4, special_reflen=None): 101 | ''' singular instance ''' 102 | 103 | self.n = n 104 | self.crefs = [] 105 | self.ctest = [] 106 | self.cook_append(test, refs) 107 | self.special_reflen = special_reflen 108 | 109 | def cook_append(self, test, refs): 110 | '''called by constructor and __iadd__ to avoid creating new instances.''' 111 | 112 | if refs is not None: 113 | self.crefs.append(cook_refs(refs)) 114 | if test is not None: 115 | cooked_test = cook_test(test, self.crefs[-1]) 116 | self.ctest.append(cooked_test) ## N.B.: -1 117 | else: 118 | self.ctest.append(None) # lens of crefs and ctest have to match 119 | 120 | self._score = None ## need to recompute 121 | 122 | def ratio(self, option=None): 123 | self.compute_score(option=option) 124 | return self._ratio 125 | 126 | def score_ratio(self, option=None): 127 | '''return (bleu, len_ratio) pair''' 128 | return (self.fscore(option=option), self.ratio(option=option)) 129 | 130 | def score_ratio_str(self, option=None): 131 | return "%.4f (%.2f)" % self.score_ratio(option) 132 | 133 | def reflen(self, option=None): 134 | self.compute_score(option=option) 135 | return self._reflen 136 | 137 | def testlen(self, option=None): 138 | self.compute_score(option=option) 139 | return self._testlen 140 | 141 | def retest(self, new_test): 142 | if type(new_test) is str: 143 | new_test = [new_test] 144 | assert len(new_test) == len(self.crefs), new_test 145 | self.ctest = [] 146 | for t, rs in zip(new_test, self.crefs): 147 | self.ctest.append(cook_test(t, rs)) 148 | self._score = None 149 | 150 | return self 151 | 152 | def rescore(self, new_test): 153 | ''' replace test(s) with new test(s), and returns the new score.''' 154 | 155 | return self.retest(new_test).compute_score() 156 | 157 | def size(self): 158 | assert len(self.crefs) == len(self.ctest), "refs/test mismatch! %d<>%d" % (len(self.crefs), len(self.ctest)) 159 | return len(self.crefs) 160 | 161 | def __iadd__(self, other): 162 | '''add an instance (e.g., from another sentence).''' 163 | 164 | if type(other) is tuple: 165 | ## avoid creating new BleuScorer instances 166 | self.cook_append(other[0], other[1]) 167 | else: 168 | assert self.compatible(other), "incompatible BLEUs." 169 | self.ctest.extend(other.ctest) 170 | self.crefs.extend(other.crefs) 171 | self._score = None ## need to recompute 172 | 173 | return self 174 | 175 | def compatible(self, other): 176 | return isinstance(other, BleuScorer) and self.n == other.n 177 | 178 | def single_reflen(self, option="average"): 179 | return self._single_reflen(self.crefs[0][0], option) 180 | 181 | def _single_reflen(self, reflens, option=None, testlen=None): 182 | 183 | if option == "shortest": 184 | reflen = min(reflens) 185 | elif option == "average": 186 | reflen = float(sum(reflens))/len(reflens) 187 | elif option == "closest": 188 | reflen = min((abs(l-testlen), l) for l in reflens)[1] 189 | else: 190 | assert False, "unsupported reflen option %s" % option 191 | 192 | return reflen 193 | 194 | def recompute_score(self, option=None, verbose=0): 195 | self._score = None 196 | return self.compute_score(option, verbose) 197 | 198 | def compute_score(self, option=None, verbose=0): 199 | n = self.n 200 | small = 1e-9 201 | tiny = 1e-15 ## so that if guess is 0 still return 0 202 | bleu_list = [[] for _ in range(n)] 203 | 204 | if self._score is not None: 205 | return self._score 206 | 207 | if option is None: 208 | option = "average" if len(self.crefs) == 1 else "closest" 209 | 210 | self._testlen = 0 211 | self._reflen = 0 212 | totalcomps = {'testlen':0, 'reflen':0, 'guess':[0]*n, 'correct':[0]*n} 213 | 214 | # for each sentence 215 | for comps in self.ctest: 216 | testlen = comps['testlen'] 217 | self._testlen += testlen 218 | 219 | if self.special_reflen is None: ## need computation 220 | reflen = self._single_reflen(comps['reflen'], option, testlen) 221 | else: 222 | reflen = self.special_reflen 223 | 224 | self._reflen += reflen 225 | 226 | for key in ['guess','correct']: 227 | for k in range(n): 228 | totalcomps[key][k] += comps[key][k] 229 | 230 | # append per image bleu score 231 | bleu = 1. 232 | for k in range(n): 233 | bleu *= (float(comps['correct'][k]) + tiny) \ 234 | /(float(comps['guess'][k]) + small) 235 | bleu_list[k].append(bleu ** (1./(k+1))) 236 | ratio = (testlen + tiny) / (reflen + small) ## N.B.: avoid zero division 237 | if ratio < 1: 238 | for k in range(n): 239 | bleu_list[k][-1] *= math.exp(1 - 1/ratio) 240 | 241 | if verbose > 1: 242 | print(comps, reflen) 243 | 244 | totalcomps['reflen'] = self._reflen 245 | totalcomps['testlen'] = self._testlen 246 | 247 | bleus = [] 248 | bleu = 1. 249 | for k in range(n): 250 | bleu *= float(totalcomps['correct'][k] + tiny) \ 251 | / (totalcomps['guess'][k] + small) 252 | bleus.append(bleu ** (1./(k+1))) 253 | ratio = (self._testlen + tiny) / (self._reflen + small) ## N.B.: avoid zero division 254 | if ratio < 1: 255 | for k in range(n): 256 | bleus[k] *= math.exp(1 - 1/ratio) 257 | 258 | if verbose > 0: 259 | print(totalcomps) 260 | print("ratio:", ratio) 261 | 262 | self._score = bleus 263 | return self._score, bleu_list 264 | -------------------------------------------------------------------------------- /pycocoevalcap/cider/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /pycocoevalcap/cider/cider.py: -------------------------------------------------------------------------------- 1 | # Filename: cider.py 2 | # 3 | # Description: Describes the class to compute the CIDEr (Consensus-Based Image Description Evaluation) Metric 4 | # by Vedantam, Zitnick, and Parikh (http://arxiv.org/abs/1411.5726) 5 | # 6 | # Creation Date: Sun Feb 8 14:16:54 2015 7 | # 8 | # Authors: Ramakrishna Vedantam and Tsung-Yi Lin 9 | 10 | from .cider_scorer import CiderScorer 11 | import pdb 12 | 13 | class Cider: 14 | """ 15 | Main Class to compute the CIDEr metric 16 | 17 | """ 18 | def __init__(self, test=None, refs=None, n=4, sigma=6.0): 19 | # set cider to sum over 1 to 4-grams 20 | self._n = n 21 | # set the standard deviation parameter for gaussian penalty 22 | self._sigma = sigma 23 | 24 | def compute_score(self, gts, res): 25 | """ 26 | Main function to compute CIDEr score 27 | :param hypo_for_image (dict) : dictionary with key and value 28 | ref_for_image (dict) : dictionary with key and value 29 | :return: cider (float) : computed CIDEr score for the corpus 30 | """ 31 | 32 | assert(list(gts.keys()) == list(res.keys())) 33 | imgIds = list(gts.keys()) 34 | 35 | cider_scorer = CiderScorer(n=self._n, sigma=self._sigma) 36 | 37 | for id in imgIds: 38 | hypo = res[id] 39 | ref = gts[id] 40 | 41 | # Sanity check. 42 | assert(type(hypo) is list) 43 | assert(len(hypo) == 1) 44 | assert(type(ref) is list) 45 | assert(len(ref) > 0) 46 | 47 | cider_scorer += (hypo[0], ref) 48 | 49 | (score, scores) = cider_scorer.compute_score() 50 | 51 | return score, scores 52 | 53 | def method(self): 54 | return "CIDEr" -------------------------------------------------------------------------------- /pycocoevalcap/cider/cider_scorer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Tsung-Yi Lin 3 | # Ramakrishna Vedantam 4 | 5 | import copy 6 | from collections import defaultdict 7 | import numpy as np 8 | import pdb 9 | import math 10 | 11 | def precook(s, n=4, out=False): 12 | """ 13 | Takes a string as input and returns an object that can be given to 14 | either cook_refs or cook_test. This is optional: cook_refs and cook_test 15 | can take string arguments as well. 16 | :param s: string : sentence to be converted into ngrams 17 | :param n: int : number of ngrams for which representation is calculated 18 | :return: term frequency vector for occuring ngrams 19 | """ 20 | words = s.split() 21 | counts = defaultdict(int) 22 | for k in range(1,n+1): 23 | for i in range(len(words)-k+1): 24 | ngram = tuple(words[i:i+k]) 25 | counts[ngram] += 1 26 | return counts 27 | 28 | def cook_refs(refs, n=4): ## lhuang: oracle will call with "average" 29 | '''Takes a list of reference sentences for a single segment 30 | and returns an object that encapsulates everything that BLEU 31 | needs to know about them. 32 | :param refs: list of string : reference sentences for some image 33 | :param n: int : number of ngrams for which (ngram) representation is calculated 34 | :return: result (list of dict) 35 | ''' 36 | return [precook(ref, n) for ref in refs] 37 | 38 | def cook_test(test, n=4): 39 | '''Takes a test sentence and returns an object that 40 | encapsulates everything that BLEU needs to know about it. 41 | :param test: list of string : hypothesis sentence for some image 42 | :param n: int : number of ngrams for which (ngram) representation is calculated 43 | :return: result (dict) 44 | ''' 45 | return precook(test, n, True) 46 | 47 | class CiderScorer(object): 48 | """CIDEr scorer. 49 | """ 50 | 51 | def copy(self): 52 | ''' copy the refs.''' 53 | new = CiderScorer(n=self.n) 54 | new.ctest = copy.copy(self.ctest) 55 | new.crefs = copy.copy(self.crefs) 56 | return new 57 | 58 | def __init__(self, test=None, refs=None, n=4, sigma=6.0): 59 | ''' singular instance ''' 60 | self.n = n 61 | self.sigma = sigma 62 | self.crefs = [] 63 | self.ctest = [] 64 | self.document_frequency = defaultdict(float) 65 | self.cook_append(test, refs) 66 | self.ref_len = None 67 | 68 | def cook_append(self, test, refs): 69 | '''called by constructor and __iadd__ to avoid creating new instances.''' 70 | 71 | if refs is not None: 72 | self.crefs.append(cook_refs(refs)) 73 | if test is not None: 74 | self.ctest.append(cook_test(test)) ## N.B.: -1 75 | else: 76 | self.ctest.append(None) # lens of crefs and ctest have to match 77 | 78 | def size(self): 79 | assert len(self.crefs) == len(self.ctest), "refs/test mismatch! %d<>%d" % (len(self.crefs), len(self.ctest)) 80 | return len(self.crefs) 81 | 82 | def __iadd__(self, other): 83 | '''add an instance (e.g., from another sentence).''' 84 | 85 | if type(other) is tuple: 86 | ## avoid creating new CiderScorer instances 87 | self.cook_append(other[0], other[1]) 88 | else: 89 | self.ctest.extend(other.ctest) 90 | self.crefs.extend(other.crefs) 91 | 92 | return self 93 | def compute_doc_freq(self): 94 | ''' 95 | Compute term frequency for reference data. 96 | This will be used to compute idf (inverse document frequency later) 97 | The term frequency is stored in the object 98 | :return: None 99 | ''' 100 | for refs in self.crefs: 101 | # refs, k ref captions of one image 102 | for ngram in set([ngram for ref in refs for (ngram,count) in ref.items()]): 103 | self.document_frequency[ngram] += 1 104 | # maxcounts[ngram] = max(maxcounts.get(ngram,0), count) 105 | 106 | def compute_cider(self): 107 | def counts2vec(cnts): 108 | """ 109 | Function maps counts of ngram to vector of tfidf weights. 110 | The function returns vec, an array of dictionary that store mapping of n-gram and tf-idf weights. 111 | The n-th entry of array denotes length of n-grams. 112 | :param cnts: 113 | :return: vec (array of dict), norm (array of float), length (int) 114 | """ 115 | vec = [defaultdict(float) for _ in range(self.n)] 116 | length = 0 117 | norm = [0.0 for _ in range(self.n)] 118 | for (ngram,term_freq) in cnts.items(): 119 | # give word count 1 if it doesn't appear in reference corpus 120 | df = np.log(max(1.0, self.document_frequency[ngram])) 121 | # ngram index 122 | n = len(ngram)-1 123 | # tf (term_freq) * idf (precomputed idf) for n-grams 124 | vec[n][ngram] = float(term_freq)*(self.ref_len - df) 125 | # compute norm for the vector. the norm will be used for computing similarity 126 | norm[n] += pow(vec[n][ngram], 2) 127 | 128 | if n == 1: 129 | length += term_freq 130 | norm = [np.sqrt(n) for n in norm] 131 | return vec, norm, length 132 | 133 | def sim(vec_hyp, vec_ref, norm_hyp, norm_ref, length_hyp, length_ref): 134 | ''' 135 | Compute the cosine similarity of two vectors. 136 | :param vec_hyp: array of dictionary for vector corresponding to hypothesis 137 | :param vec_ref: array of dictionary for vector corresponding to reference 138 | :param norm_hyp: array of float for vector corresponding to hypothesis 139 | :param norm_ref: array of float for vector corresponding to reference 140 | :param length_hyp: int containing length of hypothesis 141 | :param length_ref: int containing length of reference 142 | :return: array of score for each n-grams cosine similarity 143 | ''' 144 | delta = float(length_hyp - length_ref) 145 | # measure consine similarity 146 | val = np.array([0.0 for _ in range(self.n)]) 147 | for n in range(self.n): 148 | # ngram 149 | for (ngram,count) in vec_hyp[n].items(): 150 | # vrama91 : added clipping 151 | val[n] += min(vec_hyp[n][ngram], vec_ref[n][ngram]) * vec_ref[n][ngram] 152 | 153 | if (norm_hyp[n] != 0) and (norm_ref[n] != 0): 154 | val[n] /= (norm_hyp[n]*norm_ref[n]) 155 | 156 | assert(not math.isnan(val[n])) 157 | # vrama91: added a length based gaussian penalty 158 | val[n] *= np.e**(-(delta**2)/(2*self.sigma**2)) 159 | return val 160 | 161 | # compute log reference length 162 | self.ref_len = np.log(float(len(self.crefs))) 163 | if len(self.crefs) == 1: 164 | self.ref_len = 1 165 | scores = [] 166 | for test, refs in zip(self.ctest, self.crefs): 167 | # compute vector for test captions 168 | vec, norm, length = counts2vec(test) 169 | # compute vector for ref captions 170 | score = np.array([0.0 for _ in range(self.n)]) 171 | for ref in refs: 172 | vec_ref, norm_ref, length_ref = counts2vec(ref) 173 | score += sim(vec, vec_ref, norm, norm_ref, length, length_ref) 174 | # change by vrama91 - mean of ngram scores, instead of sum 175 | score_avg = np.mean(score) 176 | # divide by number of references 177 | score_avg /= len(refs) 178 | # multiply score by 10 179 | score_avg *= 10.0 180 | # append score of an image to the score list 181 | scores.append(score_avg) 182 | return scores 183 | 184 | def compute_score(self, option=None, verbose=0): 185 | # compute idf 186 | self.compute_doc_freq() 187 | # assert to check document frequency 188 | assert(len(self.ctest) >= max(self.document_frequency.values())) 189 | # compute cider score 190 | score = self.compute_cider() 191 | # debug 192 | # print score 193 | return np.mean(np.array(score)), np.array(score) 194 | -------------------------------------------------------------------------------- /pycocoevalcap/eval.py: -------------------------------------------------------------------------------- 1 | from .bleu.bleu import Bleu 2 | from .cider.cider import Cider 3 | from .meteor.meteor import Meteor 4 | from .rouge.rouge import Rouge 5 | from .spice.spice import Spice 6 | from .wmd.wmd import WMD 7 | 8 | def eval(gts,res): 9 | scorer = Bleu(n=4) 10 | s1, _ = scorer.compute_score(gts, res) 11 | 12 | scorer = Cider() 13 | s2, _ = scorer.compute_score(gts, res) 14 | 15 | scorer = Meteor() 16 | s3, _ = scorer.compute_score(gts, res) 17 | 18 | scorer = Rouge() 19 | s4, _ = scorer.compute_score(gts, res) 20 | 21 | scorer = Spice() 22 | s5, _ = scorer.compute_score(gts, res) 23 | 24 | scorer = WMD() 25 | s6, _ = scorer.compute_score(gts, res) 26 | 27 | return {'bleu':s1,'cider':s2,'meteor':s3,'rouge':s4,'spice':s5,'wmd':s6} 28 | 29 | def get_bleu(gts,res): 30 | scorer = Bleu(n=4) 31 | s, _ = scorer.compute_score(gts, res) 32 | return s 33 | 34 | def get_meteor(gts, res): 35 | scorer = Meteor() 36 | s, _ = scorer.compute_score(gts, res) 37 | return s 38 | 39 | def get_cider(gts, res): 40 | scorer = Cider() 41 | s, _ = scorer.compute_score(gts, res) 42 | return s 43 | 44 | def get_rouge(gts, res): 45 | scorer = Rouge() 46 | s, _ = scorer.compute_score(gts, res) 47 | return s 48 | 49 | 50 | def get_spice(gts, res): 51 | scorer = Spice() 52 | s, _ = scorer.compute_score(gts, res) 53 | return s 54 | 55 | 56 | def get_wmd(gts, res): 57 | scorer = WMD() 58 | s, _ = scorer.compute_score(gts, res) 59 | return s 60 | -------------------------------------------------------------------------------- /pycocoevalcap/meteor/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /pycocoevalcap/meteor/data/paraphrase-en.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/meteor/data/paraphrase-en.gz -------------------------------------------------------------------------------- /pycocoevalcap/meteor/meteor-1.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/meteor/meteor-1.5.jar -------------------------------------------------------------------------------- /pycocoevalcap/meteor/meteor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Python wrapper for METEOR implementation, by Xinlei Chen 4 | # Acknowledge Michael Denkowski for the generous discussion and help 5 | 6 | import os 7 | import sys 8 | import subprocess 9 | import threading 10 | 11 | # Assumes meteor-1.5.jar is in the same directory as meteor.py. Change as needed. 12 | METEOR_JAR = 'meteor-1.5.jar' 13 | # print METEOR_JAR 14 | 15 | class Meteor: 16 | 17 | def __init__(self): 18 | self.env = os.environ 19 | self.env['LC_ALL'] = 'en_US.UTF_8' 20 | self.meteor_cmd = ['java', '-jar', '-Xmx2G', METEOR_JAR, \ 21 | '-', '-', '-stdio', '-l', 'en', '-norm'] 22 | self.meteor_p = subprocess.Popen(self.meteor_cmd, \ 23 | cwd=os.path.dirname(os.path.abspath(__file__)), \ 24 | stdin=subprocess.PIPE, \ 25 | stdout=subprocess.PIPE, \ 26 | stderr=subprocess.PIPE, 27 | env=self.env, universal_newlines=True, bufsize=1) 28 | # Used to guarantee thread safety 29 | self.lock = threading.Lock() 30 | 31 | def compute_score(self, gts, res): 32 | assert(gts.keys() == res.keys()) 33 | imgIds = sorted(list(gts.keys())) 34 | scores = [] 35 | 36 | eval_line = 'EVAL' 37 | self.lock.acquire() 38 | for i in imgIds: 39 | assert(len(res[i]) == 1) 40 | stat = self._stat(res[i][0], gts[i]) 41 | eval_line += ' ||| {}'.format(stat) 42 | 43 | # Send to METEOR 44 | self.meteor_p.stdin.write(eval_line + '\n') 45 | 46 | # Collect segment scores 47 | for i in range(len(imgIds)): 48 | score = float(self.meteor_p.stdout.readline().strip()) 49 | scores.append(score) 50 | 51 | # Final score 52 | final_score = float(self.meteor_p.stdout.readline().strip()) 53 | self.lock.release() 54 | 55 | return final_score, scores 56 | 57 | def method(self): 58 | return "METEOR" 59 | 60 | def _stat(self, hypothesis_str, reference_list): 61 | # SCORE ||| reference 1 words ||| reference n words ||| hypothesis words 62 | hypothesis_str = hypothesis_str.replace('|||', '').replace(' ', ' ') 63 | score_line = ' ||| '.join(('SCORE', ' ||| '.join(reference_list), hypothesis_str)) 64 | self.meteor_p.stdin.write(score_line+'\n') 65 | return self.meteor_p.stdout.readline().strip() 66 | 67 | def __del__(self): 68 | self.lock.acquire() 69 | self.meteor_p.stdin.close() 70 | self.meteor_p.kill() 71 | self.meteor_p.wait() 72 | self.lock.release() 73 | -------------------------------------------------------------------------------- /pycocoevalcap/rouge/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'vrama91' 2 | -------------------------------------------------------------------------------- /pycocoevalcap/rouge/rouge.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # File Name : rouge.py 4 | # 5 | # Description : Computes ROUGE-L metric as described by Lin and Hovey (2004) 6 | # 7 | # Creation Date : 2015-01-07 06:03 8 | # Author : Ramakrishna Vedantam 9 | 10 | import numpy as np 11 | 12 | def my_lcs(string, sub): 13 | """ 14 | Calculates longest common subsequence for a pair of tokenized strings 15 | :param string : list of str : tokens from a string split using whitespace 16 | :param sub : list of str : shorter string, also split using whitespace 17 | :returns: length (list of int): length of the longest common subsequence between the two strings 18 | 19 | Note: my_lcs only gives length of the longest common subsequence, not the actual LCS 20 | """ 21 | if(len(string)< len(sub)): 22 | sub, string = string, sub 23 | 24 | lengths = [[0 for i in range(0,len(sub)+1)] for j in range(0,len(string)+1)] 25 | 26 | for j in range(1,len(sub)+1): 27 | for i in range(1,len(string)+1): 28 | if(string[i-1] == sub[j-1]): 29 | lengths[i][j] = lengths[i-1][j-1] + 1 30 | else: 31 | lengths[i][j] = max(lengths[i-1][j] , lengths[i][j-1]) 32 | 33 | return lengths[len(string)][len(sub)] 34 | 35 | class Rouge(): 36 | ''' 37 | Class for computing ROUGE-L score for a set of candidate sentences for the MS COCO test set 38 | 39 | ''' 40 | def __init__(self): 41 | # vrama91: updated the value below based on discussion with Hovey 42 | self.beta = 1.2 43 | 44 | def calc_score(self, candidate, refs): 45 | """ 46 | Compute ROUGE-L score given one candidate and references for an image 47 | :param candidate: str : candidate sentence to be evaluated 48 | :param refs: list of str : COCO reference sentences for the particular image to be evaluated 49 | :returns score: int (ROUGE-L score for the candidate evaluated against references) 50 | """ 51 | assert(len(candidate)==1) 52 | assert(len(refs)>0) 53 | prec = [] 54 | rec = [] 55 | 56 | # split into tokens 57 | token_c = candidate[0].split(" ") 58 | 59 | for reference in refs: 60 | # split into tokens 61 | token_r = reference.split(" ") 62 | # compute the longest common subsequence 63 | lcs = my_lcs(token_r, token_c) 64 | prec.append(lcs/float(len(token_c))) 65 | rec.append(lcs/float(len(token_r))) 66 | 67 | prec_max = max(prec) 68 | rec_max = max(rec) 69 | 70 | if(prec_max!=0 and rec_max !=0): 71 | score = ((1 + self.beta**2)*prec_max*rec_max)/float(rec_max + self.beta**2*prec_max) 72 | else: 73 | score = 0.0 74 | return score 75 | 76 | def compute_score(self, gts, res): 77 | """ 78 | Computes Rouge-L score given a set of reference and candidate sentences for the dataset 79 | Invoked by evaluate_captions.py 80 | :param hypo_for_image: dict : candidate / test sentences with "image name" key and "tokenized sentences" as values 81 | :param ref_for_image: dict : reference MS-COCO sentences with "image name" key and "tokenized sentences" as values 82 | :returns: average_score: float (mean ROUGE-L score computed by averaging scores for all the images) 83 | """ 84 | assert(list(gts.keys()) == list(res.keys())) 85 | imgIds = list(gts.keys()) 86 | 87 | score = [] 88 | for id in imgIds: 89 | hypo = res[id] 90 | ref = gts[id] 91 | 92 | score.append(self.calc_score(hypo, ref)) 93 | 94 | # Sanity check. 95 | assert(type(hypo) is list) 96 | assert(len(hypo) == 1) 97 | assert(type(ref) is list) 98 | assert(len(ref) > 0) 99 | 100 | average_score = np.mean(np.array(score)) 101 | return average_score, np.array(score) 102 | 103 | def method(self): 104 | return "Rouge" 105 | -------------------------------------------------------------------------------- /pycocoevalcap/spice/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/__init__.py -------------------------------------------------------------------------------- /pycocoevalcap/spice/cache/data.mdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/cache/data.mdb -------------------------------------------------------------------------------- /pycocoevalcap/spice/cache/lock.mdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/cache/lock.mdb -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/Meteor-1.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/Meteor-1.5.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/SceneGraphParser-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/SceneGraphParser-1.0.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/ejml-0.23.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/ejml-0.23.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/fst-2.47.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/fst-2.47.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/guava-19.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/guava-19.0.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/hamcrest-core-1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/hamcrest-core-1.3.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/jackson-core-2.5.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/jackson-core-2.5.3.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/javassist-3.19.0-GA.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/javassist-3.19.0-GA.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/json-simple-1.1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/json-simple-1.1.1.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/junit-4.12.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/junit-4.12.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/lmdbjni-0.4.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/lmdbjni-0.4.6.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/lmdbjni-linux64-0.4.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/lmdbjni-linux64-0.4.6.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/lmdbjni-osx64-0.4.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/lmdbjni-osx64-0.4.6.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/lmdbjni-win64-0.4.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/lmdbjni-win64-0.4.6.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/objenesis-2.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/objenesis-2.4.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/slf4j-api-1.7.12.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/slf4j-api-1.7.12.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/lib/slf4j-simple-1.7.21.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/lib/slf4j-simple-1.7.21.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/spice-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/spice-1.0.jar -------------------------------------------------------------------------------- /pycocoevalcap/spice/spice.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import os 3 | import sys 4 | import subprocess 5 | import threading 6 | import json 7 | import numpy as np 8 | import ast 9 | import tempfile 10 | 11 | # Assumes spice.jar is in the same directory as spice.py. Change as needed. 12 | SPICE_JAR = 'spice-1.0.jar' 13 | TEMP_DIR = 'tmp' 14 | CACHE_DIR = 'cache' 15 | 16 | class Spice: 17 | """ 18 | Main Class to compute the SPICE metric 19 | """ 20 | 21 | def float_convert(self, obj): 22 | try: 23 | return float(obj) 24 | except: 25 | return np.nan 26 | 27 | def compute_score(self, gts, res): 28 | assert(sorted(gts.keys()) == sorted(res.keys())) 29 | imgIds = sorted(gts.keys()) 30 | 31 | # Prepare temp input file for the SPICE scorer 32 | input_data = [] 33 | for id in imgIds: 34 | hypo = res[id] 35 | ref = gts[id] 36 | 37 | # Sanity check. 38 | assert(type(hypo) is list) 39 | assert(len(hypo) == 1) 40 | assert(type(ref) is list) 41 | assert(len(ref) >= 1) 42 | 43 | input_data.append({ 44 | "image_id" : id, 45 | "test" : hypo[0], 46 | "refs" : ref 47 | }) 48 | 49 | cwd = os.path.dirname(os.path.abspath(__file__)) 50 | temp_dir=os.path.join(cwd, TEMP_DIR) 51 | if not os.path.exists(temp_dir): 52 | os.makedirs(temp_dir) 53 | in_file = tempfile.NamedTemporaryFile(delete=False, dir=temp_dir) 54 | in_file.write(json.dumps(input_data, indent=2).encode('utf-8')) 55 | in_file.close() 56 | 57 | # Start job 58 | out_file = tempfile.NamedTemporaryFile(delete=False, dir=temp_dir) 59 | out_file.close() 60 | cache_dir=os.path.join(cwd, CACHE_DIR) 61 | if not os.path.exists(cache_dir): 62 | os.makedirs(cache_dir) 63 | spice_cmd = ['java', '-jar', '-Xmx8G', SPICE_JAR, in_file.name, 64 | '-cache', cache_dir, 65 | '-out', out_file.name, 66 | '-subset', 67 | '-silent' 68 | ] 69 | subprocess.check_call(spice_cmd, 70 | cwd=os.path.dirname(os.path.abspath(__file__))) 71 | 72 | # Read and process results 73 | with open(out_file.name) as data_file: 74 | results = json.load(data_file) 75 | os.remove(in_file.name) 76 | os.remove(out_file.name) 77 | 78 | imgId_to_scores = {} 79 | spice_scores = [] 80 | for item in results: 81 | imgId_to_scores[item['image_id']] = item['scores'] 82 | spice_scores.append(self.float_convert(item['scores']['All']['f'])) 83 | average_score = np.mean(np.array(spice_scores)) 84 | scores = [] 85 | for image_id in imgIds: 86 | # Convert none to NaN before saving scores over subcategories 87 | score_set = {} 88 | for category,score_tuple in imgId_to_scores[image_id].items(): 89 | score_set[category] = {k: self.float_convert(v) for k, v in score_tuple.items()} 90 | scores.append(score_set) 91 | return average_score, scores 92 | 93 | def method(self): 94 | return "SPICE" 95 | 96 | 97 | -------------------------------------------------------------------------------- /pycocoevalcap/spice/tmp/tmpy5tu8h2v: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/spice/tmp/tmpy5tu8h2v -------------------------------------------------------------------------------- /pycocoevalcap/tokenizer/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'hfang' 2 | -------------------------------------------------------------------------------- /pycocoevalcap/tokenizer/ptbtokenizer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # File Name : ptbtokenizer.py 4 | # 5 | # Description : Do the PTB Tokenization and remove punctuations. 6 | # 7 | # Creation Date : 29-12-2014 8 | # Last Modified : Thu Mar 19 09:53:35 2015 9 | # Authors : Hao Fang and Tsung-Yi Lin 10 | 11 | import os 12 | import sys 13 | import subprocess 14 | import tempfile 15 | import itertools 16 | 17 | # path to the stanford corenlp jar 18 | STANFORD_CORENLP_3_4_1_JAR = 'stanford-corenlp-3.4.1.jar' 19 | 20 | # punctuations to be removed from the sentences 21 | PUNCTUATIONS = ["''", "'", "``", "`", "-LRB-", "-RRB-", "-LCB-", "-RCB-", \ 22 | ".", "?", "!", ",", ":", "-", "--", "...", ";"] 23 | 24 | class PTBTokenizer: 25 | """Python wrapper of Stanford PTBTokenizer""" 26 | 27 | def tokenize(self, captions_for_image): 28 | cmd = ['java', '-cp', STANFORD_CORENLP_3_4_1_JAR, \ 29 | 'edu.stanford.nlp.process.PTBTokenizer', \ 30 | '-preserveLines', '-lowerCase'] 31 | 32 | # ====================================================== 33 | # prepare data for PTB Tokenizer 34 | # ====================================================== 35 | final_tokenized_captions_for_image = {} 36 | image_id = [k for k, v in list(captions_for_image.items()) for _ in range(len(v))] 37 | sentences = '\n'.join([c['caption'].replace('\n', ' ') for k, v in list(captions_for_image.items()) for c in v]) 38 | 39 | # ====================================================== 40 | # save sentences to temporary file 41 | # ====================================================== 42 | path_to_jar_dirname=os.path.dirname(os.path.abspath(__file__)) 43 | tmp_file = tempfile.NamedTemporaryFile(delete=False, dir=path_to_jar_dirname) 44 | tmp_file.write(sentences.encode('utf-8')) 45 | tmp_file.close() 46 | 47 | # ====================================================== 48 | # tokenize sentence 49 | # ====================================================== 50 | cmd.append(os.path.basename(tmp_file.name)) 51 | p_tokenizer = subprocess.Popen(cmd, cwd=path_to_jar_dirname, \ 52 | stdout=subprocess.PIPE) 53 | token_lines = p_tokenizer.communicate(input=sentences.rstrip())[0] 54 | lines = token_lines.decode("utf-8").split('\n') 55 | # remove temp file 56 | os.remove(tmp_file.name) 57 | 58 | # ====================================================== 59 | # create dictionary for tokenized captions 60 | # ====================================================== 61 | for k, line in zip(image_id, lines): 62 | if not k in final_tokenized_captions_for_image: 63 | final_tokenized_captions_for_image[k] = [] 64 | tokenized_caption = ' '.join([w for w in line.rstrip().split(' ') \ 65 | if w not in PUNCTUATIONS]) 66 | final_tokenized_captions_for_image[k].append(tokenized_caption) 67 | 68 | return final_tokenized_captions_for_image 69 | -------------------------------------------------------------------------------- /pycocoevalcap/tokenizer/stanford-corenlp-3.4.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricWWWW/image-caption-metrics/fa674067517faeec5759e3910abc046becff390e/pycocoevalcap/tokenizer/stanford-corenlp-3.4.1.jar -------------------------------------------------------------------------------- /pycocoevalcap/wmd/data/stopwords.txt: -------------------------------------------------------------------------------- 1 | i me my myself we our ours ourselves you your yours yourself yourselves he him his himself she her hers herself it its itself they them their theirs themselves what which who whom this that these those am is are was were be been being have has had having do does did doing a an the and but if or because as until while of at by for with about against between into through during before after above below to from up down in out on off over under again further then once here there when where why how all any both each few more most other some such no nor not only own same so than too very s t can will just don should now d ll m o re ve y ain aren couldn didn doesn hadn hasn haven isn ma mightn mustn needn shan shouldn wasn weren won wouldn -------------------------------------------------------------------------------- /pycocoevalcap/wmd/wmd.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Special thanks to Mert Kilickaya, first author of 'Re-evaluating Automatic Metrics for Image Captioning' [http://aclweb.org/anthology/E17-1019] for giving exact instructions on how to implement the Word Mover's Distance metric here. 3 | ''' 4 | 5 | import numpy as np 6 | import gensim 7 | import os 8 | 9 | class WMD: 10 | 11 | def __init__(self): 12 | with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'stopwords.txt'), 'r', encoding='utf-8') as f: 13 | self.stop_words = set(f.read().strip().split(' ')) #Stop words were taken from NLTK nltk.stopwords.words('english') 14 | self.model = gensim.models.KeyedVectors.load_word2vec_format(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'GoogleNews-vectors-negative300.bin'), binary=True) 15 | self.sigma = 1.0 16 | 17 | def calc_score(self, candidate, refs): 18 | scores = list() 19 | c_tokens = [ token for token in candidate[0].split(' ') if token not in self.stop_words ] 20 | for ref in refs: 21 | r_tokens = [ token for token in ref.split(' ') if token not in self.stop_words ] 22 | dist = self.model.wmdistance(c_tokens, r_tokens) 23 | score = np.exp(-dist/self.sigma) 24 | scores.append(score) 25 | return max(scores) 26 | 27 | def compute_score(self, gts, res): 28 | assert(sorted(gts.keys()) == sorted(res.keys())) 29 | imgIds = sorted(gts.keys()) 30 | 31 | score = [] 32 | for id in imgIds: 33 | hypo = res[id] 34 | ref = gts[id] 35 | 36 | score.append(self.calc_score(hypo, ref)) 37 | 38 | # Sanity check. 39 | assert(type(hypo) is list) 40 | assert(len(hypo) == 1) 41 | assert(type(ref) is list) 42 | assert(len(ref) >= 1) 43 | 44 | average_score = np.mean(np.array(score)) 45 | return average_score, np.array(score) 46 | 47 | def method(self): 48 | return "WMD" 49 | --------------------------------------------------------------------------------