├── .gitignore ├── README.md ├── bin └── unsupervised_similarity ├── doc └── developer.md ├── license.txt └── src ├── h2 ├── Dockerfile ├── create-tables.sql └── service ├── scala └── tiefvision-web │ ├── app │ ├── controllers │ │ ├── Application.scala │ │ ├── Configuration.scala │ │ └── TiefVisionResourcesAssets.scala │ ├── core │ │ ├── Accuracy.scala │ │ ├── Crop.scala │ │ ├── DatabaseProcessing.scala │ │ ├── ImageProcessing.scala │ │ ├── ImageSearchResult.scala │ │ └── TrainAndTestFiles.scala │ ├── db │ │ ├── BoundingBoxQueryActions.scala │ │ ├── BoundingBoxTable.scala │ │ ├── Dataset.scala │ │ ├── SimilarityQueryActions.scala │ │ └── SimilarityTable.scala │ └── views │ │ ├── editBoundingBox.scala.html │ │ ├── helpers │ │ └── rank.scala.html │ │ ├── layout │ │ ├── footer.scala.html │ │ ├── header.scala.html │ │ └── html.scala.html │ │ ├── similarityEditor.scala.html │ │ ├── similarityFinder.scala.html │ │ ├── similarityFinderUploadForm.scala.html │ │ └── similarityGallery.scala.html │ ├── build.sbt │ ├── conf │ ├── application.conf │ ├── logback.xml │ └── routes │ ├── project │ ├── build.properties │ └── plugins.sbt │ ├── public │ ├── css │ │ └── style.css │ └── javascripts │ │ ├── BoundingBoxEditor.js │ │ └── ImageSimilarityEditor.js │ └── test │ └── ImageProcessingSpec.scala └── torch ├── 0-tiefvision-commons ├── io │ ├── tiefvision_curl_io.lua │ ├── tiefvision_redis_io.lua │ ├── tiefvision_rest_io.lua │ └── tiefvision_torch_io.lua ├── is_gray.lua ├── tiefvision_commons.lua ├── tiefvision_config_loader.lua └── tiefvision_reduction.lua ├── 1-split-encoder-classifier ├── deploy.prototxt ├── deploy.prototxt.lua ├── split-encoder-classifier.lua └── synset_words.txt ├── 10-similarity-searcher-cnn-db ├── search.lua └── search_commons.lua ├── 11-similarity-searcher-cnn-file └── search.lua ├── 12-deeprank-train └── deeprank-train.lua ├── 13-deeprank-encoding └── deeprank-encoding.lua ├── 14-deeprank-db ├── deeprank-db.lua └── similarity_lib.lua ├── 15-deeprank-searcher-db └── search.lua ├── 2-encode-bounding-box-training-and-test-images └── encode-training-and-test-images.lua ├── 3-train-regression-bounding-box ├── locatorconv.lua ├── test-regression-bounding-box.lua └── train-regression-bounding-box.lua ├── 4-encode-classification-train-and-test-images └── encode-training-and-test-images.lua ├── 5-train-classification ├── classifier-conv.lua └── train-classification.lua ├── 6-bboxlib └── bboxlib.lua ├── 7-bboxes-images └── bboxes-images.lua ├── 8-similarity-db-cnn ├── generate-similarity-db.lua └── similarity_db_lib.lua ├── 9-similarity-db ├── similarity-db.lua └── similarity_lib.lua └── config.lua /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | db/ 3 | *.jpg 4 | *.t7 5 | *.model 6 | *.data 7 | src/scala/tiefvision-web/project/target 8 | *.class 9 | *.cache 10 | *.*~ 11 | test-data 12 | train-data 13 | out 14 | *.log 15 | *.swp 16 | *.jar 17 | *.backup* 18 | *.old* 19 | ilsvrc_2012_mean.t7 20 | *.jpeg 21 | *.database 22 | *.caffemodel 23 | resources 24 | data 25 | deprecated 26 | src/scala/tiefvision-web/public/bootstrap-3.3.6-dist 27 | src/scala/tiefvision-web/public/images 28 | src/scala/tiefvision-web/public/javascripts/bootstrap.min.js 29 | src/scala/tiefvision-web/public/javascripts/jquery-2.2.0.js 30 | src/scala/tiefvision-web/.idea 31 | src/scala/tiefvision-web/activator 32 | src/torch/models/ 33 | src/scala/tiefvision-web/target 34 | src/torch/data/db 35 | .idea 36 | 37 | # H2 database 38 | *.db 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TiefVision 2 | 3 | End-to-end deep learning image-similarity search engine. 4 | 5 | **TiefVision** is an integrated end-to-end image-based search engine based on deep learning. It covers **image classification**, **image location** ( [OverFeat](http://arxiv.org/pdf/1312.6229v4.pdf) ) and **image similarity** ( [Deep Ranking](http://users.eecs.northwestern.edu/~jwa368/pdfs/deep_ranking.pdf)). 6 | 7 | **TiefVision** is implemented in [Torch](http://torch.ch/) and [Play Framework](https://www.playframework.com/) (Scala version). It currently only supports Linux with CUDA-enabled GPU. 8 | 9 | # Results 10 | 11 | Here are some of the (best) results for a dataset of dresses: 12 | 13 | ![Flowers](https://lh3.googleusercontent.com/qHnAydafymTdspCFkQEutnVjDVBUhqeoqybH1_26KULE0nXQsOvyuELc2kj53PLB5gfEEM_tjenJ7TLAW5EFW39mIBmJgDJ0M6wpSaAwFwzAWHme-Y9ROlBES5S0s-5wQJrnEST_TzR7A55fX7URt0k8xH5GLDjEzGSaY7SCE96vSAveS_XTfp8FRxDa8HhdmSV1BZS5HmUawY3bhmHi9UyXY2XjLabXOPpvmUlgXsH4ee-1DoSBpugdoxXrct63gQVZVsDy07ikpUrwqM-JanSULL4FybjYIXgUqIEqUek6hrGGSHAKwD5xqhVtzOCQFnvtpTLya_2lAUBsZ5sssHYmaAJs-_NGiXasSpw4oy5hQi4Q50kqxdKgEaCtFizZgyq-7dK7UbeuXoQbvP43iF9hhA4_U16y03F04lMOIdlW5wN83PKQQ6TDHhgLaODYsOJ4ulla3ftYttgJ2WO6tpX7eUwR5hRwajB1s9kWN0ta1JA1JXO6ECFwReDjoqNa9LiG-MtrNoK_UzFKUqi620FkQ_y6KpblD8NvPiqtfoRKD8phMJMarEpjJS9XEqY9-uOd=w1788-h995-no) 14 | 15 | ![Gown](https://lh3.googleusercontent.com/r79vpytThecVvh8bCcRa20H3kqyACe_xAocu7CHsv4jpyH1pPbqpTzjhCQmtfvrsuqb9wnlqY6MtfJH7L7e2yQBcACD_LmgAYipVqv2CeUxnTvOIa9XsvvSKrwD6X07juIejfD7TC6PtXb47Xudyw0ZhG8-M0MCqO-Ues9ElAceZyKlVX_-ScImDxhh7a1s5UlsNFV2yiOekV9JLxgB2a0VxHQGoLg5IxVXDs4NsyWiNJWasLLbLYUjAIrVgo-n0MeikSwWTFCAPr9m5497BNWzzRDtc0s-hPm8-S7XoJQhkvIbYyk3u_x284oFrE9N_Uxeg5CSZOE6sEQPBRZhLzOPcJXZaoIHIx47VQCmnGqVTxbWM17Ci-_y9pXpqJ1lXKGos0pBuvqisnlcQiZyyKvL1lXjqT1B9Jy_FISCmjFlaSS21QqpwUx2rwvxfUv3olJLDE5SZZid_vhYr9F9z7b7Wiu2t8jhgyLnoOgNwULw0jf47uDM_PmSvQXcDK8KTy2MJ8CxOJ1bhfiF_YM6LVHZCHrCgF-0KwCieDoPctxS-2HPTLjxtg99-Id9uwF9R8CIK=w1798-h995-no) 16 | 17 | ![Long](https://lh3.googleusercontent.com/MRGJeRWwyf8YXrnr4YLdJLS8X11VFAskS7K23OBwzF7PxqZCQcFPJaBY97b6O9HqN569FKLcANTlaJFPkAwcXKxtOeH0nXGOrfR70baCGOGAjowSR_-x6a7ZFgfaSGSzKEG6OodX3zrH1Cjgrs2iAk1EJmv1QXe9wdrftMsN45K6DweIerN6RupMGxIXeEwr8mFyb9ZEfvjcnWdgTQ-uWV1Nn3OwV6UdHH0nxzyG5Q0-NW37kJV8LXgwV_zQmqlUFOf5gpa0NckdO0kWnY589g1X8A7FUcpWhRcgMpBhf3sjla_5GeBVUJjVM4tHnymjIE65H-B45ptFbGx0B0AWbI-9yT_-wcHoaQKkg2lZjw8pk1IJ2l7RCOWuuzJphepdgtX4Wr4oR-unY5WB8VvMlX0sayQBwyCGu709R-3zp7TPv3yrG09RTdGkev5hqxu4Gcolt6kyAcIK5cKjMlERAvcNm8ILEJSZDzVXOOhT7GQmNhH3EOk1WZcTmcNVSLr06HaJFVHenhfSld84Wa-s_a_xf2Z_m_t7gt0EMK6kgU-WCIyDD07kts5K1RPT874VLJn5=w1790-h995-no) 18 | 19 | ![Neck](https://lh3.googleusercontent.com/YzFR61IKk57KwvnASs5KUeeHOaGK_emoVBn09ahf0JqISN7oPb_KwYBsIIjToI04vGbLhe_676b67KqVDRAUS3-b5aZt39n4jTgdMYjuifhiB8fTuhDn_uewH8irGrLxNSfe-i__9Oqp-IP227Qq5xN94gOTutPrLcL2QiP4MOgqCU6oBZQv_ZHLm7bqBEaMNCU3Iyz4xkLEF1BoxWQ55AEZAkPKPmVP3LVMIYDtdhEBts7QQ6QrmBrFF-5ZjSNDx6wTwIXg_3xiennh471u2uuDzLyIHFhbjF_yFQO5rfXb2Zv5GHbgqd0Ehy7Agvlh5gs0FPa1CjTJaRcBPTFXghNpcfBsQmyfcHBMM8wY7xFGohkqyUI-iWzatHv1oEWBShzkbNWpci9xOEzx2uXAZYobFgA_ofJZjB8eAc64CR9Vi86syvSJYPVhQFNrE_BXiPf24PAq76vTItVsv1fSr12eT2IxLBMDKMM3RA8jUddCDwzS7fGz6_HaDyw2yICpZVdaVpiQTEWfwVUva_Rno8g_yi3PgNCnN8wO0yjFc9bu4dDMrUhtCtnEbf7DPM7sCzR2=w1796-h995-no) 20 | 21 | ![Black Party](https://lh3.googleusercontent.com/eNkvxzMgq3pw1R5JkI7Uu4MW1oOchVU3YQlA2i9VqnbNXwCosCceQn2nxr_FkPgB-11QAuR-2P9KyYaWkoXB0ugmRHCQFhuEb0wMTEikh1_ULgI2gpQLDogs8jVlE8lDxuuVHTohzmjAetdjNY5lpkkBhXx4RjvVEI4X8Br0kfdn4kmhoowhpljFSRQB71KUoIdtfDTxltCsadGNOX6h4gxFlJzxb7C13ZhuXDiBQpZjHWMhPzBs29hq2qMbnEGsyUCwd-tDKMYTirAe8Vg2bJjtVg95fywaSwwT0ZWjSR19x17VkG_xvGJKBsSnocZj0bEwjz0DUmaUbqQWGlp-VMBN2gH2JYN6HA0bGz_CEBcbAn1_C8PaBYeEIGiWkclNQzQm2Do0TMNJ0T6nVcvXssBghvoypJl0iE0qYf2v2tiaFTaLqS10H5hXYju9P9Si7YdJdiPMfYCCmoCRLFQzsV4GVfEoFpoDJG4LT9eT_ZJM0JEuEdVt9yphoMdhxtGqex__Alc99ojBlPfFoTWfBU_hPZ2lyUs4CPZkGHhHkmu_VrphcKXkjCS-NuIrx7AeZkhD=w1851-h990-no) 22 | 23 | ![Patterns](https://lh3.googleusercontent.com/SbvE1FhJr9o0gbRwHsnK02gCdMOw8KynPppZKGM4mEobtLQkwTaEz9QIpf_Ym5KO4Up_VshxaLdQ5MRmUjecATLsTQSQtXHVfKiMe0vzJZ2lqvblwRkO1AcsfBuWQRfP16W74VQlHMiEl2vl5YPwrd8LoGq16N659IwH58xTio8ex5TEU4YB9WcDIxZfYrOrGLsvdHFDjOEUyohfOHpCecmQ8oBX7MnU4yQeV_7PsrS_bOrYMtroxzXFlnBydu1ZXTbyg0LdsEDHJgIQX1oYpSnS49PW6JP8anDKzLSNlZ0EkmYuLPGjH0hVSJCHs5YJVMwehT_vtY-WgA7QrAK2X2ZvH0vGFmuyndRN_xGxErQI1foAzYYeotB5whzTxOcUVVCsJtdVx_uoHVRahIiLCcUi54ubJZazToleu7Mm4-lMKzUH3QUWRIsgN8ZbpWq88xo0k_4kyLFKZ5WoIsa3U1seK9Ubkf8n5YOtUV1lHbNypVgljUFG3pYqwcN5Mh2IHZDwvoGA4Te_aGhgu3BWvMS6eAsj-wkxca9uHd0s6HKsMS6ViDGPm-O4JSS784zSDS9-=w1815-h995-no) 24 | 25 | ![Squares](https://lh3.googleusercontent.com/wfTvPzHu4SFQiy-ACNFsKlq6EGFeABMbVmR4W0QsR2kVYd2RUS9JNyza8Hz7yPAbxje0Klj9It_S-WZGF1G7PQL9q8KAszqIwE3vYl_aIDwCut1UKQI-jSuAiHreEyNYCXhZrgcD9fA3cNMEt5Xyg8azZQ5LvuRHf0VrzK5KZ7VfSQseXIuZx8f8CxUf0W4xl0TTykkhpzP7Kv-sMlE5zoq6OUaC7ym-ok20n2pNPLB1ERWVXI5ecZUWEODbQdUUoueM5slTGBA0uJrwpxKeA2hWpogqtxSP5dySuIjh2YLH3pppFUpX8a3OuVumo2-SAseOuKx7eB9j7ekr8R6z5NGc9DYx5jWg5OGOLYSWWrbM3l2f0X37Z6WtDO13aVoR6YDOXUTZHo0M36ISfCm5JIuqZ6GB8_1U6T9ltnKVpTxkSr8e3FwIQO6dqV7t0v3-prg-rfuK1kRagUMtOERA6iqLvUCjvdO64TUfe1BL39-QlDkqr1qe6I0AmtKUQHqxgBfVieSci3oQbwEGK2PA3nheTUQrQNnF_N0elYGN94O_FWThAjFwPOPvQTuUshL7Yj7O=w1755-h995-no) 26 | 27 | ![Stripes](https://lh3.googleusercontent.com/Ue8HDZOxCL9bwBozV54WsXWI7kIy75o30blZIEf-bjzVUUtSXdjzR15zoNjgEK8A5VTa8ct5Z6_oXIphCgwWz30VX_98VDCUyIDKRfDiJzcvJpUvnE1szAPBpbOjvALZ3VbO9-BIc-PsRGGkAdfxegTjX2pgzyNsKx7wkjjDaTcWy-CLQrC_C573srngxdVJoYuPdBCGFXGDiqfRHkhDiDceOgMsyTD4nV5iAaZTJ27yLHVXUp4lkuEVn3tTxDtmSIM28bJreMW7VLM6eTpdHfcb2j-GC494DsBFJxMHkNp89lESRoo1OfqFkEgBWuVrhTViB75SwZWLtpQ6HS8PhayYrSMhnTKuHbeUtAVJPUuE3WKuGSC-_3DoxEVP9eW6vFoaWMVi6yoZegOOs7VUMBs0QifvSPz3VlAJ5ormA9ApJskvoXqBHiFUdXL88ogYgKG6RD_9dNCtRMgdkbGm0qxCAui7w46kb3KfWNoFFDtWCQ5Kl1yZkCLtTWew6hBlkOcdPsdXpAtb9AzNmUISDqqJSQyaJtmwy-nugJbcxeariVgFN3eVLiqeW4knxKDeh0Nz=w1748-h995-no) 28 | 29 | # Features 30 | 31 | The project is divided into two module groups: **Deep Learning Modules** and **Tooling Modules** 32 | 33 | ## Deep Learning Modules 34 | 35 | The deep learning modules included in **TiefVision** are the following: 36 | 37 | ### Transfer Learning 38 | **TiefVision** transfers a simplified (without grouping) **AlexNet** network that is used for encoding purposes. 39 | The steps involved in the transfer learning phase are the following: 40 | * It splits an already trained **AlexNet** neural network without grouping into two neural networks: 41 | * The lower convolutional part that acts as an encoder of high-level features (“image encoder") 42 | * The upper/top fully connected part that is discarded as it’s meant to classify images for other purposes (ImageNet classification). 43 | * It reduces the last max pooling step size from the encoder neural network (lower-part) to increase the spatial accuracy. 44 | 45 | ### Image Classification 46 | The image classification module performs the following steps: 47 | * It encodes all the crops from the target image (e.g. dresses) and its background using the encoder neural network: 48 | * **Target Image Crops**: crops of the images in such a way at least 50% of the crop is inside the target image bounding box. For a dataset of dresses, at least 50% of the crop contains a dress (it can include up to 50% of the background). 49 | * **Background Image Crops**: crops of the images in such a way at least 50% of the crop contains the target image background. For the example of dresses, at least 50% of the crop contains background. 50 | 51 | * It trains a fully connected neural network to classify the target image crops (e.g. dresses) and its background crops (e.g. photo studio background). 52 | 53 | ### Image Location (based on [OverFeat](http://arxiv.org/pdf/1312.6229v4.pdf)) 54 | The image location module perform the following steps: 55 | * It encodes the **Target Image Crops** dataset together with its normalized bounding box delta (distance between the bounding box upper-left point and the bounding box coordinates). 56 | * It trains four fully connected neural networks to predict the two relative bounding box points: 57 | * Two neural networks for the two dimensions of the upper-left point. 58 | * Two neural networks for the two dimensions of the lower-right point. 59 | * It extracts the bounding box filtering out background crops using the image classification neural 60 | network and averaging the bounding boxes using the bounding box neural network. 61 | 62 | ### Image Similarity ( based on [Deep Ranking](http://users.eecs.northwestern.edu/~jwa368/pdfs/deep_ranking.pdf) ) 63 | The similarity is based on the distance between two image encodings. 64 | **TiefVision** trains a neural network to map encoded images into a space in which the dot product acts as a similarity distance between images. As the encodings are normalized, the dot product computes the cosine of the angle between the encodings. 65 | 66 | Given the following triplets of images: 67 | * **H**: a reference image 68 | * **H+**: an image similar to the reference image (**H**). 69 | * **H-**: another image that is similar to **H** but not as similar as **H+**. 70 | 71 | It trains a neural network to make **H+** closer to **H** than **H-** using the Hinge loss: **l(H, H+, H-) = max(0, margin + D(H, H+) - D(H, H-))** where **D** is the dot product of the two images mapped into the neural network’s output space: **D(H1, H2) = NN(H1) · NN(H2)** 72 | 73 | 74 | ## Tooling Modules 75 | 76 | **TiefVision** includes a set of web tools to ease the generation of datasets and thus increase productivity. 77 | 78 | The current tools are the following: 79 | * Visual Bounding Box Database Editor 80 | 81 | ![Bounding Box Editor](https://lh3.googleusercontent.com/FgNHIA_p7YR-sIXkgyt_FEhxjE1vNdPAHfdcGcXlSNvMLCcqz3dmgJdyxw4ejpPcJTEPLzXwfkWWPHCj6RYlfa9T3bwKrtwb7-EV9y8J-PG--AKCXXbmgBisfJ7DlN-v8dH7nXa7oOgIMAxD2uqFCTAjsLSrt8U2ZJO1g5lu_FrFKY9D8FPMGneYcQhv3W55bULAteObgc7h-noaFn2pBkD4V42vgL_cUHwZqYvqLyaKXvJEsvv8B7qgqyJI3tjgHmPzPvozP1teFstUKBxVWZTweD-AwAqLpJ_VAZu2YuVDCKK5I1hb7yhPLIKNoTWPQWACmIPrayVTI4ZF04XkBl0BZuakegcphis9GXZyle6tfHgW9n0GTOM2FIJb9rMrc8hV3wizVxIiQghu2PYwT5nGOEG4VBL3dvTIheXsByg58V3lPrQZdqy0KAZEgAMRsxDN-GsLi48M1A57p7GZT6bSbOJHDzCiUGrHPPXognEfk6Gr7g5H2nS7RKERmjDeJS-MlbZnK4n6NUoYhE3ClV7eK6p6onTtRDRIJXoZqBQSPP5U6-A5JyPL5KrQ5sqUTYcy=w493-h873-no) 82 | 83 | * Visual Similarity Database Editor 84 | 85 | ![Similarity Editor](https://lh3.googleusercontent.com/sU5-eCYORgcuJKMQd8EZHkHg9MF37s2EyV3Awj_CdTkPScWdANOOxE96kK58aNG3Y43u8dVo6weL5uPwjPc8I-IlZVSL7Z93TOycN3X6iKlLP1Ppo0PRo2qNUmLZ_VrZmmEj4jn4qaqAU7-knyTXmWXV-I80oghscP1FN31TwinXvIhq0qBylZMLnOM9Nrdvy0NpOx6v3lcMjcN5jTOV1Atm_ClSLUU2onUVXe7qp81h6gRW_gUzfsC3bwHi4Ea3TzGo68wm-RC4bK9x_DH0G7vM7x7f7fKAv8Dw3Lx5yZG-NWMdkb-JRT2r8EcvPQ6OFFe3uluh4KO-aUVmevUcCZrBzVROpr_Ujy-r_PGkncnYzobGuws43X80ZeBbODGGhbhR92p77KUOxn3sI5NxXQnI0rutJpL2KW7712GBXQJMl0YSbl_JnCdeEaa46GlegEhdacw4DbW3QtX9o_AbWSX8dsnA58CbRW5rU49K2LEj6n_mU-5ez7VuSsa7A6tp1nn2LL0w-LmZeei1KjVQ61B4zr9jw-_nlGJhrJAOUHrfICuqerU7HcSzQnO_YV5G9oVW=w1056-h995-no) 86 | 87 | * Web File-based Image Search 88 | 89 | ![File Search](https://lh3.googleusercontent.com/2OmpaVRHg9crrWvSfL58NZzp_svwI_GZOt22eDuI8FIdP-o0kPz-kIllrlmwBm4z51_SEJUZc3ZMkfKk6Y2QAsE3n6XU90asuuEkqM-eA54OowGoILbjD_ADQ-nG1avyq8vmFYgYldXW3QC1Wdy7FKG3cuDiPd7zEMjZ-Pac5zIDBAnXE-dryiSShlaqcokdILnMsVRVOtIDb2IYZi-3-6WPpYO2UKLZ0FqahRVCXPF6-W-auUChT2pXSGRVFkfCABT0MSpOZbm_FpbxTcdRkpRNS8_9FHUNrTwNoRBdw0hWnDVj8-tJ9PjiKdV6lxWJQ9Oe1-8QyvgBBm9MkCyh_mha83gWQuQJCtxSjo4rzZSNreIwF8_HJtRdiCqWTJWl2ImJ91EhBWYkivgjvqA4uh7eFxUPpwUImKch4COlljfXbSgKWecece6WHhdpDkL5rHOEKSlY4ob-DEufaEI9CFB_zUL8VwuJrztTp4NFmV5a0A8s8MJfvXdUkcP-k-KMu0CodFPHNdtbNorvNyhWg_yiWjQ1BMaODvSa1vyPGPsqhkcg4eDbCietqTMGrO67EWpt=w1543-h377-no) 90 | 91 | * Image Gallery Browser (search upon click) 92 | 93 | ![Image Gallery Browser](https://lh3.googleusercontent.com/f0fa4rDOq0iA_E_8zW3M90d-QomvGyV2cbRR3zv0Un4sKODedHPVdnh1zdaQH6Z-lGGHhLoKf--JOvvjh2xKoBLNOUQNF1nQ22Ex9rsMT6ZI_BKeILrqICdU95_QncEIGrpg0ahZfU4mC0GECAPNPJ2PFzRak3GKFEOLK-v8eu0Xuk592fZ-ucdc33ZrAr-1VuFuiyST_m8Q14dOViD19iBDgTx_qtkG7s_ElIFTyXn7q2PULKCNKn1FvOlp4k2r-riBvfSSDb9-H29eiy1rw9KH3ZcOoFBHQq2s0RAKu3dgzY6P3PltRZFnDuyFz2nQ_G7cVCYwRltsa5QqKKc76OEiOm9OzHvtX2wjVwnSZQml50BWZLXWdbY-ZcZEFIeRQWh-kjGcwQM5t-K0JRSO6YLsjUI0TbC9j6rMTxNOc1sJAiET2Fb8Z1CefaScKW-wesWHid3nGzjGgesvurFQOOJLgoIVPjzRfm9CkmTWN-PoWbz0wnPvZsYKuR1yaE_5IJJ-VkueTjyxV7JdyWc_cB6nrsPgA99NCJ4Fg4tbqYF8PKj01txBQ4UmCN8qgg8aD7ey=w1751-h995-no) 94 | 95 | * Automated Train and Test dataset generation for: 96 | * Image (crop) classification 97 | * Bounding box regression 98 | * Image Similarity ([Deep Ranking](http://users.eecs.northwestern.edu/~jwa368/pdfs/deep_ranking.pdf)) 99 | 100 | ## Developer Guide 101 | Please refer to the [Developer Guide](doc/developer.md) for details. 102 | 103 | ## Copyright 104 | Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 105 | You may use, distribute and modify this code under the 106 | terms of the [Apache License v2](http://www.apache.org/licenses/LICENSE-2.0.txt). 107 | -------------------------------------------------------------------------------- /bin/unsupervised_similarity: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap 'exit 1' ERR 4 | 5 | ROOT=$(realpath $(dirname ${BASH_SOURCE[0]})/..) 6 | APP_FOLDER=$ROOT/src/torch 7 | 8 | ENCODED_IMAGES=/tmp/tiefvision/images 9 | K_NEAREST_NEIGHBORS=2500 10 | CONFIG=$APP_FOLDER/config.lua 11 | 12 | usage () { 13 | echo "usage: $0 image_folder [options]" 14 | echo "options:" 15 | echo " --encoded_images_folder folder to store encoded images (default: $ENCODED_IMAGES)" 16 | echo " --k-nearest-neighbors amount of similarities to store (default: $K_NEAREST_NEIGHBORS)" 17 | echo " --config config file to use (default: $CONFIG)" 18 | 19 | exit 1; 20 | } 21 | 22 | init_cli_args () { 23 | while [[ $# -ge 1 ]]; do 24 | case "$1" in 25 | --encoded-images) ENCODED_IMAGES="$2"; shift; ;; 26 | --k-nearest-neighbors) K_NEAREST_NEIGHBORS="$2"; shift; ;; 27 | --config) CONFIG="$2"; shift; ;; 28 | *) 29 | if [ -z "$IMAGES" ]; then IMAGES=$1 30 | else usage; fi 31 | ;; 32 | esac 33 | 34 | shift 35 | done 36 | 37 | if [ -z "$IMAGES" ]; then usage; fi 38 | } 39 | 40 | download () { 41 | url=https://s3.amazonaws.com/gilt-ml/$1 42 | output=$ROOT/$2 43 | 44 | mkdir -p $(dirname $output) 45 | 46 | curl $url --output $output --silent 47 | } 48 | 49 | export TIEFVISION_HOME=$ROOT 50 | 51 | init_cli_args $@ 52 | 53 | echo "Fetching models" 54 | download caffe-zoo-mirror/ilsvrc_2012_mean.t7 src/torch/models/ilsvrc_2012_mean.t7 55 | download caffe-zoo-mirror/synset_words.txt src/torch/models/synset_words.txt 56 | download caffe-zoo-mirror/nin_imagenet.caffemodel src/torch/1-split-encoder-classifier/nin_imagenet.caffemodel 57 | download caffe-zoo-mirror/Goldfish3.jpg src/torch/1-split-encoder-classifier/Goldfish3.jpg 58 | 59 | echo "Generating encoder" 60 | luajit $APP_FOLDER/1-split-encoder-classifier/split-encoder-classifier.lua -config $CONFIG 61 | 62 | echo "Generating image encoding" 63 | luajit $APP_FOLDER/8-similarity-db-cnn/generate-similarity-db.lua \ 64 | -sources $IMAGES \ 65 | -destinations $ENCODED_IMAGES \ 66 | -config $CONFIG 67 | 68 | echo "Generating image similarities" 69 | luajit $APP_FOLDER/9-similarity-db/similarity-db.lua \ 70 | -images $ENCODED_IMAGES \ 71 | -reprocess \ 72 | -kNearestNeighbors $K_NEAREST_NEIGHBORS \ 73 | -config $CONFIG 74 | -------------------------------------------------------------------------------- /doc/developer.md: -------------------------------------------------------------------------------- 1 | ## Developer Guide 2 | 3 | This document explains the different modules in **TiefVision** and how to use them. 4 | Even though it's developer oriented, it can also be used as a user manual. 5 | 6 | ### Requirements 7 | 8 | The current mandatory requirements to make **TiefVision** work are the following: 9 | 10 | * Development machine with an **nVidia CUDA** graphics card. 11 | Note that there so far no will to remove this requirement. I might move it to **OpenCL** at the time **Torch** fully supports it for all neural network layers, it performs as fast as **CUDA** and it's really mature. 12 | * **Linux OS** ( other Unix-like OSs *should* also work) 13 | * Latest version of **Torch** 14 | * Compatible **Java Development Kit 8** (recommended Oracle (Copyright) JDK ) 15 | * A database server. H2 is implemented, but postgresql can easily used instead 16 | 17 | ## Environment setup 18 | 19 | Once your machine is set up with all the required software and hardware, you'll 20 | have to follow these steps: 21 | 22 | * Clone TiefVision GIT repository: 23 | ``` 24 | git clone git@github.com:paucarre/tiefvision.git 25 | ``` 26 | * Create the *TIEFVISION_HOME* envrionment variable pointing to your cloned repository and add it into your bash profile. 27 | ``` 28 | export TIEFVISION_HOME= 29 | ``` 30 | 31 | ## Database setup 32 | 33 | **TiefVision** uses a database to store data regarding bounding boxes in images and also similarities between images. 34 | **TiefVision** has an easy to run **H2** docker image that can be built and run via **$TIEFVISION_HOME/src/h2/service**. 35 | **TiefVision**'s can be configured to use another database, like postgresql. 36 | Just change **slick.dbs.bounding_box.* ** from **$TIEFVISION_HOME/src/scala/tiefvision-web/conf/application.conf**. 37 | 38 | ## Transfer Learning 39 | 40 | To encode images TiefVision uses an already trained AlexNet-like neural network. The AlexNet network is split between the convolutional side and the fully connected side. 41 | The last max pooling layer of the convolutional side is reduce to a step size of one to increase spatial resolution. The images are encoded using the convolutional side. 42 | 43 | To split the network you have to follow these steps: 44 | ``` 45 | cd $TIEFVISION_HOME/src/torch/1-split-encoder-classifier 46 | luajit split-encoder-classifier.lua 47 | ``` 48 | 49 | ## Bounding Box Regression 50 | 51 | The first step in **TiefVision** is to detect where in the image you target object is located. For example, in the case of a dress style search engine, what we would be interested is on predicting the top-left and bottom-right corners of the bounding box that encompasses the dress. For a face search engine, you would a network able to detect the bounding box that encompasses the face in the image. 52 | 53 | First off, you'll have to first download a set of images and place them in 54 | **<$TIEFVISION_HOME>/resources/dresses-db/master**. There shouldn't be any major constraint regarding the image format but it's only tested with **JPG** and therefore that's the only supported format. 55 | 56 | Keep in mind that the file name will identify the image and therefore it's wise 57 | to name the image using some sort of database identifier so you can later on keep track of the image and the product it's related to. 58 | 59 | The next step is to create bounding boxes for those images. That's a pretty annoying task but at least with **TiefVision** you have a very basic web bounding box editor that'll make your life easier. To do that you'll have to start both the web server and also the database server. 60 | 61 | To start the web server do the following: 62 | ``` 63 | cd $TIEFVISION_HOME/src/scala/tiefvision-web 64 | activator run 65 | ``` 66 | 67 | To start the default database sever do the following: 68 | ``` 69 | $TIEFVISION_HOME/src/h2/service start 70 | ``` 71 | 72 | Once you are done open a web browser and open the [bounding box endpoint](http://localhost:9000/bounding_box). 73 | There you'll be able to create bounding boxes and store them into the database. 74 | Make sure after the your save the first bounding box that there is no error log in the web server console and that the entry is stored in the database by running 75 | the following SQL query: 76 | ```sql 77 | SELECT COUNT(*) FROM BOUNDING_BOX 78 | ``` 79 | This query must output the number of bounding boxes that are stored in the database. 80 | If the result of the query is zero and you are supposed to have saved a bounding box, there is a problem as nothing has been saved (check the logs!). 81 | 82 | Once you have already tagged enough bounding boxes (say, around 1.000) you'll need to generate the training and the test set to train the neural network. That's something that **TiefVision** automatically does for you by opening the [Bounding Box Test and Training endpoint](http://localhost:9000/generate_bounding_box_train_and_test_files). That endpoint will generate crops that have at least 50% of its area within the bounding box of the target object. It'll also generate training and test sets in such a way all the crops in the training set come from images that don't belong to the images from the test set crops. 83 | 84 | The crops are generated in **<$TIEFVISION_HOME>/resources/bounding-boxes/crops**. The training set file is generated in **<$TIEFVISION_HOME>/resources/bounding-boxes/TRAIN.txt** and the test set in 85 | **<$TIEFVISION_HOME>/resources/bounding-boxes/TEST.txt**. 86 | 87 | The next step is to encode each one of the crops by forwarding them throughout the encoder. On the other hand, the output is statistically normalized (substracted mean and divided by standard deviation). 88 | To encode the crops you'll have to do the following: 89 | ``` 90 | cd $TIEFVISION_HOME/src/torch/2-encode-bounding-box-training-and-test-images 91 | luajit encode-training-and-test-images.lua 92 | ``` 93 | 94 | The next step is to train the four neural networks needed to detect the bounding box: top, left, bottom and right. 95 | To do it you'll have to do the following: 96 | ``` 97 | luajit train-regression-bounding-box.lua -index 1 98 | luajit train-regression-bounding-box.lua -index 2 99 | luajit train-regression-bounding-box.lua -index 3 100 | luajit train-regression-bounding-box.lua -index 4 101 | ``` 102 | 103 | Finally, you can test the neural networks results by running: 104 | ``` 105 | luajit test-regression-bounding-box.lua 106 | ``` 107 | 108 | ## Image Classification 109 | 110 | Given an image, you'll need to detect where the object is located 111 | in the image. To do that, you'll have to train a classification neural network to detect foreground (your object) and background (everything else but the object). 112 | For example, if you are trying to make image processing on dresses, the foreground 113 | will be the dress and the background the photo studio and the model's extremities 114 | (head, legs, arms). 115 | 116 | For this step it's required to have the downloaded images and bounding boxes explained in **Bounding Box Regression** section. 117 | 118 | The first task is to generate the train and test data which is something **TiefVision** automatically does by opening the [Image Classification Test and Train dataset endpoint](http://localhost:9000/generate_classification_train_and_test_files). 119 | 120 | The next step is to encode all the crops for image classification: 121 | ``` 122 | cd $TIEFVISION_HOME/src/torch/4-encode-classification-train-and-test-images 123 | luajit encode-training-and-test-images.lua 124 | ``` 125 | 126 | Once all the crops are encoded, you'll need to train a classification nerual network for the foreground and the background classes. 127 | ``` 128 | luajit train-classification.lua 129 | ``` 130 | 131 | # Image Bounding Box Generation and Encoding (OverFeat) 132 | 133 | The next stage is to detect the bounding box for all the images stored in **<$TIEFVISION_HOME>/resources/dresses-db/master** and generate the crops as new JPG files which are stored in **<$TIEFVISION_HOME>/resources/dresses-db/bboxes/1** and its horizontally flipped versions in **<$TIEFVISION_HOME>/resources/dresses-db/bboxes-flipped/1**. The crops are generated with a fixed minimal dimension (width or height) of 224 which is the minimum dimension that the encoder can accept. 134 | For that run: 135 | ``` 136 | cd $TIEFVISION_HOME/src/torch/7-bboxes-images 137 | luajit bboxes-images.lua 138 | ``` 139 | 140 | The following step is to encode each one of the cropped JPG files passing them throughout the encoder network: 141 | ``` 142 | cd $TIEFVISION_HOME/src/torch/8-similarity-db-cnn 143 | luajit generate-similarity-db.lua 144 | ``` 145 | 146 | # Unsupervised Similarity Search 147 | 148 | To generate a database with all the distances between all the pairs of images in the master folder you should do the following: 149 | ``` 150 | cd $TIEFVISION_HOME/src/torch/9-similarity-db 151 | luajit similarity-db.lua 152 | ``` 153 | 154 | Once the database is generated, the images in master can be searched using that database using: 155 | ``` 156 | cd $TIEFVISION_HOME/src/torch/10-similarity-searcher-cnn-db 157 | luajit search.lua 158 | ``` 159 | 160 | # Supervised Image Similarity (Deep Rank) 161 | 162 | Deep Rank is a neural network that transforms the feature space of the images into another that is optimal for image similarity. 163 | The first step to train the neural is to generate a database of similar images: 164 | * A reference image **H** 165 | * An image **H+** similar to **H** 166 | * An image **H-** similar to **H** but not as similar as **H+** 167 | 168 | Ideally, the training set should be one in which generate a wrong ordering when using an unsupervised approach: **H**x**H+** < **H**x**H-**. The neural network **NN** should be one that generates a correct ordering: **NN**(**H**)x**NN**(**H+**) > **NN**(**H**)x**NN**(**H-**) 169 | 170 | 171 | The first step is to generate database by using the [Similarity Editor](http://localhost:9000/similarity_editor). The **H** (reference) image is the one in the top-center of the screen ('Image To Search'). Below there is a list of images. The user should first click on the image that will represent **H+** which will be framed in blue. Afterwards the user should click on the image that will be **H-** which will be framed in red. 172 | 173 | Once the database is generated the next step is to generate a training and a test set to train the neural network. That is done automatically in TiefVision using the [image similarity train and test generator endpoint](http://localhost:9000/generate_similarity_train_and_test_files). 174 | 175 | The next step is to train the neural network using a Hing loss criterion: 176 | ``` 177 | cd $TIEFVISION_HOME/src/torch/13-deeprank-train 178 | luajit deeprank-train.lua 179 | ``` 180 | 181 | Finally it's necessary to do the same encoding and database generation procedure done for the unsupervised case in order to use it in the search engine. 182 | * Encoding of the images into the new space generated by the trained neural network 183 | ``` 184 | cd $TIEFVISION_HOME/src/torch/14-deeprank-encoding 185 | luajit deeprank-encoding.lua 186 | ``` 187 | * Generate database of similarity between each pair of images in the master folder 188 | ``` 189 | cd $TIEFVISION_HOME/src/torch/15-deeprank-db 190 | luajit deeprank-db.lua 191 | ``` 192 | 193 | Once the database is generated, the images in master can be searched using that database using: 194 | ``` 195 | cd $TIEFVISION_HOME/src/torch/15-deeprank-searcher-db 196 | luajit search.lua 197 | ``` 198 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /src/h2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-alpine 2 | 3 | ENV H2_VERSION 1.4.192 4 | 5 | COPY create-tables.sql /root/create-tables.sql 6 | 7 | RUN \ 8 | mkdir /opt && \ 9 | wget http://repo1.maven.org/maven2/com/h2database/h2/$H2_VERSION/h2-$H2_VERSION.jar -O /opt/h2.jar -q 10 | 11 | EXPOSE 8082 9092 12 | 13 | ENTRYPOINT \ 14 | java -cp /opt/h2.jar org.h2.tools.RunScript -url jdbc:h2:~/tiefvision -user sa -script /root/create-tables.sql && \ 15 | java -cp /opt/h2.jar org.h2.tools.Server -tcp -tcpAllowOthers -web -webAllowOthers 16 | -------------------------------------------------------------------------------- /src/h2/create-tables.sql: -------------------------------------------------------------------------------- 1 | -- COPYRIGHT (C) 2016 PAU CARRÉ CARDONA - ALL RIGHTS RESERVED 2 | -- YOU MAY USE, DISTRIBUTE AND MODIFY THIS CODE UNDER THE 3 | -- TERMS OF THE APACHE LICENSE V2.0 (HTTP://WWW.APACHE.ORG/LICENSES/LICENSE-2.0.TXT). 4 | 5 | CREATE TABLE IF NOT EXISTS "BOUNDING_BOX"( 6 | "FILE_NAME" VARCHAR(255) PRIMARY KEY, 7 | "TOP" INT NOT NULL, 8 | "LEFT" INT NOT NULL, 9 | "BOTTOM" INT NOT NULL, 10 | "RIGHT" INT NOT NULL, 11 | "WIDTH" INT NOT NULL, 12 | "HEIGHT" INT NOT NULL, 13 | "DATASET" VARCHAR(255) DEFAULT 'UNKNOWN' 14 | ); 15 | 16 | CREATE TABLE IF NOT EXISTS "SIMILARITY"( 17 | "REFERENCE" VARCHAR(255), 18 | "POSITIVE" VARCHAR(255), 19 | "NEGATIVE" VARCHAR(255), 20 | "DATASET" VARCHAR(255) DEFAULT 'UNKNOWN', 21 | PRIMARY KEY("REFERENCE", "POSITIVE", "NEGATIVE") 22 | ); 23 | -------------------------------------------------------------------------------- /src/h2/service: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | trap 'exit 1' ERR 4 | 5 | ROOT=$(realpath $(dirname ${BASH_SOURCE[0]})/../..) 6 | 7 | DOCKER_IMAGE_NAME=paucarre/tiefvision-db 8 | DOCKER_SERVICE_NAME=tiefvision-db 9 | 10 | case "$1" in 11 | start) 12 | touch $ROOT/src/h2/tiefvision.mv.db 13 | 14 | echo "Building image $DOCKER_IMAGE_NAME" 15 | docker build -t $DOCKER_IMAGE_NAME $ROOT/src/h2 16 | 17 | echo "Running container $DOCKER_IMAGE_NAME" 18 | docker run \ 19 | --detach \ 20 | --name $DOCKER_SERVICE_NAME \ 21 | --publish 8082:8082 \ 22 | --publish 9092:9092 \ 23 | --volume $ROOT/src/h2/tiefvision.mv.db:/root/tiefvision.mv.db \ 24 | $DOCKER_IMAGE_NAME 25 | ;; 26 | 27 | stop) 28 | echo "Removing container $DOCKER_IMAGE_NAME" 29 | docker rm --force $DOCKER_SERVICE_NAME 30 | ;; 31 | 32 | backup) 33 | backup_file=$2 34 | if [ -z "$backup_file" ]; then 35 | echo "Missing backup file path" 1>&2 36 | exit 1 37 | fi 38 | 39 | echo "Building image $DOCKER_IMAGE_NAME" 40 | docker build -t $DOCKER_IMAGE_NAME $ROOT/src/h2 41 | 42 | echo "Backing up data from container $DOCKER_IMAGE_NAME" 43 | docker run \ 44 | -it --rm \ 45 | --volume $ROOT/src/h2/tiefvision.mv.db:/root/tiefvision.mv.db \ 46 | --entrypoint java \ 47 | $DOCKER_IMAGE_NAME \ 48 | -cp /opt/h2.jar org.h2.tools.Shell \ 49 | -url jdbc:h2:~/tiefvision \ 50 | -user sa \ 51 | -sql SCRIPT \ 52 | | sed 1d \ 53 | | sed \$d \ 54 | > $backup_file 55 | ;; 56 | 57 | *) 58 | echo $"Usage: $0 {start|stop|backup FILE}" 59 | exit 1 60 | esac 61 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/controllers/Application.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package controllers 7 | 8 | import _root_.db._ 9 | import core.{DatabaseProcessing, ImageProcessing} 10 | import ImageProcessing._ 11 | import play.api.mvc._ 12 | import scala.concurrent.ExecutionContext.Implicits.global 13 | 14 | class Application extends Controller { 15 | 16 | lazy val ImagesGrouped = Images.toList.grouped(20).toList 17 | 18 | def index = editBoundingBox(randomImage) 19 | 20 | def similarityGallery(isSupervised: Boolean, page: Int = 1, pageGroup: Int = 1) = Action { 21 | Ok(views.html.similarityGallery(ImagesGrouped(((pageGroup - 1) * 20) + (page - 1)), isSupervised, page, pageGroup, ImagesGrouped.size / 20)) 22 | } 23 | 24 | def similarityFinderUploadForm() = Action { 25 | Ok(views.html.similarityFinderUploadForm()) 26 | } 27 | 28 | def similarityFinderUploadResults(image: String) = Action { 29 | val imageSearchResult = findSimilarImages(image, findSimilarImagesFromFileFolder, 30 | Some(s"${Configuration.HomeFolder}/${Configuration.UploadedImagesFolder}")) 31 | //TODO: supervised is hardcoded to true 32 | Ok(views.html.similarityFinder(imageSearchResult, true, "uploaded_dresses_db", "uploaded_bboxes_db")) 33 | } 34 | 35 | def upload = Action(parse.multipartFormData) { request => 36 | request.body.file("picture").map { picture => 37 | import java.io.File 38 | val filename = picture.filename 39 | val contentType = picture.contentType 40 | picture.ref.moveTo(new File(s"${Configuration.HomeFolder}/resources/dresses-db/uploaded/master/${filename}"), true) 41 | Redirect(routes.Application.similarityFinderUploadResults(filename)) 42 | }.getOrElse { 43 | Redirect(routes.Application.index).flashing( 44 | "error" -> "Missing file" 45 | ) 46 | } 47 | } 48 | 49 | def similarityFinder(isSupervised: Boolean = false) = Action { 50 | val imageSearchResult = findSimilarImages(randomSmilarityImage, SimilarityFinder.getSimilarityFinder(isSupervised)) 51 | Ok(views.html.similarityFinder(imageSearchResult, isSupervised, "dresses_db", "bboxes_db")) 52 | } 53 | 54 | def supervisedSimilarityFinderFor(image: String) = similarityFinderFor(image, true) 55 | 56 | def unsupervisedSimilarityFinderFor(image: String) = similarityFinderFor(image, false) 57 | 58 | def similarityFinderFor(image: String, isSupervised: Boolean) = Action { 59 | val imageSearchResult = findSimilarImages(image, SimilarityFinder.getSimilarityFinder(isSupervised)) 60 | Ok(views.html.similarityFinder(imageSearchResult, isSupervised, "dresses_db", "bboxes_db")) 61 | } 62 | 63 | def similarityEditor(isSupervised: Boolean = false) = similarityEditorFor(randomSmilarityImage) 64 | 65 | def similarityEditorFor(image: String) = Action.async { 66 | val imageSearchResult = findSimilarImages(image, SimilarityFinder.getSimilarityFinder(true)) 67 | SimilarityQueryActions.getSimilarityByReference(image).map { similarity => 68 | Ok(views.html.similarityEditor(imageSearchResult, "dresses_db", "bboxes_db", 69 | similarity.map(_.positive), similarity.map(_.negative))) 70 | } 71 | } 72 | 73 | def saveBoundingBox(name: String, left: Int, right: Int, top: Int, bottom: Int, width: Int, height: Int) = Action { 74 | BoundingBoxQueryActions.insertOrUpdate(BoundingBox(name = name, top = top, 75 | left = left, bottom = bottom, right = right, width = width, height = height, dataset = Dataset.UndefinedSet)) 76 | Redirect(routes.Application.index) 77 | } 78 | 79 | def saveSimilarity(reference: String, positive: String, negative: String) = Action { 80 | SimilarityQueryActions.insertOrUpdate(Similarity(reference = reference, positive = positive, 81 | negative = negative, dataset = Dataset.UndefinedSet)) 82 | Redirect(routes.Application.similarityEditor()) 83 | } 84 | 85 | def editBoundingBox(name: String) = Action.async { 86 | val boundingBoxFutOpt = BoundingBoxQueryActions.getBoundingBoxByFileName(name) 87 | boundingBoxFutOpt.map { boundingBoxOpt => 88 | boundingBoxOpt.fold { 89 | Ok(views.html.editBoundingBox(name)) 90 | } { boundingBox => 91 | Ok(views.html.editBoundingBox(name, Some(boundingBox))) 92 | } 93 | } 94 | } 95 | 96 | def generateSimilarityTrainAndTestFiles() = Action.async { 97 | DatabaseProcessing.generateSimilarityTestAndTrainFiles().map { generated => 98 | Ok("Similarity Test And Train Files Generated.") 99 | } 100 | } 101 | 102 | def generateBoundingBoxesCrops() = Action.async { 103 | DatabaseProcessing.generateBoundingBoxDatabaseImages(true).map { generated => 104 | Ok("Bounding Boxes Crops Generated.") 105 | } 106 | } 107 | 108 | def generateBoundingBoxTrainAndTestFiles() = Action.async { 109 | DatabaseProcessing.generateBoundingBoxTrainAndTestFiles().map { trainAndTestFiles => 110 | Ok("Bounding Boxe Train and Test Files Generated.") 111 | } 112 | } 113 | 114 | def generateClassificationTrainAndTestFiles() = Action.async { 115 | DatabaseProcessing.generateClassificationTrainAndTestFiles().map { trainAndTestFiles => 116 | Ok("Classification Train and Test Files Generated.") 117 | } 118 | } 119 | 120 | 121 | } 122 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/controllers/Configuration.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import play.api.Play 4 | 5 | object Configuration { 6 | 7 | lazy val HomeFolder = sys.env("TIEFVISION_HOME") 8 | val CropSize = 224 9 | val NumSamples = 5 10 | val BoundingBoxesFolder = "resources/bounding-boxes" 11 | val ScaledImagesFolder = "resources/bounding-boxes/scaled-images" 12 | val CropImagesFolder = "resources/bounding-boxes/crops" 13 | val BackgroundCropImagesFolder = "resources/bounding-boxes/background-crops" 14 | val DbImagesFolder = "resources/dresses-db/master" 15 | val UploadedImagesFolder = "resources/dresses-db/uploaded/master" 16 | val scaleLevels = Seq(2, 3) 17 | val testPercentage = 0.05 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/controllers/TiefVisionResourcesAssets.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package controllers 7 | 8 | import java.io.File 9 | import com.google.inject.Inject 10 | import play.api.{Environment, Logger, Play} 11 | import play.api.mvc._ 12 | 13 | class TiefVisionResourcesAssets @Inject()(environment: Environment) extends Controller { 14 | 15 | // Request local resources within the TIEFVISION_HOME folder (for security reasons) 16 | def atResources(rootPath: String, filePath: String) = Action { 17 | val file = new File(s"${Configuration.HomeFolder}/$rootPath/$filePath") 18 | val fileAbsolutePath = file.getAbsolutePath() 19 | 20 | if(fileAbsolutePath.startsWith(Configuration.HomeFolder)) { 21 | Ok.sendFile(file) 22 | } else { 23 | Logger.error(s"Dangerous file requested: ${fileAbsolutePath} !!!") 24 | NotFound 25 | } 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/core/Accuracy.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | 7 | package core 8 | 9 | case class Accuracy(val count: Map[AccuracyType, Int] = Map.empty, 10 | val sum: Map[AccuracyType, Int] = Map.empty) { 11 | 12 | def update(isCorrect: Boolean, isTest: Boolean, isFlipped: Boolean): Accuracy = { 13 | val performanceType = AccuracyType(isTest, isFlipped) 14 | val currentCount = count.get(performanceType).getOrElse(0) + 1 15 | val currentSum = sum.get(performanceType).getOrElse(0) + (if (isCorrect) 1 else 0) 16 | Accuracy( 17 | count = count + (performanceType -> currentCount), 18 | sum = sum + (performanceType -> currentSum) 19 | ) 20 | } 21 | 22 | // def apply(performanceType: AccuracyType): Option[Double] = { 23 | // count.get(performanceType).map { currentCount => 24 | // if(currentCount == 0) None[Double] 25 | // else sum.get(performanceType).getOrElse(0).toDouble / currentCount.toDouble 26 | // } 27 | // } 28 | 29 | } 30 | 31 | case class AccuracyType(val isTest: Boolean, isFlipped: Boolean) 32 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/core/Crop.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package core 7 | 8 | case class Crop(left: Int, right: Int, top: Int, bottom: Int) { 9 | 10 | def minus(other: Crop) = Crop(left - other.left, right - other.right, top - other.top, bottom - other.bottom) 11 | 12 | def area = (right - left + 1) * (bottom - top + 1) 13 | 14 | def intersectsMoreThan50PercentWith(boundingBoxCrop: Crop) = { 15 | var intersectMoreThan50Percent = false 16 | if (intersectsWith(boundingBoxCrop)) { 17 | val interLeft = Math.max(left, boundingBoxCrop.left) 18 | val interRight = Math.min(right, boundingBoxCrop.right) 19 | val interTop = Math.max(top, boundingBoxCrop.top) 20 | val interBottom = Math.min(bottom, boundingBoxCrop.bottom) 21 | intersectMoreThan50Percent = ((interRight - interLeft) * (interBottom - interTop)) > area / 2 22 | } 23 | intersectMoreThan50Percent 24 | } 25 | 26 | def intersectsWith(boundingBoxCrop: Crop) = 27 | Math.max(left, boundingBoxCrop.left) <= Math.min(right, boundingBoxCrop.right) && 28 | Math.max(top, boundingBoxCrop.top) <= Math.min(bottom, boundingBoxCrop.bottom) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/core/DatabaseProcessing.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package core 7 | 8 | import java.io.{BufferedWriter, File, FileOutputStream, OutputStreamWriter} 9 | import java.nio.charset.StandardCharsets 10 | import java.nio.file.{Files, Paths} 11 | 12 | import controllers.Configuration 13 | import core.ImageProcessing.SimilarityFinder 14 | import db.Dataset._ 15 | import db.{Dataset, _} 16 | 17 | import scala.concurrent.ExecutionContext.Implicits.global 18 | 19 | object DatabaseProcessing { 20 | 21 | private lazy val Images = new File(s"${Configuration.HomeFolder}/resources/dresses-db/master").listFiles().map(_.getName) 22 | private lazy val SimilarityImages = new File(s"${Configuration.HomeFolder}/src/torch/data/db/similarity/img-enc-cnn").listFiles().map(_.getName) 23 | 24 | private val random = scala.util.Random 25 | 26 | def randomImage = Images(random.nextInt(Images.length)) 27 | 28 | def getUnsupervisedAccuracy() = { 29 | SimilarityQueryActions.getAllSimilarities().map { allSimilarities => 30 | allSimilarities.foldLeft(Accuracy())((accuracy, similarity) => { 31 | val imageToDist = ImageProcessing.findSimilarImages(similarity.reference, 32 | SimilarityFinder.getSimilarityFinder(false)).distanceToSimilarImages.map(e => e._2 -> e._1).toMap 33 | val isCorrect = for { 34 | positiveDist <- imageToDist.get(similarity.positive) 35 | negativeDist <- imageToDist.get(similarity.negative) 36 | } yield { 37 | positiveDist < negativeDist 38 | } 39 | accuracy.update(isCorrect = isCorrect.getOrElse(false), isFlipped = similarity.isFlipped, isTest = similarity.isTest) 40 | }) 41 | } 42 | } 43 | 44 | def generateSimilarityTestAndTrainFiles() = { 45 | generateFlippedSimilarityDbForUnflippedReferenceImages 46 | generateRandomFlippedSimilarityDb 47 | assignSimilaritiesToTestAndTrainDataset 48 | generateTextTrainAndTestFiles 49 | } 50 | 51 | def generateTextTrainAndTestFiles() = { 52 | def text(similarity: Similarity) = s"${similarity.reference},${similarity.positive},${similarity.negative}" 53 | val similaritiesFut = SimilarityQueryActions.getAllSimilarities() 54 | similaritiesFut.map { similarities => 55 | val shuffledSimilarities = random.shuffle(similarities) 56 | val testSimilaritiesDbAsString = shuffledSimilarities.filter(_.dataset == Dataset.TestSet).map(text(_)).mkString("\n") 57 | val trainSimilaritiesDbAsString = shuffledSimilarities.filter(_.dataset == Dataset.TrainingSet).map(text(_)).mkString("\n") 58 | Files.write(Paths.get(s"${Configuration.HomeFolder}/resources/dresses-db/similarity-db-train"), trainSimilaritiesDbAsString.getBytes(StandardCharsets.UTF_8)) 59 | Files.write(Paths.get(s"${Configuration.HomeFolder}/resources/dresses-db/similarity-db-test"), testSimilaritiesDbAsString.getBytes(StandardCharsets.UTF_8)) 60 | } 61 | } 62 | 63 | def assignSimilaritiesToTestAndTrainDataset() = { 64 | for { 65 | unflippedSimilarities <- SimilarityQueryActions.getUnflippedSimilarities() 66 | flippedSimilarities <- SimilarityQueryActions.getFlippedSimilarities() 67 | } yield { 68 | assignSimilaritiesToTestAndTrainSimilarities(unflippedSimilarities) 69 | assignSimilaritiesToTestAndTrainSimilarities(flippedSimilarities) 70 | } 71 | } 72 | 73 | private def assignSimilaritiesToTestAndTrainSimilarities(similarities: Seq[Similarity]) = { 74 | val undefined = similarities.filter(_.dataset == Dataset.UndefinedSet) 75 | val test = similarities.filter(_.dataset == Dataset.TestSet) 76 | val testToAddCount = (Configuration.testPercentage * similarities.size.toDouble) - test.size.toDouble 77 | val testToAdd = random.shuffle(undefined).take(testToAddCount.toInt) 78 | testToAdd.foreach { test => 79 | SimilarityQueryActions.insertOrUpdate(test.copy(dataset = Dataset.TestSet)) 80 | } 81 | val trainToAdd = undefined.toSet &~ testToAdd.toSet 82 | trainToAdd.foreach { train => 83 | SimilarityQueryActions.insertOrUpdate(train.copy(dataset = Dataset.TrainingSet)) 84 | } 85 | } 86 | 87 | def generateRandomFlippedSimilarityDb() = 88 | for { 89 | unflippedSimilarities <- SimilarityQueryActions.getUnflippedSimilarities() 90 | flippedSimilarities <- SimilarityQueryActions.getFlippedSimilarities() 91 | } yield { 92 | val currentRandomFlippedSimilarities = flippedSimilarities.map(_.reference).toSet &~ (unflippedSimilarities.map(_.reference).toSet) 93 | val randomFlippedSimilaritiesCountToAdd = unflippedSimilarities.map(_.reference).size - currentRandomFlippedSimilarities.size 94 | val possibleFlippedSimilaritiesToAdd = SimilarityImages.toSet.&~(unflippedSimilarities.map(_.reference).toSet.union(currentRandomFlippedSimilarities)) 95 | val flippedSimilaritiesToAdd = random.shuffle(possibleFlippedSimilaritiesToAdd).take(randomFlippedSimilaritiesCountToAdd) 96 | flippedSimilaritiesToAdd.foreach { flippedSimilarityToAdd => 97 | addFlippedSimilarity(flippedSimilarityToAdd) 98 | } 99 | } 100 | 101 | private def addFlippedSimilarity(flippedSimilarityToAdd: String) = { 102 | val closestImageOpt = ImageProcessing.findSimilarImages(flippedSimilarityToAdd, 103 | SimilarityFinder.getSimilarityFinder(true)).distanceToSimilarImages.headOption 104 | closestImageOpt.map { closestImage => 105 | val closestImageName = closestImage._2 106 | SimilarityQueryActions.insertOrUpdate( 107 | Similarity( 108 | reference = flippedSimilarityToAdd, 109 | positive = flippedSimilarityToAdd, 110 | negative = closestImageName, 111 | dataset = Dataset.UndefinedSet 112 | ) 113 | ) 114 | } 115 | } 116 | 117 | def generateFlippedSimilarityDbForUnflippedReferenceImages() = 118 | for { 119 | unflippedSimilarities <- SimilarityQueryActions.getUnflippedSimilarities() 120 | flippedSimilarities <- SimilarityQueryActions.getFlippedSimilarities() 121 | } yield { 122 | val flippedSimilaritiesToAdd = unflippedSimilarities.map(_.reference).toSet.&~(flippedSimilarities.map(_.reference).toSet) 123 | flippedSimilaritiesToAdd.foreach { flippedSimilarityToAdd => 124 | addFlippedSimilarity(flippedSimilarityToAdd) 125 | } 126 | } 127 | 128 | def generateBoundingBoxTrainAndTestFiles() = 129 | generateBoundingBoxDatabaseImages(true) flatMap (imageUnit => 130 | setDatasetToUnknownDatasets() flatMap (datasetUnit => 131 | generateBoundingBoxTrainAndTestSetFiles(true) 132 | ) 133 | ) 134 | 135 | def generateClassificationTrainAndTestFiles() = 136 | generateBoundingBoxDatabaseImages(false) flatMap (imageUnit => 137 | generateBackgroundBoundingBoxDatabaseImages(false) flatMap (backgroundImageUnit => 138 | setDatasetToUnknownDatasets() flatMap (datasetUnit => 139 | generateClassificationTrainAndTestSetFiles(false) 140 | ) 141 | ) 142 | ) 143 | 144 | def generateBoundingBoxTrainAndTestSetFiles(extendedBoundingBox: Boolean) = { 145 | val boundingBoxesSeqFut = BoundingBoxQueryActions.getAllBoundingBoxes() 146 | boundingBoxesSeqFut.map { boundingBoxesSeq => 147 | generateBoundingBoxDatasetFile(boundingBoxesSeq, Dataset.TrainingSet, extendedBoundingBox) 148 | generateBoundingBoxDatasetFile(boundingBoxesSeq, Dataset.TestSet, extendedBoundingBox) 149 | } 150 | } 151 | 152 | def generateClassificationTrainAndTestSetFiles(extendBoundingBox: Boolean) = { 153 | val boundingBoxesSeqFut = BoundingBoxQueryActions.getAllBoundingBoxes() 154 | boundingBoxesSeqFut.map { boundingBoxesSeq => 155 | generateClassificationDatasetFile(boundingBoxesSeq, Dataset.TrainingSet, extendBoundingBox) 156 | generateClassificationDatasetFile(boundingBoxesSeq, Dataset.TestSet, extendBoundingBox) 157 | generateClassificationBackgroundDatasetFile(boundingBoxesSeq, Dataset.TrainingSet, extendBoundingBox) 158 | generateClassificationBackgroundDatasetFile(boundingBoxesSeq, Dataset.TestSet, extendBoundingBox) 159 | } 160 | } 161 | 162 | def generateClassificationBackgroundDatasetFile(boundingBoxes: Seq[BoundingBox], dataset: Dataset, extendBoundingBox: Boolean) = { 163 | val cropsFolderName = s"${Configuration.HomeFolder}/${Configuration.BackgroundCropImagesFolder}/${ImageProcessing.boundingBoxTypeFolder(extendBoundingBox)}" 164 | val cropsDirectoryFiles = new File(cropsFolderName).listFiles() 165 | val boundingBoxesInDataset = boundingBoxes.filter(_.dataset == dataset) 166 | val content = cropsDirectoryFiles.toSet 167 | .filter(file => boundingBoxesInDataset.find(boundingBox => file.getName startsWith boundingBox.name).isDefined) 168 | .map(file => s"$cropsFolderName/${file.getName}").mkString("\n") 169 | 170 | val writer = new BufferedWriter(new OutputStreamWriter( 171 | new FileOutputStream(s"${Configuration.HomeFolder}/${Configuration.BoundingBoxesFolder}/1-${dataset.toString.toLowerCase}.txt"), "utf-8")) 172 | writer.write(content) 173 | writer.close() 174 | } 175 | 176 | def generateClassificationDatasetFile(boundingBoxes: Seq[BoundingBox], dataset: Dataset, extendBoundingBox: Boolean) = { 177 | val cropsFolderName = s"${Configuration.HomeFolder}/${Configuration.CropImagesFolder}/${ImageProcessing.boundingBoxTypeFolder(extendBoundingBox)}" 178 | val cropsDirectoryFiles = new File(cropsFolderName).listFiles() 179 | val boundingBoxesInDataset = boundingBoxes.filter(_.dataset == dataset) 180 | val content = cropsDirectoryFiles.toSet 181 | .filter(file => boundingBoxesInDataset.find(boundingBox => file.getName startsWith boundingBox.name).isDefined) 182 | .map(file => s"$cropsFolderName/${file.getName}").mkString("\n") 183 | 184 | val writer = new BufferedWriter(new OutputStreamWriter( 185 | new FileOutputStream(s"${Configuration.HomeFolder}/${Configuration.BoundingBoxesFolder}/0-${dataset.toString.toLowerCase}.txt"), "utf-8")) 186 | writer.write(content) 187 | writer.close() 188 | } 189 | 190 | def generateBoundingBoxDatasetFile(boundingBoxes: Seq[BoundingBox], dataset: Dataset, extendBoundingBox: Boolean) = { 191 | val cropsFolderName = s"${Configuration.HomeFolder}/${Configuration.CropImagesFolder}/${ImageProcessing.boundingBoxTypeFolder(extendBoundingBox)}" 192 | val cropsDirectoryFiles = new File(cropsFolderName).listFiles() 193 | val boundingBoxesInDataset = boundingBoxes.filter(_.dataset == dataset) 194 | val content = cropsDirectoryFiles.toSet 195 | .filter(file => boundingBoxesInDataset.find(boundingBox => file.getName startsWith boundingBox.name).isDefined) 196 | .map(file => s"$cropsFolderName/${file.getName}").mkString("\n") 197 | 198 | val writer = new BufferedWriter(new OutputStreamWriter( 199 | new FileOutputStream(s"${Configuration.HomeFolder}/${Configuration.BoundingBoxesFolder}/${ImageProcessing.boundingBoxTypeFolder(extendBoundingBox)}${dataset.toString}.txt"), "utf-8")) 200 | writer.write(content) 201 | writer.close() 202 | } 203 | 204 | def setDatasetToUnknownDatasets() = { 205 | val boundingBoxesSeqFut = BoundingBoxQueryActions.getAllBoundingBoxes() 206 | boundingBoxesSeqFut.map { boundingBoxesSeq => 207 | val testCount = boundingBoxesSeq.filter(_.dataset == Dataset.TestSet).size.toDouble 208 | val undefinedBoundingBoxes = boundingBoxesSeq.filter(_.dataset == Dataset.UndefinedSet) 209 | val testSamplesCountToAdd = ((boundingBoxesSeq.size.toDouble * Configuration.testPercentage) - testCount).toInt 210 | val testSamplesToAdd = undefinedBoundingBoxes.take(testSamplesCountToAdd) 211 | val trainSamplesToAdd = undefinedBoundingBoxes.toSet &~ testSamplesToAdd.toSet 212 | testSamplesToAdd.foreach { testSampleToAdd => 213 | BoundingBoxQueryActions.insertOrUpdate(testSampleToAdd.copy(dataset = Dataset.TestSet)) 214 | } 215 | trainSamplesToAdd.foreach { trainSampleToAdd => 216 | BoundingBoxQueryActions.insertOrUpdate(trainSampleToAdd.copy(dataset = Dataset.TrainingSet)) 217 | } 218 | } 219 | } 220 | 221 | def generateBoundingBoxDatabaseImages(extendBoundingBox: Boolean) = { 222 | val boundingBoxesSeqFut = BoundingBoxQueryActions.getAllBoundingBoxes() 223 | boundingBoxesSeqFut.map { boundingBoxesSeq => 224 | boundingBoxesSeq.foreach { boundingBox => 225 | if (!cropsGenerated(boundingBox)) { 226 | Configuration.scaleLevels.foreach { scaleLevel => 227 | val scale = boundingBox.width.toDouble / (Configuration.CropSize.toDouble * scaleLevel.toDouble) 228 | val scaledBoundingBox = boundingBox div scale 229 | ImageProcessing.saveImageScaled(scaledBoundingBox, scaleLevel) 230 | ImageProcessing.generateCrops(scaledBoundingBox, scaleLevel, extendBoundingBox) 231 | } 232 | } 233 | } 234 | } 235 | } 236 | 237 | def generateBackgroundBoundingBoxDatabaseImages(extendBoundingBox: Boolean) = { 238 | val boundingBoxesSeqFut = BoundingBoxQueryActions.getAllBoundingBoxes() 239 | boundingBoxesSeqFut.map { boundingBoxesSeq => 240 | boundingBoxesSeq.foreach { boundingBox => 241 | if (!backgroundCropsGenerated(boundingBox)) { 242 | Configuration.scaleLevels.foreach { scaleLevel => 243 | val scale = boundingBox.width.toDouble / (Configuration.CropSize.toDouble * scaleLevel.toDouble) 244 | val scaledBoundingBox = boundingBox div scale 245 | ImageProcessing.saveImageScaled(scaledBoundingBox, scaleLevel) 246 | ImageProcessing.generateBackgroundCrops(scaledBoundingBox, scaleLevel, extendBoundingBox) 247 | } 248 | } 249 | } 250 | } 251 | } 252 | 253 | def cropsGenerated(scaledBoundingBox: BoundingBox): Boolean = { 254 | val cropsFolderName = s"${Configuration.HomeFolder}/${Configuration.CropImagesFolder}" 255 | val cropsDirectoryFiles = new File(cropsFolderName).listFiles() 256 | cropsDirectoryFiles.foldLeft(false)((exists, file) => exists || file.getName.startsWith(scaledBoundingBox.name)) 257 | } 258 | 259 | def backgroundCropsGenerated(scaledBoundingBox: BoundingBox): Boolean = { 260 | val cropsFolderName = s"${Configuration.HomeFolder}/${Configuration.BackgroundCropImagesFolder}" 261 | val cropsDirectoryFiles = new File(cropsFolderName).listFiles() 262 | cropsDirectoryFiles.foldLeft(false)((exists, file) => exists || file.getName.startsWith(scaledBoundingBox.name)) 263 | } 264 | 265 | } 266 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/core/ImageProcessing.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | 7 | package core 8 | 9 | import java.io.File 10 | import java.nio.file.{Files, Paths} 11 | 12 | import controllers.Configuration 13 | import db.BoundingBox 14 | import scala.sys.process.Process 15 | 16 | object ImageProcessing { 17 | 18 | lazy val Images = new File(s"${Configuration.HomeFolder}/resources/dresses-db/master").listFiles().map(_.getName) 19 | lazy val SimilarityImages = new File(s"${Configuration.HomeFolder}/src/torch/data/db/similarity/img-enc-cnn").listFiles().map(_.getName) 20 | val random = scala.util.Random 21 | 22 | def randomImage = Images(random.nextInt(Images.length)) 23 | 24 | def randomSmilarityImage = SimilarityImages(random.nextInt(SimilarityImages.length)) 25 | 26 | lazy val luaConfigFlag: String = { 27 | Option(sys.props("luaConfig")) match { 28 | case Some(path) => s"-config $path" 29 | case _ => "" 30 | } 31 | } 32 | 33 | def findSimilarImages(image: String, finderFolder: String, imagesFolderOpt: Option[String] = None): ImageSearchResult = { 34 | val luaSearch = s"luajit search.lua $image ${imagesFolderOpt.getOrElse("")} $luaConfigFlag" 35 | val finderProcessBuilder = Process(Seq("bash", "-c", luaSearch), new File(finderFolder)) 36 | val finderProcessOutput: String = finderProcessBuilder.!! 37 | val data = { 38 | def reduce(lines: Seq[(String, Double)], line: String): Seq[(String, Double)] = { 39 | similarityLineToSimilarityResult(line) match { 40 | case Some((file: String, similarity: Double)) => lines :+(file: String, similarity: Double) 41 | case None => lines 42 | } 43 | } 44 | finderProcessOutput.lines.foldLeft(Seq[(String, Double)]())(reduce) 45 | } 46 | val similaritySimilarityMap = data.map(e => { 47 | val (file: String, similarity: Double) = e 48 | similarity -> file 49 | }).sortBy(-_._1) 50 | ImageSearchResult(image, similaritySimilarityMap) 51 | } 52 | 53 | object SimilarityFinder { 54 | def getSimilarityFinder(isSupervised: Boolean) = { 55 | if(isSupervised) s"${Configuration.HomeFolder}/src/torch/15-deeprank-searcher-db" 56 | else s"${Configuration.HomeFolder}/src/torch/10-similarity-searcher-cnn-db" 57 | } 58 | } 59 | 60 | val findSimilarImagesFromFileFolder = s"${Configuration.HomeFolder}/src/torch/11-similarity-searcher-cnn-file" 61 | 62 | def similarityLineToSimilarityResult(similarityLine: String) = { 63 | val pattern = "(.+)\\s(.+)".r 64 | similarityLine match { 65 | case pattern(filePath, similarity) => Some(filePath.split("\\/").last, similarity.toDouble) 66 | case _ => None 67 | } 68 | } 69 | 70 | 71 | def saveImageScaled(scaledBoundingBox: BoundingBox, scale: Int) = { 72 | import sys.process._ 73 | val destinationFile = s"${Configuration.HomeFolder}/${Configuration.ScaledImagesFolder}/${scaledBoundingBox.name}_${scale}" 74 | if (!Files.exists(Paths.get(destinationFile))) { 75 | val sourceFile = s"${Configuration.HomeFolder}/${Configuration.DbImagesFolder}/${scaledBoundingBox.name}" 76 | val scaleImagesCommand = s"convert $sourceFile -resize ${scaledBoundingBox.width}x $destinationFile" 77 | scaleImagesCommand !!; 78 | } 79 | } 80 | 81 | def boundingBoxTypeFolder(extendBoundingBox: Boolean) = if (extendBoundingBox) "extended" else "original" 82 | 83 | def generateBackgroundCrops(scaledBoundingBox: BoundingBox, scale: Int, extendBoundingBox: Boolean) = { 84 | import sys.process._ 85 | if (scaledBoundingBox.width >= Configuration.CropSize && scaledBoundingBox.height >= Configuration.CropSize) { 86 | var samples = 0 87 | var attempts = 0 88 | while (samples < Configuration.NumSamples && attempts < Configuration.NumSamples * 5) { 89 | attempts = attempts + 1 90 | val randomCrop = generateRandomCrop(scaledBoundingBox) 91 | if (!randomCrop.intersectsWith(scaledBoundingBox.toCrop)) { 92 | val diff = scaledBoundingBox.toCrop minus randomCrop 93 | val destinationFilename = 94 | s"${scaledBoundingBox.name}___${scaledBoundingBox.width}_${scaledBoundingBox.height}_" + 95 | s"${diff.left}_${diff.top}_${diff.right + Configuration.CropSize}_${diff.bottom + Configuration.CropSize}.jpg" 96 | val destinationFilePath = s"${Configuration.HomeFolder}/${Configuration.BackgroundCropImagesFolder}/${boundingBoxTypeFolder(extendBoundingBox)}/${destinationFilename}" 97 | val sourceFilePath = s"${Configuration.HomeFolder}/${Configuration.ScaledImagesFolder}/${scaledBoundingBox.name}_${scale}" 98 | samples = samples + 1 99 | s"convert ${sourceFilePath} -crop ${Configuration.CropSize}x${Configuration.CropSize}+${randomCrop.left}+${randomCrop.top} -type truecolor ${destinationFilePath}" !!; 100 | } 101 | } 102 | } 103 | } 104 | 105 | def generateCrops(scaledBoundingBox: BoundingBox, scale: Int, extendBoundingBox: Boolean) = { 106 | import sys.process._ 107 | if (scaledBoundingBox.width >= Configuration.CropSize && scaledBoundingBox.height >= Configuration.CropSize) { 108 | var samples = 0 109 | var attempts = 0 110 | while (samples < Configuration.NumSamples && attempts < Configuration.NumSamples * 5) { 111 | attempts = attempts + 1 112 | val extendedBoundingBox = { 113 | if (extendBoundingBox) 114 | scaledBoundingBox.copy( 115 | top = math.max(1, (scaledBoundingBox.top - (scaledBoundingBox.bottom - scaledBoundingBox.top) * 0.10)).toInt, 116 | bottom = math.min(scaledBoundingBox.height, (scaledBoundingBox.bottom + (scaledBoundingBox.bottom - scaledBoundingBox.top) * 0.50)).toInt) 117 | else scaledBoundingBox 118 | } 119 | val randomCrop = generateRandomCrop(scaledBoundingBox) 120 | if (randomCrop.intersectsMoreThan50PercentWith(extendedBoundingBox.toCrop)) { 121 | val diff = scaledBoundingBox.toCrop minus randomCrop 122 | val destinationFilename = 123 | s"${scaledBoundingBox.name}___${scaledBoundingBox.width}_${scaledBoundingBox.height}_" + 124 | s"${diff.left}_${diff.top}_${diff.right + Configuration.CropSize}_${diff.bottom + Configuration.CropSize}.jpg" 125 | val destinationFilePath = s"${Configuration.HomeFolder}/${Configuration.CropImagesFolder}/${boundingBoxTypeFolder(extendBoundingBox)}/${destinationFilename}" 126 | val sourceFilePath = s"${Configuration.HomeFolder}/${Configuration.ScaledImagesFolder}/${scaledBoundingBox.name}_${scale}" 127 | samples = samples + 1 128 | s"convert ${sourceFilePath} -crop ${Configuration.CropSize}x${Configuration.CropSize}+${randomCrop.left}+${randomCrop.top} -type truecolor ${destinationFilePath}" !!; 129 | } 130 | } 131 | } 132 | } 133 | 134 | def generateRandomCrop(boundingBox: BoundingBox) = { 135 | val left = random.nextInt(boundingBox.width - Configuration.CropSize + 1) 136 | val right = left + Configuration.CropSize 137 | val top = random.nextInt(boundingBox.height - Configuration.CropSize + 1) 138 | val bottom = top + Configuration.CropSize 139 | Crop(left = left, right = right, top = top, bottom = bottom) 140 | } 141 | 142 | } 143 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/core/ImageSearchResult.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package core 7 | 8 | case class ImageSearchResult(image: String, distanceToSimilarImages: Seq[(Double, String)]) 9 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/core/TrainAndTestFiles.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package core 7 | 8 | case class TrainAndTestFiles(trainFile: String, testFile: String) 9 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/db/BoundingBoxQueryActions.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package db 7 | 8 | import java.util.concurrent.TimeUnit 9 | 10 | import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfig} 11 | import slick.driver.H2Driver.api._ 12 | import slick.driver.JdbcProfile 13 | import slick.jdbc.JdbcBackend 14 | import slick.jdbc.meta.MTable 15 | import play.api.{Play, Logger} 16 | import scala.concurrent.{Future, Await} 17 | import scala.concurrent.ExecutionContext.Implicits.global 18 | import scala.concurrent.duration.Duration 19 | 20 | object BoundingBoxQueryActions extends App with HasDatabaseConfig[JdbcProfile] { 21 | 22 | lazy val dbConfig = DatabaseConfigProvider.get[JdbcProfile]("bounding_box")(Play.current) 23 | lazy val logger: Logger = Logger(this.getClass()) 24 | lazy val boundingBoxTableQuery = TableQuery[BoundingBoxTable] 25 | 26 | def getBoundingBoxByFileName(name: String) = { 27 | val selectByName = boundingBoxTableQuery.filter{ boundingBoxTable => 28 | boundingBoxTable.name === name 29 | } 30 | db.run(selectByName.result.headOption) 31 | } 32 | 33 | def getAllBoundingBoxes() = db.run(boundingBoxTableQuery.result) 34 | 35 | def insertOrUpdate(boundingBox: BoundingBox) = { 36 | val insertOrUpdateAction = boundingBoxTableQuery.insertOrUpdate(boundingBox) 37 | val insertOrUpdateResult = db.run(insertOrUpdateAction) 38 | insertOrUpdateResult.onFailure { case err => db: JdbcBackend#DatabaseDef 39 | logger.error("Unable to insert bounding box.", err) 40 | } 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/db/BoundingBoxTable.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package db 7 | 8 | import core.Crop 9 | import db.Dataset._ 10 | import slick.driver.H2Driver.api._ 11 | 12 | 13 | class BoundingBoxTable(tag: Tag) extends Table[BoundingBox](tag, "BOUNDING_BOX") { 14 | 15 | def name = column[String]("FILE_NAME", O.PrimaryKey) 16 | 17 | def top = column[Int]("TOP") 18 | 19 | def left = column[Int]("LEFT") 20 | 21 | def bottom = column[Int]("BOTTOM") 22 | 23 | def right = column[Int]("RIGHT") 24 | 25 | def width = column[Int]("WIDTH") 26 | 27 | def height = column[Int]("HEIGHT") 28 | 29 | def dataset = column[Dataset]("DATASET") 30 | 31 | def * = (name, top, left, bottom, right, width, height, dataset) <>(BoundingBox.tupled, BoundingBox.unapply) 32 | 33 | } 34 | 35 | case class BoundingBox(name: String, top: Int, 36 | left: Int, bottom: Int, right: Int, width: Int, height: Int, dataset: Dataset) { 37 | 38 | def toCrop = Crop(left, right, top, bottom) 39 | 40 | def div(denominator: Double) = BoundingBox( 41 | name = name, 42 | top = (top.toDouble / denominator).ceil.toInt, 43 | left = (left.toDouble / denominator).ceil.toInt, 44 | bottom = (bottom.toDouble / denominator).ceil.toInt, 45 | right = (right.toDouble / denominator).ceil.toInt, 46 | width = (width.toDouble / denominator).ceil.toInt, 47 | height = (height.toDouble / denominator).ceil.toInt, 48 | dataset = dataset) 49 | } 50 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/db/Dataset.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package db 7 | import slick.driver.H2Driver.api._ 8 | 9 | object Dataset extends Enumeration { 10 | type Dataset = Value 11 | val TrainingSet = Value("TRAIN") 12 | val TestSet = Value("TEST") 13 | val UndefinedSet = Value("UNKNOWN") 14 | 15 | implicit val fDatasetMapper = MappedColumnType.base[Dataset, String]( 16 | { e => e.toString }, { s => Dataset.withName(s) } 17 | ) 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/db/SimilarityQueryActions.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package db 7 | 8 | import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfig} 9 | import play.api.{Logger, Play} 10 | import slick.driver.H2Driver.api._ 11 | import slick.driver.JdbcProfile 12 | import slick.jdbc.JdbcBackend 13 | 14 | import scala.concurrent.ExecutionContext.Implicits.global 15 | 16 | object SimilarityQueryActions extends App with HasDatabaseConfig[JdbcProfile] { 17 | 18 | lazy val dbConfig = DatabaseConfigProvider.get[JdbcProfile]("bounding_box")(Play.current) 19 | lazy val logger: Logger = Logger(this.getClass()) 20 | lazy val similarityTableQuery = TableQuery[SimilarityTable] 21 | 22 | def getSimilarityByReference(name: String) = { 23 | val selectByName = similarityTableQuery.filter{ similarityTable => 24 | similarityTable.reference === name 25 | } 26 | db.run(selectByName.result.headOption) 27 | } 28 | 29 | def getUnflippedSimilarities() = { 30 | val unflippedQuery = similarityTableQuery.filter{ similarityTable => 31 | similarityTable.reference =!= similarityTable.positive 32 | } 33 | db.run(unflippedQuery.result) 34 | } 35 | 36 | def getFlippedSimilarities() = { 37 | val flippedQuery = similarityTableQuery.filter{ similarityTable => 38 | similarityTable.reference === similarityTable.positive 39 | } 40 | db.run(flippedQuery.result) 41 | } 42 | 43 | def getAllSimilarities() = db.run(similarityTableQuery.result) 44 | 45 | def insertOrUpdate(similarity: Similarity) = { 46 | val insertOrUpdateAction = similarityTableQuery.insertOrUpdate(similarity) 47 | val insertOrUpdateResult = db.run(insertOrUpdateAction) 48 | insertOrUpdateResult.onFailure { case err => db: JdbcBackend#DatabaseDef 49 | logger.error("Unable to insert similarity.", err) 50 | } 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/db/SimilarityTable.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | package db 7 | 8 | import slick.driver.H2Driver.api._ 9 | import db.Dataset._ 10 | 11 | class SimilarityTable(tag: Tag) extends Table[Similarity](tag, "SIMILARITY") { 12 | 13 | def reference = column[String]("REFERENCE") 14 | 15 | def positive = column[String]("POSITIVE") 16 | 17 | def negative = column[String]("NEGATIVE") 18 | 19 | def dataset = column[Dataset]("DATASET") 20 | 21 | def pk = primaryKey("pk", (reference, positive, negative)) 22 | 23 | def * = (reference, positive, negative, dataset) <>(Similarity.tupled, Similarity.unapply) 24 | 25 | } 26 | 27 | case class Similarity(reference: String, positive: String, negative: String, dataset: Dataset) { 28 | 29 | def isFlipped = reference == positive 30 | def isTest = dataset == Dataset.TestSet 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/views/editBoundingBox.scala.html: -------------------------------------------------------------------------------- 1 | @* 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | *@ 6 | @(image: String, boundingBoxOpt: Option[db.BoundingBox] = None) 7 | 8 | @layout.html { 9 | 10 |
11 |
12 | 13 | 14 |
15 |

@{image}

16 | 17 | 18 |
19 | 20 | 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/views/helpers/rank.scala.html: -------------------------------------------------------------------------------- 1 | @(rank: Int) 2 | 3 | @{ 4 | rank match { 5 | case 1 => 6 | case 2 => "Second" 7 | case 3 => "Third" 8 | case 4 => "Fourth" 9 | case 5 => "Fifth" 10 | case 6 => "Sixth" 11 | case 7 => "Seventh" 12 | case 8 => "Eighth" 13 | case 9 => "Ninth" 14 | case 10 => "Tenth" 15 | case 11 => "Eleventh" 16 | case 12 => "Twelfth" 17 | case 13 => "Thirteenth" 18 | case 14 => "Fourteenth" 19 | case 15 => "Fifteenth" 20 | case 16 => "Sixteenth" 21 | case 17 => "Seventeenth" 22 | case 18 => "Eighteenth" 23 | case 19 => "Nineteenth" 24 | case 20 => "Twentieth" 25 | case _=> "Not a" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/views/layout/footer.scala.html: -------------------------------------------------------------------------------- 1 | @* 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | *@ 6 | 7 |
8 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/views/layout/header.scala.html: -------------------------------------------------------------------------------- 1 | @* 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | *@ 6 | 7 | 20 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/views/layout/html.scala.html: -------------------------------------------------------------------------------- 1 | @* 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | *@ 6 | 7 | @(content: Html) 8 | 9 | 10 | 11 | TiefVision 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | @layout.header() 24 | @content 25 | @layout.footer() 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/views/similarityEditor.scala.html: -------------------------------------------------------------------------------- 1 | @* 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | *@ 6 | @(imageSearchResult: core.ImageSearchResult, masterBase: String, bboxBase: String, positive: Option[String], negative: Option[String]) 7 | @imageSearchResultDisplay = @{imageSearchResult.distanceToSimilarImages.take(15)} 8 | 9 | @layout.html { 10 | 11 |
12 |
13 |
Image To Search
14 | 15 |
16 | 17 | 18 |
19 | 20 |
21 | @imageSearchResultDisplay.zipWithIndex.map { case ((_, image), index) => 22 |
23 |
@helpers.rank(index + 1) Closest Dress
24 | 25 |
26 | } 27 |
28 | 29 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/views/similarityFinder.scala.html: -------------------------------------------------------------------------------- 1 | @* 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | *@ 6 | @(imageSearchResult: core.ImageSearchResult, isSupervised: Boolean, masterBase: String, bboxBase: String) 7 | @imageSearchResultDisplay = @{imageSearchResult.distanceToSimilarImages.take(5)} 8 | 9 | @layout.html { 10 | 11 |
12 |
13 |
Image To Search
14 | 15 |
16 |
17 | 18 |
19 | @imageSearchResultDisplay.zipWithIndex.map { case ((_, image), index) => 20 |
21 |
@helpers.rank(index + 1) Closest Dress
22 | 23 | 24 | 25 |
26 | } 27 |
28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/views/similarityFinderUploadForm.scala.html: -------------------------------------------------------------------------------- 1 | @* 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | *@ 6 | @import helper._ 7 | 8 | @layout.html { 9 | 10 |
11 |
12 | @form(action = routes.Application.upload, 'enctype -> "multipart/form-data") { 13 |
14 | 15 | 16 |
17 | 18 | } 19 |
20 |
21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/app/views/similarityGallery.scala.html: -------------------------------------------------------------------------------- 1 | @* 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | *@ 6 | @(images: Seq[String], isSupervised: Boolean, page: Int, pageGroup: Int, pageGroups: Int) 7 | 8 | @layout.html { 9 | 10 | 11 |
12 | 33 | 34 |
35 | @for(image <- images) { 36 |
37 | 38 | 39 | 40 |
@{image}
41 |
42 | } 43 |
44 |
45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/build.sbt: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | name := """tiefvision-web""" 7 | 8 | version := "1.0-SNAPSHOT" 9 | 10 | lazy val root = (project in file(".")).enablePlugins(PlayScala) 11 | 12 | scalaVersion := "2.11.6" 13 | 14 | libraryDependencies ++= Seq( 15 | jdbc, 16 | cache, 17 | ws, 18 | "com.typesafe.slick" %% "slick" % "3.1.1", 19 | "org.slf4j" % "slf4j-nop" % "1.6.4", 20 | "com.h2database" % "h2" % "1.4.190", 21 | "org.postgresql" % "postgresql" % "9.4.1211", 22 | "com.typesafe.play" %% "play-slick" % "1.1.1", 23 | "org.scalatest" %% "scalatest" % "2.2.4" % Test 24 | ) 25 | 26 | resolvers += "scalaz-bintray" at "http://dl.bintray.com/scalaz/releases" 27 | 28 | // Play provides two styles of routers, one expects its actions to be injected, the 29 | // other, legacy style, accesses its actions statically. 30 | routesGenerator := InjectedRoutesGenerator 31 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/conf/application.conf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | # You may use, distribute and modify this code under the 3 | # terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | # Secret key 6 | play.crypto.secret = "changeme" 7 | 8 | # The application languages 9 | play.i18n.langs = [ "en" ] 10 | 11 | # Database configuration 12 | slick.dbs.bounding_box.driver="slick.driver.H2Driver$" 13 | slick.dbs.bounding_box.db.driver="org.h2.Driver" 14 | slick.dbs.bounding_box.db.url="jdbc:h2:tcp://localhost/~/tiefvision" 15 | slick.dbs.bounding_box.db.user="sa" 16 | slick.dbs.bounding_box.db.password="" 17 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/conf/logback.xml: -------------------------------------------------------------------------------- 1 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | %coloredLevel - %logger - %message%n%xException 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/conf/routes: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | # You may use, distribute and modify this code under the 3 | # terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | # Bounding box editing 6 | GET /bounding_box controllers.Application.index 7 | GET /save_bounding_box controllers.Application.saveBoundingBox(name: String, left: Int, right: Int, top: Int, bottom: Int, width: Int, height: Int) 8 | GET /edit_bounding_box controllers.Application.editBoundingBox(name: String) 9 | 10 | # Bounding box training and test sets generation 11 | GET /generate_bounding_boxes_crops controllers.Application.generateBoundingBoxesCrops() 12 | GET /generate_bounding_box_train_and_test_files controllers.Application.generateBoundingBoxTrainAndTestFiles() 13 | GET /generate_classification_train_and_test_files controllers.Application.generateClassificationTrainAndTestFiles() 14 | 15 | # Similarity finder 16 | GET / controllers.Application.similarityGallery(isSupervised: Boolean = false, page: Int = 1, pageGroup : Int = 1) 17 | GET /similarity_gallery/:isSupervised/:page/:pageGroup controllers.Application.similarityGallery(isSupervised: Boolean, page: Int, pageGroup : Int) 18 | GET /similarity_finder/supervised controllers.Application.similarityFinder(isSupervised: Boolean ?= true) 19 | GET /similarity_finder/unsupervised controllers.Application.similarityFinder(isSupervised: Boolean ?= false) 20 | GET /similarity_finder_for controllers.Application.similarityFinderFor(image: String, isSupervised: Boolean) 21 | 22 | # Similarity editor 23 | GET /similarity_editor controllers.Application.similarityEditor() 24 | GET /similarity_editor_for/:image controllers.Application.similarityEditorFor(image: String) 25 | GET /save_similarity controllers.Application.saveSimilarity(reference: String, positive: String, negative: String) 26 | GET /generate_similarity_train_and_test_files controllers.Application.generateSimilarityTrainAndTestFiles() 27 | 28 | # Similarity finder upload 29 | GET /similarity_finder_upload_form controllers.Application.similarityFinderUploadForm() 30 | POST /similarity_finder_upload_action controllers.Application.upload 31 | GET /similarity_finder_upload_results/:image controllers.Application.similarityFinderUploadResults(image: String) 32 | 33 | # Resources 34 | GET /assets/*file controllers.Assets.versioned(path="/public", file: Asset) 35 | GET /dresses_db/*file controllers.TiefVisionResourcesAssets.atResources(path="resources/dresses-db/master/", file) 36 | GET /bboxes_db/*file controllers.TiefVisionResourcesAssets.atResources(path="resources/dresses-db/bboxes/1", file) 37 | GET /uploaded_dresses_db/*file controllers.TiefVisionResourcesAssets.atResources(path="resources/dresses-db/uploaded/master", file) 38 | GET /uploaded_bboxes_db/*file controllers.TiefVisionResourcesAssets.atResources(path="resources/dresses-db/uploaded/bbox", file) 39 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/project/build.properties: -------------------------------------------------------------------------------- 1 | #Activator-generated Properties 2 | #Mon Feb 01 21:10:38 GMT 2016 3 | template.uuid=37a0ffc3-739a-47a4-95a1-2100413d0180 4 | sbt.version=0.13.8 5 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | 7 | addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.6") 8 | 9 | addSbtPlugin("com.typesafe.sbt" % "sbt-coffeescript" % "1.0.0") 10 | 11 | addSbtPlugin("com.typesafe.sbt" % "sbt-less" % "1.0.6") 12 | 13 | addSbtPlugin("com.typesafe.sbt" % "sbt-jshint" % "1.0.3") 14 | 15 | addSbtPlugin("com.typesafe.sbt" % "sbt-rjs" % "1.0.7") 16 | 17 | addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.0") 18 | 19 | addSbtPlugin("com.typesafe.sbt" % "sbt-mocha" % "1.1.0") 20 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/public/css/style.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | 7 | html, 8 | body { 9 | overflow-x: hidden; /* Prevent scroll on narrow devices */ 10 | } 11 | 12 | body { 13 | padding-top: 70px; 14 | } 15 | 16 | footer { 17 | padding: 30px 0; 18 | } 19 | 20 | .result-container { 21 | margin-top: 10px; 22 | background-color: #333; 23 | color: #DBDBDB; 24 | } 25 | 26 | .img-thumbnail { 27 | width: 337px; 28 | margin: 5px; 29 | } 30 | 31 | .img-result-positive { 32 | padding: 0px; 33 | border: 5px solid #009F4F; 34 | } 35 | 36 | .img-result-negative { 37 | padding: 0px; 38 | border: 5px solid #932327; 39 | } 40 | 41 | #bbox_canvas { 42 | border:1px solid #d3d3d3; 43 | } 44 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/public/javascripts/BoundingBoxEditor.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | var BoundingBoxEditor = function (imageName, left, right, top, bottom) { 7 | var $scope = this; 8 | $scope.canvas = document.getElementById("bbox_canvas"); 9 | $scope.repaintButton = document.getElementById("repaint_button"); 10 | $scope.saveButton = document.getElementById("save_button"); 11 | $scope.ctx = $scope.canvas.getContext("2d"); 12 | $scope.scale = 0.5; 13 | $scope.boundingBox = { name: imageName } 14 | $scope.boundingBox.left = left; 15 | $scope.boundingBox.right = right; 16 | $scope.boundingBox.top = top; 17 | $scope.boundingBox.bottom = bottom; 18 | $scope.state = "bottomRight"; 19 | console.log($scope.img) 20 | $scope.ctx.scale($scope.scale, this.scale); 21 | $scope.img = new Image(); 22 | $scope.img.addEventListener('load', function() { 23 | $scope.width = $scope.img.naturalWidth; 24 | $scope.height = $scope.img.naturalHeight; 25 | $scope.ctx.drawImage($scope.img, 0, 0, $scope.width, $scope.height); 26 | $scope.drawSavedBox() 27 | }, false); 28 | $scope.img.src = 'dresses_db/' + imageName; 29 | 30 | $scope.repaintButton.addEventListener("click", function(){ 31 | window.location = "/"; 32 | }); 33 | 34 | $scope.saveButton.addEventListener("click", function(){ 35 | window.location = "/save_bounding_box?" + 36 | "name=" + $scope.boundingBox.name + 37 | "&left=" + $scope.boundingBox.left + 38 | "&right=" + $scope.boundingBox.right + 39 | "&top=" + $scope.boundingBox.top + 40 | "&bottom=" + $scope.boundingBox.bottom + 41 | "&width=" + $scope.width + 42 | "&height=" + $scope.height; 43 | }); 44 | 45 | $scope.canvas.addEventListener('mousemove', 46 | function(evt) { 47 | var mousePos = $scope.getMousePos($scope.canvas, evt); 48 | if($scope.state === "bottomRight") { 49 | $scope.drawBBox(1, 1, mousePos.x, mousePos.y); 50 | } else if($scope.state === "topLeft") { 51 | $scope.drawBBox( 52 | mousePos.x, 53 | mousePos.y, 54 | ($scope.boundingBox.right * $scope.scale ) - mousePos.x, 55 | ($scope.boundingBox.bottom * $scope.scale ) - mousePos.y 56 | ); 57 | } 58 | } 59 | , false); 60 | 61 | $scope.canvas.addEventListener('mouseup', 62 | function(evt) { 63 | var mousePos = $scope.getMousePos($scope.canvas, evt); 64 | if($scope.state === "bottomRight") { 65 | $scope.boundingBox.right = mousePos.x / $scope.scale; 66 | $scope.boundingBox.bottom = mousePos.y / $scope.scale; 67 | $scope.state = "topLeft"; 68 | } else if($scope.state === "topLeft") { 69 | $scope.boundingBox.left = mousePos.x / $scope.scale; 70 | $scope.boundingBox.top = mousePos.y / $scope.scale; 71 | $scope.state = "finish"; 72 | } 73 | } 74 | , false); 75 | 76 | }; 77 | 78 | BoundingBoxEditor.prototype.getMousePos = function(canvas, evt) { 79 | var $scope = this; 80 | var rect = $scope.canvas.getBoundingClientRect(); 81 | return { 82 | x: evt.clientX - rect.left, 83 | y: evt.clientY - rect.top 84 | }; 85 | }; 86 | 87 | BoundingBoxEditor.prototype.drawBBox = function (xi, yi, xf, yf) { 88 | var $scope = this; 89 | $scope.ctx.drawImage($scope.img, 0, 0, $scope.width, $scope.height); 90 | $scope.ctx.beginPath(); 91 | $scope.ctx.lineWidth = "1"; 92 | $scope.ctx.strokeStyle = "blue"; 93 | $scope.ctx.rect(xi / $scope.scale, yi / $scope.scale, xf / $scope.scale, yf / $scope.scale); 94 | $scope.ctx.stroke(); 95 | $scope.drawSavedBox() 96 | }; 97 | 98 | BoundingBoxEditor.prototype.drawSavedBox = function () { 99 | var $scope = this; 100 | if($scope.boundingBox.top && $scope.boundingBox.left && $scope.boundingBox.right && $scope.boundingBox.bottom) { 101 | $scope.ctx.beginPath(); 102 | $scope.ctx.lineWidth = "1"; 103 | $scope.ctx.strokeStyle = "red"; 104 | $scope.ctx.rect( 105 | $scope.boundingBox.left, 106 | $scope.boundingBox.top, 107 | $scope.boundingBox.right - ($scope.boundingBox.left), 108 | $scope.boundingBox.bottom - ($scope.boundingBox.top) 109 | ); 110 | $scope.ctx.stroke(); 111 | } 112 | } -------------------------------------------------------------------------------- /src/scala/tiefvision-web/public/javascripts/ImageSimilarityEditor.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 3 | * You may use, distribute and modify this code under the 4 | * terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 5 | */ 6 | 7 | var ImageSimilarityEditor = function (positive, negative) { 8 | var $scope = this; 9 | 10 | $scope.imageResults = $('.img-result'); 11 | $scope.reference = document.getElementById('reference'); 12 | $scope.positive = positive; 13 | $scope.negative = negative; 14 | $scope.saveButton = document.getElementById('save_button'); 15 | 16 | // add event listeners 17 | $.each($scope.imageResults, function(index, item) { 18 | item.addEventListener("click", function(){ 19 | if($scope.positive) { 20 | if($scope.negative) { 21 | //reset 22 | $scope.positive = item; 23 | $scope.negative = null; 24 | } else { 25 | $scope.negative = item; 26 | } 27 | } else { 28 | $scope.positive = item; 29 | } 30 | $scope.paint() 31 | }); 32 | }); 33 | 34 | $scope.saveButton.addEventListener("click", function(){ 35 | if($scope.positive && $scope.reference && $scope.negative) { 36 | window.location = "/save_similarity?" + 37 | "reference=" + $scope.reference.name + 38 | "&positive=" + $scope.positive.id + 39 | "&negative=" + $scope.negative.id; 40 | } 41 | }); 42 | 43 | $scope.paint() 44 | } 45 | 46 | ImageSimilarityEditor.prototype.paint = function () { 47 | var $scope = this; 48 | $.each($scope.imageResults, function(index, item) { 49 | $(item).removeClass('img-result-positive'); 50 | $(item).removeClass('img-result-negative'); 51 | }); 52 | if($scope.positive){ 53 | $($scope.positive).addClass('img-result-positive'); 54 | } 55 | if($scope.negative){ 56 | $($scope.negative).addClass('img-result-negative'); 57 | } 58 | }; 59 | -------------------------------------------------------------------------------- /src/scala/tiefvision-web/test/ImageProcessingSpec.scala: -------------------------------------------------------------------------------- 1 | import core.Crop 2 | import imageprocessing.ImageProcessing 3 | import org.scalatest._ 4 | 5 | class ImageProcessingSpec extends FlatSpec with Matchers { 6 | 7 | "intersectsMoreThan50PercentWith" should "return true when the crop and the bounding box are the same" in { 8 | val crop = Crop(1, 20, 1, 20) 9 | val boundingBoxCrop = Crop(1, 20, 1, 20) 10 | val theyIntersect50Percent = crop.intersectsMoreThan50PercentWith(boundingBoxCrop) 11 | theyIntersect50Percent should be(true) 12 | } 13 | 14 | "intersectsMoreThan50PercentWith" should "return true when the crop and the bounding box span +50%" in { 15 | val crop = Crop(95, 120, 95, 120) 16 | val boundingBoxCrop = Crop(100, 200, 100, 200) 17 | val theyIntersect50Percent = crop.intersectsMoreThan50PercentWith(boundingBoxCrop) 18 | theyIntersect50Percent should be(true) 19 | } 20 | 21 | "intersectsMoreThan50PercentWith" should "return false when the crop and the bounding box do not intersect enough area" in { 22 | val crop = Crop(1, 20, 1, 20) 23 | val boundingBoxCrop = Crop(15, 40, 15, 40) 24 | val theyIntersect50Percent = crop.intersectsMoreThan50PercentWith(boundingBoxCrop) 25 | theyIntersect50Percent should be(false) 26 | } 27 | 28 | "intersectsMoreThan50PercentWith" should "return false when the crop and the bounding box do not intersect at all" in { 29 | val crop = Crop(10, 30, 10, 30) 30 | val boundingBoxCrop = Crop(40, 80, 40, 80) 31 | val theyIntersect50Percent = crop.intersectsMoreThan50PercentWith(boundingBoxCrop) 32 | theyIntersect50Percent should be(false) 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/torch/0-tiefvision-commons/io/tiefvision_curl_io.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- 6 | -- Reader and writer to store information thanks to redis 7 | -- 8 | 9 | local paths = require('paths') 10 | local torchFolder = paths.thisfile('../..') 11 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 12 | 13 | local cURL = require "cURL" 14 | 15 | function performCurlRequest(curl) 16 | local buff = {} 17 | curl:setopt_writefunction(function(data) table.insert(buff, data) end) 18 | :perform() 19 | 20 | local response_code = curl:getinfo_response_code() 21 | local response = table.concat(buff) 22 | 23 | curl:close() 24 | 25 | if response_code ~= 200 then 26 | print(response) 27 | os.exit(1) 28 | end 29 | 30 | return response 31 | end 32 | 33 | local tiefvision_curl_io = {} 34 | 35 | function tiefvision_curl_io.keys() 36 | local request = cURL.easy { url = tiefvision_curl_io.urlKeys() } 37 | local response = performCurlRequest(request) 38 | local keys = tiefvision_curl_io.responseToKeys(response) 39 | 40 | return keys 41 | end 42 | 43 | function tiefvision_curl_io.read(key) 44 | local request = cURL.easy { url = tiefvision_curl_io.urlRead(key) } 45 | local response = performCurlRequest(request) 46 | local value = tiefvision_curl_io.responseToValue(response) 47 | 48 | return value 49 | end 50 | 51 | function tiefvision_curl_io.write(key, value) 52 | local requestObj = { url = tiefvision_curl_io.urlWrite(key) } 53 | for k,v in pairs(tiefvision_curl_io.valueToRequest(value)) do requestObj[k] = v end 54 | 55 | local request = cURL.easy(requestObj) 56 | 57 | performCurlRequest(request) 58 | end 59 | 60 | local factory = {} 61 | setmetatable(factory, { __call = function(_, urlKeys, responseToKeys, urlRead, responseToValue, urlWrite, valueToRequest) 62 | tiefvision_curl_io.urlKeys = urlKeys 63 | tiefvision_curl_io.responseToKeys = responseToKeys 64 | 65 | tiefvision_curl_io.urlRead = urlRead 66 | tiefvision_curl_io.responseToValue = responseToValue 67 | 68 | tiefvision_curl_io.urlWrite = urlWrite 69 | tiefvision_curl_io.valueToRequest = valueToRequest 70 | 71 | return tiefvision_curl_io 72 | end }) 73 | 74 | return factory 75 | -------------------------------------------------------------------------------- /src/torch/0-tiefvision-commons/io/tiefvision_redis_io.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- 6 | -- Reader and writer to store information thanks to redis 7 | -- 8 | 9 | local paths = require('paths') 10 | local torchFolder = paths.thisfile('../..') 11 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 12 | 13 | local redis = require 'redis' 14 | 15 | local function toRedisProtocol(...) 16 | local args = {...} 17 | local argsLength = #args 18 | 19 | local redisProtocol = "*" .. argsLength .. "\r\n" 20 | for i = 1, argsLength do 21 | local arg = tostring(args[i]) 22 | 23 | redisProtocol = redisProtocol .. "$" .. #arg .. "\r\n" 24 | redisProtocol = redisProtocol .. arg .. "\r\n" 25 | end 26 | 27 | return redisProtocol 28 | end 29 | 30 | local tiefvision_redis_io = {} 31 | 32 | function tiefvision_redis_io.read(key) 33 | return tiefvision_redis_io.redisClient:hgetall(key) 34 | end 35 | 36 | function tiefvision_redis_io.write(key, value) 37 | local tmpFileName = paths.tmpname() 38 | local file = io.open(tmpFileName, "w") 39 | 40 | file:write(toRedisProtocol("DEL", key)) 41 | for k, v in pairs(value) do 42 | file:write(toRedisProtocol("HSET", key, k, v)) 43 | end 44 | 45 | file:close() 46 | os.execute("cat " .. tmpFileName .. " | redis-cli --pipe -h " .. tiefvision_redis_io.host .. " -p " .. tiefvision_redis_io.port .. " -n " .. tiefvision_redis_io.database .. " 1>/dev/null &") 47 | end 48 | 49 | function tiefvision_redis_io.keys() 50 | return tiefvision_redis_io.redisClient:keys("*") 51 | end 52 | 53 | local factory = {} 54 | setmetatable(factory, { __call = function(_, host, port, database) 55 | tiefvision_redis_io.host = host 56 | tiefvision_redis_io.port = port or 6379 57 | tiefvision_redis_io.database = database or 0 58 | 59 | tiefvision_redis_io.redisClient = redis.connect( 60 | tiefvision_redis_io.host, 61 | tiefvision_redis_io.port 62 | ) 63 | 64 | tiefvision_redis_io.redisClient:select(tiefvision_redis_io.database) 65 | 66 | return tiefvision_redis_io 67 | end }) 68 | 69 | return factory 70 | -------------------------------------------------------------------------------- /src/torch/0-tiefvision-commons/io/tiefvision_rest_io.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- 6 | -- Reader and writer to store information thanks to redis 7 | -- 8 | 9 | local paths = require('paths') 10 | local torchFolder = paths.thisfile('../..') 11 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 12 | 13 | local cjson = require("cjson") 14 | local curl = require('0-tiefvision-commons/io/tiefvision_curl_io') 15 | 16 | local factory = {} 17 | setmetatable(factory, { __call = function(_, url, fileJsonKey, scoreJsonKey) 18 | local urlKeys = function() return url end 19 | local urlRead = function(key) return url .. "/" .. key end 20 | local urlWrite = function(key) return url .. "/" .. key end 21 | 22 | local responseToKeys = function(response) 23 | return cjson.decode(response) 24 | end 25 | 26 | local responseToValue = function(response) 27 | local values = {} 28 | for _, value in pairs(cjson.decode(response)) do 29 | local file = value[fileJsonKey] 30 | local score = value[scoreJsonKey] 31 | 32 | values[file] = score 33 | end 34 | 35 | return values 36 | end 37 | 38 | local valueToRequest = function(value) 39 | local buff = {} 40 | for file, score in pairs(value) do 41 | local json = string.format('{"%s":%s,"%s":%s}', fileJsonKey, file, scoreJsonKey, score) 42 | table.insert(buff, json) 43 | end 44 | 45 | return { 46 | httpheader = { "Content-Type: application/json" }, 47 | postfields = string.format('[%s]', table.concat(buff, ",")) 48 | } 49 | end 50 | 51 | return curl(urlKeys, responseToKeys, urlRead, responseToValue, urlWrite, valueToRequest) 52 | end }) 53 | 54 | return factory 55 | -------------------------------------------------------------------------------- /src/torch/0-tiefvision-commons/io/tiefvision_torch_io.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- 6 | -- Reader and writer to store information thanks to torch 7 | -- 8 | 9 | local paths = require('paths') 10 | local torchFolder = paths.thisfile('..') 11 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 12 | 13 | local torch = require 'torch' 14 | 15 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 16 | local tiefvision_torch_io = {} 17 | 18 | function tiefvision_torch_io.read(key) 19 | local file = tiefvision_commons.path(tiefvision_torch_io.folder, key) 20 | if not paths.filep(file) then 21 | return nil 22 | end 23 | 24 | return torch.load(file) 25 | end 26 | 27 | function tiefvision_torch_io.write(key, value) 28 | local file = tiefvision_commons.path(tiefvision_torch_io.folder, key) 29 | 30 | paths.mkdir(paths.dirname(file)) 31 | torch.save(file, value) 32 | end 33 | 34 | function tiefvision_torch_io.keys() 35 | local files = {} 36 | for file in paths.files(tiefvision_torch_io.folder) do 37 | files[#files + 1] = file 38 | end 39 | 40 | return files 41 | end 42 | 43 | local factory = {} 44 | setmetatable(factory, { __call = function(_, folder) 45 | tiefvision_torch_io.folder = folder 46 | 47 | return tiefvision_torch_io 48 | end }) 49 | 50 | return factory 51 | -------------------------------------------------------------------------------- /src/torch/0-tiefvision-commons/is_gray.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local image = require 'image' 6 | 7 | local function detectGrayscale(imagePath) 8 | local img = image.load(imagePath) 9 | if img:size()[1] == 3 then 10 | print("C") 11 | else 12 | print("G") 13 | end 14 | end 15 | 16 | detectGrayscale(arg[1]) 17 | -------------------------------------------------------------------------------- /src/torch/0-tiefvision-commons/tiefvision_commons.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- 6 | -- Commmon utility methods that are used thoughout the other modules. They are mostly 7 | -- related to IO. 8 | -- 9 | 10 | local image = require 'image' 11 | local lfs = require 'lfs' 12 | local torch = require 'torch' 13 | local tiefvision_commons = {} 14 | 15 | function tiefvision_commons.fileExists(name) 16 | local f = io.open(name, "r") 17 | if f ~= nil then io.close(f) return true else return false end 18 | end 19 | 20 | function tiefvision_commons.getLines(filename) 21 | local trainFile = io.open(filename) 22 | print(trainFile) 23 | local lines = {} 24 | if trainFile ~= nil then 25 | local index = 1 26 | for trainFileLine in trainFile:lines() do 27 | print(trainFileLine) 28 | if (tiefvision_commons.fileExists(trainFileLine)) then 29 | lines[index] = trainFileLine 30 | index = index + 1 31 | end 32 | end 33 | io.close(trainFile) 34 | end 35 | return lines 36 | end 37 | 38 | function tiefvision_commons.getFiles(folder) 39 | local files = {} 40 | for file in lfs.dir(folder) do 41 | if (lfs.attributes(folder .. '/' .. file, "mode") == "file") then 42 | table.insert(files, file) 43 | end 44 | end 45 | return files 46 | end 47 | 48 | -- Loads the mapping from net outputs to human readable labels 49 | function tiefvision_commons.load_synset() 50 | local file = io.open(tiefvision_commons.modelPath('synset_words.txt')) 51 | local list = {} 52 | while true do 53 | local line = file:read() 54 | if not line then break end 55 | table.insert(list, string.sub(line, 11)) 56 | end 57 | io.close(file) 58 | return list 59 | end 60 | 61 | function tiefvision_commons.img_mean() 62 | local img_mean_name = tiefvision_commons.modelPath('ilsvrc_2012_mean.t7') 63 | return torch.load(img_mean_name).img_mean:transpose(3, 1) 64 | end 65 | 66 | function tiefvision_commons.load(imagePath) 67 | local img = image.load(imagePath) 68 | img = tiefvision_commons.preprocess(img) 69 | return img 70 | end 71 | 72 | function tiefvision_commons.loadImage(img) 73 | img = tiefvision_commons.preprocess(img) 74 | return img 75 | end 76 | 77 | function tiefvision_commons.preprocess(im) 78 | local img_mean = tiefvision_commons.img_mean() 79 | local scaledImage = im * 255 80 | -- converts RGB to BGR 81 | local bgrImage = scaledImage:clone() 82 | bgrImage[{ 1, {}, {} }] = scaledImage[{ 3, {}, {} }] 83 | bgrImage[{ 3, {}, {} }] = scaledImage[{ 1, {}, {} }] 84 | 85 | local imageMinusAvg = bgrImage - image.scale(img_mean, im:size()[2], im:size()[3], 'bilinear') 86 | return imageMinusAvg:cuda() 87 | end 88 | 89 | function tiefvision_commons.tableSubtraction(t1, t2) 90 | t1 = t1 or {} 91 | t2 = t2 or {} 92 | 93 | local t = {} 94 | for i = 1, #t2 do t[t2[i]] = true end 95 | 96 | for i = #t1, 1, -1 do 97 | if t[t1[i]] then 98 | table.remove(t1, i) 99 | end 100 | end 101 | 102 | return t1 103 | end 104 | 105 | function tiefvision_commons.tableShuffle(t) 106 | local n = #t -- gets the length of the table 107 | while n > 1 do -- only run if the table has more than 1 element 108 | local k = math.random(n) -- get a random number 109 | t[n], t[k] = t[k], t[n] 110 | n = n - 1 111 | end 112 | 113 | return t 114 | end 115 | 116 | function tiefvision_commons.path(...) 117 | local file_path, _ = table.concat({...}, '/'):gsub('/+', '/') 118 | return file_path 119 | end 120 | 121 | function tiefvision_commons.rootPath() 122 | local environmentVariableName = 'TIEFVISION_HOME' 123 | local root = os.getenv(environmentVariableName) 124 | if not root then 125 | error(string.format("missing environment variable: %s", environmentVariableName)) 126 | end 127 | 128 | return root 129 | end 130 | 131 | function tiefvision_commons.dataPath(...) 132 | return tiefvision_commons.path( 133 | tiefvision_commons.rootPath(), 134 | 'src/torch/data', 135 | unpack({...})) 136 | end 137 | 138 | function tiefvision_commons.modelPath(...) 139 | return tiefvision_commons.path( 140 | tiefvision_commons.rootPath(), 141 | 'src/torch/models', 142 | unpack({...})) 143 | end 144 | 145 | function tiefvision_commons.resourcePath(...) 146 | return tiefvision_commons.path( 147 | tiefvision_commons.rootPath(), 148 | 'resources', 149 | unpack({...})) 150 | end 151 | 152 | return tiefvision_commons 153 | -------------------------------------------------------------------------------- /src/torch/0-tiefvision-commons/tiefvision_config_loader.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- 6 | -- Get config object from cli, env, or a default 7 | -- 8 | 9 | local torchFolder = require('paths').thisfile('..') 10 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 11 | 12 | local tiefvision_commons = require('0-tiefvision-commons/tiefvision_commons') 13 | 14 | local function argument(arg) 15 | local prefix = '^%-%-?config' 16 | local prefixAndEqual = prefix .. '=' 17 | 18 | for index, value in pairs(arg) do 19 | if string.match(value, prefixAndEqual) then 20 | return (string.gsub(value, prefixAndEqual, '')) 21 | elseif string.match(value, prefix) then 22 | return arg[index + 1] 23 | end 24 | end 25 | 26 | return nil 27 | end 28 | 29 | local configLoader = {} 30 | 31 | configLoader.argument = argument(arg) 32 | configLoader.environment = os.getenv('CONFIG') 33 | configLoader.default = tiefvision_commons.path( 34 | tiefvision_commons.rootPath(), 35 | 'src/torch/config.lua') 36 | 37 | configLoader.file = configLoader.argument or 38 | configLoader.environment or 39 | configLoader.default 40 | 41 | function configLoader.load() 42 | package.path = package.path .. ';' .. configLoader.file 43 | return require(configLoader.file) 44 | end 45 | 46 | if not require('paths').filep(configLoader.file) then 47 | error("configuration file doesn't exist: " .. configLoader.file) 48 | end 49 | 50 | return configLoader 51 | -------------------------------------------------------------------------------- /src/torch/0-tiefvision-commons/tiefvision_reduction.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- 6 | -- Get k nearest neighbor 7 | -- 8 | 9 | local torchFolder = require('paths').thisfile('..') 10 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 11 | 12 | local tiefvision_reduction = {} 13 | 14 | tiefvision_reduction.getNearestNeighbors = function(neighbors, k) 15 | local keysSortedByValue = function(tbl, sortFunction) 16 | local keys = {} 17 | for key in pairs(tbl) do table.insert(keys, key) end 18 | table.sort(keys, function(a, b) return sortFunction(tbl[a], tbl[b]) end) 19 | 20 | return keys 21 | end 22 | 23 | local neighborsKey = keysSortedByValue(neighbors, function(a, b) return a > b end) 24 | local kMin = math.min(#neighborsKey, k) 25 | 26 | local buff = {} 27 | for i = 1, kMin do 28 | local key = neighborsKey[i] 29 | local value = neighbors[key] 30 | 31 | buff[key] = value 32 | end 33 | 34 | return buff 35 | end 36 | 37 | return tiefvision_reduction 38 | -------------------------------------------------------------------------------- /src/torch/1-split-encoder-classifier/deploy.prototxt: -------------------------------------------------------------------------------- 1 | name: "AlexNet" 2 | input: "data" 3 | input_dim: 1 4 | input_dim: 3 5 | input_dim: 227 6 | input_dim: 227 7 | layer { 8 | name: "conv1" 9 | type: "Convolution" 10 | bottom: "data" 11 | top: "conv1" 12 | param { 13 | lr_mult: 1 14 | decay_mult: 1 15 | } 16 | param { 17 | lr_mult: 2 18 | decay_mult: 0 19 | } 20 | convolution_param { 21 | num_output: 96 22 | kernel_size: 11 23 | stride: 4 24 | } 25 | } 26 | layer { 27 | name: "relu1" 28 | type: "ReLU" 29 | bottom: "conv1" 30 | top: "conv1" 31 | } 32 | layer { 33 | name: "norm1" 34 | type: "LRN" 35 | bottom: "conv1" 36 | top: "norm1" 37 | lrn_param { 38 | local_size: 5 39 | alpha: 0.0001 40 | beta: 0.75 41 | } 42 | } 43 | layer { 44 | name: "pool1" 45 | type: "Pooling" 46 | bottom: "norm1" 47 | top: "pool1" 48 | pooling_param { 49 | pool: MAX 50 | kernel_size: 3 51 | stride: 2 52 | } 53 | } 54 | layer { 55 | name: "conv2" 56 | type: "Convolution" 57 | bottom: "pool1" 58 | top: "conv2" 59 | param { 60 | lr_mult: 1 61 | decay_mult: 1 62 | } 63 | param { 64 | lr_mult: 2 65 | decay_mult: 0 66 | } 67 | convolution_param { 68 | num_output: 256 69 | pad: 2 70 | kernel_size: 5 71 | group: 2 72 | } 73 | } 74 | layer { 75 | name: "relu2" 76 | type: "ReLU" 77 | bottom: "conv2" 78 | top: "conv2" 79 | } 80 | layer { 81 | name: "norm2" 82 | type: "LRN" 83 | bottom: "conv2" 84 | top: "norm2" 85 | lrn_param { 86 | local_size: 5 87 | alpha: 0.0001 88 | beta: 0.75 89 | } 90 | } 91 | layer { 92 | name: "pool2" 93 | type: "Pooling" 94 | bottom: "norm2" 95 | top: "pool2" 96 | pooling_param { 97 | pool: MAX 98 | kernel_size: 3 99 | stride: 2 100 | } 101 | } 102 | layer { 103 | name: "conv3" 104 | type: "Convolution" 105 | bottom: "pool2" 106 | top: "conv3" 107 | param { 108 | lr_mult: 1 109 | decay_mult: 1 110 | } 111 | param { 112 | lr_mult: 2 113 | decay_mult: 0 114 | } 115 | convolution_param { 116 | num_output: 384 117 | pad: 1 118 | kernel_size: 3 119 | } 120 | } 121 | layer { 122 | name: "relu3" 123 | type: "ReLU" 124 | bottom: "conv3" 125 | top: "conv3" 126 | } 127 | layer { 128 | name: "conv4" 129 | type: "Convolution" 130 | bottom: "conv3" 131 | top: "conv4" 132 | param { 133 | lr_mult: 1 134 | decay_mult: 1 135 | } 136 | param { 137 | lr_mult: 2 138 | decay_mult: 0 139 | } 140 | convolution_param { 141 | num_output: 384 142 | pad: 1 143 | kernel_size: 3 144 | group: 2 145 | } 146 | } 147 | layer { 148 | name: "relu4" 149 | type: "ReLU" 150 | bottom: "conv4" 151 | top: "conv4" 152 | } 153 | layer { 154 | name: "conv5" 155 | type: "Convolution" 156 | bottom: "conv4" 157 | top: "conv5" 158 | param { 159 | lr_mult: 1 160 | decay_mult: 1 161 | } 162 | param { 163 | lr_mult: 2 164 | decay_mult: 0 165 | } 166 | convolution_param { 167 | num_output: 256 168 | pad: 1 169 | kernel_size: 3 170 | group: 2 171 | } 172 | } 173 | layer { 174 | name: "relu5" 175 | type: "ReLU" 176 | bottom: "conv5" 177 | top: "conv5" 178 | } 179 | layer { 180 | name: "pool5" 181 | type: "Pooling" 182 | bottom: "conv5" 183 | top: "pool5" 184 | pooling_param { 185 | pool: MAX 186 | kernel_size: 3 187 | stride: 2 188 | } 189 | } 190 | layer { 191 | name: "fc6" 192 | type: "InnerProduct" 193 | bottom: "pool5" 194 | top: "fc6" 195 | param { 196 | lr_mult: 1 197 | decay_mult: 1 198 | } 199 | param { 200 | lr_mult: 2 201 | decay_mult: 0 202 | } 203 | inner_product_param { 204 | num_output: 4096 205 | } 206 | } 207 | layer { 208 | name: "relu6" 209 | type: "ReLU" 210 | bottom: "fc6" 211 | top: "fc6" 212 | } 213 | layer { 214 | name: "fc7" 215 | type: "InnerProduct" 216 | bottom: "fc6" 217 | top: "fc7" 218 | param { 219 | lr_mult: 1 220 | decay_mult: 1 221 | } 222 | param { 223 | lr_mult: 2 224 | decay_mult: 0 225 | } 226 | inner_product_param { 227 | num_output: 4096 228 | } 229 | } 230 | layer { 231 | name: "relu7" 232 | type: "ReLU" 233 | bottom: "fc7" 234 | top: "fc7" 235 | } 236 | layer { 237 | name: "fc8-light" 238 | type: "InnerProduct" 239 | bottom: "fc7" 240 | top: "fc8-light" 241 | param { 242 | lr_mult: 1 243 | decay_mult: 1 244 | } 245 | param { 246 | lr_mult: 2 247 | decay_mult: 0 248 | } 249 | inner_product_param { 250 | num_output: 16 251 | } 252 | } 253 | layer { 254 | name: "prob" 255 | type: "Softmax" 256 | bottom: "fc8-light" 257 | top: "prob" 258 | } 259 | -------------------------------------------------------------------------------- /src/torch/1-split-encoder-classifier/deploy.prototxt.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | local model = {} 3 | -- warning: module 'data [type 5]' not found 4 | table.insert(model, {'conv1', nn.SpatialConvolution(3, 96, 11, 11, 4, 4, 0, 0)}) 5 | table.insert(model, {'relu0', nn.ReLU(true)}) 6 | table.insert(model, {'cccp1', nn.SpatialConvolution(96, 96, 1, 1, 1, 1, 0, 0)}) 7 | table.insert(model, {'relu1', nn.ReLU(true)}) 8 | table.insert(model, {'cccp2', nn.SpatialConvolution(96, 96, 1, 1, 1, 1, 0, 0)}) 9 | table.insert(model, {'relu2', nn.ReLU(true)}) 10 | table.insert(model, {'pool0', nn.SpatialMaxPooling(3, 3, 2, 2, 0, 0):ceil()}) 11 | table.insert(model, {'conv2', nn.SpatialConvolution(96, 256, 5, 5, 1, 1, 2, 2)}) 12 | table.insert(model, {'relu3', nn.ReLU(true)}) 13 | table.insert(model, {'cccp3', nn.SpatialConvolution(256, 256, 1, 1, 1, 1, 0, 0)}) 14 | table.insert(model, {'relu5', nn.ReLU(true)}) 15 | table.insert(model, {'cccp4', nn.SpatialConvolution(256, 256, 1, 1, 1, 1, 0, 0)}) 16 | table.insert(model, {'relu6', nn.ReLU(true)}) 17 | table.insert(model, {'pool2', nn.SpatialMaxPooling(3, 3, 2, 2, 0, 0):ceil()}) 18 | table.insert(model, {'conv3', nn.SpatialConvolution(256, 384, 3, 3, 1, 1, 1, 1)}) 19 | table.insert(model, {'relu7', nn.ReLU(true)}) 20 | table.insert(model, {'cccp5', nn.SpatialConvolution(384, 384, 1, 1, 1, 1, 0, 0)}) 21 | table.insert(model, {'relu8', nn.ReLU(true)}) 22 | table.insert(model, {'cccp6', nn.SpatialConvolution(384, 384, 1, 1, 1, 1, 0, 0)}) 23 | table.insert(model, {'relu9', nn.ReLU(true)}) 24 | table.insert(model, {'pool3', nn.SpatialMaxPooling(3, 3, 2, 2, 0, 0):ceil()}) 25 | table.insert(model, {'drop', nn.Dropout(0.500000)}) 26 | table.insert(model, {'conv4-1024', nn.SpatialConvolution(384, 1024, 3, 3, 1, 1, 1, 1)}) 27 | table.insert(model, {'relu10', nn.ReLU(true)}) 28 | table.insert(model, {'cccp7-1024', nn.SpatialConvolution(1024, 1024, 1, 1, 1, 1, 0, 0)}) 29 | table.insert(model, {'relu11', nn.ReLU(true)}) 30 | table.insert(model, {'cccp8-1024', nn.SpatialConvolution(1024, 1000, 1, 1, 1, 1, 0, 0)}) 31 | table.insert(model, {'relu12', nn.ReLU(true)}) 32 | table.insert(model, {'pool4', nn.SpatialAveragePooling(6, 6, 1, 1, 0, 0):ceil()}) 33 | table.insert(model, {'loss', nn.SoftMax()}) 34 | return model -------------------------------------------------------------------------------- /src/torch/1-split-encoder-classifier/split-encoder-classifier.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- 6 | -- It Splits the (Alexnet-like) network into an encoder and a classifier 7 | -- The classifier is discarded and the encoder is used by other modules 8 | -- to encode images. 9 | -- 10 | 11 | local paths = require('paths') 12 | local torchFolder = paths.thisfile('..') 13 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 14 | 15 | require 'image' 16 | require 'inn' 17 | local loadcaffe = require 'loadcaffe' 18 | local nn = require 'nn' 19 | local torch = require 'torch' 20 | 21 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 22 | 23 | local proto_name = paths.thisfile('deploy.prototxt') 24 | local model_name = paths.thisfile('nin_imagenet.caffemodel') 25 | local image_name = paths.thisfile('Goldfish3.jpg'); 26 | 27 | local net = loadcaffe.load(proto_name, model_name):cuda() 28 | net.modules[#net.modules] = nil -- remove the top softmax 29 | 30 | net:evaluate() 31 | local synset_words = tiefvision_commons.load_synset() 32 | 33 | local im = tiefvision_commons.load(image_name) 34 | 35 | local loss, output = net:forward(im):view(-1):float():sort(true) 36 | 37 | -- create classification encoder 38 | local encoder = net:clone() 39 | for _ = 1, 9 do 40 | encoder:remove(21) 41 | end 42 | local concat = nn.ConcatTable() 43 | concat:add(nn.SpatialMaxPooling(3, 3, 2, 2, 0, 0):ceil():cuda()) 44 | concat:add(nn.SpatialMaxPooling(3, 3, 1, 1, 0, 0):ceil():cuda()) 45 | encoder:add(concat) 46 | encoder:evaluate() 47 | print(encoder) 48 | 49 | -- create classifier 50 | local classifier = net:clone() 51 | for _ = 1, 21 do 52 | classifier:remove(1) 53 | end 54 | 55 | local outputEnc = encoder:forward(im) 56 | local lossClassifier, outputClassifier = classifier:forward(outputEnc[1]):view(-1):float():sort(true) 57 | 58 | local outputSize = output:size()[1] 59 | assert(outputSize == 1000, 'Output size should be 1000') 60 | assert(torch.eq(outputClassifier, output), 'the output of the network should be the same as the output of the classifier using the encoder') 61 | 62 | -- test expected output 63 | local expectedOutput = torch.Tensor(4) 64 | expectedOutput[1] = 2 65 | expectedOutput[2] = 89 66 | expectedOutput[3] = 91 67 | expectedOutput[4] = 131 68 | for i = 1, expectedOutput:size()[1] do 69 | print(outputClassifier[i] .. ' ' .. expectedOutput[i] .. ' ' .. synset_words[outputClassifier[i]] .. ' || ' .. synset_words[output[i]]) 70 | assert(outputClassifier[i] == expectedOutput[i], 'the network predicted an unexpected class') 71 | end 72 | 73 | -- Test encoder reduction 74 | local fakeIm = torch.Tensor(3, 224, 224):cuda() 75 | outputEnc = encoder:forward(fakeIm) 76 | assert(outputEnc[1]:size()[3] == 6, 'the encoder size for 224 input size should be 6') 77 | assert(outputEnc[2]:size()[3] == (outputEnc[1]:size()[3] * 2) - 1, 'the regression encoder size for 2240 input size should be the double of the classification one') 78 | 79 | fakeIm = torch.Tensor(3, 224, 224 * 10):cuda() 80 | outputEnc = encoder:forward(fakeIm) 81 | assert(outputEnc[1]:size()[3] == 69, 'the encoder size for 2240 input size should be 69') 82 | assert(outputEnc[2]:size()[3] == (outputEnc[1]:size()[3] * 2) - 1, 'the regression encoder size for 2240 input size should be the double of the classification one') 83 | 84 | fakeIm = torch.Tensor(3, 224, 224 * 15):cuda() 85 | outputEnc = encoder:forward(fakeIm) 86 | assert(outputEnc[1]:size()[3] == 104, 'the encoder size for 4480 input size should be 104') 87 | assert(outputEnc[2]:size()[3] == (outputEnc[1]:size()[3] * 2) - 1, 'the regression encoder size for 224 * 15 input size should be the double of the classification one') 88 | 89 | print("Saving models...") 90 | 91 | torch.save(tiefvision_commons.modelPath('net.model'), net) 92 | torch.save(tiefvision_commons.modelPath('encoder.model'), encoder) 93 | torch.save(tiefvision_commons.modelPath('classifier-original.model'), classifier) 94 | 95 | print("Finished!") 96 | -------------------------------------------------------------------------------- /src/torch/10-similarity-searcher-cnn-db/search.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'optim' 10 | require 'xlua' 11 | require 'lfs' 12 | local torch = require 'torch' 13 | 14 | local tiefvision_config_loader = require '0-tiefvision-commons/tiefvision_config_loader' 15 | local search_commons = require '10-similarity-searcher-cnn-db/search_commons' 16 | local database = tiefvision_config_loader.load().database.unsupervised_similarity 17 | 18 | local function getTestError(reference) 19 | local similarities = database.read(reference) 20 | 21 | local comparisonTable = {} 22 | for file, sim in pairs(similarities) do 23 | table.insert(comparisonTable, { file, sim }) 24 | end 25 | 26 | table.sort(comparisonTable, search_commons.sortCmpTable) 27 | search_commons.printCmpTable(comparisonTable) 28 | end 29 | 30 | local function getOptions() 31 | local cmd = torch.CmdLine() 32 | cmd:text() 33 | cmd:text('Unsupervised image search from precomputed database of distances between each pair of images in the master folder.') 34 | cmd:text('Returns a descending sorted list of filenames concatenated with a similarity metric.') 35 | cmd:text('Both the filename to search and the result filenames come from the folder $TIEFVISION_HOME/resources/dresses-db/master.') 36 | cmd:text() 37 | cmd:text('Options:') 38 | cmd:argument('image', 'Filename (not full path, just the filename) from $TIEFVISION_HOME/resources/dresses-db/master.', 'string') 39 | cmd:text() 40 | cmd:option('-config', tiefvision_config_loader.default, 'Configuration file to use.') 41 | cmd:text() 42 | return cmd:parse(arg) 43 | end 44 | 45 | local options = getOptions() 46 | getTestError(options.image) 47 | -------------------------------------------------------------------------------- /src/torch/10-similarity-searcher-cnn-db/search_commons.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local search_commons = {} 6 | 7 | function search_commons.sortCmpTable(a, b) 8 | return a[2] > b[2] 9 | end 10 | 11 | function search_commons.printCmpTable(cmpTable) 12 | for i = 1, #cmpTable do 13 | print(cmpTable[i][1] .. ' ' .. cmpTable[i][2]) 14 | end 15 | end 16 | 17 | return search_commons 18 | -------------------------------------------------------------------------------- /src/torch/11-similarity-searcher-cnn-file/search.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'optim' 10 | require 'xlua' 11 | require 'lfs' 12 | local torch = require 'torch' 13 | local image = require 'image' 14 | 15 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 16 | local tiefvision_config_loader = require '0-tiefvision-commons/tiefvision_config_loader' 17 | local bboxlib = require '6-bboxlib/bboxlib' 18 | local similarity_db_lib = require '8-similarity-db-cnn/similarity_db_lib' 19 | local similarity_lib = require '9-similarity-db/similarity_lib' 20 | local search_commons = require '10-similarity-searcher-cnn-db/search_commons' 21 | 22 | local function getTestError(referenceEncoding) 23 | local dataFolder = tiefvision_commons.dataPath('encoded-images') 24 | local testLines = tiefvision_commons.getFiles(dataFolder) 25 | local comparisonTable = {} 26 | for testIndex = 1, #testLines do 27 | local file = testLines[testIndex] 28 | local imageEncoding = torch.load(dataFolder .. '/' .. file):double() 29 | local dist = similarity_lib.similarity(referenceEncoding, imageEncoding) 30 | table.insert(comparisonTable, { file, dist }) 31 | end 32 | table.sort(comparisonTable, search_commons.sortCmpTable) 33 | search_commons.printCmpTable(comparisonTable) 34 | end 35 | 36 | local function getImage(fileName, imagesFolder) 37 | local input = bboxlib.loadImageFromFile(imagesFolder .. '/' .. fileName) 38 | local bboxes = bboxlib.getImageBoundingBoxesTable(input, 1) 39 | local xmin = bboxes[1][1] 40 | local ymin = bboxes[1][2] 41 | local xmax = bboxes[1][3] 42 | local ymax = bboxes[1][4] 43 | input = image.crop(input, xmin, ymin, xmax, ymax) 44 | image.save(tiefvision_commons.resourcePath('dresses-db/uploaded/bbox', fileName), input) 45 | local encoder = similarity_db_lib.getEncoder() 46 | local encodedOutput = similarity_db_lib.encodeImage(tiefvision_commons.resourcePath('dresses-db/uploaded/bbox', fileName), encoder) 47 | return encodedOutput:double() 48 | end 49 | 50 | local function getOptions() 51 | local cmd = torch.CmdLine() 52 | cmd:text() 53 | cmd:text('Unsupervised image search from an image file.') 54 | cmd:text('Returns a descending sorted list of filenames concatenated with a similarity metric.') 55 | cmd:text('The result filenames come from the folder $TIEFVISION_HOME/resources/dresses-db/master.') 56 | cmd:text() 57 | cmd:text('Options:') 58 | cmd:argument('image', 'Filename of the query image to search.', 'string') 59 | cmd:argument('imagesFolder', 'Folder where the images are contained.', 'string') 60 | cmd:text() 61 | cmd:option('-config', tiefvision_config_loader.default, 'Configuration file to use.') 62 | cmd:text() 63 | return cmd:parse(arg) 64 | end 65 | 66 | local options = getOptions() 67 | local referenceDress = getImage(options.image, options.imagesFolder) 68 | getTestError(referenceDress) 69 | -------------------------------------------------------------------------------- /src/torch/12-deeprank-train/deeprank-train.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'image' 10 | local nn = require 'nn' 11 | local torch = require 'torch' 12 | 13 | local inputSize = 11 * 11 * 384 14 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 15 | 16 | local function getSimilarityModel() 17 | 18 | local leftImage = nn.Sequential() 19 | --leftImage:add(nn.SpatialConvolutionMM(11, 11, 1, 1, 1, 1, 0, 0):cuda()) 20 | 21 | local convLayer = nn.SpatialConvolutionMM(384, 384, 1, 1, 1, 1, 0, 0):cuda() 22 | convLayer.weight = torch.eye(384):cuda() 23 | convLayer.bias = torch.zeros(384):cuda() 24 | leftImage:add(convLayer:cuda()) 25 | --leftImage:add(nn.ReLU(true):cuda()) 26 | --leftImage:add(nn.Dropout(0.5):cuda()) 27 | --leftImage:add(nn.SpatialConvolutionMM(512, 126, 1, 1, 1, 1, 0, 0):cuda()) 28 | --leftImage:add(nn.ReLU(true):cuda()) 29 | --leftImage:add(nn.Dropout(0.5):cuda()) 30 | leftImage:add(nn.Reshape(inputSize):cuda()) 31 | 32 | local rightImage = leftImage:clone('weight', 'bias') 33 | 34 | local imagesParallel = nn.ParallelTable() 35 | imagesParallel:add(leftImage) 36 | imagesParallel:add(rightImage) 37 | 38 | local imagesDistSimilar = nn.Sequential() 39 | imagesDistSimilar:add(imagesParallel) 40 | imagesDistSimilar:add(nn.DotProduct():cuda()) 41 | 42 | local imagesDistDifferent = imagesDistSimilar:clone('weight', 'bias') 43 | 44 | local similarityModel = nn.Sequential() 45 | local distanceParallel = nn.ParallelTable() 46 | distanceParallel:add(imagesDistSimilar) 47 | distanceParallel:add(imagesDistDifferent) 48 | similarityModel:add(distanceParallel) 49 | 50 | return similarityModel:cuda() 51 | end 52 | 53 | local function getCriterion() 54 | local criterion = nn.MarginRankingCriterion(0.1):cuda() 55 | return criterion 56 | end 57 | 58 | local function gradUpdate(similarityModel, x, criterion, learningRate) 59 | local pred = similarityModel:forward(x) 60 | local err = criterion:forward(pred, torch.ones(32):cuda()) 61 | local gradCriterion = criterion:backward(pred, torch.ones(32):cuda()) 62 | similarityModel:zeroGradParameters() 63 | similarityModel:backward(x, gradCriterion) 64 | similarityModel:updateParameters(learningRate) 65 | return err 66 | end 67 | 68 | local function getDataSet(file) 69 | local lines = tiefvision_commons.getLines(tiefvision_commons.resourcePath('dresses-db', file)) 70 | local trainingSet = {} 71 | for i = 1, #lines do 72 | local reference, similar, different = string.match(lines[i], "(.+),(.+),(.+)") 73 | trainingSet[i] = { reference, similar, different } 74 | end 75 | return trainingSet 76 | end 77 | 78 | local function getTestSet() 79 | return getDataSet('similarity-db-test') 80 | end 81 | 82 | local function getTrainingSet() 83 | return getDataSet('similarity-db-train') 84 | end 85 | 86 | local criterion = getCriterion() 87 | local learningRate = 0.0001 88 | 89 | local function getHeightWindow(input, heightStart) 90 | local windowInput = torch.Tensor(11, 11, 384):cuda() 91 | local trInput = input:transpose(1, 3) 92 | for w = 1, 11 do 93 | for h = 1, 11 do 94 | windowInput[w][h] = trInput[w][h + heightStart - 1] 95 | end 96 | end 97 | return windowInput:transpose(1, 3) 98 | end 99 | 100 | local function getReferenceSimilarDifferentRaw(datasource) 101 | local encodedFolder = tiefvision_commons.dataPath('db/similarity/img-enc-cnn-encoder') 102 | local flippedEncodedFolder = tiefvision_commons.dataPath('db/similarity/img-enc-cnn-encoder-flipped') 103 | local reference = torch.load(encodedFolder .. '/' .. datasource[1]):cuda() 104 | local similar 105 | if (datasource[1] == datasource[2]) then 106 | similar = torch.load(flippedEncodedFolder .. '/' .. datasource[2]):cuda() 107 | else 108 | similar = torch.load(encodedFolder .. '/' .. datasource[2]) 109 | end 110 | local different = torch.load(encodedFolder .. '/' .. datasource[3]) 111 | return reference:transpose(1, 3):cuda(), similar:transpose(1, 3):cuda(), different:transpose(1, 3):cuda() 112 | end 113 | 114 | local function getReferenceSimilarDifferent(datasource) 115 | local reference, similar, different = getReferenceSimilarDifferentRaw(datasource) 116 | 117 | local minHeight = math.min(math.min(reference:size()[2], similar:size()[2]), different:size()[2]) - 11 + 1 118 | local heightStart = math.random(minHeight) 119 | local referenceW = getHeightWindow(reference, heightStart) 120 | local similarW = getHeightWindow(similar, heightStart) 121 | local differentW = getHeightWindow(different, heightStart) 122 | 123 | return referenceW, similarW, differentW 124 | end 125 | 126 | local function trainBatchGradUpdate(similarityModel, batchSize, initialTrainingIndex, trainingSet, batchSet) 127 | for batchIndex = 1, batchSize do 128 | local trainingIndex = initialTrainingIndex + batchIndex - 1 129 | -- print(trainingSet[trainingIndex][1] .. ' ' .. trainingSet[trainingIndex][2] .. ' ' .. trainingSet[trainingIndex][3]) 130 | local reference, similar, different = getReferenceSimilarDifferent(trainingSet[trainingIndex]) 131 | batchSet[1][1][batchIndex] = reference 132 | batchSet[1][2][batchIndex] = similar 133 | batchSet[2][1][batchIndex] = reference 134 | batchSet[2][2][batchIndex] = different 135 | end 136 | local trainError = gradUpdate(similarityModel, batchSet, criterion, learningRate) 137 | return trainError 138 | end 139 | 140 | local function test(similarityModel) 141 | local imagesDist = similarityModel.modules[1].modules[1] 142 | -- local linear1 = imagesDistSimilar.modules[1].modules[1].modules[5] 143 | -- local linear2 = imagesDistDifferent.modules[1].modules[2].modules[5] 144 | -- local weightDiff = linear1.weight - linear2.weight 145 | -- print(torch.mean(weightDiff), torch.std(weightDiff)) 146 | 147 | similarityModel:evaluate() 148 | local testSet = getTestSet() 149 | local batchSetSimilar = torch.Tensor(2, 384, 11, 11):cuda() 150 | local batchSetDifferent = torch.Tensor(2, 384, 11, 11):cuda() 151 | local correctRank = 0 152 | local successfulTestAttempts = 0 153 | for testIndex = 1, #testSet do 154 | local similarOutput = 0.0 155 | local differentOutput = 0.0 156 | local currentCorrectRank = 0.0 157 | --if(testSet[testIndex][1] ~= testSet[testIndex][2]) then 158 | local reference, similar, different = getReferenceSimilarDifferentRaw(testSet[testIndex]) 159 | for h = 1, 3 do 160 | local referenceW = getHeightWindow(reference, h) 161 | local similarW = getHeightWindow(similar, h) 162 | local differentW = getHeightWindow(different, h) 163 | 164 | batchSetSimilar[1] = referenceW 165 | batchSetSimilar[2] = similarW 166 | batchSetDifferent[1] = referenceW 167 | batchSetDifferent[2] = differentW 168 | 169 | local currSimilarOutput = imagesDist:forward(batchSetSimilar)[1] 170 | similarOutput = similarOutput + currSimilarOutput 171 | local currDifferentOutput = imagesDist:forward(batchSetDifferent)[1] 172 | differentOutput = differentOutput + currDifferentOutput 173 | if (currSimilarOutput > currDifferentOutput) then 174 | currentCorrectRank = currentCorrectRank + 1 175 | else 176 | currentCorrectRank = currentCorrectRank - 1 177 | end 178 | end 179 | --if(currentCorrectRank > 0) then 180 | if (similarOutput > differentOutput) then 181 | correctRank = correctRank + 1 182 | end 183 | successfulTestAttempts = successfulTestAttempts + 1 184 | --end 185 | end 186 | return correctRank, successfulTestAttempts 187 | end 188 | 189 | local function loadModel() 190 | local modelPath = tiefvision_commons.modelPath('similarity.model') 191 | if (tiefvision_commons.fileExists(modelPath)) then 192 | return torch.load(modelPath) 193 | else 194 | return getSimilarityModel() 195 | end 196 | end 197 | 198 | local function saveModel(model) 199 | torch.save(tiefvision_commons.modelPath('similarity.model'), model) 200 | end 201 | 202 | local function testModel(similarityModel) 203 | local testOk, testNum = test(similarityModel) 204 | print('Test Rate:' .. testOk / testNum, testOk .. ' out of ' .. testNum) 205 | end 206 | 207 | local function train(similarityModel) 208 | math.randomseed(os.time()) 209 | local batchSize = 32 210 | local trainingSet = getTrainingSet() 211 | local trainIndex = 1 212 | local batchSet = torch.Tensor(2, 2, batchSize, 384, 11, 11):cuda() 213 | while trainIndex <= #trainingSet - batchSize do 214 | similarityModel:training() 215 | local trainError = trainBatchGradUpdate(similarityModel, batchSize, trainIndex, trainingSet, batchSet) 216 | print('TRAIN_ERROR:' .. trainError) 217 | if (((trainIndex - 1) / batchSize) % 10 == 0) then 218 | print("Saving model...") 219 | saveModel(similarityModel) 220 | print("Model saved") 221 | testModel(similarityModel) 222 | end 223 | trainIndex = trainIndex + batchSize 224 | end 225 | end 226 | 227 | local function getOptions() 228 | local cmd = torch.CmdLine() 229 | cmd:text() 230 | cmd:text('Deep Rank Hinge Loss Training') 231 | cmd:text() 232 | cmd:text('Options:') 233 | cmd:option('-reset', false, 'Reset the saved model (if any) and use a new model.') 234 | cmd:option('-epochs', 20, 'Epochs during training (iterations over all the batches in the training set).') 235 | cmd:text() 236 | return cmd:parse(arg) 237 | end 238 | 239 | local function getModel(reset) 240 | if (reset) then 241 | return getSimilarityModel() 242 | else 243 | return loadModel() 244 | end 245 | end 246 | 247 | local options = getOptions() 248 | local similarityModel = getModel(options.reset) 249 | for epoch = 1, options.epochs do 250 | print("Epoch: " .. epoch) 251 | train(similarityModel) 252 | end 253 | 254 | print("Similarity Model Successfully Trained") 255 | -------------------------------------------------------------------------------- /src/torch/13-deeprank-encoding/deeprank-encoding.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'nn' 9 | require 'inn' 10 | require 'image' 11 | require 'lfs' 12 | local torch = require 'torch' 13 | 14 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 15 | 16 | local function saveEncoding(dataFolder, destFolder, imageName, imageEncoder) 17 | local imageInput = torch.load(dataFolder .. '/' .. imageName) 18 | imageInput = imageInput:transpose(1, 3) 19 | local encodedImage = imageEncoder:forward(imageInput):double() 20 | torch.save(destFolder .. '/' .. imageName, encodedImage) 21 | end 22 | 23 | local function generateDatabaseForFolders(dataFolder, destFolder, imageEncoder) 24 | local imageFiles = tiefvision_commons.getFiles(dataFolder) 25 | for imageIndex = 1, #imageFiles do 26 | local imageFile = imageFiles[imageIndex] 27 | print(imageFile) 28 | saveEncoding(dataFolder, destFolder, imageFile, imageEncoder) 29 | end 30 | end 31 | 32 | local function generateDatabase(imageEncoder) 33 | -- unflipped 34 | local dataFolder = tiefvision_commons.dataPath('db/similarity/img-enc-cnn-encoder') 35 | local destFolder = tiefvision_commons.dataPath('db/similarity/img-similarity-deeprank') 36 | generateDatabaseForFolders(dataFolder, destFolder, imageEncoder) 37 | 38 | -- flipped 39 | local dataFolderFlipped = tiefvision_commons.dataPath('db/similarity/img-enc-cnn-encoder-flipped') 40 | local destFolderFlipped = tiefvision_commons.dataPath('db/similarity/img-flipped-similarity-deeprank') 41 | generateDatabaseForFolders(dataFolderFlipped, destFolderFlipped, imageEncoder) 42 | end 43 | 44 | local function loadModel() 45 | return torch.load(tiefvision_commons.modelPath('similarity.model')) 46 | end 47 | 48 | local function removeReshapeModule(imageEncoder) 49 | imageEncoder:remove(2) 50 | return imageEncoder 51 | end 52 | 53 | local similarityModel = loadModel() 54 | local imageEncoder = similarityModel.modules[1].modules[1].modules[1].modules[1] 55 | print(imageEncoder) 56 | imageEncoder = removeReshapeModule(imageEncoder) 57 | generateDatabase(imageEncoder) 58 | 59 | print("Database for Deeprank searcher generated") 60 | -------------------------------------------------------------------------------- /src/torch/14-deeprank-db/deeprank-db.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'optim' 10 | require 'xlua' 11 | require 'lfs' 12 | local torch = require 'torch' 13 | 14 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 15 | local similarity_lib = require '14-deeprank-db/similarity_lib' 16 | local database = require('0-tiefvision-commons/tiefvision_config_loader').load().database.supervised_similarity 17 | 18 | local function similarityDb() 19 | local dataFolder = tiefvision_commons.dataPath('db/similarity/img-similarity-deeprank') 20 | local files = tiefvision_commons.getFiles(dataFolder) 21 | local filesAlreadyProcessed = database.keys() 22 | local filesRemaining = tiefvision_commons.tableSubtraction(files, filesAlreadyProcessed) 23 | 24 | for referenceIndex = 1, #filesRemaining do 25 | local reference = filesRemaining[referenceIndex] 26 | print(reference) 27 | 28 | local similarities = {} 29 | 30 | local referenceEncoding = torch.load(dataFolder .. '/' .. reference):double() 31 | for testIndex = 1, #files do 32 | local test = files[testIndex] 33 | local imageEncoding = torch.load(dataFolder .. '/' .. test):double() 34 | local similarity = similarity_lib.similarity(referenceEncoding, imageEncoding) 35 | similarities[test] = similarity or -1 36 | end 37 | 38 | -- compare itself with its mirror 39 | local flipped = tiefvision_commons.dataPath('db/similarity/img-flipped-similarity-deeprank', reference) 40 | local flippedEncoding = torch.load(flipped):double() 41 | local similarity = similarity_lib.similarity(referenceEncoding, flippedEncoding) 42 | similarities[reference] = similarity or -1 43 | 44 | database.write(reference, similarities) 45 | 46 | if referenceIndex % 10 == 0 then 47 | collectgarbage() 48 | end 49 | end 50 | end 51 | 52 | similarityDb() 53 | -------------------------------------------------------------------------------- /src/torch/14-deeprank-db/similarity_lib.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | require 'torch' 6 | 7 | local similarity_lib = {} 8 | 9 | function similarity_lib.similarity(referenceEncoding, imageEncoding) 10 | local sumSimilarity = 0.0 11 | if (referenceEncoding:size():size() == 3 and imageEncoding:size():size() == 3) then 12 | --print(referenceEncoding:size()) 13 | --print(imageEncoding:size()) 14 | referenceEncoding = referenceEncoding:transpose(1, 3) 15 | imageEncoding = imageEncoding:transpose(1, 3) 16 | local minHeight = math.min(referenceEncoding:size()[2], imageEncoding:size()[2]) 17 | local maxHeight = math.max(referenceEncoding:size()[2], imageEncoding:size()[2]) 18 | if (maxHeight - minHeight < 5) then 19 | for h = 1, minHeight do 20 | for w = 1, referenceEncoding:size()[1] do 21 | local similarityLoc = imageEncoding[w][h] * referenceEncoding[w][h] 22 | sumSimilarity = sumSimilarity + similarityLoc 23 | end 24 | end 25 | local similarity = sumSimilarity / (minHeight * referenceEncoding:size()[1]) 26 | return similarity 27 | end 28 | end 29 | return -1.0 30 | end 31 | 32 | return similarity_lib 33 | -------------------------------------------------------------------------------- /src/torch/15-deeprank-searcher-db/search.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'optim' 10 | require 'xlua' 11 | require 'lfs' 12 | local torch = require 'torch' 13 | 14 | local tiefvision_config_loader = require '0-tiefvision-commons/tiefvision_config_loader' 15 | local search_commons = require '10-similarity-searcher-cnn-db/search_commons' 16 | local database = tiefvision_config_loader.load().database.supervised_similarity 17 | 18 | local function getTestError(reference) 19 | local similarities = database.read(reference) 20 | 21 | local comparisonTable = {} 22 | for file, sim in pairs(similarities) do 23 | table.insert(comparisonTable, { file, sim }) 24 | end 25 | 26 | table.sort(comparisonTable, search_commons.sortCmpTable) 27 | search_commons.printCmpTable(comparisonTable) 28 | end 29 | 30 | local function getOptions() 31 | local cmd = torch.CmdLine() 32 | cmd:text() 33 | cmd:text('Supervised image search from precomputed database of distances between each pair of images in the master folder.') 34 | cmd:text('Returns a descending sorted list of filenames concatenated with a similarity metric.') 35 | cmd:text('Both the filename to search and the result filenames come from the folder $TIEFVISION_HOME/resources/dresses-db/master.') 36 | cmd:text() 37 | cmd:text('Options:') 38 | cmd:argument('image', 'Filename (not full path, just the filename) from $TIEFVISION_HOME/resources/dresses-db/master.', 'string') 39 | cmd:text() 40 | cmd:option('-config', tiefvision_config_loader.default, 'Configuration file to use.') 41 | cmd:text() 42 | return cmd:parse(arg) 43 | end 44 | 45 | local options = getOptions() 46 | getTestError(options.image) 47 | -------------------------------------------------------------------------------- /src/torch/2-encode-bounding-box-training-and-test-images/encode-training-and-test-images.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- Uses the image encoder to encode the train and test 6 | -- data sets for the bounding box regression 7 | 8 | local torchFolder = require('paths').thisfile('..') 9 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 10 | 11 | require 'nn' 12 | require 'inn' 13 | require 'image' 14 | require 'lfs' 15 | local torch = require 'torch' 16 | 17 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 18 | 19 | 20 | local function encodedInputOutput(trainFile, encoder) 21 | local name, widht, height, xmin, ymin, xmax, ymax = string.match(trainFile, "(.+)___(%d+)_(%d+)_(-?%d+)_(-?%d+)_(-?%d+)_(-?%d+).jpg") 22 | local input = tiefvision_commons.load(trainFile) 23 | local encodedInput = encoder:forward(input)[2] 24 | local target = torch.CudaTensor(4) 25 | target[1] = tonumber(xmin) 26 | target[2] = tonumber(ymin) 27 | target[3] = tonumber(xmax) 28 | target[4] = tonumber(ymax) 29 | return encodedInput, target 30 | end 31 | 32 | local function loadData(encoder, filename) 33 | local outputsBatch = {} 34 | local inputsBatch = {} 35 | local lines = tiefvision_commons.getLines(filename) 36 | local batchSize = 320 37 | local batches = math.ceil(#lines / batchSize) 38 | local linesIndex = 1 39 | for i = 1, batches do 40 | local currentBatchSize = batchSize 41 | if i == batches and #lines % batchSize > 0 then 42 | currentBatchSize = #lines % batchSize 43 | end 44 | local inputs = torch.Tensor(currentBatchSize, 384, 11, 11):cuda() 45 | local outputs = torch.Tensor(currentBatchSize, 4):cuda() 46 | for li = 1, currentBatchSize do 47 | local fileIndex = li + linesIndex - 1 48 | local trainFileLine = lines[fileIndex] 49 | local encodedInput, target = encodedInputOutput(trainFileLine, encoder) 50 | inputs[li] = inputs[li]:set(encodedInput) 51 | outputs[li] = outputs[li]:set(target) 52 | end 53 | linesIndex = linesIndex + currentBatchSize 54 | outputsBatch[i] = outputs 55 | inputsBatch[i] = inputs 56 | end 57 | return inputsBatch, outputsBatch 58 | end 59 | 60 | local function loadDataFromFolder(bboxFolder, i) 61 | local filePath = tiefvision_commons.dataPath(bboxFolder, i .. '.data') 62 | return torch.load(filePath) 63 | end 64 | 65 | local function testStdDevNonZero(size, folder) 66 | for ig = 1, size do 67 | local data = loadDataFromFolder(folder, ig) 68 | for i = 1, data:size()[1] do 69 | assert(torch.std(data[i]) > 5.0, 'the standard deviation of the data should not be zero') 70 | end 71 | end 72 | end 73 | 74 | local function stats(output) 75 | local globalMean = torch.zeros(4):cuda() 76 | local globalStd = torch.zeros(4):cuda() 77 | for i = 1, #output do 78 | local mean = torch.mean(output[i], 1):cuda() 79 | local std = torch.std(output[i], 1):cuda() 80 | globalMean = torch.add(globalMean, mean[1]) 81 | globalStd = torch.add(globalStd, std[1]) 82 | end 83 | return globalMean / #output, globalStd / #output 84 | end 85 | 86 | local function postprocessOutput(output, mean, std) 87 | local globalOutZeroMeanOneStd = {} 88 | for i = 1, #output do 89 | local meanRep = torch.repeatTensor(mean, output[i]:size()[1], 1) 90 | local stdRep = torch.repeatTensor(std, output[i]:size()[1], 1) 91 | local outZeroMeanOneStd = torch.add(output[i], -meanRep) 92 | outZeroMeanOneStd:cdiv(stdRep) 93 | globalOutZeroMeanOneStd[i] = outZeroMeanOneStd 94 | end 95 | return globalOutZeroMeanOneStd 96 | end 97 | 98 | local function getBoundingBoxes(output) 99 | local avg = torch.load(tiefvision_commons.modelPath('bbox-train-mean')) 100 | local std = torch.load(tiefvision_commons.modelPath('bbox-train-std')) 101 | local avgExt = torch.repeatTensor(avg, output:size()[1], 1) 102 | local stdExt = torch.repeatTensor(std, output:size()[1], 1) 103 | local outputTrans = torch.cmul(output, stdExt) + avgExt 104 | return outputTrans 105 | end 106 | 107 | local function testinLoadTest(testin) 108 | testStdDevNonZero(#testin, 'bbox-test-in') 109 | for i = 1, #testin do 110 | local testinLoad = loadDataFromFolder('bbox-test-in', i) 111 | assert(torch.eq(testinLoad, testin[i]), 'test input not properly saved') 112 | end 113 | end 114 | 115 | local function testEq(actual, savedFolder) 116 | for i = 1, #actual do 117 | local saved = loadDataFromFolder(savedFolder, i) 118 | assert(torch.eq(actual[i], saved), 'tensor not properly saved') 119 | end 120 | end 121 | 122 | local function traininLoadTest(trainin) 123 | testStdDevNonZero(#trainin, 'bbox-train-in') 124 | testEq(trainin, 'bbox-train-in') 125 | end 126 | 127 | local function testBoundingBoxes(trainout, trainoutProc) 128 | for i = 1, #trainout do 129 | local reconstructedBoundingBox = getBoundingBoxes(trainoutProc[i]) 130 | assert(torch.mean(torch.abs(reconstructedBoundingBox - trainout[i])) < 0.0001, 'the reconstructed bounding box should be the original one') 131 | end 132 | end 133 | 134 | local function saveEncodedData() 135 | local encoder = torch.load(tiefvision_commons.modelPath('encoder.model')) 136 | local trainin, trainout = loadData(encoder, tiefvision_commons.resourcePath('bounding-boxes/extendedTRAIN.txt')) 137 | local mean, std = stats(trainout) 138 | torch.save(tiefvision_commons.modelPath('bbox-train-mean'), mean) 139 | torch.save(tiefvision_commons.modelPath('bbox-train-std'), std) 140 | 141 | local trainoutProc = postprocessOutput(trainout, mean, std) 142 | local meantest, stdtest = stats(trainoutProc) 143 | assert(torch.mean(meantest) < 0.0001, 'mean should be zero') 144 | assert(math.abs(torch.mean(stdtest) - 1) < 0.0001, 'std should be one') 145 | 146 | 147 | local testin, testout = loadData(encoder, tiefvision_commons.resourcePath('bounding-boxes/extendedTEST.txt')) 148 | local testoutProc = postprocessOutput(testout, mean, std) 149 | meantest, stdtest = stats(testoutProc) 150 | assert(torch.mean(meantest) < 0.2, 'test mean should be close to zero') 151 | assert(math.abs(torch.mean(stdtest) - 1) < 0.3, 'test std should be close to one') 152 | 153 | for i = 1, #testin do 154 | local testoutTr = testoutProc[i]:transpose(1, 2) 155 | torch.save(tiefvision_commons.dataPath('bbox-test-in', i .. '.data'), testin[i]) 156 | torch.save(tiefvision_commons.dataPath('bbox-test-out', i .. '.data'), testoutTr) 157 | end 158 | for i = 1, #trainin do 159 | local trainoutTr = trainoutProc[i]:transpose(1, 2) 160 | torch.save(tiefvision_commons.dataPath('bbox-train-in', i .. '.data'), trainin[i]) 161 | torch.save(tiefvision_commons.dataPath('bbox-train-out', i .. '.data'), trainoutTr) 162 | end 163 | return testin, testoutProc, trainin, trainoutProc, trainout 164 | end 165 | 166 | local testin, testoutProc, trainin, trainoutProc, trainout = saveEncodedData() 167 | 168 | testinLoadTest(testin) 169 | traininLoadTest(trainin) 170 | 171 | testEq(testoutProc, 'bbox-test-out') 172 | testEq(trainoutProc, 'bbox-train-out') 173 | 174 | testBoundingBoxes(trainout, trainoutProc) 175 | 176 | print('Data encoding finished') 177 | -------------------------------------------------------------------------------- /src/torch/3-train-regression-bounding-box/locatorconv.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | require "inn" 6 | require 'optim' 7 | require 'torch' 8 | require 'xlua' 9 | local nn = require 'nn' 10 | 11 | local locatorconv = {} 12 | 13 | function locatorconv.loadModel() 14 | local nhiddens1 = 512 15 | local nhiddens2 = 128 16 | local noutputs = 1 17 | local model = nn.Sequential() 18 | model:add(nn.SpatialConvolutionMM(384, nhiddens1, 11, 11, 1, 1, 0, 0)) 19 | model:add(nn.Tanh()) 20 | model:add(nn.SpatialConvolutionMM(nhiddens1, nhiddens2, 1, 1, 1, 1, 0, 0)) 21 | model:add(nn.Tanh()) 22 | model:add(nn.SpatialConvolutionMM(nhiddens2, noutputs, 1, 1, 1, 1, 0, 0)) 23 | return model:cuda() 24 | end 25 | 26 | return locatorconv 27 | -------------------------------------------------------------------------------- /src/torch/3-train-regression-bounding-box/test-regression-bounding-box.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'optim' 10 | require 'xlua' 11 | local torch = require 'torch' 12 | 13 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 14 | 15 | local function getTestError(model, index) 16 | local testIn = torch.load(tiefvision_commons.dataPath('bbox-test-in/1.data')) 17 | local testOut = torch.load(tiefvision_commons.dataPath('bbox-test-out/1.data')) 18 | local mean = torch.load(tiefvision_commons.modelPath('bbox-train-mean')) 19 | local std = torch.load(tiefvision_commons.modelPath('bbox-train-std')) 20 | local errRandVec = 0.0 21 | local countRand = 0 22 | local errVec = 0.0 23 | local count = 0 24 | for i = 1, testIn:size()[1] do 25 | local output = (model:forward(testIn[i])[1][1][1] * std[index]) + mean[index] 26 | local target = (testOut[index][i] * std[index]) + mean[index] 27 | errVec = errVec + math.abs(target - output) 28 | count = count + 1 29 | local outputRand = model:forward(testIn[((i + 2) % testIn:size()[1]) + 1])[1][1][1] * std[index] 30 | errRandVec = errRandVec + math.abs((outputRand - (testOut[index][i] * std[index]))) 31 | countRand = countRand + 1 32 | end 33 | errVec = errVec / count 34 | errRandVec = errRandVec / countRand 35 | return errVec, errRandVec 36 | end 37 | 38 | local function loadSavedModelConv(index) 39 | return torch.load(tiefvision_commons.modelPath('locatorconv-' .. index .. '.model')) 40 | end 41 | 42 | for index = 1, 4 do 43 | local model = loadSavedModelConv(index) 44 | local error, errorRand = getTestError(model, index) 45 | print('Index: ' .. index) 46 | print(' Error Actual Images: ' .. error) 47 | print(' Error Random Input : ' .. errorRand) 48 | assert(error * 1.25 < errorRand, 'the error predicting images should be higher than the error from random inputs') 49 | end 50 | 51 | print('Test passed') 52 | -------------------------------------------------------------------------------- /src/torch/3-train-regression-bounding-box/train-regression-bounding-box.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'xlua' 10 | local lfs = require 'lfs' 11 | local nn = require 'nn' 12 | local optim = require 'optim' 13 | local sys = require 'sys' 14 | local torch = require 'torch' 15 | 16 | local locatorconv = require '3-train-regression-bounding-box/locatorconv' 17 | 18 | local batchSize = 32 19 | 20 | local inputsBatch = torch.Tensor(batchSize, 384, 11, 11):cuda() 21 | local outputsBatch = torch.Tensor(batchSize, 1):cuda() 22 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 23 | 24 | local function saveModel(model, index) 25 | local filename = tiefvision_commons.modelPath('locatorconv-' .. index .. '.model') 26 | print('==> Saving Model: ' .. filename) 27 | torch.save(filename, model) 28 | print('==> Model Saved: ' .. filename) 29 | end 30 | 31 | local function trainBatch(model, criterion, inputsBatchesLocal, outputsBatchLocal, optimState) 32 | local parameters, gradParameters = model:getParameters() 33 | local feval = function(x) 34 | if x ~= parameters then 35 | parameters:copy(x) 36 | end 37 | gradParameters:zero() 38 | local outputs = model:forward(inputsBatchesLocal) 39 | local f = criterion:forward(outputs, outputsBatchLocal) 40 | local df_do = criterion:backward(outputs, outputsBatchLocal) 41 | model:backward(inputsBatchesLocal, df_do) 42 | return f, gradParameters 43 | end 44 | 45 | local x, fx = optim.sgd(feval, parameters, optimState) 46 | return fx[1] 47 | end 48 | 49 | local function train(trainIn, trainOut, model, criterion, index, optimState) 50 | local trainIndex = 1 51 | local trainingLoss = 0.0 52 | local trainBatches = math.floor(trainIn:size()[1] / batchSize) 53 | for b = 1, trainBatches do 54 | for batchIndex = 1, batchSize do 55 | if batchIndex + trainIndex - 1 <= trainIn:size()[1] then 56 | inputsBatch[batchIndex] = trainIn[batchIndex + trainIndex - 1] 57 | outputsBatch[batchIndex] = trainOut[batchIndex + trainIndex - 1] 58 | end 59 | end 60 | trainIndex = trainIndex + batchSize 61 | trainingLoss = trainBatch(model, criterion, inputsBatch, outputsBatch, optimState) 62 | print("Batch: " .. b .. " out of " .. trainBatches .. ". Train Loss: " .. trainingLoss) 63 | end 64 | saveModel(model, index) 65 | return trainingLoss 66 | end 67 | 68 | local function loadDataFromFolder(dataFolder) 69 | local folder = tiefvision_commons.dataPath(dataFolder) 70 | local fileCount = 0 71 | for file in lfs.dir(folder) do 72 | if (lfs.attributes(folder .. '/' .. file, "mode") == "file") then 73 | fileCount = fileCount + 1 74 | end 75 | end 76 | local data = { n = fileCount } 77 | for file in lfs.dir(folder) do 78 | if (lfs.attributes(folder .. '/' .. file, "mode") == "file") then 79 | local dataInFile = torch.load(folder .. '/' .. file) 80 | local indexStr = string.match(file, '(%d+)%.data') 81 | local i = math.floor(indexStr) 82 | data[i] = dataInFile 83 | end 84 | end 85 | return data 86 | end 87 | 88 | local function getTestError(model, criterion, index) 89 | local testIn = torch.load(tiefvision_commons.dataPath('bbox-test-in/1.data')) 90 | local testOut = torch.load(tiefvision_commons.dataPath('bbox-test-out/1.data')) 91 | local output = model:forward(testIn) 92 | local err = criterion:forward(output, testOut[index]) 93 | return err 94 | end 95 | 96 | local function loadCriterion() 97 | local criterion = nn.MSECriterion() 98 | criterion.sizeAverage = false 99 | return criterion:cuda() 100 | end 101 | 102 | local function loadSavedModel(index) 103 | local modelPath = tiefvision_commons.modelPath('locatorconv-' .. index .. '.model') 104 | if(tiefvision_commons.fileExists(modelPath)) then 105 | return torch.load(modelPath) 106 | else 107 | return locatorconv.loadModel() 108 | end 109 | end 110 | 111 | local function getOptions() 112 | local cmd = torch.CmdLine() 113 | cmd:text() 114 | cmd:text('Bounding Box Regression Training') 115 | cmd:text() 116 | cmd:text('Options:') 117 | cmd:option('-reset', false, 'Reset the saved model (if any) and use a new model.') 118 | cmd:option('-index', 1, 'Index of the bunding box point to train (1:x-min, 2: y-min, 3: x-max, 4:y-max)') 119 | -- optim state 120 | cmd:option('-learningRate', 1e-6, 'Learning rate.') 121 | cmd:option('-weightDecay', 1.0, 'Weight Decay (L1 regularization).') 122 | cmd:option('-momentum', 0.1, 'Momentum.') 123 | cmd:option('-learningRateDecay', 1e-7, 'Learning Rate Decay.') 124 | cmd:text() 125 | return cmd:parse(arg) 126 | end 127 | 128 | local function getModel(options) 129 | if (options.reset) then 130 | return locatorconv.loadModel() 131 | else 132 | return loadSavedModel(options.index) 133 | end 134 | end 135 | 136 | local function getIndexLabel(index) 137 | if (index == 1) then 138 | return "x-min" 139 | elseif (index == 2) then 140 | return "y-min" 141 | elseif (index == 3) then 142 | return "x-max" 143 | else 144 | return "y-max" 145 | end 146 | end 147 | 148 | local function trainIndex(index, model, optimState) 149 | local indexLabel = getIndexLabel(index) 150 | print("Training " .. indexLabel) 151 | local criterion = loadCriterion() 152 | model:training() 153 | local epochs = 30 154 | for epoch = 1, epochs do 155 | local trainInData = loadDataFromFolder("bbox-train-in") 156 | local trainOutData = loadDataFromFolder("bbox-train-out") 157 | local time = sys.clock() 158 | for i = 1, #trainInData do 159 | local trainIn = trainInData[i] 160 | local trainOuts = trainOutData[i] 161 | local trainOut = trainOuts[index] 162 | print("-------------------------------------------------------------------") 163 | print("Training index " .. index .. " for epoch " .. epoch .. " and batch " .. i) 164 | local trainingLoss = train(trainIn, trainOut, model, criterion, index, optimState) 165 | local testError = getTestError(model, criterion, index) 166 | print("Test Loss: " .. testError .. ". Train Loss: " .. trainingLoss) 167 | end 168 | time = sys.clock() - time 169 | print("Time to learn full batch = " .. (time / (60 * 60)) .. " hours\n") 170 | end 171 | end 172 | 173 | local function getOptimSatate(options) 174 | local optimState = { 175 | learningRate = options.learningRate, 176 | weightDecay = options.weightDecay, 177 | momentum = options.momentum, 178 | learningRateDecay = options.learningRateDecay 179 | } 180 | return optimState 181 | end 182 | 183 | local options = getOptions() 184 | local optimState = getOptimSatate(options) 185 | local model = getModel(options) 186 | trainIndex(options.index, model, optimState) 187 | -------------------------------------------------------------------------------- /src/torch/4-encode-classification-train-and-test-images/encode-training-and-test-images.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'nn' 9 | require 'inn' 10 | require 'image' 11 | local torch = require 'torch' 12 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 13 | 14 | local batch_size = 64 15 | 16 | local function getEncodedInput(name, encoder) 17 | print(name) 18 | local input = tiefvision_commons.load(name) 19 | local encodedInput = encoder:forward(input)[2] 20 | collectgarbage() 21 | return encodedInput 22 | end 23 | 24 | -- TODO: we are wasting at most 63 samples due to batching 25 | local function loadData(encoder, lines) 26 | local batches = #lines / batch_size 27 | local inputs = torch.Tensor(batches, batch_size, 384, 11, 11):cuda() 28 | for batch = 1, batches do 29 | for batch_el = 1, batch_size do 30 | local lineIndex = ((batch - 1) * batch_size) + batch_el 31 | local fileName = lines[lineIndex] 32 | local encodedInput = getEncodedInput(fileName, encoder) 33 | inputs[batch][batch_el] = inputs[batch][batch_el]:set(encodedInput) 34 | end 35 | end 36 | return inputs 37 | end 38 | 39 | local function getFilesAsTable(prefix) 40 | local trainingFiles = {} 41 | for cl = 0, 1 do 42 | local file_path = tiefvision_commons.resourcePath('bounding-boxes', cl .. "-" .. prefix .. ".txt") 43 | local lines = tiefvision_commons.getLines(file_path) 44 | print(file_path .. ': ' .. #lines) 45 | trainingFiles[cl + 1] = lines 46 | end 47 | return trainingFiles 48 | end 49 | 50 | local function testSavedData(data) 51 | for i = 1, data:size()[1] do 52 | assert(torch.mean(data[i]) > 0.01, 'the mean of the data should no be zero') 53 | end 54 | end 55 | 56 | local function getFile(type, cl, i) 57 | return tiefvision_commons.dataPath('classification', cl, type, i .. '.data') 58 | end 59 | 60 | local function encodeData(type, encoder) 61 | for cl = 0, 1 do 62 | local files = getFilesAsTable(type) 63 | print('Class ' .. cl .. ' with files ' .. #files[cl + 1]) 64 | local input = loadData(encoder, files[cl + 1]) 65 | input = input:double() 66 | for i = 1, input:size()[1] do 67 | print("Saving batch " .. i .. " for " .. type) 68 | torch.save(getFile(type, cl, i), input[i]:clone()) -- use clone as otherwise it saves the whole tensor 69 | local inputLoaded = torch.load(getFile(type, cl, i)) 70 | testSavedData(inputLoaded) 71 | assert(torch.eq(input[i], inputLoaded), 'test input not properly saved') 72 | collectgarbage() 73 | end 74 | end 75 | end 76 | 77 | local encoder = torch.load(tiefvision_commons.modelPath('encoder.model')) 78 | 79 | encodeData('train', encoder) 80 | encodeData('test', encoder) 81 | 82 | print('Data encoding finished') 83 | -------------------------------------------------------------------------------- /src/torch/5-train-classification/classifier-conv.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | require "inn" 6 | require 'optim' 7 | require 'torch' 8 | require 'xlua' 9 | local nn = require 'nn' 10 | 11 | local classifierconv = {} 12 | 13 | function classifierconv.loadModel() 14 | local nhiddens1 = 1024 15 | local nhiddens2 = 256 16 | local noutputs = 2 17 | local model = nn.Sequential() 18 | model:add(nn.SpatialConvolutionMM(384, nhiddens1, 11, 11, 1, 1, 0, 0)) 19 | model:add(nn.ReLU()) 20 | model:add(nn.SpatialConvolutionMM(nhiddens1, nhiddens2, 1, 1, 1, 1, 0, 0)) 21 | model:add(nn.ReLU()) 22 | model:add(nn.SpatialConvolutionMM(nhiddens2, noutputs, 1, 1, 1, 1, 0, 0)) 23 | model:add(nn.Sigmoid()) 24 | return model:cuda() 25 | end 26 | 27 | return classifierconv 28 | -------------------------------------------------------------------------------- /src/torch/5-train-classification/train-classification.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'xlua' 10 | local nn = require 'nn' 11 | local optim = require 'optim' 12 | local torch = require 'torch' 13 | 14 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 15 | local classifier = require '5-train-classification/classifier-conv' 16 | 17 | local batchSize = 64 18 | 19 | local inputsBatch = torch.Tensor(batchSize, 384, 11, 11):cuda() 20 | local targetsBatch = torch.Tensor(batchSize):cuda() 21 | 22 | local function getBatchesInClassAndType(class, type) 23 | local folder = tiefvision_commons.dataPath('classification', (class - 1), type) 24 | local lines = tiefvision_commons.getFiles(folder) 25 | return #lines 26 | end 27 | 28 | local function getFilename(type, cl, i) 29 | return tiefvision_commons.dataPath('classification', (cl - 1), type, i .. '.data') 30 | end 31 | 32 | local function getDataFromClassAndType(class, type) 33 | local batches = getBatchesInClassAndType(class, type) 34 | local tensor = torch.Tensor(batches, 64, 384, 11, 11) 35 | for l = 1, batches do 36 | local loadedTensor = torch.load(getFilename(type, class, l)) 37 | tensor[l] = loadedTensor 38 | collectgarbage() 39 | end 40 | return tensor:cuda() 41 | end 42 | 43 | local function getMaxIndex(outputs) 44 | local maxIndex = torch.Tensor(outputs:size()[1]) 45 | for e = 1, outputs:size()[1] do 46 | local output = outputs[e] 47 | local index = 1 48 | for i = 1, output:size()[1] do 49 | if (output[i] > output[index]) then 50 | index = i 51 | end 52 | end 53 | maxIndex[e] = index 54 | end 55 | return maxIndex 56 | end 57 | 58 | local function correctClassNum(maxIndex, cl) 59 | local correctClass = 0 60 | for e = 1, maxIndex:size()[1] do 61 | if (maxIndex[e] == cl) then 62 | correctClass = correctClass + 1 63 | end 64 | end 65 | return correctClass 66 | end 67 | 68 | local function getTestError(model) 69 | local testIn = {} 70 | testIn[1] = getDataFromClassAndType(1, 'test') 71 | testIn[2] = getDataFromClassAndType(2, 'test') 72 | local classified = 0 73 | local elements = 0 74 | for cl = 1, 2 do 75 | local testInCl = testIn[cl] 76 | for batch = 1, testInCl:size()[1] do 77 | local output = model:forward(testInCl[batch]) 78 | output = torch.squeeze(output) 79 | local firstIndex = getMaxIndex(output) 80 | classified = classified + correctClassNum(firstIndex, cl) 81 | elements = elements + testInCl[batch]:size()[1] 82 | end 83 | end 84 | classified = classified / elements 85 | return classified 86 | end 87 | 88 | local function saveModelConv(model) 89 | local filename = tiefvision_commons.modelPath('classifier.model') 90 | print('==> Saving Model: ' .. filename) 91 | torch.save(filename, model) 92 | end 93 | 94 | local function trainBatch(model, criterion, inputsBatchLocal, targetsBatchLocal, optimState) 95 | local parameters, gradParameters = model:getParameters() 96 | local feval = function(x) 97 | if x ~= parameters then 98 | parameters:copy(x) 99 | end 100 | gradParameters:zero() 101 | local outputs = model:forward(inputsBatchLocal) 102 | local f = criterion:forward(outputs, targetsBatchLocal) 103 | local df_do = criterion:backward(outputs, targetsBatchLocal) 104 | model:backward(inputsBatchLocal, df_do) 105 | collectgarbage() 106 | return f, gradParameters 107 | end 108 | local x, fx = optim.sgd(feval, parameters, optimState) 109 | return fx[1] 110 | end 111 | 112 | local function train(model, criterion, epochs, optimState) 113 | model:training() 114 | for epoch = 1, epochs do 115 | math.randomseed(os.time()) 116 | local batchesIn1 = getBatchesInClassAndType(1, 'train') 117 | local batchesIn2 = getBatchesInClassAndType(2, 'train') 118 | for iter = 1, batchesIn1 + batchesIn2 do 119 | local batchIndexClass1 = math.random(batchesIn1) 120 | local batchIndexClass2 = math.random(batchesIn2) 121 | local batchClass1 = torch.load(getFilename('train', 1, batchIndexClass1)):cuda() 122 | local batchClass2 = torch.load(getFilename('train', 2, batchIndexClass2)):cuda() 123 | local batches = { batchClass1, batchClass2 } 124 | for batchIndex = 1, batchSize do 125 | -- select random class 126 | local cl = math.random(2) 127 | -- select random sample in batch 128 | local sampleIndex = math.random(64) 129 | inputsBatch[batchIndex] = batches[cl][sampleIndex] 130 | targetsBatch[batchIndex] = cl 131 | end 132 | local trainingLoss = trainBatch(model, criterion, inputsBatch, targetsBatch, optimState) 133 | if (iter % 10 == 0) then 134 | local meanClass = getTestError(model) 135 | print("Epoch: " .. epoch .. ". Batch: " .. iter .. ". Train Loss: " .. trainingLoss .. ". Test Accuracy: " .. meanClass) 136 | saveModelConv(model) 137 | else 138 | print("Epoch " .. epoch .. " out of " .. epochs .. ". Batch Iteration: " .. iter .. ". Train Loss: " .. trainingLoss) 139 | end 140 | collectgarbage() 141 | end 142 | end 143 | end 144 | 145 | local function loadCriterion() 146 | local criterion = nn.CrossEntropyCriterion() 147 | criterion.sizeAverage = true 148 | return criterion:cuda() 149 | end 150 | 151 | local function loadSavedModelConv() 152 | local modelPath = tiefvision_commons.modelPath('classifier.model') 153 | if(tiefvision_commons.fileExists(modelPath)) then 154 | return torch.load(modelPath) 155 | else 156 | return classifier.loadModel() 157 | end 158 | end 159 | 160 | local function getOptions() 161 | local cmd = torch.CmdLine() 162 | cmd:text() 163 | cmd:text('Foreground and Background Classification Training') 164 | cmd:text() 165 | cmd:text('Options:') 166 | cmd:option('-reset', false, 'Reset the saved model (if any) and use a new model.') 167 | cmd:option('-epochs', 10, 'Number of epochs to train.') 168 | -- optim state 169 | cmd:option('-learningRate', 1e-2, 'Learning rate.') 170 | cmd:option('-weightDecay', 0.0, 'Weight Decay (L1 regularization).') 171 | cmd:option('-momentum', 0.1, 'Momentum.') 172 | cmd:option('-learningRateDecay', 1e-7, 'Learning Rate Decay.') 173 | cmd:text() 174 | return cmd:parse(arg) 175 | end 176 | 177 | local function getModel(options) 178 | if (options.reset) then 179 | return classifier.loadModel() 180 | else 181 | return loadSavedModelConv() 182 | end 183 | end 184 | 185 | local function getOptimSatate(options) 186 | local optimState = { 187 | learningRate = options.learningRate, 188 | weightDecay = options.weightDecay, 189 | momentum = options.momentum, 190 | learningRateDecay = options.learningRateDecay 191 | } 192 | return optimState 193 | end 194 | 195 | local options = getOptions() 196 | local optimState = getOptimSatate(options) 197 | local model = getModel(options) 198 | local meanClass = getTestError(model) 199 | print("Test Accuracy:" .. meanClass) 200 | local criterion = loadCriterion() 201 | train(model, criterion, options.epochs, optimState) 202 | -------------------------------------------------------------------------------- /src/torch/6-bboxlib/bboxlib.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'optim' 10 | require 'xlua' 11 | local image = require 'image' 12 | local torch = require 'torch' 13 | 14 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 15 | 16 | local bboxlib = {} 17 | 18 | local function loadLocator(index) 19 | return torch.load(tiefvision_commons.modelPath('locatorconv-' .. index .. '.model')) 20 | end 21 | 22 | local function loadClassifier() 23 | return torch.load(tiefvision_commons.modelPath('classifier.model')) 24 | end 25 | 26 | local function loadEncoder() 27 | return torch.load(tiefvision_commons.modelPath('encoder.model')) 28 | end 29 | 30 | local function getBoundingBoxes(model, encodedInputs, index) 31 | local avg = torch.load(tiefvision_commons.modelPath('bbox-train-mean'))[index] 32 | local avgTensor = torch.Tensor(1):cuda() 33 | avgTensor[1] = avg 34 | local std = torch.load(tiefvision_commons.modelPath('bbox-train-std'))[index] 35 | local stdTensor = torch.Tensor(1):cuda() 36 | stdTensor[1] = std 37 | local output = model:forward(encodedInputs) 38 | output = output:transpose(1, 3) 39 | output = output:transpose(1, 2) 40 | local avgExt = torch.repeatTensor(avgTensor, output:size()[1], output:size()[2], 1) 41 | local stdExt = torch.repeatTensor(stdTensor, output:size()[1], output:size()[2], 1) 42 | local outputTrans = torch.cmul(output, stdExt) + avgExt 43 | return outputTrans 44 | end 45 | 46 | local function toOutputCoordinate(coord) 47 | return math.max(math.floor((math.floor((math.floor((coord - 10) / 4) - 2) / 2) - 2) / 2) - 2 + 1 - 10, 1) 48 | end 49 | 50 | local function toOutputCoordinates(x, y) --, reduction, xdelta, ydelta) 51 | -- local xo = math.floor((x - (xdelta / 2.0)) / reduction) 52 | -- local yo = math.floor((y - (ydelta / 2.0)) / reduction) 53 | -- if xo <= 0 then xo = 1 end 54 | -- if yo <= 0 then yo = 1 end 55 | -- return xo, yo 56 | return toOutputCoordinate(x), toOutputCoordinate(y) 57 | end 58 | 59 | local function toImageCoordinate(coord) 60 | return ((((((coord - 1 + 2 + 10) * 2) + 2) * 2) + 2) * 4) + 10 + 7 - 224 61 | end 62 | 63 | local function toImageCoordinates(x, y) 64 | return toImageCoordinate(x), toImageCoordinate(y) 65 | end 66 | 67 | local function meanBoundingBox(boundingBoxes) 68 | local sum = { 0.0, 0.0, 0.0, 0.0 } 69 | for i = 1, #boundingBoxes do 70 | local boundingBox = boundingBoxes[i] 71 | sum = { sum[1] + boundingBox[1], sum[2] + boundingBox[2], sum[3] + boundingBox[3], sum[4] + boundingBox[4] } 72 | end 73 | local mean = { sum[1] / #boundingBoxes, sum[2] / #boundingBoxes, sum[3] / #boundingBoxes, sum[4] / #boundingBoxes } 74 | return mean 75 | end 76 | 77 | local function boundingBoxOutputCenter(boundingBox) 78 | local xcenter = boundingBox[1] + ((boundingBox[3] - boundingBox[1]) / 2.0) 79 | local ycenter = boundingBox[2] + ((boundingBox[4] - boundingBox[2]) / 2.0) 80 | return toOutputCoordinates(xcenter, ycenter) 81 | end 82 | 83 | local function getExpectedBoundingBox(boundingBoxes) 84 | local mean = meanBoundingBox(boundingBoxes) 85 | local xcenter, ycenter = boundingBoxOutputCenter(mean) 86 | local eminx, eminy, emaxx, emaxy = 0.0, 0.0, 0.0, 0.0 87 | local eminxCount, eminyCount, emaxxCount, emaxyCount = 0, 0, 0, 0 88 | for i = 1, #boundingBoxes do 89 | local boundingBox = boundingBoxes[i] 90 | local minx, miny, maxx, maxy, xo, yo = boundingBox[1], boundingBox[2], boundingBox[3], boundingBox[4], boundingBox[6], boundingBox[7] 91 | if xo < xcenter then 92 | eminx = eminx + minx 93 | eminxCount = eminxCount + 1 94 | end 95 | if xo > xcenter then 96 | emaxx = emaxx + maxx 97 | emaxxCount = emaxxCount + 1 98 | end 99 | if yo < ycenter then 100 | eminy = eminy + miny 101 | eminyCount = eminyCount + 1 102 | end 103 | if yo > ycenter then 104 | emaxy = emaxy + maxy 105 | emaxyCount = emaxyCount + 1 106 | end 107 | end 108 | 109 | if eminxCount == 0 then 110 | eminx = mean[1] 111 | eminxCount = 1 112 | end 113 | if eminyCount == 0 then 114 | eminy = mean[2] 115 | eminyCount = 1 116 | end 117 | if emaxxCount == 0 then 118 | emaxx = mean[3] 119 | emaxxCount = 1 120 | end 121 | if emaxyCount == 0 then 122 | emaxy = mean[4] 123 | emaxyCount = 1 124 | end 125 | 126 | return eminx / eminxCount, eminy / eminyCount, emaxx / emaxxCount, emaxy / emaxyCount 127 | end 128 | 129 | local function cleanBoundingBox(x, y, imageW, imageH) 130 | if (x < 1) then 131 | x = 1 132 | elseif (x > imageW) then 133 | x = imageW 134 | end 135 | if (y < 1) then 136 | y = 1 137 | elseif (y > imageH) then 138 | y = imageH 139 | end 140 | return math.floor(x), math.floor(y) 141 | end 142 | 143 | local function getImageBoundingBox(x, y, bboxMinx, bboxMiny, bboxMaxx, bboxMaxy, imageW, imageH) 144 | local xbi, ybi = toImageCoordinates(x, y) 145 | local minx = xbi + bboxMinx[y][x][1] 146 | local miny = ybi + bboxMiny[y][x][1] 147 | local maxx = xbi + bboxMaxx[y][x][1] 148 | local maxy = ybi + bboxMaxy[y][x][1] 149 | minx, miny = cleanBoundingBox(minx, miny, imageW, imageH) 150 | maxx, maxy = cleanBoundingBox(maxx, maxy, imageW, imageH) 151 | return minx, miny, maxx, maxy 152 | end 153 | 154 | local function getEncodedInput(input) 155 | local encoder = loadEncoder() 156 | local encodedInput = encoder:forward(input)[2] 157 | return encodedInput, input:size()[3], input:size()[2] 158 | end 159 | 160 | local function locate(encodedInput, index) 161 | local locator = loadLocator(index) 162 | local boundingBoxes = getBoundingBoxes(locator, encodedInput, index) 163 | return boundingBoxes 164 | end 165 | 166 | local function getProbabilities(encodedInput) 167 | local classifier = loadClassifier() 168 | local classes = classifier:forward(encodedInput) 169 | return classes[1] 170 | end 171 | 172 | function bboxlib.loadImageFromFile(imagePath) 173 | local input = image.load(imagePath) 174 | return input 175 | end 176 | 177 | local function getInitialScales(input) 178 | local scaleBase 179 | if input:size()[2] > input:size()[3] then 180 | scaleBase = (1 * 224) / input:size()[3] 181 | else 182 | scaleBase = (1 * 224) / input:size()[2] 183 | end 184 | local scales = {} 185 | local numScales = 4 186 | for s = 1, numScales do 187 | scales[s] = s * scaleBase 188 | end 189 | return scales 190 | end 191 | 192 | local function getScaledImages(input, scales) 193 | local pyramidProc = {} 194 | local pyramid = image.gaussianpyramid(input, scales) 195 | for s = 1, #scales do 196 | pyramidProc[s] = tiefvision_commons.loadImage(pyramid[s]) 197 | end 198 | return pyramidProc 199 | end 200 | 201 | local function getBboxes(input) 202 | local encodedInput = getEncodedInput(input) 203 | local bboxMinx = locate(encodedInput, 1) 204 | local bboxMiny = locate(encodedInput, 2) 205 | local bboxMaxx = locate(encodedInput, 3) 206 | local bboxMaxy = locate(encodedInput, 4) 207 | local probabilities = getProbabilities(encodedInput) 208 | local bboxes = {} 209 | local width = input:size()[3] 210 | local height = input:size()[2] 211 | local i = 1 212 | for x = 1, bboxMinx:size()[2] do 213 | for y = 1, bboxMinx:size()[1] do 214 | if (probabilities[y][x] > 0.85) then 215 | local xmin, ymin, xmax, ymax = getImageBoundingBox(x, y, bboxMinx, bboxMiny, bboxMaxx, bboxMaxy, width, height) 216 | bboxes[i] = { xmin, ymin, xmax, ymax, probabilities[y][x], x, y } 217 | i = i + 1 218 | end 219 | end 220 | end 221 | return bboxes, probabilities 222 | end 223 | 224 | local function getCroppedImage(input) 225 | local scale = getInitialScales(input)[2] 226 | local scaledImage = getScaledImages(input, { scale })[1] 227 | local bboxes, probabilities = getBboxes(scaledImage) 228 | local xminNew, yminNew, xmaxNew, ymaxNew = getExpectedBoundingBox(bboxes) 229 | return xminNew / scale, yminNew / scale, xmaxNew / scale, ymaxNew / scale 230 | end 231 | 232 | function bboxlib.getImageBoundingBoxesTable(input) 233 | local boundingBoxes = {} 234 | local xmin, ymin, xmax, ymax = getCroppedImage(input) 235 | table.insert(boundingBoxes, { xmin, ymin, xmax, ymax }) 236 | return boundingBoxes 237 | end 238 | 239 | return bboxlib 240 | -------------------------------------------------------------------------------- /src/torch/7-bboxes-images/bboxes-images.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local paths = require('paths') 6 | local torchFolder = paths.thisfile('..') 7 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 8 | 9 | require 'inn' 10 | require 'optim' 11 | require 'torch' 12 | require 'xlua' 13 | local lfs = require 'lfs' 14 | 15 | local image = require 'image' 16 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 17 | local bboxlib = require '6-bboxlib/bboxlib' 18 | 19 | local function getFiles(folder) 20 | local files = {} 21 | for file in lfs.dir(folder) do 22 | if (lfs.attributes(folder .. '/' .. file, "mode") == "file") then 23 | table.insert(files, file) 24 | end 25 | end 26 | return files 27 | end 28 | 29 | local folder = tiefvision_commons.resourcePath('dresses-db/master') 30 | local bboxesFolder = tiefvision_commons.resourcePath('dresses-db/bboxes') 31 | local flippedBboxesFolder = tiefvision_commons.resourcePath('dresses-db/bboxes-flipped') 32 | local files = getFiles(folder) 33 | for fileIndex = 1, #files do 34 | if not tiefvision_commons.fileExists(bboxesFolder .. '/1/' .. files[fileIndex]) then 35 | print(files[fileIndex]) 36 | local fileName = folder .. '/' .. files[fileIndex] 37 | local input = bboxlib.loadImageFromFile(fileName) 38 | local bboxes = bboxlib.getImageBoundingBoxesTable(input, 1) 39 | for i = 1, #bboxes do 40 | local xmin = bboxes[i][1] 41 | local ymin = bboxes[i][2] 42 | local xmax = bboxes[i][3] 43 | local ymax = bboxes[i][4] 44 | print(xmin, ymin, xmax, ymax) 45 | 46 | local inputCropped = image.crop(input, xmin, ymin, xmax, ymax) 47 | paths.mkdir(bboxesFolder .. '/' .. i) 48 | image.save(bboxesFolder .. '/' .. i .. '/' .. files[fileIndex], inputCropped) 49 | 50 | -- generate flipped images 51 | local flippedInput = image.hflip(inputCropped) 52 | paths.mkdir(flippedBboxesFolder .. '/' .. i) 53 | image.save(flippedBboxesFolder .. '/' .. i .. '/' .. files[fileIndex], flippedInput) 54 | 55 | collectgarbage() 56 | end 57 | end 58 | end 59 | 60 | print('Bounding Box Dresses DB Generated') 61 | -------------------------------------------------------------------------------- /src/torch/8-similarity-db-cnn/generate-similarity-db.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local paths = require('paths') 6 | local torchFolder = paths.thisfile('..') 7 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 8 | 9 | require 'inn' 10 | require 'optim' 11 | require 'xlua' 12 | require 'lfs' 13 | require 'image' 14 | local torch = require 'torch' 15 | 16 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 17 | local similarity_db_lib = require '8-similarity-db-cnn/similarity_db_lib' 18 | local tiefvision_config_loader = require('0-tiefvision-commons/tiefvision_config_loader') 19 | 20 | local function createDb(sourceFolder, destinationFolder) 21 | local files = tiefvision_commons.getFiles(sourceFolder) 22 | local encoder = similarity_db_lib.getEncoder() 23 | for fileIndex = 1, #files do 24 | local file = files[fileIndex] 25 | local destPath = destinationFolder .. '/' .. file 26 | 27 | paths.mkdir(destinationFolder) 28 | if(not tiefvision_commons.fileExists(destPath)) then 29 | print('Encoding ' .. file) 30 | local encoderOutput = similarity_db_lib.encodeImage(sourceFolder .. '/' .. file, encoder) 31 | torch.save(destPath, encoderOutput) 32 | collectgarbage() 33 | end 34 | end 35 | end 36 | 37 | function getOptions() 38 | local function extract_table(value) 39 | local tmp = {} 40 | for w in string.gmatch(value, "[^ ]+") do table.insert(tmp, w) end 41 | 42 | return tmp 43 | end 44 | 45 | local cmd = torch.CmdLine() 46 | local sources = tiefvision_commons.resourcePath('dresses-db/bboxes/1') .. ' ' .. tiefvision_commons.resourcePath('dresses-db/bboxes-flipped/1') 47 | cmd:option('-sources', sources, 'Source directory to load images') 48 | 49 | local destinations = tiefvision_commons.dataPath('encoded-images') .. ' ' .. tiefvision_commons.dataPath('encoded-images-flipped') 50 | cmd:option('-destinations', destinations, 'Source directory to load images') 51 | 52 | cmd:text('') 53 | cmd:option('-config', tiefvision_config_loader.default, 'Configuration file to use.') 54 | 55 | local args = cmd:parse(arg) 56 | args.sources = extract_table(args.sources) 57 | args.destinations = extract_table(args.destinations) 58 | 59 | assert(#args.sources == #args.destinations) 60 | 61 | return args 62 | end 63 | 64 | local options = getOptions() 65 | for i = 1, #options.sources do 66 | createDb(options.sources[i], options.destinations[i]) 67 | end 68 | -------------------------------------------------------------------------------- /src/torch/8-similarity-db-cnn/similarity_db_lib.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | local torch = require 'torch' 10 | local image = require 'image' 11 | 12 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 13 | 14 | local similarity_db_lib = {} 15 | 16 | function similarity_db_lib.getEncoder() 17 | return torch.load(tiefvision_commons.modelPath('encoder.model')) 18 | end 19 | 20 | function similarity_db_lib.encodeImage(imagePath, encoder) 21 | local input = image.load(imagePath) 22 | -- scale 23 | local inputWidth, inputHeight = input:size()[3], input:size()[2] 24 | local inputScaled = image.scale(input, 224, inputHeight * 224.0 / inputWidth) 25 | -- encode 26 | local encoderInput = tiefvision_commons.loadImage(inputScaled) 27 | local encoderOutput = encoder:forward(encoderInput)[2] 28 | encoderOutput = encoderOutput:transpose(1, 3):clone() -- make it contiguous by cloning 29 | for w = 1, encoderOutput:size()[1] do 30 | for h = 1, encoderOutput:size()[2] do 31 | encoderOutput[w][h] = encoderOutput[w][h] / torch.norm(encoderOutput[w][h]) 32 | end 33 | end 34 | return encoderOutput 35 | end 36 | 37 | return similarity_db_lib 38 | -------------------------------------------------------------------------------- /src/torch/9-similarity-db/similarity-db.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | local torchFolder = require('paths').thisfile('..') 6 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 7 | 8 | require 'inn' 9 | require 'optim' 10 | require 'xlua' 11 | require 'lfs' 12 | local torch = require 'torch' 13 | 14 | local tiefvision_commons = require '0-tiefvision-commons/tiefvision_commons' 15 | local tiefvision_reduction = require '0-tiefvision-commons/tiefvision_reduction' 16 | local similarity_lib = require '9-similarity-db/similarity_lib' 17 | local tiefvision_config_loader = require('0-tiefvision-commons/tiefvision_config_loader') 18 | local database = tiefvision_config_loader.load().database.unsupervised_similarity 19 | 20 | function similarityDb(imageFolder, reprocess, kNearestNeighbors) 21 | local files = tiefvision_commons.getFiles(imageFolder) 22 | files = tiefvision_commons.tableShuffle(files) 23 | 24 | local filesToProcess = files 25 | if not reprocess then 26 | local filesAlreadyProcessed = database.keys() 27 | filesToProcess = tiefvision_commons.tableSubtraction(files, filesAlreadyProcessed) 28 | end 29 | 30 | for referenceIndex = 1, #filesToProcess do 31 | local reference = filesToProcess[referenceIndex] 32 | print(reference) 33 | 34 | local similarities = {} 35 | 36 | local referenceEncoding = torch.load(imageFolder .. '/' .. reference):double() 37 | for testIndex = 1, #files do 38 | local test = files[testIndex] 39 | local imageEncoding = torch.load(imageFolder .. '/' .. test):double() 40 | local similarity = similarity_lib.similarity(referenceEncoding, imageEncoding) 41 | similarities[test] = similarity or -1 42 | end 43 | 44 | if kNearestNeighbors > 0 then 45 | similarities = tiefvision_reduction.getNearestNeighbors(similarities, kNearestNeighbors) 46 | end 47 | 48 | database.write(reference, similarities) 49 | 50 | if referenceIndex % 5 == 0 then 51 | collectgarbage() 52 | end 53 | end 54 | end 55 | 56 | function getOptions() 57 | local cmd = torch.CmdLine() 58 | cmd:text('Compare images to one another to identify which are the most similar') 59 | cmd:text('Options:') 60 | cmd:option('-images', tiefvision_commons.dataPath('encoded-images'), 'Directory to load images') 61 | cmd:option('-reprocess', false, 'Reprocess images that were already processed') 62 | cmd:option('-kNearestNeighbors', -1, 'Amount of nearest neighbors to save') 63 | cmd:text('') 64 | cmd:option('-config', tiefvision_config_loader.default, 'Configuration file to use.') 65 | 66 | return cmd:parse(arg) 67 | end 68 | 69 | local options = getOptions() 70 | similarityDb(options.images, options.reprocess, options.kNearestNeighbors) 71 | -------------------------------------------------------------------------------- /src/torch/9-similarity-db/similarity_lib.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | require 'torch' 6 | 7 | local similarity_lib = {} 8 | 9 | function similarity_lib.similarity(referenceEncoding, imageEncoding) 10 | local sumSimilarity = 0.0 11 | local minHeight = math.min(referenceEncoding:size()[2], imageEncoding:size()[2]) 12 | local maxHeight = math.max(referenceEncoding:size()[2], imageEncoding:size()[2]) 13 | if (maxHeight - minHeight < 5) then 14 | for w = 1, referenceEncoding:size()[1] do 15 | for h = 1, minHeight do 16 | local similarityLoc = imageEncoding[w][h] * referenceEncoding[w][h] 17 | sumSimilarity = sumSimilarity + similarityLoc 18 | end 19 | end 20 | local similarity = sumSimilarity / (referenceEncoding:size()[1] * minHeight) 21 | return similarity 22 | else 23 | return -1.0 24 | end 25 | end 26 | 27 | return similarity_lib 28 | -------------------------------------------------------------------------------- /src/torch/config.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2016 Pau Carré Cardona - All Rights Reserved 2 | -- You may use, distribute and modify this code under the 3 | -- terms of the Apache License v2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt). 4 | 5 | -- 6 | -- Default configuration file 7 | -- 8 | 9 | local torchFolder = require('paths').thisfile('.') 10 | package.path = string.format("%s;%s/?.lua", os.getenv("LUA_PATH"), torchFolder) 11 | 12 | local tiefvision_commons = require('0-tiefvision-commons/tiefvision_commons') 13 | local database_factory = require('0-tiefvision-commons/io/tiefvision_torch_io') 14 | 15 | return { 16 | database = { 17 | supervised_similarity = database_factory(tiefvision_commons.dataPath('image-supervised-similarity-database')), 18 | unsupervised_similarity = database_factory(tiefvision_commons.dataPath('image-unsupervised-similarity-database')) 19 | } 20 | } 21 | --------------------------------------------------------------------------------