├── Fine-tuning ├── nets │ ├── __init__.py │ ├── nets_factory.py │ ├── lenetsound_lenetfashion.py │ └── vgg16avg_zfnet.py ├── datasets │ ├── __init__.py │ ├── dataset_factory.py │ ├── gender.py │ ├── mnist_fashion.py │ ├── sound20.py │ └── mvc_clothing.py ├── weight_loader │ ├── __init__.py │ ├── read_bin.py │ ├── weight_factory.py │ ├── lenetsound_lenetfashion_weight.py │ └── vgg16avg_zfnet_weight.py ├── requirements.txt ├── README.md ├── config.py ├── download.py ├── saver.py ├── loss.py └── finetuning.py ├── Inference └── README.md ├── LICENSE └── README.md /Fine-tuning/nets/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Fine-tuning/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Fine-tuning/weight_loader/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Inference/README.md: -------------------------------------------------------------------------------- 1 | # Coming Soon ... 2 | -------------------------------------------------------------------------------- /Fine-tuning/requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm==4.11.2 2 | requests==2.18.1 3 | 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Fine-tuning/weight_loader/read_bin.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | # ==================================================================================================== 23 | import numpy as np 24 | def load_bin(data_path,data_type="float32"): 25 | 26 | if data_type != "float32" and data_type != "uint8": 27 | raise ValueError('Data_type should be float32 or uint8') 28 | dimension = np.fromfile(data_path, dtype=np.int32) 29 | dim = dimension[0] 30 | data_shape =[] 31 | for i in range(dim): 32 | data_shape.append(dimension[i+1]) 33 | 34 | if data_type == "float32": 35 | num = dim+1 36 | data = np.fromfile(data_path, dtype=np.float32) 37 | data = data[num:] 38 | elif data_type is "uint8": 39 | num = (dim+1)*4 40 | data = np.fromfile(data_path, dtype=np.uint8) 41 | data = data[num:] 42 | 43 | data = data.reshape(data_shape) 44 | return data 45 | 46 | def load_txt(path): 47 | with open (path, "r") as f: 48 | data = f.readlines() 49 | data = [x.strip() for x in data] 50 | return data 51 | 52 | -------------------------------------------------------------------------------- /Fine-tuning/README.md: -------------------------------------------------------------------------------- 1 | # Fine-tuning 2 | ## Prerequisition 3 | - Python 2.7 4 | - [TensorFlow 1.4.0 or higher](https://github.com/tensorflow/tensorflow) 5 | 6 | ## Datasets 7 | - Sound20 (https://github.com/ivclab/Sound20) 8 | - MNIST-Fashion (https://github.com/zalandoresearch/fashion-mnist) 9 | - Multi-View Clothing (https://github.com/MVC-Datasets/MVC) 10 | - The OUI-Adience (https://www.openu.ac.il/home/hassner/Adience/publications.html) 11 | 12 | ## How to Run 13 | - Install required packages: 14 | ```bash 15 | $ pip install -r requirements.txt 16 | ``` 17 | 18 | - Download TFRecords Data and Well-trained Neural Network weight: 19 | ```bash 20 | $ python download.py 21 | ``` 22 | 23 | - Fine-tune the merged model of **Lenet-Sound** and **Lenet-Fashion**: 24 | ```bash 25 | # $ python finetuning.py --net=TASK_NAME --merger_dir=MERGED_MODEL_DIR 26 | $ python finetuning.py --net=lenetsound_lenetfashion --merger_dir=./weight_loader/weight/lenetsound_lenetfashion/merge_ACCU/ --batch_size=64 --save_model=True 27 | ``` 28 | 29 | - Fine-tune the merged model of **VGGAvg-Clothing** and **ZF-Gender**: 30 | ```bash 31 | $ python finetuning.py --net=vggclothing_zfgender --merger_dir=./weight_loader/weight/vggclothing_zfgender/merge_ACCU/ --save_model=True --lr_rate=0.00006 --batch_size=16 32 | ``` 33 | 34 | - The merged model (after fine-tuning) will be saved in `./logs/FOLDER_NAME/` 35 | 36 | ## Experimental Results 37 | 38 | The experimental results (mean values over five repetition) of mergning **Lenet-Sound** and **Lenet-Fashion** : 39 | 40 | Network Model | Sound Accuracy | Fashion Accuracy | Model Size 41 | ----------------- | ---------------- | ---------------- | ---------------- 42 | Lenet-Sound |**78.08%** | - | 17.1 MB 43 | Lenet-Fashion | - |**91.57%** | 17.0 MB 44 | Merged ACCU | 78.06% | 91.08% | 3.3 MB 45 | Merged LIGHT | 77.66% | 90.89% | 2.3 MB 46 | 47 | The experimental results (mean values over five repetition) of mergning **VGGAvg-Clothing** and **ZF-Gender** : 48 | 49 | Network Model | Clothing Accuracy | Gender Accuracy | Model Size 50 | ----------------- | ---------------- | ---------------- | ---------------- 51 | VGG-Clothing |89.97% | - | 134.7 MB 52 | ZF-Gender | - |**83.43%** | 233.1 MB 53 | Merged ACCU | **90.69%** | 82.71% | 30.4 MB 54 | Merged LIGHT | 90.02% | 81.53% | 18.3 MB 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NeuralMerger 2 | Official implementation of [Unifying and Merging Well-trained Deep Neural Networks for Inference Stage](https://arxiv.org/abs/1805.04980). 3 | 4 | Created by [Yi-Min Chou](https://github.com/yyyjoe) , [Yi-Ming Chan](https://github.com/yimingchan), [Jia-Hong Lee](https://github.com/Jia-HongHenryLee), Chih-Yi Chiu, Chu-Song Chen 5 | 6 | ## Usage 7 | **Fine-tuning**: Finetune the merged model of two well-trained neural networks (Tensorflow implementation). 8 | 9 | **Inference**: Test the speed of the merged model (C implementation). 10 | 11 | NeuralMerger 12 | ├─────── Fine-tuning 13 | └─────── Inference 14 | 15 | 16 | 1.Clone the NeuralMerger repository: 17 | 18 | $ git clone --recursive https://github.com/ivclab/NeuralMerger.git 19 | 20 | 21 | 2.Follow the instruction in [Fine-tuning](https://github.com/ivclab/NeuralMerger/tree/master/Fine-tuning) and get the well-trained merged model. 22 | 23 | 24 | 3.Test the well-trained merged model on [Inference](https://github.com/ivclab/NeuralMerger/tree/master/Inference). 25 | 26 | 27 | ## Citation 28 | Please cite following paper if these codes help your research: 29 | 30 | @inproceedings{chou2018unifying, 31 | title={Unifying and merging well-trained deep neural networks for inference stage}, 32 | author={Chou, Yi-Min and Chan, Yi-Ming and Lee, Jia-Hong and Chiu, Chih-Yi and Chen, Chu-Song}, 33 | booktitle={Proceedings of the 27th International Joint Conference on Artificial Intelligence}, 34 | pages={2049--2056}, 35 | year={2018}, 36 | organization={AAAI Press} 37 | } 38 | 39 | @inproceedings{chou2018merging, 40 | title={Merging Deep Neural Networks for Mobile Devices}, 41 | author={Chou, Yi-Min and Chan, Yi-Ming and Lee, Jia-Hong and Chiu, Chih-Yi and Chen, Chu-Song}, 42 | booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition Workshops}, 43 | pages={1686--1694}, 44 | year={2018} 45 | } 46 | 47 | ## Contact 48 | Please feel free to leave suggestions or comments to [Yi-Min Chou](https://github.com/yyyjoe)(chou@iis.sinica.edu.tw) , Yi-Ming Chan(yiming@iis.sinica.edu.tw), [Jia-Hong Lee](https://github.com/Jia-HongHenryLee)(honghenry.lee@gmail.com), [Chih-Yi Chiu](https://sites.google.com/site/chihyichiu/)(chihyi.chiu@gmail.com), [Chu-Song Chen](https://www.iis.sinica.edu.tw/pages/song/)(song@iis.sinica.edu.tw) 49 | 50 | -------------------------------------------------------------------------------- /Fine-tuning/weight_loader/weight_factory.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # MIT License 16 | # 17 | # Modifications copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy 20 | # of this software and associated documentation files (the "Software"), to deal 21 | # in the Software without restriction, including without limitation the rights 22 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | # copies of the Software, and to permit persons to whom the Software is 24 | # furnished to do so, subject to the following conditions: 25 | # 26 | # The above copyright notice and this permission notice shall be included in all 27 | # copies or substantial portions of the Software. 28 | # 29 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | # SOFTWARE. 36 | # =================================================================================================================== 37 | import tensorflow as tf 38 | import vgg16avg_zfnet_weight 39 | import lenetsound_lenetfashion_weight 40 | 41 | model_map = { 'lenetsound_lenetfashion': lenetsound_lenetfashion_weight, 42 | 'vggclothing_zfgender': vgg16avg_zfnet_weight, 43 | } 44 | 45 | 46 | def get_weight(config,name): 47 | if name not in model_map: 48 | raise ValueError('Name of network unknown %s' % name) 49 | return model_map[name].weight_loader(config) -------------------------------------------------------------------------------- /Fine-tuning/nets/nets_factory.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # MIT License 16 | # 17 | # Modifications copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy 20 | # of this software and associated documentation files (the "Software"), to deal 21 | # in the Software without restriction, including without limitation the rights 22 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | # copies of the Software, and to permit persons to whom the Software is 24 | # furnished to do so, subject to the following conditions: 25 | # 26 | # The above copyright notice and this permission notice shall be included in all 27 | # copies or substantial portions of the Software. 28 | # 29 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | # SOFTWARE. 36 | # =================================================================================================================== 37 | import tensorflow as tf 38 | 39 | import vgg16avg_zfnet 40 | import lenetsound_lenetfashion 41 | networks_map = {'lenetsound_lenetfashion': lenetsound_lenetfashion.lenetsound_lenetfashion, 42 | 'vggclothing_zfgender': vgg16avg_zfnet.vgg16avg_zfnet, 43 | } 44 | 45 | 46 | 47 | def get_network(name,shared_codebook,weight1,bias1,index1,outlayer1,weight2,bias2,index2,outlayer2): 48 | 49 | if name not in networks_map: 50 | raise ValueError('Name of network unknown %s' % name) 51 | 52 | return networks_map[name]( 53 | shared_codebook, 54 | weight1, 55 | bias1, 56 | index1, 57 | outlayer1, 58 | weight2, 59 | bias2, 60 | index2, 61 | outlayer2, 62 | ) -------------------------------------------------------------------------------- /Fine-tuning/datasets/dataset_factory.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # MIT License 16 | # 17 | # Modifications copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy 20 | # of this software and associated documentation files (the "Software"), to deal 21 | # in the Software without restriction, including without limitation the rights 22 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | # copies of the Software, and to permit persons to whom the Software is 24 | # furnished to do so, subject to the following conditions: 25 | # 26 | # The above copyright notice and this permission notice shall be included in all 27 | # copies or substantial portions of the Software. 28 | # 29 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | # SOFTWARE. 36 | # =================================================================================================================== 37 | 38 | import sound20 39 | import mnist_fashion 40 | import mvc_clothing 41 | import gender 42 | import numpy as np 43 | import tensorflow as tf 44 | slim = tf.contrib.slim 45 | datasets_map = { 46 | 'lenetsound': sound20, 47 | 'lenetfashion': mnist_fashion, 48 | 'vggclothing': mvc_clothing, 49 | 'zfgender': gender, 50 | } 51 | 52 | 53 | def get_dataset(config,name, split_name, dataset_dir, file_pattern=None, reader=None): 54 | """Given a dataset name and a split_name returns a Dataset. 55 | Args: 56 | name: String, the name of the dataset. 57 | split_name: A train/test split name. 58 | dataset_dir: The directory where the dataset files are stored. 59 | file_pattern: The file pattern to use for matching the dataset source files. 60 | reader: The subclass of tf.ReaderBase. If left as `None`, then the default 61 | reader defined by each dataset is used. 62 | Returns: 63 | A `Dataset` class. 64 | Raises: 65 | ValueError: If the dataset `name` is unknown. 66 | """ 67 | if name not in datasets_map: 68 | raise ValueError('Name of dataset unknown %s' % name) 69 | return datasets_map[name].get_split( 70 | config, 71 | split_name, 72 | dataset_dir, 73 | file_pattern, 74 | reader, 75 | ) 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /Fine-tuning/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The BEGAN-tensorflow Authors(Taehoon Kim). All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # MIT License 16 | # 17 | # Modifications copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy 20 | # of this software and associated documentation files (the "Software"), to deal 21 | # in the Software without restriction, including without limitation the rights 22 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | # copies of the Software, and to permit persons to whom the Software is 24 | # furnished to do so, subject to the following conditions: 25 | # 26 | # The above copyright notice and this permission notice shall be included in all 27 | # copies or substantial portions of the Software. 28 | # 29 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | # SOFTWARE. 36 | # ============================================================================== 37 | import argparse 38 | 39 | arg_lists = [] 40 | parser = argparse.ArgumentParser() 41 | def str2bool(v): 42 | return v.lower() in ('true', '1') 43 | 44 | def add_argument_group(name): 45 | arg = parser.add_argument_group(name) 46 | arg_lists.append(arg) 47 | return arg 48 | 49 | def setting(): 50 | net_arg = add_argument_group('Calibration Setting') 51 | net_arg.add_argument('--merger_dir', type=str, default="./weight_loader/weight/lenetsound_lenetfashion/merge_ACCU/") 52 | net_arg.add_argument('--net', type=str, default="lenetsound_lenetfashion", choices=['lenetsound_lenetfashion', 'vggclothing_zfgender']) 53 | net_arg.add_argument('--save_model', type=str2bool, default="False") 54 | net_arg.add_argument('--max_step', type=int, default=20000) 55 | net_arg.add_argument('--decay_step', type=int, default=18000) 56 | net_arg.add_argument('--decay_rate', type=float, default=0.1) 57 | net_arg.add_argument('--log_step', type=int, default=500) 58 | net_arg.add_argument('--save_step', type=int, default=5000) 59 | net_arg.add_argument('--random_seed', type=int, default=100) 60 | net_arg.add_argument('--batch_size', type=int, default=64) 61 | net_arg.add_argument('--lr_rate', type=float, default=0.0002) 62 | net_arg.add_argument('--data_split_num', type=int, default=5,choices=[1,2,3,4,5], 63 | help='data is divided into five groups. 5 means using 100% data') 64 | net_arg.add_argument('--data_path', type=str, default="./TFRecord/") 65 | net_arg.add_argument('--weight_dir', type=str, default="./weight_loader/weight/") 66 | config, unparsed = parser.parse_known_args() 67 | 68 | return config 69 | 70 | -------------------------------------------------------------------------------- /Fine-tuning/download.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The BEGAN-tensorflow Authors(Taehoon Kim). All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # MIT License 16 | # 17 | # Modifications copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy 20 | # of this software and associated documentation files (the "Software"), to deal 21 | # in the Software without restriction, including without limitation the rights 22 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | # copies of the Software, and to permit persons to whom the Software is 24 | # furnished to do so, subject to the following conditions: 25 | # 26 | # The above copyright notice and this permission notice shall be included in all 27 | # copies or substantial portions of the Software. 28 | # 29 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | # SOFTWARE. 36 | # ============================================================================== 37 | import requests 38 | import zipfile 39 | import os 40 | def download_file_from_google_drive(id, destination): 41 | URL = "https://docs.google.com/uc?export=download" 42 | 43 | session = requests.Session() 44 | 45 | response = session.get(URL, params = { 'id' : id }, stream = True) 46 | token = get_confirm_token(response) 47 | 48 | if token: 49 | params = { 'id' : id, 'confirm' : token } 50 | response = session.get(URL, params = params, stream = True) 51 | 52 | save_response_content(response, destination) 53 | 54 | def get_confirm_token(response): 55 | for key, value in response.cookies.items(): 56 | if key.startswith('download_warning'): 57 | return value 58 | 59 | return None 60 | 61 | def save_response_content(response, destination): 62 | CHUNK_SIZE = 32768 63 | 64 | with open(destination, "wb") as f: 65 | for chunk in response.iter_content(CHUNK_SIZE): 66 | if chunk: # filter out keep-alive new chunks 67 | f.write(chunk) 68 | 69 | if __name__ == "__main__": 70 | path = './logs' 71 | if not os.path.exists(path): 72 | os.makedirs(path) 73 | print('Download merged model ...') 74 | file_id = '1clKF1zxd2lToejzKvzuubWJIkt-xpoIk' 75 | destination = './weight_loader/weight.zip' 76 | download_file_from_google_drive(file_id, destination) 77 | with zipfile.ZipFile(destination) as zf: 78 | zip_dir = zf.namelist()[0] 79 | zf.extractall('./weight_loader') 80 | os.remove(destination) 81 | 82 | print('Download calibration data ...') 83 | file_id = '1y0gfVkg44BlN8gbJgr8azJq115CaiLJy' 84 | destination = 'tfdata.zip' 85 | download_file_from_google_drive(file_id, destination) 86 | with zipfile.ZipFile(destination) as zf: 87 | zip_dir = zf.namelist()[0] 88 | zf.extractall('./') 89 | os.remove(destination) -------------------------------------------------------------------------------- /Fine-tuning/saver.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | # ==================================================================================================== 23 | import numpy as np 24 | from datetime import datetime 25 | import os 26 | import shutil 27 | from weight_loader import read_bin 28 | 29 | 30 | def saver(config,shared_codebook,folder_name): 31 | NEW_PATH = "./logs/" + folder_name + '/' 32 | PATH_MERGE = config.merger_dir 33 | 34 | # copy txtfile 35 | i1_name = read_bin.load_txt(PATH_MERGE + 'model1.txt') 36 | i2_name = read_bin.load_txt(PATH_MERGE + 'model2.txt') 37 | shutil.copy2(PATH_MERGE + 'model1.txt', NEW_PATH + 'model1.txt') 38 | shutil.copy2(PATH_MERGE + 'model2.txt', NEW_PATH + 'model2.txt') 39 | shutil.copy2(PATH_MERGE + 'merged_codebook.txt', NEW_PATH + 'merged_codebook.txt') 40 | 41 | # copy bias 42 | MERGER_NAME = config.net 43 | task1,task2 = MERGER_NAME.split("_") 44 | base_path = config.weight_dir 45 | 46 | bias1_path = base_path + config.net + '/' + task1 + '/' 47 | b1_name = read_bin.load_txt(bias1_path + "/bias.txt") 48 | shutil.copy2(bias1_path + 'bias.txt', NEW_PATH + 'bias1.txt') 49 | for i in range(len(b1_name)): 50 | shutil.copy2(bias1_path + b1_name[i], NEW_PATH + b1_name[i]) 51 | 52 | bias2_path = base_path + config.net + '/' + task2 + '/' 53 | b2_name = read_bin.load_txt(bias2_path + "/bias.txt") 54 | shutil.copy2(bias2_path + 'bias.txt', NEW_PATH + 'bias2.txt') 55 | for i in range(len(b2_name)): 56 | shutil.copy2(bias2_path + b2_name[i], NEW_PATH + b2_name[i]) 57 | 58 | # copy asmt 59 | for i in range(len(i1_name)): 60 | shutil.copy2(PATH_MERGE + i1_name[i], NEW_PATH + i1_name[i]) 61 | for i in range(len(i2_name)): 62 | shutil.copy2(PATH_MERGE + i2_name[i], NEW_PATH + i2_name[i]) 63 | 64 | # save ctrd 65 | for i in range(len(shared_codebook)-2): 66 | file_path = NEW_PATH + "/" + config.net+'.ctrd.%02d'%(i+1) 67 | data_np = shared_codebook[i] 68 | write_bin(file_path,data_np) 69 | 70 | # save outputlayer 71 | file_path = NEW_PATH + "/" + 'M1_outputlayer' 72 | data_np = shared_codebook[len(shared_codebook)-2].transpose(1,0) 73 | write_bin(file_path,data_np) 74 | 75 | file_path = NEW_PATH + "/" + 'M2_outputlayer' 76 | data_np = shared_codebook[len(shared_codebook)-1].transpose(1,0) 77 | write_bin(file_path,data_np) 78 | 79 | 80 | def write_bin(file_path,data_np): 81 | file_path = file_path + '.bin' 82 | dim_np = np.array((data_np.shape),dtype=np.int32) 83 | dim_num = np.int32(len(dim_np)) 84 | 85 | output_file = open(file_path, 'wb') 86 | output_file.write(dim_num) 87 | output_file.write(np.array(dim_np)) 88 | c_order = data_np.flags['C_CONTIGUOUS'] 89 | if(c_order): 90 | output_file.write((data_np)) 91 | else: 92 | data_np1 = data_np.copy(order='C') 93 | 94 | output_file.write((data_np1)) 95 | output_file.close() 96 | return 97 | 98 | -------------------------------------------------------------------------------- /Fine-tuning/loss.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | # ==================================================================================================== 23 | import tensorflow as tf 24 | 25 | 26 | 27 | def loss_function(config,merged_net): 28 | ################## 29 | ## Define Input ## 30 | ################## 31 | im1_dim, im2_dim= merged_net.get_imshape() 32 | image1 = tf.placeholder(tf.float32,[None,im1_dim[0],im1_dim[1],im1_dim[2]]) 33 | image2 = tf.placeholder(tf.float32,[None,im2_dim[0],im2_dim[1],im2_dim[2]]) 34 | 35 | 36 | #################### 37 | ## Network Output ## 38 | #################### 39 | net1_out = merged_net.model1(image1) 40 | net1_target = merged_net.model1_target(image1) 41 | net2_out = merged_net.model2(image2) 42 | net2_target = merged_net.model2_target(image2) 43 | 44 | 45 | ################# 46 | ## Define Loss ## 47 | ################# 48 | # Model 1 loss 49 | label1 = tf.placeholder(tf.int64) 50 | layer_loss1 = 0 51 | for i in range(len(net1_out)): 52 | layer_loss1 += tf.reduce_mean(tf.abs(net1_out[i] - net1_target[i])) 53 | label_loss1 = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits (labels = label1,logits = net1_out[len(net1_out)-1])) 54 | 55 | # Model 2 loss 56 | label2 = tf.placeholder(tf.int64) 57 | layer_loss2 = 0 58 | for i in range(len(net2_out)): 59 | layer_loss2 += tf.reduce_mean(tf.abs(net2_out[i] - net2_target[i])) 60 | label_loss2 = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits (labels = label2,logits = net2_out[len(net2_out)-1])) 61 | 62 | gamma = 10 63 | M1_loss = layer_loss1 + gamma*label_loss1 64 | M2_loss = layer_loss2 + gamma*label_loss2 65 | 66 | 67 | ############## 68 | ## Training ## 69 | ############## 70 | global_step = tf.Variable(0, trainable=False) 71 | add_global = global_step.assign_add(1) 72 | initial_learning_rate = config.lr_rate 73 | learning_rate = tf.train.exponential_decay( initial_learning_rate, 74 | global_step=global_step, 75 | decay_steps=config.decay_step ,decay_rate=config.decay_rate,staircase=True) 76 | 77 | with tf.control_dependencies([add_global]): 78 | trainer1 = tf.train.AdamOptimizer(learning_rate).minimize(M1_loss) 79 | trainer2 = tf.train.AdamOptimizer(learning_rate).minimize(M2_loss) 80 | 81 | 82 | ############# 83 | ## Testing ## 84 | ############# 85 | _, acc1_op = tf.metrics.accuracy(label1, tf.argmax(tf.nn.softmax(net1_out[len(net1_out)-1]),1)) 86 | _, acc1_op_ = tf.metrics.accuracy(label1, tf.argmax(tf.nn.softmax(net1_target[len(net1_out)-1]),1)) 87 | _, acc2_op = tf.metrics.accuracy(label2, tf.argmax(tf.nn.softmax(net2_out[len(net2_out)-1]),1)) 88 | _, acc2_op_ = tf.metrics.accuracy(label2, tf.argmax(tf.nn.softmax(net2_target[len(net2_out)-1]),1)) 89 | 90 | 91 | ############### 92 | ## Operation ## 93 | ############### 94 | input_op = { 95 | 'x1':image1, 96 | 'x2':image2, 97 | 'y1':label1, 98 | 'y2':label2, 99 | } 100 | 101 | trainer_op = { 102 | 'trainer1' : trainer1, 103 | 'trainer2' : trainer2, 104 | } 105 | 106 | loss_op = { 107 | 'M1_loss':M1_loss, 108 | 'M2_loss':M2_loss, 109 | } 110 | 111 | test1_op = { 112 | 'acc1_op' : acc1_op, 113 | 'acc1_op_': acc1_op_, 114 | } 115 | 116 | test2_op = { 117 | 'acc2_op' : acc2_op, 118 | 'acc2_op_': acc2_op_, 119 | } 120 | 121 | 122 | return trainer_op,test1_op,test2_op,input_op,loss_op -------------------------------------------------------------------------------- /Fine-tuning/datasets/gender.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # MIT License 16 | # 17 | # Modifications copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy 20 | # of this software and associated documentation files (the "Software"), to deal 21 | # in the Software without restriction, including without limitation the rights 22 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | # copies of the Software, and to permit persons to whom the Software is 24 | # furnished to do so, subject to the following conditions: 25 | # 26 | # The above copyright notice and this permission notice shall be included in all 27 | # copies or substantial portions of the Software. 28 | # 29 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | # SOFTWARE. 36 | # =================================================================================================================== 37 | import os 38 | import tensorflow as tf 39 | 40 | 41 | 42 | slim = tf.contrib.slim 43 | 44 | _FILE_PATTERN = 'gender_%s_%d.tfrecords' 45 | 46 | SPLITS_TO_SIZES = {'train': 16000, 'test': 3000 }#origin 3968 47 | 48 | _NUM_CLASSES = 2 49 | 50 | 51 | IMAGE_HEIGHT = 227 52 | IMAGE_WIDTH = 227 53 | def get_split(config, split_name, dataset_dir, batch_size,file_pattern=None, reader=None): 54 | """Gets a dataset tuple with instructions for reading flowers. 55 | Args: 56 | split_name: A train/validation split name. 57 | dataset_dir: The base directory of the dataset sources. 58 | file_pattern: The file pattern to use when matching the dataset sources. 59 | It is assumed that the pattern contains a '%s' string so that the split 60 | name can be inserted. 61 | reader: The TensorFlow reader type. 62 | Returns: 63 | A `Dataset` namedtuple. 64 | Raises: 65 | ValueError: if `split_name` is not a valid train/validation split. 66 | """ 67 | all_file = [] 68 | reader = tf.TFRecordReader() 69 | batch_size = config.batch_size 70 | data_splitnum = config.data_split_num 71 | file_pattern = _FILE_PATTERN 72 | 73 | if split_name == 'train': 74 | num_epochs = None 75 | for i in range(data_splitnum): 76 | all_file.append(os.path.join(dataset_dir, 'gender/', file_pattern%(split_name,i))) 77 | elif split_name == 'test': 78 | num_epochs, batch_size = 1, 1 79 | all_file.append(os.path.join(dataset_dir, 'gender/', file_pattern%(split_name,0))) 80 | elif split_name not in SPLITS_TO_SIZES: 81 | raise ValueError('split name %s was not recognized.' % split_name) 82 | 83 | filename_queue = tf.train.string_input_producer( 84 | all_file, num_epochs=num_epochs, shuffle=False) 85 | 86 | _, serialized_example = reader.read(filename_queue) 87 | features = tf.parse_single_example( 88 | serialized_example, 89 | features={ 90 | 'height': tf.FixedLenFeature([], tf.int64), 91 | 'width': tf.FixedLenFeature([], tf.int64), 92 | 'image_string': tf.FixedLenFeature([], tf.string), 93 | 'label': tf.FixedLenFeature([], tf.float32) 94 | }) 95 | 96 | image = tf.decode_raw(features['image_string'], tf.float32) 97 | label = tf.cast(features['label'], tf.float32) 98 | height = tf.cast(features['height'], tf.int32) 99 | width = tf.cast(features['width'], tf.int32) 100 | 101 | image = tf.reshape(image, [height, width, 3]) 102 | 103 | resized_image = tf.image.resize_images(images=image, 104 | size=[IMAGE_HEIGHT,IMAGE_WIDTH]) 105 | 106 | min_after_dequeue = 10000 107 | capacity = min_after_dequeue +3*batch_size 108 | 109 | images, labels = tf.train.shuffle_batch( 110 | [resized_image, label], 111 | batch_size = batch_size, 112 | capacity = capacity, 113 | num_threads=1, 114 | min_after_dequeue= min_after_dequeue, 115 | seed=config.random_seed) 116 | 117 | return images, labels, SPLITS_TO_SIZES[split_name] -------------------------------------------------------------------------------- /Fine-tuning/datasets/mnist_fashion.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # MIT License 16 | # 17 | # Modifications copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy 20 | # of this software and associated documentation files (the "Software"), to deal 21 | # in the Software without restriction, including without limitation the rights 22 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | # copies of the Software, and to permit persons to whom the Software is 24 | # furnished to do so, subject to the following conditions: 25 | # 26 | # The above copyright notice and this permission notice shall be included in all 27 | # copies or substantial portions of the Software. 28 | # 29 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | # SOFTWARE. 36 | # =================================================================================================================== 37 | import os 38 | import tensorflow as tf 39 | 40 | 41 | 42 | slim = tf.contrib.slim 43 | 44 | _FILE_PATTERN = 'mnist_fashion_%s_%d.tfrecords' 45 | 46 | SPLITS_TO_SIZES = {'train': 55000, 'test': 10000} 47 | 48 | _NUM_CLASSES = 10 49 | 50 | 51 | IMAGE_HEIGHT = 32 #origin:28 52 | IMAGE_WIDTH = 32 #origin:28 53 | def get_split(config, split_name, dataset_dir, batch_size, file_pattern=None, reader=None): 54 | """Gets a dataset tuple with instructions for reading flowers. 55 | Args: 56 | split_name: A train/validation split name. 57 | dataset_dir: The base directory of the dataset sources. 58 | file_pattern: The file pattern to use when matching the dataset sources. 59 | It is assumed that the pattern contains a '%s' string so that the split 60 | name can be inserted. 61 | reader: The TensorFlow reader type. 62 | Returns: 63 | A `Dataset` namedtuple. 64 | Raises: 65 | ValueError: if `split_name` is not a valid train/validation split. 66 | """ 67 | all_file = [] 68 | reader = tf.TFRecordReader() 69 | batch_size = config.batch_size 70 | data_splitnum = config.data_split_num 71 | file_pattern = _FILE_PATTERN 72 | 73 | if split_name == 'train': 74 | num_epochs = None 75 | for i in range(data_splitnum): 76 | all_file.append(os.path.join(dataset_dir, 'mnist_fashion/', file_pattern%(split_name,i))) 77 | elif split_name == 'test': 78 | num_epochs, batch_size = 1, 1 79 | all_file.append(os.path.join(dataset_dir, 'mnist_fashion/', file_pattern%(split_name,0))) 80 | elif split_name not in SPLITS_TO_SIZES: 81 | raise ValueError('split name %s was not recognized.' % split_name) 82 | 83 | filename_queue = tf.train.string_input_producer( 84 | all_file, num_epochs=num_epochs, shuffle=False) 85 | 86 | _, serialized_example = reader.read(filename_queue) 87 | features = tf.parse_single_example( 88 | serialized_example, 89 | features={ 90 | 'height': tf.FixedLenFeature([], tf.int64), 91 | 'width': tf.FixedLenFeature([], tf.int64), 92 | 'image_string': tf.FixedLenFeature([], tf.string), 93 | 'label': tf.FixedLenFeature([], tf.float32) 94 | }) 95 | 96 | image = tf.decode_raw(features['image_string'], tf.uint8) 97 | label = tf.cast(features['label'], tf.float32) 98 | height = tf.cast(features['height'], tf.int32) 99 | width = tf.cast(features['width'], tf.int32) 100 | 101 | image = tf.reshape(image, [height, width, 1]) 102 | resized_image = tf.image.resize_images(images=image, 103 | size=[IMAGE_HEIGHT,IMAGE_WIDTH]) 104 | 105 | min_after_dequeue = 10000 106 | capacity = min_after_dequeue +3*batch_size 107 | 108 | images, labels = tf.train.shuffle_batch( 109 | [resized_image, label], 110 | batch_size=batch_size, 111 | capacity=capacity, 112 | num_threads=1, 113 | min_after_dequeue=min_after_dequeue, 114 | seed=config.random_seed) 115 | 116 | return images, labels, SPLITS_TO_SIZES[split_name] -------------------------------------------------------------------------------- /Fine-tuning/datasets/sound20.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # MIT License 16 | # 17 | # Modifications copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy 20 | # of this software and associated documentation files (the "Software"), to deal 21 | # in the Software without restriction, including without limitation the rights 22 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | # copies of the Software, and to permit persons to whom the Software is 24 | # furnished to do so, subject to the following conditions: 25 | # 26 | # The above copyright notice and this permission notice shall be included in all 27 | # copies or substantial portions of the Software. 28 | # 29 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | # SOFTWARE. 36 | # =================================================================================================================== 37 | import os 38 | import tensorflow as tf 39 | 40 | 41 | 42 | slim = tf.contrib.slim 43 | 44 | _FILE_PATTERN = 'sound20_%s_%d.tfrecords' 45 | 46 | SPLITS_TO_SIZES = {'train': 16636, 'test': 3727} 47 | 48 | _NUM_CLASSES = 20 49 | 50 | _ITEMS_TO_DESCRIPTIONS = { 51 | 'image': 'A color image of varying size.', 52 | 'label': 'A single integer between 0 and 4', 53 | } 54 | 55 | IMAGE_HEIGHT = 32 56 | IMAGE_WIDTH = 32 57 | def get_split(config,split_name, dataset_dir, file_pattern=None, reader=None): 58 | """Gets a dataset tuple with instructions for reading flowers. 59 | Args: 60 | split_name: A train/validation split name. 61 | dataset_dir: The base directory of the dataset sources. 62 | file_pattern: The file pattern to use when matching the dataset sources. 63 | It is assumed that the pattern contains a '%s' string so that the split 64 | name can be inserted. 65 | reader: The TensorFlow reader type. 66 | Returns: 67 | A `Dataset` namedtuple. 68 | Raises: 69 | ValueError: if `split_name` is not a valid train/validation split. 70 | """ 71 | all_file = [] 72 | reader = tf.TFRecordReader() 73 | batch_size = config.batch_size 74 | data_splitnum = config.data_split_num 75 | file_pattern = _FILE_PATTERN 76 | 77 | if split_name == 'train': 78 | num_epochs = None 79 | for i in range(data_splitnum): 80 | all_file.append(os.path.join(dataset_dir, 'sound20/', file_pattern%(split_name,i))) 81 | elif split_name == 'test': 82 | num_epochs, batch_size = 1, 1 83 | all_file.append(os.path.join(dataset_dir, 'sound20/', file_pattern%(split_name,0))) 84 | elif split_name not in SPLITS_TO_SIZES: 85 | raise ValueError('split name %s was not recognized.' % split_name) 86 | 87 | filename_queue = tf.train.string_input_producer( 88 | all_file, num_epochs=num_epochs, shuffle=False) 89 | 90 | _, serialized_example = reader.read(filename_queue) 91 | features = tf.parse_single_example( 92 | serialized_example, 93 | features={ 94 | 'height': tf.FixedLenFeature([], tf.int64), 95 | 'width': tf.FixedLenFeature([], tf.int64), 96 | 'image_string': tf.FixedLenFeature([], tf.string), 97 | 'label': tf.FixedLenFeature([], tf.float32) 98 | }) 99 | 100 | image = tf.decode_raw(features['image_string'], tf.uint8) 101 | label = tf.cast(features['label'], tf.float32) 102 | height = tf.cast(features['height'], tf.int32) 103 | width = tf.cast(features['width'], tf.int32) 104 | 105 | image = tf.reshape(image, [height, width, 1]) 106 | resized_image = tf.image.resize_images(image, 107 | size=[IMAGE_HEIGHT,IMAGE_WIDTH]) 108 | 109 | min_after_dequeue = 10000 110 | capacity = min_after_dequeue +3*batch_size 111 | 112 | images, labels = tf.train.shuffle_batch( 113 | [resized_image, label], 114 | batch_size=batch_size, 115 | capacity=capacity, 116 | num_threads=1, 117 | min_after_dequeue=min_after_dequeue, 118 | seed=config.random_seed) 119 | 120 | return images, labels, SPLITS_TO_SIZES[split_name] 121 | -------------------------------------------------------------------------------- /Fine-tuning/datasets/mvc_clothing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # MIT License 16 | # 17 | # Modifications copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy 20 | # of this software and associated documentation files (the "Software"), to deal 21 | # in the Software without restriction, including without limitation the rights 22 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | # copies of the Software, and to permit persons to whom the Software is 24 | # furnished to do so, subject to the following conditions: 25 | # 26 | # The above copyright notice and this permission notice shall be included in all 27 | # copies or substantial portions of the Software. 28 | # 29 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | # SOFTWARE. 36 | # =================================================================================================================== 37 | import os 38 | import tensorflow as tf 39 | import numpy as np 40 | 41 | 42 | slim = tf.contrib.slim 43 | 44 | _FILE_PATTERN = 'mvc_clothing_%s_%d.tfrecords' 45 | 46 | SPLITS_TO_SIZES = {'train': 37485, 'test': 3000 }#origin 86423 47 | 48 | _NUM_CLASSES = 13 49 | 50 | 51 | IMAGE_HEIGHT = 224 52 | IMAGE_WIDTH = 224 53 | def get_split(config, split_name, dataset_dir, batch_size,file_pattern=None, reader=None): 54 | """Gets a dataset tuple with instructions for reading flowers. 55 | Args: 56 | split_name: A train/validation split name. 57 | dataset_dir: The base directory of the dataset sources. 58 | file_pattern: The file pattern to use when matching the dataset sources. 59 | It is assumed that the pattern contains a '%s' string so that the split 60 | name can be inserted. 61 | reader: The TensorFlow reader type. 62 | Returns: 63 | A `Dataset` namedtuple. 64 | Raises: 65 | ValueError: if `split_name` is not a valid train/validation split. 66 | """ 67 | all_file = [] 68 | reader = tf.TFRecordReader() 69 | batch_size = config.batch_size 70 | data_splitnum = config.data_split_num 71 | file_pattern = _FILE_PATTERN 72 | 73 | if split_name == 'train': 74 | num_epochs = None 75 | for i in range(data_splitnum): 76 | all_file.append(os.path.join(dataset_dir, 'mvc_clothing/', file_pattern%(split_name,i))) 77 | elif split_name == 'test': 78 | num_epochs, batch_size = 1, 1 79 | all_file.append(os.path.join(dataset_dir, 'mvc_clothing/', file_pattern%(split_name,0))) 80 | elif split_name not in SPLITS_TO_SIZES: 81 | raise ValueError('split name %s was not recognized.' % split_name) 82 | 83 | filename_queue = tf.train.string_input_producer( 84 | all_file, num_epochs=num_epochs, shuffle=False) 85 | 86 | _, serialized_example = reader.read(filename_queue) 87 | features = tf.parse_single_example( 88 | serialized_example, 89 | features={ 90 | 'height': tf.FixedLenFeature([], tf.int64), 91 | 'width': tf.FixedLenFeature([], tf.int64), 92 | 'image_string': tf.FixedLenFeature([], tf.string), 93 | 'label': tf.FixedLenFeature([], tf.float32) 94 | }) 95 | 96 | image = tf.decode_raw(features['image_string'], tf.uint8) 97 | label = tf.cast(features['label'], tf.float32) 98 | height = tf.cast(features['height'], tf.int32) 99 | width = tf.cast(features['width'], tf.int32) 100 | 101 | image = tf.reshape(image, [height, width, 3]) 102 | 103 | resized_image = tf.image.resize_images(images=image, 104 | size=[IMAGE_HEIGHT,IMAGE_WIDTH]) 105 | 106 | mean = tf.constant(np.array(np.load(dataset_dir + 'mvc_clothing/mvc_mean.npy').transpose(1,2,0).reshape(224,224,3),dtype=np.float32)) 107 | resized_image = resized_image - mean 108 | min_after_dequeue = 10000 109 | capacity = min_after_dequeue +3*batch_size 110 | 111 | images, labels = tf.train.shuffle_batch( 112 | [resized_image, label], 113 | batch_size = batch_size, 114 | capacity = capacity, 115 | num_threads=1, 116 | min_after_dequeue= min_after_dequeue, 117 | seed=config.random_seed) 118 | 119 | return images, labels, SPLITS_TO_SIZES[split_name] -------------------------------------------------------------------------------- /Fine-tuning/weight_loader/lenetsound_lenetfashion_weight.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | # ==================================================================================================== 23 | import numpy as np 24 | import sys 25 | np.set_printoptions(threshold=np.nan) 26 | from read_bin import load_bin,load_txt 27 | def weight_loader(config): 28 | MERGER_NAME = config.net 29 | task1,task2 = MERGER_NAME.split("_") 30 | PATH = config.weight_dir 31 | 32 | # Load Vgg Well-trained 33 | conv_num = 2 34 | all_num = 4 35 | sound_w = [] 36 | sound_b = [] 37 | PATH_M1 = PATH + config.net + '/' + task1 + '/' 38 | w1_name = load_txt(PATH_M1 + "/weight.txt") 39 | b1_name = load_txt(PATH_M1 + "/bias.txt") 40 | for i in range(conv_num): 41 | sound_w.append(load_bin(PATH_M1 + w1_name[i]).transpose(2,3,1,0)) 42 | sound_b.append(load_bin(PATH_M1 + b1_name[i])) 43 | for i in range(conv_num,all_num): 44 | sound_w.append(load_bin(PATH_M1 + w1_name[i]).transpose(1,0)) 45 | sound_b.append(load_bin(PATH_M1 + b1_name[i])) 46 | sound_w[2] = sound_w[2].reshape(64,8,8,1024).transpose(1,2,0,3).reshape(4096,1024) #caffe(ch*h*w,out) -> tensorflow(h*w*ch,out) 47 | 48 | # Load ZF Well-trained 49 | conv_num = 2 50 | all_num = 4 51 | fashion_w = [] 52 | fashion_b = [] 53 | PATH_M2 = PATH + config.net + '/' + task2 + '/' 54 | w2_name = load_txt(PATH_M2 + "/weight.txt") 55 | b2_name = load_txt(PATH_M2 + "/bias.txt") 56 | for i in range(conv_num): 57 | fashion_w.append(load_bin(PATH_M2 + w2_name[i]).transpose(2,3,1,0)) 58 | fashion_b.append(load_bin(PATH_M2 + b2_name[i])) 59 | for i in range(conv_num,all_num): 60 | fashion_w.append(load_bin(PATH_M2 + w2_name[i]).transpose(1,0)) 61 | fashion_b.append(load_bin(PATH_M2 + b2_name[i])) 62 | fashion_w[2] = fashion_w[2].reshape(64,8,8,1024).transpose(1,2,0,3).reshape(4096,1024) #caffe(ch*h*w,out) -> tensorflow(h*w*ch,out) 63 | 64 | # Load Merged Model 65 | conv_num = 2 66 | all_num = 3 67 | M_codebook = [] 68 | M1_index = [] 69 | M2_index = [] 70 | 71 | PATH_MERGE = config.merger_dir 72 | c_name = load_txt(PATH_MERGE + 'merged_codebook.txt') 73 | i1_name = load_txt(PATH_MERGE + 'model1.txt') 74 | i2_name = load_txt(PATH_MERGE + 'model2.txt') 75 | for i in range(conv_num): 76 | M_codebook.append(load_bin(PATH_MERGE + c_name[i])) 77 | 78 | M1_index.append(np.array(load_bin(PATH_MERGE + i1_name[i],data_type="uint8")-1,dtype=np.int32)) 79 | M1_index[i] = M1_index[i].transpose(3,1,2,0).reshape([M1_index[i].shape[3],M1_index[i].shape[1]*M1_index[i].shape[2]*M1_index[i].shape[0]]) 80 | 81 | M2_index.append(np.array(load_bin(PATH_MERGE + i2_name[i],data_type="uint8")-1,dtype=np.int32)) 82 | M2_index[i] = M2_index[i].transpose(3,1,2,0).reshape([M2_index[i].shape[3],M2_index[i].shape[1]*M2_index[i].shape[2]*M2_index[i].shape[0]]) 83 | 84 | for i in range(conv_num,all_num): 85 | M_codebook.append(load_bin(PATH_MERGE + c_name[i])) 86 | 87 | M1_index.append(np.array(load_bin(PATH_MERGE + i1_name[i],data_type="uint8")-1,dtype=np.int32)) 88 | M1_index[i] = M1_index[i].transpose(1,0) 89 | 90 | M2_index.append(np.array(load_bin(PATH_MERGE + i2_name[i],data_type="uint8")-1,dtype=np.int32)) 91 | M2_index[i] = M2_index[i].transpose(1,0) 92 | 93 | 94 | M1_output_layer = load_bin(PATH_MERGE + 'M1_outputlayer.bin').transpose(1,0) 95 | M2_output_layer = load_bin(PATH_MERGE + 'M2_outputlayer.bin').transpose(1,0) 96 | 97 | print('----- Codebook Parameter Setting -----') 98 | print('Codebook subspace: [%3d, %3d, %3d]'%(M_codebook[0].shape[2],M_codebook[1].shape[2],M_codebook[2].shape[2])) 99 | print('Codeworkd numbers : [%3d, %3d, %3d]'%(M_codebook[0].shape[1],M_codebook[1].shape[1],M_codebook[2].shape[1])) 100 | print('Max Iteration : %d'%config.max_step) 101 | sys.stdout.write('Learning Rate : ') 102 | print(config.lr_rate) 103 | print('Batch Size : %d'%config.batch_size) 104 | 105 | return M_codebook,sound_w,sound_b,M1_index,M1_output_layer,fashion_w,fashion_b,M2_index,M2_output_layer 106 | 107 | -------------------------------------------------------------------------------- /Fine-tuning/weight_loader/vgg16avg_zfnet_weight.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | # ==================================================================================================== 23 | import numpy as np 24 | import sys 25 | from read_bin import load_bin,load_txt 26 | def weight_loader(config): 27 | MERGER_NAME = config.net 28 | task1,task2 = MERGER_NAME.split("_") 29 | PATH = config.weight_dir 30 | 31 | # Load Vgg Well-trained 32 | conv_num = 13 33 | all_num = 16 34 | vgg_w =[] 35 | vgg_b =[] 36 | PATH_M1 = PATH + config.net + '/' + task1 + '/' 37 | w1_name = load_txt(PATH_M1 + "weight.txt") 38 | b1_name = load_txt(PATH_M1 + "bias.txt") 39 | for i in range(conv_num): 40 | vgg_w.append(load_bin(PATH_M1 + w1_name[i]).transpose(2,3,1,0)) 41 | vgg_b.append(load_bin(PATH_M1 + b1_name[i])) 42 | for i in range(conv_num,all_num): 43 | vgg_w.append(load_bin(PATH_M1 + w1_name[i]).transpose(1,0)) 44 | vgg_b.append(load_bin(PATH_M1 + b1_name[i])) 45 | 46 | # Load ZF Well-trained 47 | conv_num = 5 48 | all_num = 8 49 | zf_w = [] 50 | zf_b = [] 51 | PATH_M2 = PATH + config.net + '/' + task2 + '/' 52 | w2_name = load_txt(PATH_M2 + "/weight.txt") 53 | b2_name = load_txt(PATH_M2 + "/bias.txt") 54 | for i in range(conv_num): 55 | zf_w.append(load_bin(PATH_M2 + w2_name[i]).transpose(2,3,1,0)) 56 | zf_b.append(load_bin(PATH_M2 + b2_name[i])) 57 | for i in range(conv_num,all_num): 58 | zf_w.append(load_bin(PATH_M2 + w2_name[i]).transpose(1,0)) 59 | zf_b.append(load_bin(PATH_M2 + b2_name[i])) 60 | zf_w[5] = zf_w[5].reshape(256,6,6,4096).transpose(1,2,0,3).reshape(9216,4096) #caffe(ch*h*w,out) -> tensorflow(h*w*ch,out) 61 | 62 | # Load Merged Model 63 | conv_num = 13 64 | all_num = 15 65 | M_codebook = [] 66 | M1_index = [] 67 | M2_index = [] 68 | 69 | 70 | PATH_MERGE = config.merger_dir 71 | c_name = load_txt(PATH_MERGE + 'merged_codebook.txt') 72 | i1_name = load_txt(PATH_MERGE + 'model1.txt') 73 | i2_name = load_txt(PATH_MERGE + 'model2.txt') 74 | 75 | for i in range(conv_num): 76 | M_codebook.append(load_bin(PATH_MERGE + c_name[i])) 77 | 78 | M1_index.append(np.array(load_bin(PATH_MERGE + i1_name[i],data_type="uint8")-1,dtype=np.int32)) 79 | M1_index[i] = M1_index[i].transpose(3,1,2,0).reshape([M1_index[i].shape[3],M1_index[i].shape[1]*M1_index[i].shape[2]*M1_index[i].shape[0]]) 80 | 81 | for i in range(conv_num,all_num): 82 | M_codebook.append(load_bin(PATH_MERGE + c_name[i])) 83 | 84 | M1_index.append(np.array(load_bin(PATH_MERGE + i1_name[i],data_type="uint8")-1,dtype=np.int32)) 85 | M1_index[i] = M1_index[i].transpose(1,0) 86 | 87 | conv_num = 5 88 | all_num = 7 89 | for i in range(conv_num): 90 | M2_index.append(np.array(load_bin(PATH_MERGE + i2_name[i],data_type="uint8")-1,dtype=np.int32)) 91 | M2_index[i] = M2_index[i].transpose(3,1,2,0).reshape([M2_index[i].shape[3],M2_index[i].shape[1]*M2_index[i].shape[2]*M2_index[i].shape[0]]) 92 | 93 | for i in range(conv_num,all_num): 94 | M2_index.append(np.array(load_bin(PATH_MERGE + i2_name[i],data_type="uint8")-1,dtype=np.int32)) 95 | M2_index[i] = M2_index[i].transpose(1,0) 96 | 97 | print_r = [] 98 | print_c = [] 99 | shared = [0,3,6,9,12,13,14] # VGG&ZF Shared Layer in paper 100 | for i in range(15): 101 | print_r.append(M_codebook[i].shape[2]) 102 | print_c.append(M_codebook[i].shape[1]) 103 | print('----- Codebook Parameter Setting -----') 104 | sys.stdout.write('Codebook Subspace : ') 105 | print(print_r) 106 | sys.stdout.write('Codeworkd Numbers : ') 107 | print(print_c) 108 | sys.stdout.write('Shared Layers : ') 109 | print(shared) 110 | print('Max Iteration : %d'%config.max_step) 111 | sys.stdout.write('Learning Rate : ') 112 | print(config.lr_rate) 113 | print('Batch Size : %d'%config.batch_size) 114 | 115 | M1_output_layer = load_bin(PATH_MERGE + 'M1_outputlayer.bin').transpose(1,0) 116 | M2_output_layer = load_bin(PATH_MERGE + 'M2_outputlayer.bin').transpose(1,0) 117 | 118 | return M_codebook, vgg_w, vgg_b, M1_index,M1_output_layer, zf_w, zf_b, M2_index, M2_output_layer 119 | 120 | 121 | -------------------------------------------------------------------------------- /Fine-tuning/nets/lenetsound_lenetfashion.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | # ==================================================================================================== 23 | import tensorflow as tf 24 | import numpy as np 25 | import sys 26 | import os 27 | 28 | 29 | def embedding_idx(index,k_num): 30 | for i in range(index.shape[0]): 31 | index[i] = index[i] + k_num*i 32 | return index 33 | 34 | class lenetsound_lenetfashion: 35 | def __init__(self,c,w1,b1,i1,outlayer1,w2,b2,i2,outlayer2): 36 | with tf.variable_scope("lenet_lenet"): 37 | self.c = [] 38 | codebook = [] 39 | for i in range(3): 40 | codebook.append(tf.Variable(c[i],dtype=tf.float32)) 41 | self.c.append(tf.reshape(codebook[i],[c[i].shape[0]*c[i].shape[1],c[i].shape[2]])) 42 | 43 | self.w1=[] 44 | self.b1=[] 45 | for i in range(4): 46 | self.w1.append(tf.constant(w1[i],tf.float32)) 47 | self.b1.append(tf.constant(b1[i],tf.float32)) 48 | 49 | self.i1 =[] 50 | for i in range(len(i1)): 51 | self.i1.append(tf.constant(embedding_idx(i1[i],c[i].shape[1]),dtype=tf.int32)) 52 | 53 | self.w2=[] 54 | self.b2=[] 55 | for i in range(4): 56 | self.w2.append(tf.constant(w2[i],tf.float32)) 57 | self.b2.append(tf.constant(b2[i],tf.float32)) 58 | 59 | self.i2=[] 60 | for i in range(len(i2)): 61 | self.i2.append(tf.constant(embedding_idx(i2[i],c[i].shape[1]),dtype=tf.int32)) 62 | 63 | self.outlayer1 = outlayer1 64 | self.outlayer2 = outlayer2 65 | 66 | 67 | def model1_target(self,image): 68 | output=[] 69 | output.append(max_pool_2x2(tf.nn.relu(conv2d(image, self.w1[0]) + self.b1[0]))) 70 | output.append(max_pool_2x2(tf.nn.relu(conv2d(output[0], self.w1[1]) + self.b1[1]))) 71 | output.append(tf.nn.relu(tf.matmul(tf.contrib.layers.flatten(output[1]), self.w1[2]) + self.b1[2])) 72 | output.append(tf.matmul(output[2], self.w1[3]) + self.b1[3]) 73 | return output 74 | 75 | def model1(self,image): 76 | w1 = [] 77 | for i in range(2): 78 | w1.append(tf.transpose(tf.reshape(tf.transpose(tf.nn.embedding_lookup(self.c[i],self.i1[i]),[0,2,1]),[self.w1[i].shape[2],self.w1[i].shape[0] ,self.w1[i].shape[1] ,self.w1[i].shape[3]]),(1,2,0,3))) 79 | for i in range(2,3): 80 | w1.append(tf.reshape(tf.transpose(tf.nn.embedding_lookup(self.c[i],self.i1[i]),[0,2,1]),[self.w1[i].shape[0],self.w1[i].shape[1]])) 81 | w1.append(tf.Variable(self.outlayer1,dtype=tf.float32)) 82 | 83 | output=[] 84 | output.append(max_pool_2x2(tf.nn.relu(conv2d(image, w1[0]) + self.b1[0]))) 85 | output.append(max_pool_2x2(tf.nn.relu(conv2d(output[0], w1[1]) + self.b1[1]))) 86 | output.append(tf.nn.relu(tf.matmul(tf.contrib.layers.flatten(output[1]),w1[2]) + self.b1[2])) 87 | output.append(tf.nn.relu(tf.matmul(output[2], w1[3]) + self.b1[3])) 88 | 89 | return output 90 | 91 | 92 | def model2_target(self,image): 93 | output=[] 94 | output.append(max_pool_2x2(tf.nn.relu(conv2d(image, self.w2[0]) + self.b2[0]))) 95 | output.append(max_pool_2x2(tf.nn.relu(conv2d(output[0], self.w2[1]) + self.b2[1]))) 96 | output.append(tf.nn.relu(tf.matmul(tf.contrib.layers.flatten(output[1]), self.w2[2]) + self.b2[2])) 97 | output.append(tf.matmul(output[2], self.w2[3]) + self.b2[3]) 98 | 99 | return output 100 | 101 | def model2(self,image): 102 | w2 = [] 103 | for i in range(2): 104 | w2.append(tf.transpose(tf.reshape(tf.transpose(tf.nn.embedding_lookup(self.c[i],self.i2[i]),[0,2,1]),[self.w2[i].shape[2],self.w2[i].shape[0] ,self.w2[i].shape[1] ,self.w2[i].shape[3]]),(1,2,0,3))) 105 | for i in range(2,3): 106 | w2.append(tf.reshape(tf.transpose(tf.nn.embedding_lookup(self.c[i],self.i2[i]),[0,2,1]),[self.w2[i].shape[0],self.w2[i].shape[1]])) 107 | w2.append(tf.Variable(self.outlayer2,dtype=tf.float32)) 108 | 109 | output=[] 110 | output.append(max_pool_2x2(tf.nn.relu(conv2d(image, w2[0]) + self.b2[0]))) 111 | output.append(max_pool_2x2(tf.nn.relu(conv2d(output[0], w2[1]) + self.b2[1]))) 112 | output.append(tf.nn.relu(tf.matmul(tf.contrib.layers.flatten(output[1]),w2[2]) + self.b2[2])) 113 | output.append(tf.nn.relu(tf.matmul(output[2], w2[3]) + self.b2[3])) 114 | 115 | return output 116 | def get_imshape(self): 117 | imshape={ 118 | 'model1':[32,32,1], 119 | 'model2':[32,32,1], 120 | } 121 | return imshape['model1'],imshape['model2'] 122 | 123 | def conv2d(x, W): 124 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 125 | 126 | def max_pool_2x2(x): 127 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME') 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | -------------------------------------------------------------------------------- /Fine-tuning/finetuning.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | # ==================================================================================================== 23 | import tensorflow as tf 24 | import numpy as np 25 | from config import setting 26 | from datasets import dataset_factory 27 | from nets import nets_factory 28 | from weight_loader import weight_factory 29 | from loss import loss_function 30 | from tqdm import trange 31 | from saver import saver 32 | import os 33 | from datetime import datetime 34 | tf.logging.set_verbosity(tf.logging.ERROR) 35 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 36 | 37 | def train_test(config): 38 | ############### 39 | ## Task Name ## 40 | ############### 41 | DATA_PATH = config.data_path 42 | MERGER_NAME = config.net 43 | task1,task2 = MERGER_NAME.split("_") 44 | 45 | ###################### 46 | ## Check Log Folder ## 47 | ###################### 48 | today = datetime.now() 49 | folder_name = config.net + today.strftime('_%m%d') + str(today.hour) + str(today.minute) 50 | if config.save_model : 51 | path = "logs/"+folder_name 52 | if not os.path.exists(path): 53 | os.makedirs(path) 54 | else: 55 | raise ValueError('Exist folder name %s' % folder_name) 56 | 57 | ############################################### 58 | ## Load Well-Trained Weight and Merged Model ## 59 | ############################################### 60 | shared_codebook, w1, b1, index1,outlayer1, w2, b2, index2,outlayer2 = weight_factory.get_weight(config, MERGER_NAME) 61 | 62 | #################### 63 | ## Select Network ## 64 | #################### 65 | merged_net = nets_factory.get_network(MERGER_NAME,shared_codebook,w1, b1,index1,outlayer1, w2, b2, index2,outlayer2) 66 | 67 | ############################## 68 | ## Load Train and Test Data ## 69 | ############################## 70 | train_x1, train_y1, _ = dataset_factory.get_dataset(config,task1,'train',DATA_PATH) 71 | train_x2, train_y2, _ = dataset_factory.get_dataset(config,task2,'train',DATA_PATH) 72 | test_x1, test_y1, data_num3 = dataset_factory.get_dataset(config,task1,'test', DATA_PATH) 73 | test_x2, test_y2, data_num4 = dataset_factory.get_dataset(config,task2,'test', DATA_PATH) 74 | 75 | ############################## 76 | ## Train and Test Operation ## 77 | ############################## 78 | trainer, test1_op, test2_op, input_op, loss_op = loss_function(config,merged_net) 79 | 80 | # Initialize 81 | init_op = tf.group( tf.global_variables_initializer(), 82 | tf.local_variables_initializer()) 83 | variables = tf.trainable_variables() 84 | 85 | with tf.Session() as sess: 86 | 87 | log_step = config.log_step 88 | sess.run(init_op) 89 | coord = tf.train.Coordinator() 90 | threads = tf.train.start_queue_runners(coord=coord) 91 | 92 | ####################### 93 | ## Start Calibration ## 94 | ####################### 95 | max_step = config.max_step 96 | for i in trange(max_step): 97 | x1, y1, x2, y2 = sess.run([train_x1, train_y1, train_x2, train_y2]) 98 | sess.run(trainer,feed_dict={input_op['x1']:x1, 99 | input_op['y1']:y1, 100 | input_op['x2']:x2, 101 | input_op['y2']:y2}) 102 | if (i% log_step == 0): 103 | loss = sess.run(loss_op,feed_dict={ input_op['x1']:x1, 104 | input_op['y1']:y1, 105 | input_op['x2']:x2, 106 | input_op['y2']:y2}) 107 | print('[{:6d}/{}] M1 loss:{:.3f} M2 Loss:{:.3f}'.format(i,max_step,loss['M1_loss'],loss['M2_loss'])) 108 | 109 | 110 | if (i%config.save_step == config.save_step-1 or i==(max_step-1)) and config.save_model == True: 111 | saver(config,sess.run(variables),folder_name) 112 | 113 | 114 | print('-------- Testing Result --------') 115 | print('Merged Model : {}'.format(config.net)) 116 | ##################### 117 | ## Model 1 Testing ## 118 | ##################### 119 | for i in range(data_num3): 120 | x1, y1 = sess.run([test_x1, test_y1]) 121 | accuracy1 = sess.run(test1_op,feed_dict={ input_op['x1']:x1, 122 | input_op['y1']:y1,}) 123 | 124 | ##################### 125 | ## Model 2 Testing ## 126 | ##################### 127 | for i in range(data_num4): 128 | x2, y2 = sess.run([test_x2, test_y2]) 129 | accuracy2 = sess.run(test2_op,feed_dict={ input_op['x2']:x2, 130 | input_op['y2']:y2,}) 131 | 132 | print('Model1 Origin Accuracy : %.4f'%accuracy1['acc1_op_']) 133 | print('NeuralMerger Accuracy : %.4f'%accuracy1['acc1_op']) 134 | print('Model2 Origin Accuracy : %.4f'%accuracy2['acc2_op_']) 135 | print('NeuralMerger Accuracy : %.4f'%accuracy2['acc2_op']) 136 | 137 | 138 | coord.request_stop() 139 | coord.join(threads) 140 | 141 | if __name__ == "__main__": 142 | config = setting() 143 | train_test(config) 144 | -------------------------------------------------------------------------------- /Fine-tuning/nets/vgg16avg_zfnet.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Image & Vision Computing Lab, Institute of Information Science, Academia Sinica 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | # ==================================================================================================== 23 | import tensorflow as tf 24 | import numpy as np 25 | def embedding_idx(index,k_num): 26 | index = np.copy(index) 27 | for i in range(index.shape[0]): 28 | index[i] = index[i] + k_num*i 29 | return index 30 | 31 | class vgg16avg_zfnet: 32 | def __init__(self,c,w1,b1,i1,outlayer1,w2,b2,i2,outlayer2): 33 | with tf.variable_scope("vgg16avg_zfnet"): 34 | 35 | self.c = [] 36 | codebook = [] 37 | for i in range(15): 38 | codebook.append(tf.Variable(c[i],dtype=tf.float32)) 39 | self.c.append(tf.reshape(codebook[i],[c[i].shape[0]*c[i].shape[1],c[i].shape[2]])) 40 | 41 | self.w1 = [] 42 | self.b1 = [] 43 | for i in range(16): 44 | self.w1.append(tf.constant(w1[i],tf.float32)) 45 | self.b1.append(tf.constant(b1[i],tf.float32)) 46 | 47 | self.i1 =[] 48 | for i in range(len(i1)): 49 | self.i1.append(tf.constant(embedding_idx(i1[i],c[i].shape[1]),dtype=tf.int32)) 50 | 51 | self.w2 = [] 52 | self.b2 = [] 53 | for i in range(8): 54 | self.w2.append(tf.constant(w2[i],tf.float32)) 55 | self.b2.append(tf.constant(b2[i],tf.float32)) 56 | 57 | self.shared2_index=[0,3,6,9,12,13,14] 58 | self.i2 = [] 59 | for i in range(len(i2)): 60 | self.i2.append(tf.constant(embedding_idx(i2[i],c[self.shared2_index[i]].shape[1]),dtype=tf.int32)) 61 | 62 | self.outlayer1 = outlayer1 63 | self.outlayer2 = outlayer2 64 | 65 | def model1_target(self,image): 66 | 67 | output=[] 68 | output.append(tf.nn.relu(conv2d(image,self.w1[0]) + self.b1[0])) 69 | output.append(tf.nn.relu(conv2d(output[0],self.w1[1]) + self.b1[1])) 70 | output.append(max_pool_2x2(output[1])) 71 | 72 | output.append(tf.nn.relu(conv2d(output[2],self.w1[2]) + self.b1[2])) 73 | output.append(tf.nn.relu(conv2d(output[3],self.w1[3]) + self.b1[3])) 74 | output.append(max_pool_2x2(output[4])) 75 | 76 | output.append(tf.nn.relu(conv2d(output[5],self.w1[4]) + self.b1[4])) 77 | output.append(tf.nn.relu(conv2d(output[6],self.w1[5]) + self.b1[5])) 78 | output.append(tf.nn.relu(conv2d(output[7],self.w1[6]) + self.b1[6])) 79 | output.append(max_pool_2x2(output[8])) 80 | 81 | output.append(tf.nn.relu(conv2d(output[9],self.w1[7]) + self.b1[7])) 82 | output.append(tf.nn.relu(conv2d(output[10],self.w1[8]) + self.b1[8])) 83 | output.append(tf.nn.relu(conv2d(output[11],self.w1[9]) + self.b1[9])) 84 | output.append(max_pool_2x2(output[12])) 85 | 86 | output.append(tf.nn.relu(conv2d(output[13],self.w1[10]) + self.b1[10])) 87 | output.append(tf.nn.relu(conv2d(output[14],self.w1[11]) + self.b1[11])) 88 | output.append(tf.nn.relu(conv2d(output[15],self.w1[12]) + self.b1[12])) 89 | output.append(avg_pool(output[16])) 90 | 91 | output.append(tf.nn.relu(tf.matmul(tf.contrib.layers.flatten(output[17]),self.w1[13]) +self.b1[13])) 92 | output.append(tf.nn.relu(tf.matmul(output[18],self.w1[14]) +self.b1[14])) 93 | output.append(tf.matmul(output[19],self.w1[15]) +self.b1[15]) 94 | return output 95 | 96 | def model1(self,image): 97 | w1 = [] 98 | for i in range(13): 99 | w1.append(tf.transpose(tf.reshape(tf.transpose(tf.nn.embedding_lookup(self.c[i],self.i1[i]),[0,2,1]),[self.w1[i].shape[2],self.w1[i].shape[0] ,self.w1[i].shape[1] ,self.w1[i].shape[3]]),(1,2,0,3))) 100 | for i in range(13,15): 101 | w1.append(tf.reshape(tf.transpose(tf.nn.embedding_lookup(self.c[i],self.i1[i]),[0,2,1]),[self.w1[i].shape[0],self.w1[i].shape[1]])) 102 | w1.append(tf.Variable(self.outlayer1,dtype=tf.float32)) 103 | 104 | output=[] 105 | output.append(tf.nn.relu(conv2d(image,w1[0]) + self.b1[0])) 106 | output.append(tf.nn.relu(conv2d(output[0],w1[1]) + self.b1[1])) 107 | output.append(max_pool_2x2(output[1])) 108 | 109 | output.append(tf.nn.relu(conv2d(output[2],w1[2]) + self.b1[2])) 110 | output.append(tf.nn.relu(conv2d(output[3],w1[3]) + self.b1[3])) 111 | output.append(max_pool_2x2(output[4])) 112 | 113 | output.append(tf.nn.relu(conv2d(output[5],w1[4]) + self.b1[4])) 114 | output.append(tf.nn.relu(conv2d(output[6],w1[5]) + self.b1[5])) 115 | output.append(tf.nn.relu(conv2d(output[7],w1[6]) + self.b1[6])) 116 | output.append(max_pool_2x2(output[8])) 117 | 118 | output.append(tf.nn.relu(conv2d(output[9],w1[7]) + self.b1[7])) 119 | output.append(tf.nn.relu(conv2d(output[10],w1[8]) + self.b1[8])) 120 | output.append(tf.nn.relu(conv2d(output[11],w1[9]) + self.b1[9])) 121 | output.append(max_pool_2x2(output[12])) 122 | 123 | output.append(tf.nn.relu(conv2d(output[13],w1[10]) + self.b1[10])) 124 | output.append(tf.nn.relu(conv2d(output[14],w1[11]) + self.b1[11])) 125 | output.append(tf.nn.relu(conv2d(output[15],w1[12]) + self.b1[12])) 126 | output.append(avg_pool(output[16])) 127 | 128 | output.append(tf.nn.relu(tf.matmul(tf.contrib.layers.flatten(output[17]),w1[13]) +self.b1[13])) 129 | output.append(tf.nn.relu(tf.matmul(output[18],w1[14]) +self.b1[14])) 130 | output.append(tf.matmul(output[19],w1[15]) +self.b1[15]) 131 | 132 | return output 133 | 134 | 135 | def model2_target(self,image): 136 | 137 | output=[] 138 | output.append(tf.nn.relu(conv2d_2(image,self.w2[0]) + self.b2[0])) 139 | output.append(max_pool_3(spatial_lrn(output[0],local_size=3,alpha=0.00005*9,beta=0.75))) 140 | 141 | output.append(tf.nn.relu(conv2d_2(output[1],self.w2[1]) + self.b2[1])) 142 | output.append(max_pool_3(spatial_lrn(output[2],local_size=3,alpha=0.00005*9,beta=0.75))) 143 | 144 | output.append(tf.nn.relu(conv2d(output[3],self.w2[2]) + self.b2[2])) 145 | 146 | output.append(tf.nn.relu(conv2d(output[4],self.w2[3]) + self.b2[3])) 147 | 148 | output.append(max_pool_v(tf.nn.relu(conv2d(output[5],self.w2[4]) + self.b2[4]))) 149 | 150 | output.append(tf.nn.relu(tf.matmul(tf.contrib.layers.flatten(output[6]),self.w2[5]) +self.b2[5])) 151 | output.append(tf.nn.relu(tf.matmul(output[7],self.w2[6]) +self.b2[6])) 152 | output.append(tf.matmul(output[8],self.w2[7]) +self.b2[7]) 153 | 154 | return output 155 | 156 | def model2(self,image): 157 | w2 = [] 158 | for i in range(5): 159 | w2.append(tf.transpose(tf.reshape(tf.transpose(tf.nn.embedding_lookup(self.c[self.shared2_index[i]],self.i2[i]),[0,2,1]),[self.w2[i].shape[2],self.w2[i].shape[0] ,self.w2[i].shape[1] ,self.w2[i].shape[3]]),(1,2,0,3))) 160 | for i in range(5,7): 161 | w2.append(tf.reshape(tf.transpose(tf.nn.embedding_lookup(self.c[self.shared2_index[i]],self.i2[i]),[0,2,1]),[self.w2[i].shape[0],self.w2[i].shape[1]])) 162 | w2.append(tf.Variable(self.outlayer2,dtype=tf.float32)) 163 | 164 | output=[] 165 | output.append(tf.nn.relu(conv2d_2(image,w2[0]) + self.b2[0])) 166 | output.append(max_pool_3(spatial_lrn(output[0],local_size=3,alpha=0.00005*9,beta=0.75))) 167 | 168 | output.append(tf.nn.relu(conv2d_2(output[1],w2[1]) + self.b2[1])) 169 | output.append(max_pool_3(spatial_lrn(output[2],local_size=3,alpha=0.00005*9,beta=0.75))) 170 | 171 | output.append(tf.nn.relu(conv2d(output[3],w2[2]) + self.b2[2])) 172 | 173 | output.append(tf.nn.relu(conv2d(output[4],w2[3]) + self.b2[3])) 174 | 175 | output.append(max_pool_v(tf.nn.relu(conv2d(output[5],w2[4]) + self.b2[4]))) 176 | 177 | output.append(tf.nn.relu(tf.matmul(tf.contrib.layers.flatten(output[6]),w2[5]) +self.b2[5])) 178 | output.append(tf.nn.relu(tf.matmul(output[7],w2[6]) +self.b2[6])) 179 | output.append(tf.matmul(output[8],w2[7]) +self.b2[7]) 180 | 181 | return output 182 | 183 | def get_imshape(self): 184 | imshape={ 185 | 'model1':[224,224,3], 186 | 'model2':[227,227,3], 187 | } 188 | return imshape['model1'],imshape['model2'] 189 | 190 | 191 | def conv2d(x, W): 192 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 193 | 194 | def conv2d_2(x, W): 195 | return tf.nn.conv2d(x, W, strides=[1, 2, 2, 1], padding='VALID') 196 | 197 | def max_pool_2x2(x): 198 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME') 199 | 200 | def avg_pool(x): 201 | return tf.nn.avg_pool(x, ksize=[1, 14, 14, 1], strides=[1, 1, 1, 1], padding='VALID') 202 | 203 | def max_pool_3(x): 204 | return tf.nn.max_pool(x, ksize=[1, 3, 3, 1],strides=[1, 2, 2, 1], padding='SAME') 205 | 206 | def max_pool_v(x): 207 | return tf.nn.max_pool(x, ksize=[1, 3, 3, 1],strides=[1, 2, 2, 1], padding='VALID') 208 | 209 | 210 | def spatial_lrn(tensor, local_size=5, bias=1.0, alpha=1.0, beta=0.5): 211 | # Implement caffe Local Response Normalization(WITHIN_CHANNEL) 212 | squared = tf.square(tensor) 213 | in_channels = tensor.get_shape().as_list()[3] 214 | kernel = tf.constant(1.0, shape=[local_size, local_size, in_channels,1]) 215 | squared_sum = tf.nn.depthwise_conv2d(squared, kernel, [1,1,1,1], padding='SAME') 216 | bias = tf.constant(bias, dtype=tf.float32) 217 | alpha = tf.constant(alpha, dtype=tf.float32) 218 | alpha = alpha/(local_size*local_size) 219 | beta = tf.constant(beta, dtype=tf.float32) 220 | 221 | return tensor / ((bias+alpha*squared_sum)**beta) --------------------------------------------------------------------------------