├── environment.yml ├── data ├── REFIT │ ├── after_align │ │ └── .gitkeep │ ├── after_culling │ │ └── .gitkeep │ ├── appliance_data │ │ └── .gitkeep │ ├── original_data │ │ └── .gitkeep │ └── description.txt └── UK_DALE │ ├── house_1 │ └── .gitkeep │ ├── house_2 │ └── .gitkeep │ ├── house_3 │ └── .gitkeep │ ├── house_4 │ └── .gitkeep │ ├── house_5 │ └── .gitkeep │ └── description.txt ├── image └── ukdale_washingmachine.png ├── .idea ├── misc.xml ├── vcs.xml ├── modules.xml └── seq2subseq.iml ├── preprocess ├── uk_dale_cfg.py ├── refit_cfg.py ├── uk_dale_process.py └── refit_process.py ├── README.md └── model ├── ukdale_u.py └── refit_u.py /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DLZRMR/seq2subseq/HEAD/environment.yml -------------------------------------------------------------------------------- /data/REFIT/after_align/.gitkeep: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file !.gitkeep -------------------------------------------------------------------------------- /data/UK_DALE/house_1/.gitkeep: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file !.gitkeep -------------------------------------------------------------------------------- /data/UK_DALE/house_2/.gitkeep: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file !.gitkeep -------------------------------------------------------------------------------- /data/UK_DALE/house_3/.gitkeep: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file !.gitkeep -------------------------------------------------------------------------------- /data/UK_DALE/house_4/.gitkeep: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file !.gitkeep -------------------------------------------------------------------------------- /data/UK_DALE/house_5/.gitkeep: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file !.gitkeep -------------------------------------------------------------------------------- /data/REFIT/after_culling/.gitkeep: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file !.gitkeep -------------------------------------------------------------------------------- /data/REFIT/appliance_data/.gitkeep: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file !.gitkeep -------------------------------------------------------------------------------- /data/REFIT/original_data/.gitkeep: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file !.gitkeep -------------------------------------------------------------------------------- /image/ukdale_washingmachine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DLZRMR/seq2subseq/HEAD/image/ukdale_washingmachine.png -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /data/REFIT/description.txt: -------------------------------------------------------------------------------- 1 | The original data should be put in folder 'original_data'. In the pre-process stage, a lot of intermediate data will generate, you can delete it manually or modify the code that we provided. -------------------------------------------------------------------------------- /data/UK_DALE/description.txt: -------------------------------------------------------------------------------- 1 | The house and appliance information that we used for training and test can be found in folder 'preprocess/uk_dale.cfg'. You should put the corresponding appliance data files in the correct house. -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/seq2subseq.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /preprocess/uk_dale_cfg.py: -------------------------------------------------------------------------------- 1 | # House_id : Channel_id 2 | Kettle = { 3 | 1: 10, 2: 8, 3: 2, 5: 18 4 | } 5 | Microwave = { 6 | 1: 13, 2: 15, 5: 23 7 | } 8 | Fridge = { 9 | 1: 12, 2: 14, 4: 5, 5: 19 10 | } 11 | Dishwasher = { 12 | 1: 6, 2: 13, 5: 22 13 | } 14 | WashingMachine = { 15 | 1: 5, 2: 12, 5: 24 16 | } 17 | 18 | train = { 19 | 'WashingMachine': { 20 | 1: 5, 5: 24 21 | }, 22 | 'Kettle': { 23 | 1: 10, 3: 2, 5: 18 24 | }, 25 | 'Fridge': { 26 | 1: 12, 4: 5, 5: 19 27 | }, 28 | 'Dishwasher': { 29 | 1: 6, 5: 22 30 | }, 31 | 'Microwave': { 32 | 1: 13, 5: 23 33 | } 34 | } 35 | 36 | test = { 37 | 'WashingMachine': { 38 | 2: 12 39 | }, 40 | 'Kettle': { 41 | 2: 8 42 | }, 43 | 'Fridge': { 44 | 2: 14 45 | }, 46 | 'Dishwasher': { 47 | 2: 13 48 | }, 49 | 'Microwave': { 50 | 2: 15 51 | } 52 | } 53 | 54 | window_width = { 55 | 'WashingMachine': 1024, 56 | 'Kettle': 1024, 57 | 'Fridge': 1024, 58 | 'Dishwasher': 1024, 59 | 'Microwave': 1024 60 | } 61 | negative_ratio = { 62 | 'WashingMachine': 1, 63 | 'Kettle': 1, 64 | 'Fridge': 1, 65 | 'Dishwasher': 1, 66 | 'Microwave': 1 67 | } 68 | 69 | outlier_threshold = { 70 | 'WashingMachine': 3, 71 | 'Kettle': 3, 72 | 'Fridge': 3, 73 | 'Dishwasher': 3, 74 | 'Microwave': 3 75 | } 76 | 77 | positive_negative_threshold = { 78 | 'WashingMachine': 1500, 79 | 'Kettle': 1500, 80 | 'Fridge': 1500, 81 | 'Dishwasher': 1500, 82 | 'Microwave': 1500 83 | } 84 | on_power_threshold = { 85 | 'WashingMachine': 20, 86 | 'Kettle': 1000, 87 | 'Fridge': 50, 88 | 'Dishwasher': 10, 89 | 'Microwave': 200 90 | } 91 | random_clip = { 92 | 'WashingMachine': 5000, 93 | 'Kettle': 5000, 94 | 'Fridge': 5000, 95 | 'Dishwasher': 5000, 96 | 'Microwave': 5000 97 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # SEQUENCE-TO-SUBSEQUENCE LEARNING WITH CONDITIONAL GAN FOR POWER DISAGGREGATION 3 | This code implements the seqence-to-subseqence (seq2subseq) learning model. This model makes a trade-off between sequence-to-sequence (seq2seq) and sequence-to-point (seq2point) method, thus, balancing the convergence difficulty in deep neural networks and the amount of computation during the inference period. In addition, we apply U-Net [1] and Instance Normalization [2] techniques to our model. We build our model based on pix2pix [3] under Tensorflow framework [4]. Thanks a lot! 4 | 5 | 6 | References: 7 | 8 | [1] Olaf Ronneberger, Philipp Fischer, and Thomas Brox, “U-net: Convolutional networks for biomedical image segmentation,” in International Conference on Medical Image Computing & Computer-assisted Intervention, 2015. 9 | 10 | [2] Dmitry Ulyanov, Andrea Vedaldi, and Victor S. Lempitsky, “Instance normalization: The missing ingredient for fast stylization,” CoRR, vol. abs/1607.08022, 2016. 11 | 12 | [3] https://phillipi.github.io/pix2pix/ 13 | 14 | [4] https://github.com/affinelayer/pix2pix-tensorflow 15 | 16 | ## Setup 17 | - Create your own virtual environment with Python > 3.5 18 | - Configure deep learning environment with Tensorflow (GPU edition) > 1.4.1 + cuDNN 19 | - Install other necessary softwares, such as Matplotlib, Scikit-learn etc. 20 | - Clone this repository 21 | 22 | The environments we used are listed in the file `environment.yml`. If you use `conda`, you can use `conda env create -f environment.yml` to set up the environment. 23 | 24 | 25 | ## Datasets and preprocessing 26 | 27 | ### REFIT 28 | Download the REFIT raw data from the original website (https://pureportal.strath.ac.uk/en/datasets/refit-electrical-load-measurements-cleaned). 29 | 30 | 31 | ### UK-DALE 32 | Download the UK-DALE raw data from the original website (https://ukerc.rl.ac.uk/DC/cgi-bin/edc_search.pl?GoButton=Detail&WantComp=41). 33 | 34 | 35 | ## Training & Test 36 | We recommend importing the project into Pycharm (https://www.jetbrains.com/pycharm/) for your future research. 37 | 38 | ### Tips 39 | You can look at the loss and computation graph using tensorboard: 40 | ```sh 41 | tensorboard --logdir= 'the output path' 42 | ``` 43 | 44 | 45 | ## Code validation and experiment results 46 | We evaluate our model on a Linux machine with a Nvidia RTX 2080 GPU and Core i7 9700K CPU. 47 | 48 | Example outputs of Washing Machine on ”unseen” house 2 from UK-DALE (We enlarge the right part of the figure to make it more clear): 49 | ![](/image/ukdale_washingmachine.png) 50 | 51 | 52 | ## Acknowledgments 53 | This research is sponsored by National Key R&D Program of China(2017YFB0902600); State Grid Corporation of China Project (SGJS0000DKJS1700840) Research and Application of Key Technology for Intelligent Dispatching and Security Early-warning of Large Power Grid. 54 | -------------------------------------------------------------------------------- /preprocess/refit_cfg.py: -------------------------------------------------------------------------------- 1 | 2 | missing_house = [13, 14] 3 | 4 | kettle = { 5 | 2: 8, 3: 9, 4: 9, 5: 8, 6: 7, 7: 9, 8: 9, 9: 7, 11: 7, 17: 8, 19: 5, 20: 9 6 | } 7 | kettle_train = { 8 | 2: 8, 3: 9, 4: 9, 5: 8, 6: 7, 8: 9, 9: 7, 17: 8, 19: 5 9 | } 10 | kettle_validation = { 11 | 7: 9 12 | } 13 | kettle_tv = { 14 | 11: 7, 20: 9, 3: 9, 4: 9, 5: 8, 6: 7, 8: 9, 9: 7, 17: 8, 19: 5, 7: 9 15 | } 16 | kettle_test = { 17 | 2: 8 18 | } 19 | 20 | 21 | microwave = { 22 | 2: 5, 3: 8, 4: 8, 5: 7, 6: 6, 10: 8, 11: 6, 12: 5, 17: 7, 19: 4, 20: 8 23 | } 24 | microwave_train = { 25 | 2: 5, 3: 8, 4: 8, 6: 6, 11: 6, 12: 5, 17: 7, 20: 8 26 | } 27 | microwave_validation = { 28 | 10: 8 29 | } 30 | microwave_tv = { 31 | 2: 5, 3: 8, 6: 6, 11: 6, 12: 5,4: 8 , 20: 8, 10: 8, 5: 7, 19: 4 32 | } 33 | microwave_test = { 34 | 17: 7 35 | } 36 | 37 | 38 | fridge = { 39 | 2: 1, 5: 1, 9: 1, 12: 1, 15: 1, 19: 1 40 | } 41 | fridge_train = { 42 | 5: 1, 9: 1, 2: 1, 19: 1 43 | } 44 | fridge_validation = { 45 | 15: 1 46 | } 47 | fridge_tv = { 48 | 5: 1, 9: 1, 2: 1, 19: 1, 15: 1 49 | } 50 | fridge_test = { 51 | 12: 1 52 | } 53 | 54 | 55 | dishwasher = { 56 | 1: 6, 2: 3, 3: 5, 5: 4, 7: 6, 9: 4, 10: 6, 16: 6, 18: 6, 20: 5 57 | } 58 | dishwasher_train = { 59 | 1: 6, 3: 5, 5: 4, 7: 6, 10: 6, 16: 6, 20: 5 60 | } 61 | dishwasher_validation = { 62 | 18: 6 63 | } 64 | dishwasher_tv = { 65 | 1: 6, 3: 5, 5: 4, 7: 6, 10: 6, 16: 6, 20: 5, 18: 6 66 | } 67 | dishwasher_test = { 68 | 2: 3, 9: 4 69 | } 70 | 71 | 72 | washingmachine = { 73 | 1: 5, 2: 2, 3: 6, 5: 3, 7: 5, 8: 4, 9: 3, 10: 5, 15: 3, 16: 5, 17: 4, 18: 5, 19: 2, 20: 4 74 | } 75 | washingmachine_train = { 76 | 1: 5, 2: 2, 3: 6, 7: 5, 8: 4, 9: 3, 10: 5, 15: 3, 17: 4, 18: 5, 20: 4 77 | } 78 | washingmachine_validation = { 79 | 19: 2 80 | } 81 | washingmachine_tv = { 82 | 1: 5, 2: 2, 3: 6, 7: 5, 8: 4, 9: 3, 10: 5, 15: 3, 17: 4, 18: 5, 20: 4, 19: 2 83 | } 84 | washingmachine_test = { 85 | 5: 3, 16: 5 86 | } 87 | 88 | separation_threshold = 400 89 | 90 | window_width = { 91 | 'WashingMachine': 1024, 92 | 'Kettle': 256, 93 | 'Microwave': 256, 94 | 'Fridge': 1024, 95 | 'Dishwasher': 1024 96 | } 97 | 98 | random_clip = { 99 | 'WashingMachine': 10000, 100 | 'Kettle': 10000, 101 | 'Microwave': 10000, 102 | 'Fridge': 10000, 103 | 'Dishwasher': 10000 104 | } 105 | 106 | negative_ratio = { 107 | 'WashingMachine': 1, 108 | 'Kettle': 0.2, 109 | 'Microwave': 0.2, 110 | 'Fridge': 1, 111 | 'Dishwasher': 1 112 | } 113 | 114 | train_validation = { 115 | 'WashingMachine': { 116 | 1: 5, 2: 2, 3: 6, 7: 5, 9: 3, 10: 5, 15: 3, 17: 4, 18: 5, 20: 4, 19: 2, 5: 3, 16: 5 117 | }, 118 | 'Kettle': { 119 | 3: 9, 4: 9, 5: 8, 6: 7, 8: 9, 9: 7, 17: 8, 19: 5, 7: 9, 11: 7, 20: 9 120 | }, 121 | 'Microwave': { 122 | 2: 5, 3: 8, 6: 6, 11: 6, 12: 5, 17: 7, 20: 8, 10: 8, 5: 7, 19: 4 123 | }, 124 | 'Fridge': { 125 | 5: 1, 9: 1, 2: 1, 19: 1, 15: 1 126 | }, 127 | 'Dishwasher': { 128 | 1: 6, 3: 5, 5: 4, 7: 6, 10: 6, 16: 6, 18: 6, 2: 3, 9: 4 129 | } 130 | } 131 | test = { 132 | 'WashingMachine': { 133 | 8: 4 134 | }, 135 | 'Kettle': { 136 | 2: 8 137 | }, 138 | 'Microwave': { 139 | 4: 8 140 | }, 141 | 'Fridge': { 142 | 15: 1 143 | }, 144 | 'Dishwasher': { 145 | 20: 5 146 | } 147 | } 148 | 149 | cull_dict = { 150 | 'WashingMachine': { 151 | 1: [[0, 130000], [1120000, 1240000], [1800000, 1920000], [4970000, 5110000], [5750000, 5900000]], 152 | 2: [[2300000, 2400000], [4100000, 4180000]], 153 | 3: [[50000, 250000]], 154 | 10: [[770000, 1070000], [4250000, 4550000]], 155 | 18: [[1450000, 1750000], [3400000, 3600000]], 156 | 20: [[3550000, 3850000]] 157 | }, 158 | 'Kettle': { 159 | 3: [[50000, 260000]], 160 | 4: [[1180000, 1360000]], 161 | 5: [[2910000, 3080000], [5020000, 5110000]], 162 | 9: [[480000, 680000], [1920000, 2050000]], 163 | 20: [[3670000, 3810000]] 164 | }, 165 | 'Microwave': { 166 | 4: [[1180000, 1360000], [2880000, 3400000]], 167 | 10: [[770000, 1080000]], 168 | 20: [[3700000, 3800000]] 169 | }, 170 | 'Fridge': { 171 | 9: [[21000, 73100], [478000, 678400]] 172 | }, 173 | 'Dishwasher': { 174 | 10: [[757500, 1091000], [4250000, 4550000]], 175 | 18: [[650000, 840000], [1450000, 1780000], [3400000, 3630000]], 176 | 20: [[460000, 750000], [3530000, 3870000]] 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /preprocess/uk_dale_process.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('preprocess') 3 | import matplotlib 4 | matplotlib.use("TkAgg") 5 | import matplotlib.pyplot as plt 6 | import matplotlib.gridspec as gridspec 7 | from matplotlib.pyplot import savefig 8 | import numpy as np 9 | import uk_dale_cfg 10 | import os 11 | import random 12 | from os.path import join 13 | 14 | name = ['WashingMachine', 'Kettle', 'Microwave', 'Fridge', 'Dishwasher'] 15 | appliance_dict = { 16 | 'WashingMachine': uk_dale_cfg.WashingMachine, 17 | 'Kettle': uk_dale_cfg.Kettle, 18 | 'Microwave': uk_dale_cfg.Microwave, 19 | 'Fridge': uk_dale_cfg.Fridge, 20 | 'Dishwasher': uk_dale_cfg.Dishwasher 21 | } 22 | base_path = 'data/UK_DALE' 23 | 24 | 25 | # step1: convert .dat to .npy 26 | def convertnpy(): 27 | for house_id in range(1, 6): 28 | aggregate_path = join(base_path, 'house_%d/channel_%d.dat' % (house_id, 1)) 29 | aggregate_data = [] 30 | with open(aggregate_path, 'r') as f: 31 | lines = f.readlines() 32 | for line in lines: 33 | s = line.split() 34 | aggregate_data.append([int(s[0]), int(s[1])]) 35 | np.save(join(base_path, 'house_%s\\main.npy' % house_id), aggregate_data) 36 | print('House: %d finished!' % house_id) 37 | 38 | for appliance_name in name: 39 | for house_id, channel_id in appliance_dict[appliance_name].items(): 40 | appliance_path = join(base_path, 'house_%d/channel_%d.dat' % (house_id, channel_id)) 41 | print('Appliance: %s in house %s Load!' % (appliance_name, house_id)) 42 | appliance_data = [] 43 | 44 | with open(appliance_path, 'r') as f: 45 | lines = f.readlines() 46 | for line in lines: 47 | s = line.split() 48 | appliance_data.append([int(s[0]), int(s[1])]) 49 | np.save(join(base_path, 'house_%d\\%s_appliance.npy' % (house_id, appliance_name)), appliance_data) 50 | print('Appliance: %s House: %d finished!' % (appliance_name, house_id)) 51 | 52 | 53 | # step2: align according to timestamp 54 | def align(): 55 | for appliance_name in name: 56 | aggregate_new = [] 57 | appliance_new = [] 58 | for house_id, c in uk_dale_cfg.train[appliance_name].items(): 59 | aggregate_data = np.load(join(base_path, 'house_%d\\main.npy' % house_id)) 60 | appliance_data = np.load(join(base_path, 'house_%d\\%s_appliance.npy' % (house_id, appliance_name))) 61 | 62 | aggregate_index = 0 63 | appliance_index = 0 64 | if appliance_data[0][0] < aggregate_data[0][0]: 65 | print('Appliance time is ahead of aggregate time!') 66 | for i in range(0, len(appliance_data)): 67 | if appliance_data[i][0] > aggregate_data[0][0]: 68 | appliance_index = i 69 | 70 | for i in range(appliance_index, len(appliance_data)): 71 | appliance_time = appliance_data[i][0] 72 | front = aggregate_index 73 | behind = -1 74 | for j in range(aggregate_index, len(aggregate_data)): 75 | if aggregate_data[j][0] <= appliance_time: 76 | front = j 77 | else: 78 | behind = j 79 | d_1 = abs(appliance_time - aggregate_data[front][0]) 80 | d_2 = abs(appliance_time - aggregate_data[behind][0]) 81 | if d_1 <= d_2: 82 | aggregate_new.append(aggregate_data[front][1]) 83 | appliance_new.append(appliance_data[i][1]) 84 | else: 85 | aggregate_new.append(aggregate_data[behind][1]) 86 | appliance_new.append(appliance_data[i][1]) 87 | aggregate_index = front 88 | break 89 | if i % 10000 == 0: 90 | print('Appliance: %s, House: %s, processing: %f' % (appliance_name, house_id, i/len(appliance_data))) 91 | print('Appliance: %s, House: %s finished!' % (appliance_name, house_id)) 92 | np.save(join(base_path, '%s_main_train.npy' % appliance_name), aggregate_new) 93 | np.save(join(base_path, '%s_appliance_train.npy' % appliance_name), appliance_new) 94 | print('Appliance - train : %s finished!' % appliance_name) 95 | 96 | aggregate_new = [] 97 | appliance_new = [] 98 | house_id = 2 99 | aggregate_data = np.load(join(base_path, 'house_%d\\main.npy' % house_id)) 100 | appliance_data = np.load(join(base_path, 'house_%d\\%s_appliance.npy' % (house_id, appliance_name))) 101 | 102 | aggregate_index = 0 103 | appliance_index = 0 104 | if appliance_data[0][0] < aggregate_data[0][0]: 105 | print('Appliance time is ahead of aggregate time!') 106 | for i in range(0, len(appliance_data)): 107 | if appliance_data[i][0] < aggregate_data[0][0]: 108 | appliance_index = i 109 | 110 | for i in range(appliance_index, len(appliance_data)): 111 | appliance_time = appliance_data[i][0] 112 | front = aggregate_index 113 | behind = -1 114 | for j in range(aggregate_index, len(aggregate_data)): 115 | if aggregate_data[j][0] <= appliance_time: 116 | front = j 117 | else: 118 | behind = j 119 | d_1 = abs(appliance_time - aggregate_data[front][0]) 120 | d_2 = abs(appliance_time - aggregate_data[behind][0]) 121 | if d_1 <= d_2: 122 | aggregate_new.append(aggregate_data[front][1]) 123 | appliance_new.append(appliance_data[i][1]) 124 | else: 125 | aggregate_new.append(aggregate_data[behind][1]) 126 | appliance_new.append(appliance_data[i][1]) 127 | aggregate_index = front 128 | break 129 | if i % 10000 == 0: 130 | print('Appliance: %s, House: %s, processing: %f' % ( 131 | appliance_name, house_id, i / len(appliance_data))) 132 | print('Appliance: %s, House: %s finished!' % (appliance_name, house_id)) 133 | np.save(join(base_path, '%s_main_test.npy' % appliance_name), aggregate_new) 134 | np.save(join(base_path, '%s_appliance_test.npy' % appliance_name), appliance_new) 135 | print('Appliance - test : %s finished!' % appliance_name) 136 | 137 | 138 | # step3: observe the total sequence 139 | def view_total(): 140 | if not os.path.exists(join(base_path, 'total_view')): 141 | os.mkdir(join(base_path, 'total_view')) 142 | for appliance_name in name: 143 | aggregate_data = np.load(join(base_path, '%s_main.npy' % appliance_name)) 144 | appliance_data = np.load(join(base_path, '%s_appliance.npy' % appliance_name)) 145 | 146 | plt.figure(('Appliance: %s' % appliance_name), figsize=(30, 20), dpi=200) 147 | plt.subplot(211) 148 | plt.plot(aggregate_data) 149 | plt.subplot(212) 150 | plt.plot(appliance_data) 151 | savefig('data\\UK_DALE\\total_view\\%s' % appliance_name) 152 | 153 | 154 | def separate(): 155 | for appliance_name in name: 156 | window_width = uk_dale_cfg.window_width[appliance_name] 157 | scale_up = int(1024 / window_width) 158 | print('Scale up %d' % scale_up) 159 | data_path = 'data\\UK_DALE' 160 | count = 0 161 | negative_ratio = uk_dale_cfg.negative_ratio[appliance_name] 162 | positive_negative_threshold = uk_dale_cfg.positive_negative_threshold[appliance_name] 163 | outlier_length = uk_dale_cfg.outlier_threshold[appliance_name] 164 | on_power_threshold = uk_dale_cfg.on_power_threshold[appliance_name] 165 | 166 | appliance_new = [] 167 | main_new = [] 168 | 169 | def exception_process(d): 170 | one = 0 171 | current_length = 0 172 | clip_index = 0 173 | negative = 1 174 | for i in range(len(d)): 175 | current_number = d[i] 176 | if current_number > on_power_threshold: 177 | negative = 0 178 | if one == 0: 179 | if current_number > 0: 180 | one = 1 181 | current_length += 1 182 | clip_index = i 183 | else: 184 | if current_number > 0: 185 | current_length += 1 186 | else: 187 | if current_length <= outlier_length: 188 | for j in range(clip_index, i): 189 | d[j] = 0 190 | one = 0 191 | current_length = 0 192 | if negative == 1: 193 | for j in range(len(d)): 194 | d[j] = 0 195 | return d 196 | 197 | appliance_data = np.load(join(base_path, '%s_appliance_train.npy' % appliance_name)) 198 | appliance_data_test = np.load(join(base_path, '%s_appliance_test.npy' % appliance_name)) 199 | main_data = np.load(join(data_path, '%s_main_train.npy' % appliance_name)) 200 | main_data_test = np.load(join(data_path, '%s_main_test.npy' % appliance_name)) 201 | print('Appliance: %s data load complete!' % appliance_name) 202 | 203 | # train process 204 | current_head = 1 205 | data_length = len(appliance_data) 206 | end = data_length - window_width - 1 207 | while current_head < end: 208 | temp_main = [] 209 | temp_appliance = [] 210 | t_a = [] 211 | data_ = exception_process(appliance_data[current_head:current_head + window_width]) 212 | for i in range(current_head, current_head+window_width): 213 | for k in range(scale_up): 214 | temp_main.append(main_data[i]) 215 | temp_appliance.append(data_[i-current_head]) 216 | t_a.append(data_[i-current_head]) 217 | current_head += int(window_width/2) 218 | sum_ = np.sum(t_a) 219 | if sum_ < positive_negative_threshold: 220 | r = random.random() 221 | if r > negative_ratio: 222 | continue 223 | appliance_new.append(temp_appliance) 224 | main_new.append(temp_main) 225 | count += 1 226 | if count % 1000 == 0: 227 | print('Train - Type 1, Appliance: %s processing: %f' % (appliance_name, (current_head / data_length))) 228 | 229 | data_length = len(appliance_data) - window_width-1 230 | random_clip = uk_dale_cfg.random_clip[appliance_name] 231 | for j in range(random_clip): 232 | r = random.random() 233 | start = int(r * data_length) 234 | temp_main = [] 235 | temp_appliance = [] 236 | t_a = [] 237 | data_ = exception_process(appliance_data[start:start + window_width]) 238 | for i in range(start, start+window_width): 239 | for k in range(scale_up): 240 | temp_main.append(main_data[i]) 241 | temp_appliance.append(data_[i-start]) 242 | t_a.append(data_[i-start]) 243 | sum_ = np.sum(t_a) 244 | if sum_ < positive_negative_threshold: 245 | r = random.random() 246 | if r > negative_ratio: 247 | continue 248 | appliance_new.append(temp_appliance) 249 | main_new.append(temp_main) 250 | count += 1 251 | if count % 1000 == 0: 252 | print('Train - Type 2, Appliance: %s processing: %f' % (appliance_name, (j / random_clip))) 253 | print('Appliance: %s complete!' % appliance_name) 254 | np.save(os.path.join(data_path, '%s\\appliance_train.npy' % appliance_name), appliance_new) 255 | np.save(os.path.join(data_path, '%s\\main_train.npy' % appliance_name), main_new) 256 | 257 | # test process 258 | current_head = 1 259 | data_length = len(appliance_data_test) 260 | end = data_length - window_width - 1 261 | while current_head < end: 262 | temp_main = [] 263 | temp_appliance = [] 264 | t_a = [] 265 | data_ = exception_process(appliance_data_test[current_head:current_head + window_width]) 266 | for i in range(current_head, current_head+window_width): 267 | for k in range(scale_up): 268 | temp_main.append(main_data_test[i]) 269 | temp_appliance.append(data_[i-current_head]) 270 | t_a.append(data_[i-current_head]) 271 | current_head += int(window_width/2) 272 | sum_ = np.sum(t_a) 273 | if sum_ < positive_negative_threshold: 274 | r = random.random() 275 | if r > negative_ratio: 276 | continue 277 | appliance_new.append(temp_appliance) 278 | main_new.append(temp_main) 279 | count += 1 280 | if count % 1000 == 0: 281 | print('Test - Type 1, Appliance: %s processing: %f' % (appliance_name, (current_head / data_length))) 282 | 283 | data_length = len(appliance_data_test) - window_width-1 284 | random_clip = uk_dale_cfg.random_clip[appliance_name] 285 | for j in range(random_clip): 286 | r = random.random() 287 | start = int(r * data_length) 288 | temp_main = [] 289 | temp_appliance = [] 290 | t_a = [] 291 | data_ = exception_process(appliance_data_test[start:start + window_width]) 292 | for i in range(start, start+window_width): 293 | for k in range(scale_up): 294 | temp_main.append(main_data_test[i]) 295 | temp_appliance.append(data_[i-start]) 296 | t_a.append(data_[i-start]) 297 | sum_ = np.sum(t_a) 298 | if sum_ < positive_negative_threshold: 299 | r = random.random() 300 | if r > negative_ratio: 301 | continue 302 | appliance_new.append(temp_appliance) 303 | main_new.append(temp_main) 304 | count += 1 305 | if count % 1000 == 0: 306 | print('Test - Type 2, Appliance: %s processing: %f' % (appliance_name, (j / random_clip))) 307 | print('Appliance: %s complete!' % appliance_name) 308 | np.save(os.path.join(data_path, '%s\\appliance_test.npy' % appliance_name), appliance_new) 309 | np.save(os.path.join(data_path, '%s\\main_test.npy' % appliance_name), main_new) 310 | 311 | 312 | def convert512(): 313 | for appliance_name in name: 314 | appliance_train = np.load(join(base_path, '%s\\appliance_train.npy' % appliance_name)) 315 | appliance_test = np.load(join(base_path, '%s\\appliance_test.npy' % appliance_name)) 316 | train_new = [] 317 | test_new = [] 318 | for i in range(len(appliance_train)): 319 | train_new.append(appliance_train[i][256: 768]) 320 | for i in range(len(appliance_test)): 321 | test_new.append(appliance_test[i][256: 768]) 322 | 323 | np.save(join(base_path, '%s\\appliance_train_512.npy' % appliance_name), train_new) 324 | np.save(join(base_path, '%s\\appliance_test_512.npy' % appliance_name), test_new) 325 | print('512: %s finished!' % appliance_name) 326 | 327 | 328 | def generateBalancedDataset(thres, ratio): 329 | for appliance_name in name: 330 | appliance_data = np.load(join(base_path, '%s\\appliance_train_512.npy' % appliance_name)) 331 | main_data = np.load(join(base_path, '%s\\main_train.npy' % appliance_name)) 332 | print('Appliance: %s data load!' % appliance_name) 333 | 334 | appliance_positve = [] 335 | main_positive = [] 336 | appliance_negative = [] 337 | main_negative = [] 338 | appliance_new = [] 339 | main_new = [] 340 | 341 | for i in range(len(appliance_data)): 342 | if np.sum(appliance_data[i]) > thres: 343 | appliance_positve.append(appliance_data[i]) 344 | main_positive.append(main_data[i]) 345 | else: 346 | appliance_negative.append(appliance_data[i]) 347 | main_negative.append(main_data[i]) 348 | 349 | print('Appliance: %s positive: %d negative: %d' % 350 | (appliance_name, len(appliance_positve), len(appliance_negative))) 351 | 352 | if len(appliance_positve)*ratio < len(appliance_negative): 353 | negative_length = len(appliance_positve)*ratio 354 | else: 355 | negative_length = len(appliance_negative) 356 | negative_index = np.linspace(0, negative_length-1, negative_length).astype(int) 357 | random.shuffle(negative_index) 358 | positive_index = np.linspace(0, len(appliance_positve)-1, len(appliance_positve)).astype(int) 359 | random.shuffle(positive_index) 360 | 361 | for i in positive_index: 362 | appliance_new.append(appliance_positve[i]) 363 | main_new.append(main_positive[i]) 364 | for i in negative_index: 365 | appliance_new.append(appliance_negative[i]) 366 | main_new.append(main_negative[i]) 367 | 368 | print('Appliance: %s length: %d' % (appliance_name, len(appliance_new))) 369 | np.save(join(base_path, '%s\\appliance_train_balanced_512.npy' % appliance_name), appliance_new) 370 | np.save(join(base_path, '%s\\main_train_balanced.npy' % appliance_name), main_new) 371 | print('Appliance: %s finished!' % appliance_name) 372 | 373 | 374 | def shrink(scale): 375 | for appliance_name in name: 376 | appliance_train = np.load(join(base_path, '%s\\appliance_train_balanced_512.npy' % appliance_name)) 377 | appliance_test = np.load(join(base_path, '%s\\appliance_test_512.npy' % appliance_name)) 378 | main_train = np.load(join(base_path, '%s\\main_train_balanced.npy' % appliance_name)) 379 | main_test = np.load(join(base_path, '%s\\main_test.npy' % appliance_name)) 380 | atr_new = [] 381 | ate_new = [] 382 | mtr_new = [] 383 | mte_new = [] 384 | print('Scale - Appliance: %s data load!' % appliance_name) 385 | 386 | for i in range(len(appliance_train)): 387 | atr_temp = [] 388 | mtr_temp = [] 389 | for j in range(512): 390 | atr_temp.append(float(appliance_train[i][j]/scale)) 391 | for j in range(1024): 392 | mtr_temp.append(float(main_train[i][j]/scale)) 393 | atr_new.append(atr_temp) 394 | mtr_new.append(mtr_temp) 395 | 396 | for i in range(len(appliance_test)): 397 | ate_temp = [] 398 | mte_temp = [] 399 | for j in range(512): 400 | ate_temp.append(float(appliance_test[i][j]/scale)) 401 | for j in range(1024): 402 | mte_temp.append(float(main_test[i][j]/scale)) 403 | ate_new.append(ate_temp) 404 | mte_new.append(mte_temp) 405 | 406 | np.save(join(base_path, '%s\\appliance_train_%d.npy' % (appliance_name, scale)), atr_new) 407 | np.save(join(base_path, '%s\\main_train_%d.npy' % (appliance_name, scale)), mtr_new) 408 | np.save(join(base_path, '%s\\appliance_test_%d.npy' % (appliance_name, scale)), ate_new) 409 | np.save(join(base_path, '%s\\main_test_%d.npy' % (appliance_name, scale)), mte_new) 410 | 411 | print('Scale: %s finished! ' % appliance_name) 412 | 413 | 414 | def clip_view(): 415 | clip_number = 300 416 | for appliance_name in name: 417 | print('Clip - Appliance: %s' % appliance_name) 418 | if not os.path.exists(join(base_path, '%s\\visual_train' % appliance_name)): 419 | os.mkdir(join(base_path, '%s\\visual_train' % appliance_name)) 420 | if not os.path.exists(join(base_path, '%s\\visual_test' % appliance_name)): 421 | os.mkdir(join(base_path, '%s\\visual_test' % appliance_name)) 422 | 423 | appliance_data = np.load(join(base_path, '%s\\appliance_train_1000.npy' % appliance_name)) 424 | main_data = np.load(join(base_path, '%s\\main_train_1000.npy' % appliance_name)) 425 | print('Appliance length: %d, main length: %d' % (len(appliance_data), len(main_data))) 426 | 427 | for i in range(clip_number): 428 | fig = plt.figure(figsize=(16, 16), dpi=100) 429 | gs = gridspec.GridSpec(2, 1) 430 | gs.update(wspace=0.05, hspace=0.05) 431 | x = np.linspace(256, 768, 512) 432 | appliance_temp = [] 433 | main_temp = [] 434 | r = int(random.random()*len(appliance_data)) 435 | for j in range(512): 436 | appliance_temp.append(int(appliance_data[r][j]*1000)) 437 | for j in range(1024): 438 | main_temp.append(int(main_data[r][j]*1000)) 439 | 440 | plt.subplot(2, 1, 1) 441 | plt.xlim(0, 1024) 442 | plt.plot(main_temp) 443 | 444 | plt.subplot(2, 1, 2) 445 | plt.xlim(0, 1024) 446 | plt.plot(x, appliance_temp) 447 | 448 | plt.savefig(join(base_path, '%s\\visual_train\\%d.jpg' % (appliance_name, i))) 449 | plt.close(fig) 450 | 451 | appliance_data = np.load(join(base_path, '%s\\appliance_test_1000.npy' % appliance_name)) 452 | main_data = np.load(join(base_path, '%s\\main_test_1000.npy' % appliance_name)) 453 | 454 | # test section 455 | for i in range(clip_number): 456 | fig = plt.figure(figsize=(16, 16), dpi=100) 457 | gs = gridspec.GridSpec(2, 1) 458 | gs.update(wspace=0.05, hspace=0.05) 459 | x = np.linspace(256, 768, 512) 460 | appliance_temp = [] 461 | main_temp = [] 462 | r = int(random.random()*len(appliance_data)) 463 | for j in range(512): 464 | appliance_temp.append(int(appliance_data[r][j]*1000)) 465 | for j in range(1024): 466 | main_temp.append(int(main_data[r][j]*1000)) 467 | 468 | plt.subplot(2, 1, 1) 469 | plt.xlim(0, 1024) 470 | plt.plot(main_temp) 471 | 472 | plt.subplot(2, 1, 2) 473 | plt.xlim(0, 1024) 474 | plt.plot(x, appliance_temp) 475 | 476 | plt.savefig(join(base_path, '%s\\visual_test\\%d.jpg' % (appliance_name, i))) 477 | plt.close(fig) 478 | 479 | 480 | if __name__ == '__main__': 481 | align() 482 | separate() 483 | convert512() 484 | generateBalancedDataset(1500, 1) 485 | shrink(1) 486 | -------------------------------------------------------------------------------- /model/ukdale_u.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import tensorflow as tf 6 | import numpy as np 7 | import argparse 8 | import os 9 | import random 10 | import collections 11 | import math 12 | import time 13 | import matplotlib.pyplot as plt 14 | import matplotlib.gridspec as gridspec 15 | from sklearn.metrics import mean_squared_error 16 | from sklearn.metrics import mean_absolute_error 17 | from sklearn.metrics import r2_score 18 | from tensorflow.contrib.layers import xavier_initializer_conv2d 19 | 20 | 21 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 22 | os.environ['CUDA_VISIBLE_DEVICES'] = "0" 23 | gpu_options = tf.GPUOptions(allow_growth=True) 24 | 25 | current_version = '1' 26 | dataset_name = 'UKDALE' 27 | appliance_name = 'WashingMachine' 28 | appliance_name_folder = '%s_%s' %(dataset_name, appliance_name) 29 | base_root = 'outnilm\\%s\\%s' % (appliance_name_folder, current_version) 30 | folder_list = { 31 | 'base_root': base_root, 32 | 'image': '%s\\image' % base_root, 33 | 'model': '%s\\model' % base_root, 34 | 'test_validation': '%s\\test_validation' % base_root, 35 | 'input': 'data\\UK_DALE\\%s' % appliance_name, 36 | } 37 | 38 | # if there's no checkpoint, it will be 1 39 | isFirst = 1 40 | parser = argparse.ArgumentParser() 41 | if isFirst == 1: 42 | parser.add_argument("--checkpoint", 43 | default=None, 44 | help="directory with checkpoint to resume training from or use for testing") 45 | else: 46 | parser.add_argument("--checkpoint", 47 | default=folder_list['model'], 48 | help="directory with checkpoint to resume training from or use for testing") 49 | parser.add_argument("--output_dir", 50 | default=folder_list['base_root'], 51 | help="where to put output files") 52 | parser.add_argument("--seed", type=int) 53 | parser.add_argument("--max_steps", type=int, help="number of training steps (0 to disable)") 54 | parser.add_argument("--max_epochs", default=120, type=int, help="number of training epochs") 55 | parser.add_argument("--summary_freq", type=int, default=1000, help="update summaries every summary_freq steps") 56 | parser.add_argument("--progress_freq", type=int, default=5000, help="display progress every progress_freq steps") 57 | parser.add_argument("--trace_freq", type=int, default=0, help="trace execution every trace_freq steps") 58 | parser.add_argument("--display_freq", type=int, 59 | default=500, 60 | help="write current training images every display_freq steps") 61 | parser.add_argument("--save_freq", type=int, 62 | default=50000, 63 | help="save model every save_freq steps, 0 to disable") 64 | parser.add_argument('--test_procedure', type=int, default=50000) 65 | parser.add_argument('--validation_procedure', type=int, default=50000) 66 | parser.add_argument("--batch_size", type=int, default=1, help="number of images in batch") 67 | parser.add_argument("--ngf", type=int, default=64, help="number of generator filters in first conv layer") 68 | parser.add_argument("--ndf", type=int, default=64, help="number of discriminator filters in first conv layer") 69 | parser.add_argument("--lr_d", type=float, default=0.001, help="initial learning rate for sgd") 70 | parser.add_argument("--lr_g", type=float, default=0.0005, help="initial learning rate for adam") 71 | parser.add_argument("--beta1", type=float, default=0.5, help="momentum term of adam") 72 | parser.add_argument("--l1_weight", type=float, default=100.0, help="weight on L1 term for generator gradient") 73 | parser.add_argument("--gan_weight", type=float, default=1.0, help="weight on GAN term for generator gradient") 74 | 75 | a = parser.parse_args() 76 | EPS = 1e-12 77 | gggg = 0 78 | scale = 1 79 | 80 | Examples = collections.namedtuple("Examples", "inputs, targets, count," 81 | "inputs_test, targets_test, count_test," 82 | "steps_per_epoch") 83 | Model = collections.namedtuple("Model", "outputs, discrim_loss," 84 | "gen_loss_GAN, gen_loss_L1, train") 85 | 86 | 87 | def discrim_conv(batch_input, out_channels, stride): 88 | return tf.layers.conv2d(batch_input, out_channels, kernel_size=(4, 1), strides=stride, padding="same", 89 | kernel_initializer=xavier_initializer_conv2d()) 90 | 91 | 92 | def gen_conv(batch_input, out_channels): 93 | return tf.layers.conv2d(batch_input, out_channels, kernel_size=(4, 1), strides=(2, 1), padding="same", 94 | kernel_initializer=xavier_initializer_conv2d()) 95 | 96 | 97 | def gen_deconv(batch_input, out_channels): 98 | return tf.layers.conv2d_transpose(batch_input, out_channels, kernel_size=(4, 1), strides=(2, 1), padding="same", 99 | kernel_initializer=xavier_initializer_conv2d()) 100 | 101 | 102 | def lrelu(x, a): 103 | with tf.name_scope("lrelu"): 104 | x = tf.identity(x) 105 | return (0.5 * (1 + a)) * x + (0.5 * (1 - a)) * tf.abs(x) 106 | 107 | 108 | def batchnorm(x, scope='batch_instance_norm'): 109 | with tf.variable_scope(scope): 110 | ch = x.shape[-1] 111 | eps = 1e-5 112 | 113 | batch_mean, batch_sigma = tf.nn.moments(x, axes=[0, 1, 2], keep_dims=True) 114 | x_batch = (x - batch_mean) / (tf.sqrt(batch_sigma + eps)) 115 | 116 | ins_mean, ins_sigma = tf.nn.moments(x, axes=[1, 2], keep_dims=True) 117 | x_ins = (x - ins_mean) / (tf.sqrt(ins_sigma + eps)) 118 | 119 | rho = tf.get_variable("rho", [ch], initializer=tf.constant_initializer(1.0), 120 | constraint=lambda x: tf.clip_by_value(x, clip_value_min=0.0, clip_value_max=1.0)) 121 | gamma = tf.get_variable("gamma", [ch], initializer=tf.constant_initializer(1.0)) 122 | beta = tf.get_variable("beta", [ch], initializer=tf.constant_initializer(0.0)) 123 | 124 | x_hat = rho * x_batch + (1 - rho) * x_ins 125 | x_hat = x_hat * gamma + beta 126 | 127 | return x_hat 128 | 129 | 130 | def create_generator(generator_inputs, generator_outputs_channels): 131 | layers = [] 132 | with tf.variable_scope("encoder_1"): 133 | output = gen_conv(generator_inputs, a.ngf) 134 | layers.append(output) 135 | 136 | layer_specs = [ 137 | a.ngf * 2, 138 | a.ngf * 4, 139 | a.ngf * 8, 140 | a.ngf * 8, 141 | a.ngf * 8, 142 | a.ngf * 8, 143 | a.ngf * 8, 144 | ] 145 | 146 | for out_channels in layer_specs: 147 | with tf.variable_scope("encoder_%d" % (len(layers) + 1)): 148 | rectified = lrelu(layers[-1], 0.2) 149 | convolved = gen_conv(rectified, out_channels) 150 | output = batchnorm(convolved) 151 | layers.append(output) 152 | 153 | layer_specs = [ 154 | (a.ngf * 8, 0.5), 155 | (a.ngf * 8, 0.5), 156 | (a.ngf * 8, 0.5), 157 | (a.ngf * 8, 0.0), 158 | (a.ngf * 4, 0.0), 159 | (a.ngf * 2, 0.0), 160 | (a.ngf, 0.0), 161 | ] 162 | 163 | num_encoder_layers = len(layers) 164 | for decoder_layer, (out_channels, dropout) in enumerate(layer_specs): 165 | skip_layer = num_encoder_layers - decoder_layer - 1 166 | with tf.variable_scope("decoder_%d" % (skip_layer + 1)): 167 | if decoder_layer == 0: 168 | input = layers[-1] 169 | else: 170 | input = tf.concat([layers[-1], layers[skip_layer]], axis=3) 171 | rectified = tf.nn.relu(input) 172 | output = gen_deconv(rectified, out_channels) 173 | output = batchnorm(output) 174 | if dropout > 0.0: 175 | output = tf.nn.dropout(output, keep_prob=1 - dropout) 176 | layers.append(output) 177 | 178 | with tf.variable_scope("decoder_1"): 179 | input = tf.concat([layers[-1], layers[0]], axis=3) 180 | rectified = tf.nn.relu(input) 181 | output = gen_deconv(rectified, generator_outputs_channels) 182 | layers.append(output) 183 | 184 | return layers[-1] 185 | 186 | 187 | def create_model(inputs, targets, step): 188 | learning_rate_d = tf.train.exponential_decay(a.lr_d, step, 50000, 0.95, staircase=True) 189 | learning_rate_g = tf.train.exponential_decay(a.lr_g, step, 50000, 0.95, staircase=True) 190 | 191 | # def create_discriminator(discrim_inputs, discrim_targets): 192 | # n_layers = 3 193 | # layers = [] 194 | # 195 | # input = tf.concat([discrim_inputs, discrim_targets], axis=1) 196 | # with tf.variable_scope("layer_1"): 197 | # convolved = discrim_conv(input, a.ndf, stride=(2, 1)) 198 | # rectified = lrelu(convolved, 0.2) 199 | # layers.append(rectified) 200 | # for i in range(n_layers): 201 | # with tf.variable_scope("layer_%d" % (len(layers) + 1)): 202 | # out_channels = a.ndf * min(2**(i+1), 8) 203 | # stride = 1 if i == n_layers - 1 else (2, 1) 204 | # convolved = discrim_conv(layers[-1], out_channels, stride=stride) 205 | # normalized = batchnorm(convolved) 206 | # rectified = lrelu(normalized, 0.2) 207 | # with tf.variable_scope("layer_%d" % (len(layers) + 1)): 208 | # convolved = discrim_conv(rectified, out_channels=1, stride=1) 209 | # output = tf.sigmoid(convolved) 210 | # layers.append(output) 211 | # 212 | # return layers[-1] 213 | def create_discriminator(discrim_inputs, discrim_targets): 214 | n_layers = 3 215 | layers = [] 216 | 217 | # 2x [batch, height, width, in_channels] => [batch, height, width, in_channels * 2] 218 | input = tf.concat([discrim_inputs, discrim_targets], axis=1) 219 | 220 | # layer_1: [batch, 256, 256, in_channels * 2] => [batch, 128, 128, ndf] 221 | with tf.variable_scope("layer_1"): 222 | convolved = discrim_conv(input, a.ndf, stride=(2, 1)) 223 | rectified = lrelu(convolved, 0.2) 224 | layers.append(rectified) 225 | 226 | # layer_2: [batch, 128, 128, ndf] => [batch, 64, 64, ndf * 2] 227 | # layer_3: [batch, 64, 64, ndf * 2] => [batch, 32, 32, ndf * 4] 228 | # layer_4: [batch, 32, 32, ndf * 4] => [batch, 31, 31, ndf * 8] 229 | for i in range(n_layers): 230 | with tf.variable_scope("layer_%d" % (len(layers) + 1)): 231 | out_channels = a.ndf * min(2**(i+1), 8) 232 | stride = 1 if i == n_layers - 1 else (2, 1) # last layer here has stride 1 233 | convolved = discrim_conv(layers[-1], out_channels, stride=stride) 234 | normalized = batchnorm(convolved) 235 | rectified = lrelu(normalized, 0.2) 236 | layers.append(rectified) 237 | 238 | # layer_5: [batch, 31, 31, ndf * 8] => [batch, 30, 30, 1] 239 | with tf.variable_scope("layer_%d" % (len(layers) + 1)): 240 | convolved = discrim_conv(rectified, out_channels=1, stride=1) 241 | output = tf.sigmoid(convolved) 242 | layers.append(output) 243 | 244 | return layers[-1] 245 | 246 | with tf.variable_scope("generator"): 247 | out_channels = int(targets.get_shape()[-1]) 248 | outputs = create_generator(inputs, out_channels)[:, 256:768, :, :] 249 | 250 | with tf.name_scope("real_discriminator"): 251 | with tf.variable_scope("discriminator"): 252 | predict_real = create_discriminator(inputs, targets) 253 | 254 | with tf.name_scope("fake_discriminator"): 255 | with tf.variable_scope("discriminator", reuse=True): 256 | predict_fake = create_discriminator(inputs, outputs) 257 | 258 | with tf.name_scope("discriminator_loss"): 259 | # minimizing -tf.log will try to get inputs to 1 260 | # predict_real => 1 261 | # predict_fake => 0 262 | discrim_loss = tf.reduce_mean(-(tf.log(predict_real + EPS) + tf.log(1 - predict_fake + EPS))) 263 | 264 | with tf.name_scope("generator_loss"): 265 | # predict_fake => 1 266 | # abs(targets - outputs) => 0 267 | gen_loss_GAN = tf.reduce_mean(-tf.log(predict_fake + EPS)) 268 | gen_loss_L1 = tf.reduce_mean(tf.abs(targets - outputs)) 269 | gen_loss = gen_loss_GAN * a.gan_weight + gen_loss_L1 * a.l1_weight 270 | 271 | with tf.name_scope("discriminator_train"): 272 | discrim_tvars = [var for var in tf.trainable_variables() if var.name.startswith("discriminator")] 273 | discrim_optim = tf.train.GradientDescentOptimizer(learning_rate=learning_rate_d) 274 | discrim_grads_and_vars = discrim_optim.compute_gradients(discrim_loss, var_list=discrim_tvars) 275 | discrim_train = discrim_optim.apply_gradients(discrim_grads_and_vars) 276 | 277 | with tf.name_scope("generator_train"): 278 | with tf.control_dependencies([discrim_train]): 279 | gen_tvars = [var for var in tf.trainable_variables() if var.name.startswith("generator")] 280 | gen_optim = tf.train.AdamOptimizer(learning_rate=learning_rate_g, beta1=a.beta1) 281 | gen_grads_and_vars = gen_optim.compute_gradients(gen_loss, var_list=gen_tvars) 282 | gen_train = gen_optim.apply_gradients(gen_grads_and_vars) 283 | 284 | ema = tf.train.ExponentialMovingAverage(decay=0.99) 285 | update_losses = ema.apply([discrim_loss, gen_loss_GAN, gen_loss_L1]) 286 | 287 | global_step = tf.train.get_or_create_global_step() 288 | incr_global_step = tf.assign(global_step, global_step+1) 289 | 290 | return Model( 291 | discrim_loss=ema.average(discrim_loss), 292 | gen_loss_GAN=ema.average(gen_loss_GAN), 293 | gen_loss_L1=ema.average(gen_loss_L1), 294 | outputs=outputs, 295 | train=tf.group(update_losses, incr_global_step, gen_train), 296 | ) 297 | 298 | 299 | def display_image(inputs, targets, outputs): 300 | global gggg 301 | fig = plt.figure(figsize=(16, 16), dpi=100) 302 | gs = gridspec.GridSpec(3, 1) 303 | gs.update(wspace=0.05, hspace=0.05) 304 | agg = np.reshape(inputs, [1024]) 305 | app = np.reshape(targets, [512]) 306 | out = np.reshape(outputs, [512]) 307 | agg_new = [] 308 | app_new = [] 309 | out_new = [] 310 | x = np.linspace(256, 768, 512) 311 | 312 | for i in agg: 313 | agg_new.append(int(i*scale)) 314 | for i in range(512): 315 | app_new.append(int(app[i]*scale)) 316 | out_new.append(int(out[i]*scale)) 317 | 318 | plt.subplot(3, 1, 1) 319 | plt.xlim(0, 1024) 320 | plt.plot(agg_new) 321 | 322 | plt.subplot(3, 1, 2) 323 | plt.xlim(0, 1024) 324 | plt.plot(x, app_new) 325 | 326 | plt.subplot(3, 1, 3) 327 | plt.xlim(0, 1024) 328 | plt.plot(x, out_new) 329 | 330 | plt.savefig(('%s/{}.png' % folder_list['image']) 331 | .format(str(gggg).zfill(3)), bbox_inches='tight') 332 | gggg += 1 333 | plt.close(fig) 334 | 335 | 336 | def metric(target_data, output_data, step, epoch): 337 | tar_new = [] 338 | out_new = [] 339 | mae = [] 340 | 341 | for i in range(len(output_data)): 342 | for j in range(len(output_data[i])): 343 | if output_data[i][j] < 0: 344 | o = 0 345 | else: 346 | o = output_data[i][j] 347 | out_new.append(o) 348 | tar_new.append(target_data[i][j]) 349 | 350 | for i in range(len(out_new)): 351 | r = abs(tar_new[i] - out_new[i]) 352 | mae.append(r) 353 | 354 | accuracy = (1.0-(np.sum(mae, dtype=np.int64)/(2*np.sum(tar_new, dtype=np.int64))))*100 355 | print('np.sum(mae):') 356 | print(np.sum(mae)) 357 | print('np.sum(tar_new):') 358 | print(np.sum(tar_new)) 359 | sae = abs(np.sum(tar_new, dtype=np.int64)-np.sum(out_new, dtype=np.int64))/np.sum(tar_new, dtype=np.int64) 360 | mse = mean_squared_error(tar_new, out_new) 361 | mae = mean_absolute_error(tar_new, out_new) 362 | r2 = r2_score(tar_new, out_new) 363 | 364 | with open('%s\\test.txt' % folder_list['test_validation'], 'a+') as f: 365 | print('STEP: %d' % (step+1), file=f) 366 | print('Epoch: %d' % epoch, file=f) 367 | print('Accuracy: %f' % accuracy, file=f) 368 | print('SAE: %f' % sae, file=f) 369 | print('MSE: %f' % mse, file=f) 370 | print('MAE: %f' % mae, file=f) 371 | print('R^2: %f' % r2, file=f) 372 | print('', file=f) 373 | 374 | 375 | def load_examples(): 376 | base_path = folder_list['input'] 377 | 378 | ap_data = np.load(os.path.join(base_path, 'appliance_train_1.npy')) 379 | ag_data = np.load(os.path.join(base_path, 'main_train_1.npy')) 380 | ap_data_test = np.load(os.path.join(base_path, 'appliance_test_1.npy')) 381 | ag_data_test = np.load(os.path.join(base_path, 'main_test_1.npy')) 382 | print('Data load complete!') 383 | 384 | appliance_data = tf.cast(ap_data, tf.float32) 385 | aggregate_data = tf.cast(ag_data, tf.float32) 386 | appliance_data_test = tf.cast(ap_data_test, tf.float32) 387 | aggregate_data_test = tf.cast(ag_data_test, tf.float32) 388 | print('cast complete!') 389 | 390 | queue = tf.train.slice_input_producer([aggregate_data, appliance_data], shuffle=True) 391 | inputs_batch, targets_batch = tf.train.batch(queue, batch_size=a.batch_size) 392 | queue_test = tf.train.slice_input_producer([aggregate_data_test, appliance_data_test], shuffle=False) 393 | inputs_batch_test, targets_batch_test = tf.train.batch(queue_test, batch_size=a.batch_size) 394 | 395 | steps_per_epoch = int(math.ceil(len(ap_data) / a.batch_size)) 396 | 397 | return Examples( 398 | inputs=inputs_batch, 399 | targets=targets_batch, 400 | count=len(ap_data), 401 | inputs_test=inputs_batch_test, 402 | targets_test=targets_batch_test, 403 | count_test=len(ap_data_test), 404 | steps_per_epoch=steps_per_epoch, 405 | ) 406 | 407 | 408 | def main(): 409 | if a.seed is None: 410 | a.seed = random.randint(0, 2**31 - 1) 411 | tf.set_random_seed(a.seed) 412 | np.random.seed(a.seed) 413 | random.seed(a.seed) 414 | 415 | for folder_name, folder_path in folder_list.items(): 416 | if not os.path.exists(folder_path): 417 | os.makedirs(folder_path) 418 | print('Folder creation complete!') 419 | 420 | with open(os.path.join(folder_list['base_root'], 'config.txt'), 'w') as f: 421 | for k, v in a._get_kwargs(): 422 | print(k, "=", v, file=f) 423 | print('Config save complete!') 424 | 425 | examples = load_examples() 426 | global_step = tf.placeholder(tf.float32, None) 427 | inputs = tf.placeholder(tf.float32, [a.batch_size, 1024, 1, 1]) 428 | targets = tf.placeholder(tf.float32, [a.batch_size, 512, 1, 1]) 429 | i_ = tf.reshape(examples.inputs, [a.batch_size, 1024, 1, 1]) 430 | t_ = tf.reshape(examples.targets, [a.batch_size, 512, 1, 1]) 431 | i_t = tf.reshape(examples.inputs_test, [a.batch_size, 1024, 1, 1]) 432 | t_t = tf.reshape(examples.targets_test, [a.batch_size, 512, 1, 1]) 433 | model = create_model(inputs, targets, global_step) 434 | print('model complete!') 435 | 436 | with tf.name_scope("display_images"): 437 | display_fetches = { 438 | 'output': model.outputs, 439 | } 440 | 441 | tf.summary.scalar("discriminator_loss", model.discrim_loss) 442 | tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN) 443 | tf.summary.scalar("generator_loss_L1", model.gen_loss_L1) 444 | 445 | with tf.name_scope("parameter_count"): 446 | parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()]) 447 | 448 | saver = tf.train.Saver(max_to_keep=50) 449 | 450 | logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None 451 | sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None) 452 | with sv.managed_session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 453 | print("parameter_count =", sess.run(parameter_count)) 454 | if a.checkpoint is not None: 455 | print("loading model from checkpoint") 456 | checkpoint = tf.train.latest_checkpoint(a.checkpoint) 457 | saver.restore(sess, checkpoint) 458 | 459 | max_steps = 2**32 460 | if a.max_epochs is not None: 461 | max_steps = examples.steps_per_epoch * a.max_epochs 462 | if a.max_steps is not None: 463 | max_steps = a.max_steps 464 | 465 | start = time.time() 466 | output_test = [] 467 | target_test = [] 468 | 469 | for step in range(max_steps): 470 | 471 | def should(freq): 472 | return freq > 0 and ((step + 1) % freq == 0 or step == max_steps - 1) 473 | 474 | options = None 475 | run_metadata = None 476 | if should(a.trace_freq): 477 | options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) 478 | run_metadata = tf.RunMetadata() 479 | 480 | fetches = { 481 | "train": model.train, 482 | "global_step": sv.global_step, 483 | } 484 | if should(a.progress_freq): 485 | fetches["discrim_loss"] = model.discrim_loss 486 | fetches["gen_loss_GAN"] = model.gen_loss_GAN 487 | fetches["gen_loss_L1"] = model.gen_loss_L1 488 | 489 | if should(a.summary_freq): 490 | fetches["summary"] = sv.summary_op 491 | 492 | if should(a.display_freq): 493 | fetches["display"] = display_fetches 494 | 495 | inputs_, targets_ = sess.run([i_, t_]) 496 | results = sess.run(fetches, 497 | options=options, 498 | run_metadata=run_metadata, 499 | feed_dict={global_step: step, 500 | inputs: inputs_, 501 | targets: targets_}) 502 | 503 | if should(a.display_freq): 504 | data = results['display'] 505 | display_image(inputs_, targets_, data['output']) 506 | 507 | if should(a.summary_freq): 508 | sv.summary_writer.add_summary(results["summary"], results["global_step"]) 509 | 510 | if should(a.progress_freq): 511 | train_epoch = math.ceil(results["global_step"] / examples.steps_per_epoch) 512 | train_step = (results["global_step"] - 1) % examples.steps_per_epoch + 1 513 | rate = (step + 1) * a.batch_size / (time.time() - start) 514 | print("progress epoch %d step %d image/sec %0.1f" % 515 | (train_epoch, train_step+1, rate)) 516 | 517 | print("discrim_loss", results["discrim_loss"]) 518 | print("gen_loss_GAN", results["gen_loss_GAN"]) 519 | print("gen_loss_L1", results["gen_loss_L1"]) 520 | 521 | if should(a.save_freq): 522 | print("saving model") 523 | saver.save(sess, os.path.join(folder_list['model'], "model"), global_step=sv.global_step) 524 | 525 | if should(a.test_procedure): 526 | print('Test procedure!') 527 | target_test.clear() 528 | output_test.clear() 529 | max_steps_test = examples.count_test 530 | for step_test in range(max_steps_test): 531 | inputs_, targets_= sess.run([i_t, t_t]) 532 | data = sess.run({ 533 | 'output': model.outputs, 534 | }, feed_dict={ 535 | global_step: results['global_step'], 536 | inputs: inputs_, 537 | targets: targets_ 538 | }) 539 | _target = np.reshape(targets_, [a.batch_size, 512]) 540 | _output = np.reshape(data['output'], [a.batch_size, 512]) 541 | for i in range(a.batch_size): 542 | tar_temp = [] 543 | out_temp = [] 544 | for j in range(512): 545 | tar_temp.append(int(_target[i][j] * scale)) 546 | out_temp.append(int(_output[i][j] * scale)) 547 | target_test.append(tar_temp) 548 | output_test.append(out_temp) 549 | if step_test % 1000 == 0: 550 | print('Process Test: %f Rate: %f' % (step_test / max_steps_test, 551 | (time.time() - start) / max_steps_test)) 552 | np.save('%s\\test_target_%d.npy' % (folder_list['test_validation'], results['global_step']+1), target_test) 553 | np.save('%s\\test_output_%d.npy' % (folder_list['test_validation'], results['global_step']+1), output_test) 554 | train_epoch = math.ceil(results["global_step"] / examples.steps_per_epoch) 555 | metric(target_test, output_test, results['global_step'],train_epoch) 556 | print('Test procedure complete!') 557 | 558 | if sv.should_stop(): 559 | break 560 | 561 | 562 | main() 563 | -------------------------------------------------------------------------------- /preprocess/refit_process.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('preprocess') 3 | import matplotlib 4 | matplotlib.use("TkAgg") 5 | import matplotlib.pyplot as plt 6 | from matplotlib.pyplot import savefig 7 | import numpy as np 8 | import refit_cfg 9 | import os 10 | import random 11 | from sklearn.model_selection import train_test_split 12 | 13 | 14 | name = ['WashingMachine', 'Kettle', 'Microwave', 'Fridge', 'Dishwasher'] 15 | appliance_dict = { 16 | 'WashingMachine': refit_cfg.washingmachine, 17 | 'Kettle': refit_cfg.kettle, 18 | 'Microwave': refit_cfg.microwave, 19 | 'Fridge': refit_cfg.fridge, 20 | 'Dishwasher': refit_cfg.dishwasher 21 | } 22 | 23 | 24 | def align_process(house_id): 25 | data = np.load('data\\REFIT\\original_data\\%d.npy' % house_id) 26 | new_data = [] 27 | current_index = 0 28 | current_time = int(data[0][0]) 29 | end_time = int(data[-1][0]) + 8 30 | interval_threshold = refit_cfg.separation_threshold 31 | isend = 0 32 | data_length = len(data) 33 | 34 | while current_time <= end_time: 35 | current_interval = int(data[current_index+1][0]) - int(data[current_index][0]) 36 | if current_interval < interval_threshold: # small interval 37 | if current_time > int(data[current_index][0]): 38 | temp_index = current_index + 1 39 | while current_time > int(data[temp_index][0]): 40 | temp_index += 1 41 | if temp_index > (data_length-1): 42 | temp_index -= 1 43 | break 44 | 45 | if abs(current_time - int(data[temp_index-1][0])) > abs(int(data[temp_index][0])-current_time): 46 | current_index = temp_index 47 | if temp_index == (data_length-1): 48 | print('The end!') 49 | isend = 1 50 | else: 51 | current_index = temp_index - 1 52 | t = [] 53 | for element in data[current_index]: 54 | t.append(element) 55 | t[0] = current_time 56 | new_data.append(t) 57 | if isend == 1: 58 | break 59 | current_time += 8 60 | if current_index % 1000 == 0: 61 | print('House %d processing: %f' % (house_id, current_index/data_length)) 62 | else: # big interval 63 | current_index += 1 64 | current_time = int(data[current_index][0]) 65 | 66 | np.save('data\\REFIT\\after_align\\%d.npy' % house_id, new_data) 67 | 68 | 69 | def visual(house_id, channel_id, start, length): 70 | data = np.load('data\\REFIT\\after_align\\%d.npy' % house_id) 71 | print(len(data)) 72 | target = [] 73 | c = channel_id+1 74 | for r in data: 75 | target.append(int(r[c])) 76 | y = target[start:start+length] 77 | plt.plot(y) 78 | plt.show() 79 | 80 | 81 | def diff(house_id): 82 | data = np.load('data\\REFIT\\after_align\\%d.npy' % house_id) 83 | d = [] 84 | for i in range(len(data)-1): 85 | d.append(int(data[i+1][0])-int(data[i][0])) 86 | plt.plot(d) 87 | plt.show() 88 | plt.close() 89 | 90 | 91 | def appliance_separation(dict, appliance_name): 92 | path = 'data\\REFIT\\appliance_data\\%s' % appliance_name 93 | if not os.path.exists(path): 94 | os.mkdir(path) 95 | 96 | for house_id, channel_id in dict.items(): 97 | data = np.load('data\\REFIT\\after_align\\%d.npy' % house_id) 98 | appliance_data = [] 99 | for row in data: 100 | appliance_data.append([row[1], row[channel_id+1]]) 101 | np.save(os.path.join(path, '%d_%d.npy' % (house_id, channel_id)), appliance_data) 102 | print('Appliance %s House %d complete!' % (appliance_name, house_id)) 103 | 104 | 105 | def show_appliance(house_id, appliance_name): 106 | channel_id = appliance_dict[appliance_name][house_id] 107 | data = np.load('data\\REFIT\\after_align\\%s\\%d_%d.npy' % (appliance_name, house_id, channel_id)) 108 | print(len(data)) 109 | mains = [] 110 | app = [] 111 | for i in data: 112 | mains.append(int(i[0])) 113 | app.append(int(i[1])) 114 | plt.figure(figsize=(20, 8)) 115 | plt.plot(mains) 116 | plt.plot(app) 117 | plt.show() 118 | 119 | 120 | def cull(cull_dict): 121 | for appliance_name, _dict in cull_dict.items(): 122 | path = 'data\\REFIT\\after_culling\\%s' % appliance_name 123 | if not os.path.exists(path): 124 | os.mkdir(path) 125 | for house_id, cull_list in _dict.items(): 126 | channel_id = appliance_dict[appliance_name][house_id] 127 | data = np.load('data\\REFIT\\after_align\\%s\\%d_%d.npy' % (appliance_name, house_id, channel_id)) 128 | new_data = [] 129 | _cull_list = [[0, cull_list[0][0]]] 130 | for i in range(len(cull_list)-1): 131 | _cull_list.append([cull_list[i][1], cull_list[i+1][0]]) 132 | _cull_list.append([cull_list[-1][1], (len(data)-1)]) 133 | 134 | for i in _cull_list: 135 | if i[1] - i[0] != 0: 136 | for j in range(i[0], i[1]): 137 | new_data.append(data[j]) 138 | np.save('data\\REFIT\\after_culling\\%s\\%d_%d.npy' % (appliance_name, house_id, channel_id), new_data) 139 | print('House %d %s complete!' % (house_id, appliance_name)) 140 | 141 | 142 | def appliance_separation(dict, appliance_name): 143 | """ 144 | 将各个电器的数据进行分解，放置到appliance_data文件夹下对应电器的文件夹中，以house_id和channel_id进行命名 145 | :param dict: 电器数据来源 146 | :param appliance_name: 当前电器的名称，用以创建文件夹 147 | :return: 148 | """ 149 | path = 'data\\REFIT\\appliance_data\\%s' % appliance_name 150 | if not os.path.exists(path): 151 | os.mkdir(path) 152 | 153 | for house_id, channel_id in dict.items(): 154 | data = np.load('data\\REFIT\\after_align\\%d.npy' % house_id) 155 | appliance_data = [] 156 | for row in data: 157 | appliance_data.append([row[1], row[channel_id+1]]) # 将mains 和 appliance 作为一条单独的记录 158 | np.save(os.path.join(path, '%d_%d.npy' % (house_id, channel_id)), appliance_data) 159 | print('Appliance %s House %d complete!' % (appliance_name, house_id)) 160 | 161 | 162 | def show_appliance(house_id, appliance_name): 163 | """ 164 | 具体观察每个电器的图形表示，将大段的数据缺失或者数据错误进行标注，构造cull_dict字典，在cull进行片段删除 165 | :param house_id: 166 | :param appliance_name: 167 | :return: 168 | """ 169 | channel_id = appliance_dict[appliance_name][house_id] 170 | data = np.load('data\\REFIT\\after_culling\\%s\\%d_%d.npy' % (appliance_name, house_id, channel_id)) 171 | print(len(data)) 172 | mains = [] 173 | app = [] 174 | for i in data: 175 | mains.append(int(i[0])) 176 | app.append(int(i[1])) 177 | plt.figure(figsize=(20, 8)) 178 | plt.plot(mains) 179 | plt.plot(app) 180 | plt.show() 181 | 182 | 183 | def cull(cull_dict): 184 | """ 185 | 根据画的图，将大段的空缺段进行删除，删除之后，需要进行比对 186 | :param cull_dict: 187 | :return: 188 | """ 189 | for appliance_name, _dict in cull_dict.items(): 190 | path = 'data\\REFIT\\after_culling_2\\%s' % appliance_name 191 | if not os.path.exists(path): 192 | os.mkdir(path) 193 | for house_id, cull_list in _dict.items(): 194 | channel_id = appliance_dict[appliance_name][house_id] 195 | data = np.load('data\\REFIT\\after_culling_2\\%s\\%d_%d.npy' % (appliance_name, house_id, channel_id)) 196 | new_data = [] 197 | # 对cull_list进行变形，变成表征合理数据的区间 198 | _cull_list = [[0, cull_list[0][0]]] 199 | for i in range(len(cull_list)-1): 200 | _cull_list.append([cull_list[i][1], cull_list[i+1][0]]) 201 | _cull_list.append([cull_list[-1][1], (len(data)-1)]) 202 | 203 | for i in _cull_list: 204 | if i[1] - i[0] != 0: 205 | for j in range(i[0], i[1]): 206 | new_data.append(data[j]) 207 | np.save('data\\REFIT\\after_culling_2\\%s\\%d_%d.npy' % (appliance_name, house_id, channel_id), new_data) 208 | print('House %d %s complete!' % (house_id, appliance_name)) 209 | 210 | 211 | def separate(appliance_name): 212 | window_width = refit_cfg.window_width[appliance_name] 213 | data_path = 'data\\REFIT\\after_culling\\%s' % appliance_name 214 | count = 0 215 | appliance_train_validation = [] 216 | appliance_test = [] 217 | main_train_validation = [] 218 | main_test = [] 219 | 220 | for house_id, channel_id in refit_cfg.train_validation[appliance_name].items(): 221 | # train & validation 222 | appliance_train_validation.clear() 223 | main_train_validation.clear() 224 | data = np.load(os.path.join(data_path, '%s_%s.npy' % (house_id, channel_id))) 225 | current_head = 0 226 | data_length = len(data) 227 | end = data_length - window_width - 1 228 | while current_head < end: 229 | temp_main = [] 230 | temp_appliance = [] 231 | for i in range(current_head, current_head+window_width): 232 | temp_main.append(data[i][0]) 233 | temp_appliance.append(data[i][1]) 234 | r = random.random() 235 | current_head += int(window_width*r) 236 | appliance_train_validation.append(temp_appliance) 237 | main_train_validation.append(temp_main) 238 | count += 1 239 | if count % 1000 == 0: 240 | print('T & V 1： House %d %f' % (house_id, (current_head / data_length))) 241 | 242 | data_length -= window_width 243 | random_clip = refit_cfg.random_clip[appliance_name] 244 | for i in range(random_clip): 245 | r = random.random() 246 | start = int(r*data_length) 247 | temp_main = [] 248 | temp_appliance = [] 249 | for j in range(start, start + window_width): 250 | temp_main.append(data[j][0]) 251 | temp_appliance.append(data[j][1]) 252 | appliance_train_validation.append(temp_appliance) 253 | main_train_validation.append(temp_main) 254 | count += 1 255 | if count % 1000 == 0: 256 | print('T & V 2： House %d %f' % (house_id, (i / random_clip))) 257 | print('Train & Validation: House %d %s complete!' % (house_id, appliance_name)) 258 | np.save(os.path.join(data_path, '1024\\appliance_train_validation_%d.npy' % house_id), appliance_train_validation) 259 | np.save(os.path.join(data_path, '1024\\main_train_validation_%d.npy' % house_id), main_train_validation) 260 | 261 | 262 | # test 263 | count = 0 264 | for house_id, channel_id in refit_cfg.test[appliance_name].items(): 265 | appliance_test.clear() 266 | main_test.clear() 267 | data = np.load(os.path.join(data_path, '%s_%s.npy' % (house_id, channel_id))) 268 | current_head = 0 269 | data_length = len(data) 270 | end = data_length - window_width - 1 271 | while current_head < end: 272 | temp_main = [] 273 | temp_appliance = [] 274 | for i in range(current_head, current_head+window_width): 275 | temp_main.append(data[i][0]) 276 | temp_appliance.append(data[i][1]) 277 | r = random.random() 278 | current_head += int(r*window_width) 279 | appliance_test.append(temp_appliance) 280 | main_test.append(temp_main) 281 | count += 1 282 | if count % 1000 == 0: 283 | print('Test 1： House %d %f' % (house_id, (current_head / data_length))) 284 | 285 | data_length -= window_width 286 | for i in range(refit_cfg.random_clip[appliance_name]): 287 | r = random.random() 288 | start = int(r*data_length) 289 | temp_main = [] 290 | temp_appliance = [] 291 | for j in range(start, start + window_width): 292 | temp_main.append(data[j][0]) 293 | temp_appliance.append(data[j][1]) 294 | appliance_test.append(temp_appliance) 295 | main_test.append(temp_main) 296 | count += 1 297 | if count % 1000 == 0: 298 | print('Test 2： House %d %f' % (house_id, (i / data_length))) 299 | print('Test 2: House %d %s complete!' % (house_id, appliance_name)) 300 | np.save(os.path.join(data_path, '1024\\appliance_test_%d.npy' % house_id), appliance_test) 301 | np.save(os.path.join(data_path, '1024\\main_test_%d.npy' % house_id), main_test) 302 | 303 | 304 | def clip_visual(appliance_name): 305 | base_path = 'data\\REFIT\\after_culling\\%s' % appliance_name 306 | appliance_data = np.load(os.path.join(base_path, 'appliance_train_.npy')) 307 | main_data = np.load(os.path.join(base_path, 'main_train_.npy')) 308 | print('Data load complete!') 309 | loop = 1000 310 | x = np.linspace(256, 768, 512) 311 | length = len(appliance_data) 312 | for i in range(loop): 313 | r = int(random.random()*length) 314 | plt.figure(figsize=(25, 10), dpi=100) 315 | plt.subplot(211) 316 | plt.xlim(0, 1024) 317 | plt.plot(main_data[r]) 318 | plt.subplot(212) 319 | plt.xlim(0, 1024) 320 | plt.plot(x, appliance_data[r]) 321 | savefig(os.path.join(base_path, 'clip_view\\%d.jpg' % i)) 322 | plt.close() 323 | 324 | 325 | def train_validation_split(appliance_name): 326 | data_path = 'data\\REFIT\\after_culling\\%s\\1024' % appliance_name 327 | appliance = np.load(os.path.join(data_path, 'appliance_train_validation.npy')) 328 | main = np.load(os.path.join(data_path, 'main_train_validation.npy')) 329 | appliance_train, appliance_validation, main_train, main_validation = \ 330 | train_test_split(appliance, main, test_size=0.2) 331 | print(len(appliance_train)) 332 | print(len(main_train)) 333 | 334 | np.save(os.path.join(data_path, 'appliance_train.npy'), appliance_train) 335 | np.save(os.path.join(data_path, 'main_train.npy'), main_train) 336 | np.save(os.path.join(data_path, 'appliance_validation.npy'), appliance_validation) 337 | np.save(os.path.join(data_path, 'main_validation.npy'), main_validation) 338 | 339 | 340 | def data_integration(appliance_name): 341 | data_path = 'data\\REFIT\\after_culling\\%s\\1024' % appliance_name 342 | appliance = [] 343 | main = [] 344 | for house_id, channel_id in refit_cfg.train_validation[appliance_name].items(): 345 | appliance_data = np.load(os.path.join(data_path, 'appliance_train_validation_%d.npy' % house_id)) 346 | main_data = np.load(os.path.join(data_path, 'main_train_validation_%d.npy' % house_id)) 347 | for i in appliance_data: 348 | appliance.append(i) 349 | for i in main_data: 350 | main.append(i) 351 | 352 | print(len(appliance)) 353 | print(len(main)) 354 | np.save(os.path.join(data_path, 'appliance_train_validation.npy'), appliance) 355 | np.save(os.path.join(data_path, 'main_train_validation.npy'), main) 356 | 357 | appliance_test = [] 358 | main_test = [] 359 | for house_id, channel_id in refit_cfg.test[appliance_name].items(): 360 | appliance_data = np.load(os.path.join(data_path, 'appliance_test_%d.npy' % house_id)) 361 | main_data = np.load(os.path.join(data_path, 'main_test_%d.npy' % house_id)) 362 | for i in appliance_data: 363 | appliance_test.append(i) 364 | for i in main_data: 365 | main_test.append(i) 366 | 367 | print(len(appliance_test)) 368 | print(len(main_test)) 369 | np.save(os.path.join(data_path, 'appliance_test.npy'), appliance_test) 370 | np.save(os.path.join(data_path, 'main_test.npy'), main_test) 371 | 372 | 373 | def positive_negative(appliance_name): 374 | base_path = 'data\\REFIT\\after_culling\\%s' % appliance_name 375 | appliance_data = np.load(os.path.join(base_path, 'appliance_train.npy')) 376 | count = 0 377 | threshold = [0, 50, 100, 200, 500, 1000, 2000, 5000, 10000] 378 | d = {} 379 | for i in range(len(threshold)): 380 | d[threshold[i]] = 0 381 | print(d) 382 | 383 | for th in threshold: 384 | for i in appliance_data: 385 | sum = 0 386 | for j in i: 387 | sum += int(j) 388 | if sum > th: 389 | d[th] += 1 390 | print('Thres %d complete!' % th) 391 | 392 | for thres, count in d.items(): 393 | print('Thres: %d %d/%d %f' % (thres, count, len(appliance_data), count/len(appliance_data))) 394 | 395 | 396 | def clip_view(appliance_name, thres): 397 | base_path = 'data\\REFIT\\after_culling\\%s' % appliance_name 398 | appliance_data = np.load(os.path.join(base_path, 'appliance_train.npy')) 399 | count = 0 400 | 401 | for i in appliance_data: 402 | sum = 0 403 | for j in i: 404 | sum += int(j) 405 | if sum > thres: 406 | plt.figure(figsize=(25, 10), dpi=100) 407 | plt.plot(i.astype(int)) 408 | savefig(os.path.join(base_path, 'clip_view\\%d.jpg' % count)) 409 | plt.close() 410 | count += 1 411 | 412 | 413 | def test_process(appliance_name): 414 | base_path = 'data\\REFIT\\after_culling\\%s\\1024' % appliance_name 415 | appliance_data = np.load(os.path.join(base_path, 'appliance_test_512.npy')) 416 | temp = [0.0]*512 417 | new_app = [] 418 | for i in range(len(appliance_data)): 419 | max = np.max(appliance_data[i]) 420 | if max < 0.05: 421 | print(max) 422 | new_app.append(temp) 423 | else: 424 | new_app.append(appliance_data[i]) 425 | np.save(os.path.join(base_path, 'appliance_test_512.npy'), new_app) 426 | 427 | 428 | def separate_positive_negative(appliance_name, thres, peak): 429 | base_path = 'data\\REFIT\\after_culling\\%s\\1024' % appliance_name 430 | appliance_data = np.load(os.path.join(base_path, 'appliance_train.npy')) 431 | main_data = np.load(os.path.join(base_path, 'main_train.npy')) 432 | count = 0 433 | appliance_positive = [] 434 | appliance_negative = [] 435 | main_positive = [] 436 | main_negative = [] 437 | appliance_temp = [0] * 1024 438 | 439 | for i in range(len(appliance_data)): 440 | sum = 0 441 | max = 0 442 | for j in appliance_data[i]: 443 | sum += int(j) 444 | for j in range(512): 445 | if int(appliance_data[i][j+256]) > max: 446 | max = int(appliance_data[i][j+256]) 447 | if max < peak: 448 | sum = 0 449 | if sum > thres: 450 | appliance_positive.append(appliance_data[i]) 451 | main_positive.append(main_data[i]) 452 | else: 453 | appliance_negative.append(appliance_temp) 454 | main_negative.append(main_data[i]) 455 | if i % 1000 == 0: 456 | print('Processing: %f' % (i/len(appliance_data))) 457 | 458 | np.save(os.path.join(base_path, 'appliance_positive.npy'), appliance_positive) 459 | np.save(os.path.join(base_path, 'main_positive.npy'), main_positive) 460 | np.save(os.path.join(base_path, 'appliance_negative.npy'), appliance_negative) 461 | np.save(os.path.join(base_path, 'main_negative.npy'), main_negative) 462 | 463 | 464 | def generate_balanced_dataset(appliance_name, negative_ratio): 465 | base_path = 'data\\REFIT\\after_culling\\%s\\1024' % appliance_name 466 | appliance_positive = list(np.load(os.path.join(base_path, 'appliance_positive.npy'))) 467 | appliance_negative = np.load(os.path.join(base_path, 'appliance_negative.npy')) 468 | main_positive = list(np.load(os.path.join(base_path, 'main_positive.npy'))) 469 | main_negative = np.load(os.path.join(base_path, 'main_negative.npy')) 470 | print('Data load complete!') 471 | 472 | positive_length = len(appliance_positive) 473 | negative_length = len(appliance_negative) 474 | print('Postive length: %d negative length: %d' % (positive_length, negative_length)) 475 | for i in range(int(positive_length*negative_ratio)): 476 | r = int(random.random()*negative_length) 477 | appliance_positive.append(appliance_negative[r]) 478 | main_positive.append(main_negative[r]) 479 | print('Data generate complete! length: %d' % (len(appliance_positive))) 480 | 481 | index = np.linspace(0, len(appliance_positive)-1, len(appliance_positive)).astype(int) 482 | random.shuffle(index) 483 | appliance_new = [] 484 | main_new = [] 485 | 486 | for i in index: 487 | appliance_new.append(appliance_positive[i]) 488 | main_new.append(main_positive[i]) 489 | print('Data shuffle complete!') 490 | 491 | np.save(os.path.join(base_path, 'appliance_train_balanced.npy'), appliance_new) 492 | np.save(os.path.join(base_path, 'main_train_balanced.npy'), main_new) 493 | print('Data save complete!') 494 | 495 | 496 | def shrink(appliance_name, scale): 497 | base_path = 'data\\REFIT\\after_culling\\%s\\1024' % appliance_name 498 | appliance_data = np.load(os.path.join(base_path, 'appliance_train_balanced.npy')) 499 | main_data = np.load(os.path.join(base_path, 'main_train_balanced.npy')) 500 | appliance_new = [] 501 | main_new = [] 502 | print('Data load complete!') 503 | 504 | for i in range(len(appliance_data)): 505 | appliance_temp = [] 506 | main_temp = [] 507 | for j in range(len(appliance_data[i])): 508 | appliance_temp.append(float(int(appliance_data[i][j])/scale)) 509 | for j in range(len(main_data[i])): 510 | main_temp.append(float(int(main_data[i][j])/scale)) 511 | appliance_new.append(appliance_temp) 512 | main_new.append(main_temp) 513 | print('Process complete!') 514 | 515 | np.save(os.path.join(base_path, 'appliance_train_%d.npy' % scale), appliance_new) 516 | np.save(os.path.join(base_path, 'main_train_%d.npy' % scale), main_new) 517 | 518 | 519 | def shrink_validation(appliance_name, scale): 520 | base_path = 'data\\REFIT\\after_culling\\%s\\1024' % appliance_name 521 | appliance_data = np.load(os.path.join(base_path, 'appliance_validation.npy')) 522 | main_data = np.load(os.path.join(base_path, 'main_validation.npy')) 523 | appliance_new = [] 524 | main_new = [] 525 | print('Data load complete!') 526 | 527 | for i in range(len(appliance_data)): 528 | appliance_temp = [] 529 | main_temp = [] 530 | for j in range(len(appliance_data[i])): 531 | appliance_temp.append(float(int(appliance_data[i][j])/scale)) 532 | for j in range(len(main_data[i])): 533 | main_temp.append(float(int(main_data[i][j])/scale)) 534 | appliance_new.append(appliance_temp) 535 | main_new.append(main_temp) 536 | print('Process complete!') 537 | 538 | np.save(os.path.join(base_path, 'appliance_validation_%d.npy' % scale), appliance_new) 539 | np.save(os.path.join(base_path, 'main_validation_%d.npy' % scale), main_new) 540 | 541 | 542 | def appliance_1024to512(appliance_name): 543 | base_path = 'data\\REFIT\\after_culling\\%s\\1024' % appliance_name 544 | appliance_train = np.load(os.path.join(base_path, 'appliance_train_1000.npy')) 545 | appliance_validation = np.load(os.path.join(base_path, 'appliance_validation_1000.npy')) 546 | appliance_test = np.load(os.path.join(base_path, 'appliance_test_1000.npy')) 547 | at_new = [] 548 | av_new = [] 549 | ae_new = [] 550 | 551 | for i in range(len(appliance_train)): 552 | at_temp = [] 553 | for j in range(256, 768): 554 | at_temp.append(float(appliance_train[i][j])) 555 | at_new.append(at_temp) 556 | for i in range(len(appliance_validation)): 557 | av_temp = [] 558 | for j in range(256, 768): 559 | av_temp.append(float(appliance_validation[i][j])) 560 | av_new.append(av_temp) 561 | for i in range(len(appliance_test)): 562 | ae_temp = [] 563 | for j in range(256, 768): 564 | ae_temp.append(float(appliance_test[i][j])) 565 | ae_new.append(ae_temp) 566 | 567 | np.save(os.path.join(base_path, 'appliance_train_512.npy'), at_new) 568 | np.save(os.path.join(base_path, 'appliance_validation_512.npy'), av_new) 569 | np.save(os.path.join(base_path, 'appliance_test_512.npy'), ae_new) 570 | 571 | 572 | def shrink_test(appliance_name, scale): 573 | base_path = 'data\\REFIT\\after_culling\\%s\\1024' % appliance_name 574 | appliance_data = np.load(os.path.join(base_path, 'appliance_test.npy')) 575 | main_data = np.load(os.path.join(base_path, 'main_test.npy')) 576 | appliance_new = [] 577 | main_new = [] 578 | print('Data load complete!') 579 | 580 | for i in range(len(appliance_data)): 581 | appliance_temp = [] 582 | main_temp = [] 583 | for j in range(len(appliance_data[i])): 584 | appliance_temp.append(float(int(appliance_data[i][j])/scale)) 585 | for j in range(len(main_data[i])): 586 | main_temp.append(float(int(main_data[i][j])/scale)) 587 | appliance_new.append(appliance_temp) 588 | main_new.append(main_temp) 589 | print('Process complete!') 590 | 591 | np.save(os.path.join(base_path, 'appliance_test_1000.npy'), appliance_new) 592 | np.save(os.path.join(base_path, 'main_test_1000.npy'), main_new) 593 | 594 | 595 | if __name__ == '__main__': 596 | appliance_name = 'WashingMachine' 597 | separate(appliance_name) 598 | data_integration(appliance_name) 599 | train_validation_split(appliance_name) 600 | separate_positive_negative(appliance_name, 1500, 20) 601 | generate_balanced_dataset(appliance_name, 1) 602 | shrink(appliance_name, 1000) 603 | shrink_validation(appliance_name, 1000) 604 | shrink_test(appliance_name, 1000) 605 | appliance_1024to512(appliance_name) 606 | # test_process(appliance_name) 607 | print('Process complete!!!') 608 | -------------------------------------------------------------------------------- /model/refit_u.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import tensorflow as tf 6 | import numpy as np 7 | import argparse 8 | import os 9 | import random 10 | import collections 11 | import math 12 | import time 13 | import matplotlib.pyplot as plt 14 | import matplotlib.gridspec as gridspec 15 | from sklearn.metrics import mean_squared_error 16 | from sklearn.metrics import mean_absolute_error 17 | from sklearn.metrics import r2_score 18 | from tensorflow.contrib.layers import xavier_initializer_conv2d 19 | 20 | 21 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 22 | os.environ['CUDA_VISIBLE_DEVICES'] = "0" 23 | 24 | current_version = '1' 25 | dataset_name = 'REFIT' 26 | appliance_name = 'WashingMachine' 27 | appliance_name_folder = '%s_%s' % (dataset_name, appliance_name) 28 | base_root = 'outnilm\\%s\\%s' % (appliance_name_folder, current_version) 29 | folder_list = { 30 | 'base_root': base_root, 31 | 'image': '%s\\image' % base_root, 32 | 'model': '%s\\model' % base_root, 33 | 'test_validation': '%s\\test_validation' % base_root, 34 | 'input': 'data\\REFIT\\after_culling\\%s\\1024\\1' % appliance_name, 35 | } 36 | 37 | 38 | isFirst = 1 39 | parser = argparse.ArgumentParser() 40 | if isFirst == 1: 41 | parser.add_argument("--checkpoint", 42 | default=None, 43 | help="directory with checkpoint to resume training from or use for testing") 44 | else: 45 | parser.add_argument("--checkpoint", 46 | default=folder_list['model'], 47 | help="directory with checkpoint to resume training from or use for testing") 48 | parser.add_argument("--output_dir", 49 | default=folder_list['base_root'], 50 | help="where to put output files") 51 | parser.add_argument("--seed", type=int) 52 | parser.add_argument("--max_steps", type=int, help="number of training steps (0 to disable)") 53 | parser.add_argument("--max_epochs", default=100, type=int, help="number of training epochs") 54 | parser.add_argument("--summary_freq", type=int, default=1000, help="update summaries every summary_freq steps") 55 | parser.add_argument("--progress_freq", type=int, default=5000, help="display progress every progress_freq steps") 56 | parser.add_argument("--trace_freq", type=int, default=0, help="trace execution every trace_freq steps") 57 | # 修改这里使得每隔100展示一次训练的图片 58 | parser.add_argument("--display_freq", type=int, 59 | default=500, 60 | help="write current training images every display_freq steps") 61 | parser.add_argument("--save_freq", type=int, 62 | default=50000, 63 | help="save model every save_freq steps, 0 to disable") 64 | parser.add_argument('--test_procedure', type=int, default=50000) 65 | parser.add_argument('--validation_procedure', type=int, default=50000) 66 | parser.add_argument("--batch_size", type=int, default=1, help="number of images in batch") 67 | parser.add_argument("--ngf", type=int, default=64, help="number of generator filters in first conv layer") 68 | parser.add_argument("--ndf", type=int, default=64, help="number of discriminator filters in first conv layer") 69 | parser.add_argument("--lr_d", type=float, default=0.001, help="initial learning rate for sgd") 70 | parser.add_argument("--lr_g", type=float, default=0.0005, help="initial learning rate for adam") 71 | parser.add_argument("--beta1", type=float, default=0.5, help="momentum term of adam") 72 | parser.add_argument("--l1_weight", type=float, default=100.0, help="weight on L1 term for generator gradient") 73 | parser.add_argument("--gan_weight", type=float, default=1.0, help="weight on GAN term for generator gradient") 74 | 75 | a = parser.parse_args() 76 | EPS = 1e-12 77 | gggg = 0 78 | scale = 1000 79 | 80 | Examples = collections.namedtuple("Examples", "inputs, targets, count," 81 | "inputs_validation, targets_validation, count_validation, " 82 | "inputs_test, targets_test, count_test," 83 | "steps_per_epoch") 84 | Model = collections.namedtuple("Model", "outputs, discrim_loss," 85 | "gen_loss_GAN, gen_loss_L1, train") 86 | 87 | 88 | def discrim_conv(batch_input, out_channels, stride): 89 | return tf.layers.conv2d(batch_input, out_channels, kernel_size=(4, 1), strides=stride, padding="same", 90 | kernel_initializer=xavier_initializer_conv2d()) 91 | 92 | 93 | def gen_conv(batch_input, out_channels): 94 | return tf.layers.conv2d(batch_input, out_channels, kernel_size=(4, 1), strides=(2, 1), padding="same", 95 | kernel_initializer=xavier_initializer_conv2d()) 96 | 97 | 98 | def gen_deconv(batch_input, out_channels): 99 | return tf.layers.conv2d_transpose(batch_input, out_channels, kernel_size=(4, 1), strides=(2, 1), padding="same", 100 | kernel_initializer=xavier_initializer_conv2d()) 101 | 102 | 103 | def lrelu(x, a): 104 | with tf.name_scope("lrelu"): 105 | x = tf.identity(x) 106 | return (0.5 * (1 + a)) * x + (0.5 * (1 - a)) * tf.abs(x) 107 | 108 | 109 | def batchnorm(x, scope='batch_instance_norm'): 110 | with tf.variable_scope(scope): 111 | ch = x.shape[-1] 112 | eps = 1e-5 113 | 114 | batch_mean, batch_sigma = tf.nn.moments(x, axes=[0, 1, 2], keep_dims=True) 115 | x_batch = (x - batch_mean) / (tf.sqrt(batch_sigma + eps)) 116 | 117 | ins_mean, ins_sigma = tf.nn.moments(x, axes=[1, 2], keep_dims=True) 118 | x_ins = (x - ins_mean) / (tf.sqrt(ins_sigma + eps)) 119 | 120 | rho = tf.get_variable("rho", [ch], initializer=tf.constant_initializer(1.0), 121 | constraint=lambda x: tf.clip_by_value(x, clip_value_min=0.0, clip_value_max=1.0)) 122 | gamma = tf.get_variable("gamma", [ch], initializer=tf.constant_initializer(1.0)) 123 | beta = tf.get_variable("beta", [ch], initializer=tf.constant_initializer(0.0)) 124 | 125 | x_hat = rho * x_batch + (1 - rho) * x_ins 126 | x_hat = x_hat * gamma + beta 127 | 128 | return x_hat 129 | 130 | 131 | def create_generator(generator_inputs, generator_outputs_channels): 132 | layers = [] 133 | with tf.variable_scope("encoder_1"): 134 | output = gen_conv(generator_inputs, a.ngf) 135 | layers.append(output) 136 | 137 | layer_specs = [ 138 | a.ngf * 2, 139 | a.ngf * 4, 140 | a.ngf * 8, 141 | a.ngf * 8, 142 | a.ngf * 8, 143 | a.ngf * 8, 144 | a.ngf * 8, 145 | ] 146 | 147 | for out_channels in layer_specs: 148 | with tf.variable_scope("encoder_%d" % (len(layers) + 1)): 149 | rectified = lrelu(layers[-1], 0.2) 150 | convolved = gen_conv(rectified, out_channels) 151 | output = batchnorm(convolved) 152 | layers.append(output) 153 | 154 | layer_specs = [ 155 | (a.ngf * 8, 0.5), 156 | (a.ngf * 8, 0.5), 157 | (a.ngf * 8, 0.5), 158 | (a.ngf * 8, 0.0), 159 | (a.ngf * 4, 0.0), 160 | (a.ngf * 2, 0.0), 161 | (a.ngf, 0.0), 162 | ] 163 | 164 | num_encoder_layers = len(layers) 165 | for decoder_layer, (out_channels, dropout) in enumerate(layer_specs): 166 | skip_layer = num_encoder_layers - decoder_layer - 1 167 | with tf.variable_scope("decoder_%d" % (skip_layer + 1)): 168 | if decoder_layer == 0: 169 | input = layers[-1] 170 | else: 171 | input = tf.concat([layers[-1], layers[skip_layer]], axis=3) 172 | rectified = tf.nn.relu(input) 173 | output = gen_deconv(rectified, out_channels) 174 | output = batchnorm(output) 175 | if dropout > 0.0: 176 | output = tf.nn.dropout(output, keep_prob=1 - dropout) 177 | layers.append(output) 178 | 179 | with tf.variable_scope("decoder_1"): 180 | input = tf.concat([layers[-1], layers[0]], axis=3) 181 | rectified = tf.nn.relu(input) 182 | output = gen_deconv(rectified, generator_outputs_channels) 183 | layers.append(output) 184 | 185 | return layers[-1] 186 | 187 | 188 | def create_model(inputs, targets, step): 189 | learning_rate_d = tf.train.exponential_decay(a.lr_d, step, 50000, 0.95, staircase=True) 190 | learning_rate_g = tf.train.exponential_decay(a.lr_g, step, 50000, 0.95, staircase=True) 191 | 192 | def create_discriminator(discrim_inputs, discrim_targets): 193 | n_layers = 3 194 | layers = [] 195 | 196 | input = tf.concat([discrim_inputs, discrim_targets], axis=1) 197 | 198 | with tf.variable_scope("layer_1"): 199 | convolved = discrim_conv(input, a.ndf, stride=(2, 1)) 200 | rectified = lrelu(convolved, 0.2) 201 | layers.append(rectified) 202 | 203 | for i in range(n_layers): 204 | with tf.variable_scope("layer_%d" % (len(layers) + 1)): 205 | out_channels = a.ndf * min(2**(i+1), 8) 206 | stride = 1 if i == n_layers - 1 else (2, 1) 207 | convolved = discrim_conv(layers[-1], out_channels, stride=stride) 208 | normalized = batchnorm(convolved) 209 | rectified = lrelu(normalized, 0.2) 210 | layers.append(rectified) 211 | 212 | with tf.variable_scope("layer_%d" % (len(layers) + 1)): 213 | convolved = discrim_conv(rectified, out_channels=1, stride=1) 214 | output = tf.sigmoid(convolved) 215 | layers.append(output) 216 | 217 | return layers[-1] 218 | 219 | with tf.variable_scope("generator"): 220 | out_channels = int(targets.get_shape()[-1]) 221 | outputs = create_generator(inputs, out_channels)[:, 256:768, :, :] 222 | 223 | with tf.name_scope("real_discriminator"): 224 | with tf.variable_scope("discriminator"): 225 | predict_real = create_discriminator(inputs, targets) 226 | 227 | with tf.name_scope("fake_discriminator"): 228 | with tf.variable_scope("discriminator", reuse=True): 229 | predict_fake = create_discriminator(inputs, outputs) 230 | 231 | with tf.name_scope("discriminator_loss"): 232 | # minimizing -tf.log will try to get inputs to 1 233 | # predict_real => 1 234 | # predict_fake => 0 235 | discrim_loss = tf.reduce_mean(-(tf.log(predict_real + EPS) + tf.log(1 - predict_fake + EPS))) 236 | 237 | with tf.name_scope("generator_loss"): 238 | # predict_fake => 1 239 | # abs(targets - outputs) => 0 240 | gen_loss_GAN = tf.reduce_mean(-tf.log(predict_fake + EPS)) 241 | gen_loss_L1 = tf.reduce_mean(tf.abs(targets - outputs)) 242 | gen_loss = gen_loss_GAN * a.gan_weight + gen_loss_L1 * a.l1_weight 243 | 244 | with tf.name_scope("discriminator_train"): 245 | discrim_tvars = [var for var in tf.trainable_variables() if var.name.startswith("discriminator")] 246 | discrim_optim = tf.train.GradientDescentOptimizer(learning_rate=learning_rate_d) 247 | discrim_grads_and_vars = discrim_optim.compute_gradients(discrim_loss, var_list=discrim_tvars) 248 | discrim_train = discrim_optim.apply_gradients(discrim_grads_and_vars) 249 | 250 | with tf.name_scope("generator_train"): 251 | with tf.control_dependencies([discrim_train]): 252 | gen_tvars = [var for var in tf.trainable_variables() if var.name.startswith("generator")] 253 | gen_optim = tf.train.AdamOptimizer(learning_rate=learning_rate_g, beta1=a.beta1) 254 | gen_grads_and_vars = gen_optim.compute_gradients(gen_loss, var_list=gen_tvars) 255 | gen_train = gen_optim.apply_gradients(gen_grads_and_vars) 256 | 257 | ema = tf.train.ExponentialMovingAverage(decay=0.99) 258 | update_losses = ema.apply([discrim_loss, gen_loss_GAN, gen_loss_L1]) 259 | 260 | global_step = tf.train.get_or_create_global_step() 261 | incr_global_step = tf.assign(global_step, global_step+1) 262 | 263 | return Model( 264 | discrim_loss=ema.average(discrim_loss), 265 | gen_loss_GAN=ema.average(gen_loss_GAN), 266 | gen_loss_L1=ema.average(gen_loss_L1), 267 | outputs=outputs, 268 | train=tf.group(update_losses, incr_global_step, gen_train), 269 | ) 270 | 271 | 272 | def display_image(inputs, targets, outputs): 273 | global gggg 274 | fig = plt.figure(figsize=(16, 16), dpi=100) 275 | gs = gridspec.GridSpec(3, 1) 276 | gs.update(wspace=0.05, hspace=0.05) 277 | agg = np.reshape(inputs, [1024]) 278 | app = np.reshape(targets, [512]) 279 | out = np.reshape(outputs, [512]) 280 | agg_new = [] 281 | app_new = [] 282 | out_new = [] 283 | x = np.linspace(256, 768, 512) 284 | 285 | for i in agg: 286 | agg_new.append(int(i*scale)) 287 | for i in range(512): 288 | app_new.append(int(app[i]*scale)) 289 | out_new.append(int(out[i]*scale)) 290 | 291 | plt.subplot(3, 1, 1) 292 | plt.xlim(0, 1024) 293 | plt.plot(agg_new) 294 | 295 | plt.subplot(3, 1, 2) 296 | plt.xlim(0, 1024) 297 | plt.plot(x, app_new) 298 | 299 | plt.subplot(3, 1, 3) 300 | plt.xlim(0, 1024) 301 | plt.plot(x, out_new) 302 | 303 | plt.savefig(('%s/{}.png' % folder_list['image']) 304 | .format(str(gggg).zfill(3)), bbox_inches='tight') 305 | gggg += 1 306 | plt.close(fig) 307 | 308 | 309 | def metric(target_data, output_data, t_v, step): 310 | tar_new = [] 311 | out_new = [] 312 | mae = [] 313 | 314 | for i in range(len(output_data)): 315 | for j in range(len(output_data[i])): 316 | if output_data[i][j] < 0: 317 | o = 0 318 | else: 319 | o = output_data[i][j] 320 | out_new.append(o) 321 | tar_new.append(target_data[i][j]) 322 | 323 | for i in range(len(out_new)): 324 | r = abs(tar_new[i] - out_new[i]) 325 | mae.append(r) 326 | 327 | accuracy = (1.0-(np.sum(mae, dtype=np.int64)/(2*np.sum(tar_new, dtype=np.int64))))*100 328 | print('np.sum(mae):') 329 | print(np.sum(mae)) 330 | print('np.sum(tar_new):') 331 | print(np.sum(tar_new)) 332 | sae = abs(np.sum(tar_new, dtype=np.int64)-np.sum(out_new, dtype=np.int64))/np.sum(tar_new, dtype=np.int64) 333 | mse = mean_squared_error(tar_new, out_new) 334 | mae = mean_absolute_error(tar_new, out_new) 335 | r2 = r2_score(tar_new, out_new) 336 | 337 | if t_v == 'validation': 338 | with open('%s\\validation.txt' % folder_list['test_validation'], 'a+') as f: 339 | print('STEP: %d' % (step+1), file=f) 340 | print('Accuracy: %f' % accuracy, file=f) 341 | print('SAE: %f' % sae, file=f) 342 | print('MSE: %f' % mse, file=f) 343 | print('MAE: %f' % mae, file=f) 344 | print('R^2: %f' % r2, file=f) 345 | print('', file=f) 346 | else: 347 | with open('%s\\test.txt' % folder_list['test_validation'], 'a+') as f: 348 | print('STEP: %d' % (step+1), file=f) 349 | print('Accuracy: %f' % accuracy, file=f) 350 | print('SAE: %f' % sae, file=f) 351 | print('MSE: %f' % mse, file=f) 352 | print('MAE: %f' % mae, file=f) 353 | print('R^2: %f' % r2, file=f) 354 | print('', file=f) 355 | 356 | 357 | def load_examples(): 358 | base_path = folder_list['input'] 359 | 360 | ap_data = np.load(os.path.join(base_path, 'appliance_train_512.npy')) 361 | ag_data = np.load(os.path.join(base_path, 'main_train_1000.npy')) 362 | ap_data_validation = np.load(os.path.join(base_path, 'appliance_validation_512.npy')) 363 | ag_data_validation = np.load(os.path.join(base_path, 'main_validation_1000.npy')) 364 | ap_data_test = np.load(os.path.join(base_path, 'appliance_test_512.npy')) 365 | ag_data_test = np.load(os.path.join(base_path, 'main_test_1000.npy')) 366 | print('Data load complete!') 367 | 368 | appliance_data = tf.cast(ap_data, tf.float32) 369 | aggregate_data = tf.cast(ag_data, tf.float32) 370 | appliance_data_validation = tf.cast(ap_data_validation, tf.float32) 371 | aggregate_data_validation = tf.cast(ag_data_validation, tf.float32) 372 | appliance_data_test = tf.cast(ap_data_test, tf.float32) 373 | aggregate_data_test = tf.cast(ag_data_test, tf.float32) 374 | print('cast complete!') 375 | 376 | queue = tf.train.slice_input_producer([aggregate_data, appliance_data], shuffle=True) 377 | inputs_batch, targets_batch = tf.train.batch(queue, batch_size=a.batch_size) 378 | queue_validation = tf.train.slice_input_producer([aggregate_data_validation, appliance_data_validation], shuffle=False) 379 | inputs_batch_validation, targets_batch_validation = tf.train.batch(queue_validation, batch_size=a.batch_size) 380 | queue_test = tf.train.slice_input_producer([aggregate_data_test, appliance_data_test], shuffle=False) 381 | inputs_batch_test, targets_batch_test = tf.train.batch(queue_test, batch_size=a.batch_size) 382 | 383 | steps_per_epoch = int(math.ceil(len(ap_data) / a.batch_size)) 384 | 385 | return Examples( 386 | inputs=inputs_batch, 387 | targets=targets_batch, 388 | count=len(ap_data), 389 | inputs_validation=inputs_batch_validation, 390 | targets_validation=targets_batch_validation, 391 | count_validation=len(ap_data_validation), 392 | inputs_test=inputs_batch_test, 393 | targets_test=targets_batch_test, 394 | count_test=len(ap_data_test), 395 | steps_per_epoch=steps_per_epoch, 396 | ) 397 | 398 | 399 | def main(): 400 | if a.seed is None: 401 | a.seed = random.randint(0, 2**31 - 1) 402 | tf.set_random_seed(a.seed) 403 | np.random.seed(a.seed) 404 | random.seed(a.seed) 405 | 406 | for folder_name, folder_path in folder_list.items(): 407 | if not os.path.exists(folder_path): 408 | os.makedirs(folder_path) 409 | print('Folder creation complete!') 410 | 411 | with open(os.path.join(folder_list['base_root'], 'config.txt'), 'w') as f: 412 | for k, v in a._get_kwargs(): 413 | print(k, "=", v, file=f) 414 | print('Config save complete!') 415 | 416 | examples = load_examples() 417 | trainable = tf.placeholder(tf.bool) 418 | global_step = tf.placeholder(tf.float32, None) 419 | inputs = tf.placeholder(tf.float32, [a.batch_size, 1024, 1, 1]) 420 | targets = tf.placeholder(tf.float32, [a.batch_size, 512, 1, 1]) 421 | i_ = tf.reshape(examples.inputs, [a.batch_size, 1024, 1, 1]) 422 | t_ = tf.reshape(examples.targets, [a.batch_size, 512, 1, 1]) 423 | i_t = tf.reshape(examples.inputs_test, [a.batch_size, 1024, 1, 1]) 424 | t_t = tf.reshape(examples.targets_test, [a.batch_size, 512, 1, 1]) 425 | i_v = tf.reshape(examples.inputs_validation, [a.batch_size, 1024, 1, 1]) 426 | t_v = tf.reshape(examples.targets_validation, [a.batch_size, 512, 1, 1]) 427 | model = create_model(inputs, targets, global_step) 428 | print('model complete!') 429 | 430 | with tf.name_scope("display_images"): 431 | display_fetches = { 432 | 'output': model.outputs, 433 | } 434 | 435 | tf.summary.scalar("discriminator_loss", model.discrim_loss) 436 | tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN) 437 | tf.summary.scalar("generator_loss_L1", model.gen_loss_L1) 438 | 439 | with tf.name_scope("parameter_count"): 440 | parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()]) 441 | 442 | saver = tf.train.Saver(max_to_keep=50) 443 | 444 | logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None 445 | sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None) 446 | with sv.managed_session() as sess: 447 | print("parameter_count =", sess.run(parameter_count)) 448 | if a.checkpoint is not None: 449 | print("loading model from checkpoint") 450 | checkpoint = tf.train.latest_checkpoint(a.checkpoint) 451 | saver.restore(sess, checkpoint) 452 | 453 | max_steps = 2**32 454 | if a.max_epochs is not None: 455 | max_steps = examples.steps_per_epoch * a.max_epochs 456 | if a.max_steps is not None: 457 | max_steps = a.max_steps 458 | 459 | start = time.time() 460 | output_validation = [] 461 | target_validation = [] 462 | output_test = [] 463 | target_test = [] 464 | 465 | for step in range(max_steps): 466 | 467 | def should(freq): 468 | return freq > 0 and ((step + 1) % freq == 0 or step == max_steps - 1) 469 | 470 | options = None 471 | run_metadata = None 472 | if should(a.trace_freq): 473 | options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) 474 | run_metadata = tf.RunMetadata() 475 | 476 | fetches = { 477 | "train": model.train, 478 | "global_step": sv.global_step, 479 | } 480 | if should(a.progress_freq): 481 | fetches["discrim_loss"] = model.discrim_loss 482 | fetches["gen_loss_GAN"] = model.gen_loss_GAN 483 | fetches["gen_loss_L1"] = model.gen_loss_L1 484 | 485 | if should(a.summary_freq): 486 | fetches["summary"] = sv.summary_op 487 | 488 | if should(a.display_freq): 489 | fetches["display"] = display_fetches 490 | 491 | inputs_, targets_ = sess.run([i_, t_]) 492 | results = sess.run(fetches, 493 | options=options, 494 | run_metadata=run_metadata, 495 | feed_dict={global_step: step, 496 | inputs: inputs_, 497 | targets: targets_}) 498 | 499 | if should(a.display_freq): 500 | data = results['display'] 501 | display_image(inputs_, targets_, data['output']) 502 | 503 | if should(a.summary_freq): 504 | sv.summary_writer.add_summary(results["summary"], results["global_step"]) 505 | 506 | if should(a.progress_freq): 507 | train_epoch = math.ceil(results["global_step"] / examples.steps_per_epoch) 508 | train_step = (results["global_step"] - 1) % examples.steps_per_epoch + 1 509 | rate = (step + 1) * a.batch_size / (time.time() - start) 510 | print("progress epoch %d step %d image/sec %0.1f" % 511 | (train_epoch, train_step+1, rate)) 512 | print("discrim_loss", results["discrim_loss"]) 513 | print("gen_loss_GAN", results["gen_loss_GAN"]) 514 | print("gen_loss_L1", results["gen_loss_L1"]) 515 | 516 | if should(a.save_freq): 517 | print("saving model") 518 | saver.save(sess, os.path.join(folder_list['model'], "model"), global_step=sv.global_step) 519 | 520 | if should(a.test_procedure): 521 | print('Test procedure!') 522 | target_test.clear() 523 | output_test.clear() 524 | max_steps_test = examples.count_test 525 | for step_test in range(max_steps_test): 526 | inputs_, targets_= sess.run([i_t, t_t]) 527 | data = sess.run({ 528 | 'output': model.outputs, 529 | }, feed_dict={ 530 | global_step: results['global_step'], 531 | inputs: inputs_, 532 | targets: targets_ 533 | }) 534 | _target = np.reshape(targets_, [a.batch_size, 512]) 535 | _output = np.reshape(data['output'], [a.batch_size, 512]) 536 | for i in range(a.batch_size): 537 | tar_temp = [] 538 | out_temp = [] 539 | for j in range(512): 540 | tar_temp.append(int(_target[i][j] * scale)) 541 | out_temp.append(int(_output[i][j] * scale)) 542 | target_test.append(tar_temp) 543 | output_test.append(out_temp) 544 | if step_test % 1000 == 0: 545 | print('Process Test: %f Rate: %f' % (step_test / max_steps_test, 546 | (time.time() - start) / max_steps_test)) 547 | np.save('%s\\test_target_%d.npy' % (folder_list['test_validation'], results['global_step']+1), target_test) 548 | np.save('%s\\test_output_%d.npy' % (folder_list['test_validation'], results['global_step']+1), output_test) 549 | metric(target_test, output_test, 'test', results['global_step']) 550 | print('Test procedure complete!') 551 | 552 | if should(a.validation_procedure): 553 | print('Validation procedure!') 554 | target_validation.clear() 555 | output_validation.clear() 556 | max_steps_validation = examples.count_validation 557 | for step_validation in range(max_steps_validation): 558 | inputs_, targets_ = sess.run([i_v, t_v]) 559 | data = sess.run({ 560 | 'output': model.outputs, 561 | }, feed_dict={ 562 | global_step: results['global_step'], 563 | inputs: inputs_, 564 | targets: targets_ 565 | }) 566 | _target = np.reshape(targets_, [a.batch_size, 512]) 567 | _output = np.reshape(data['output'], [a.batch_size, 512]) 568 | for i in range(a.batch_size): 569 | tar_temp = [] 570 | out_temp = [] 571 | for j in range(512): 572 | tar_temp.append(int(_target[i][j] * scale)) 573 | out_temp.append(int(_output[i][j] * scale)) 574 | target_validation.append(tar_temp) 575 | output_validation.append(out_temp) 576 | if step_validation % 1000 == 0: 577 | print('Process Validation: %f Rate: %f' % (step_validation / max_steps_validation, 578 | (time.time() - start) / max_steps_validation)) 579 | np.save('%s\\validation_target_%d.npy' % (folder_list['test_validation'], results['global_step']+1), target_validation) 580 | np.save('%s\\validation_output_%d.npy' % (folder_list['test_validation'], results['global_step']+1), output_validation) 581 | metric(target_validation, output_validation, 'validation', results['global_step']) 582 | print('Validation procedure complete!') 583 | 584 | if sv.should_stop(): 585 | break 586 | 587 | 588 | main() 589 | --------------------------------------------------------------------------------