├── README.md ├── appendix1.png ├── dataset ├── authentic │ └── 50 │ │ ├── name_list50.txt │ │ ├── name_list50_1.txt │ │ ├── name_list50_2.txt │ │ └── test │ │ ├── U1025_mal50_undirected.npy │ │ ├── U1289_mal50.npy │ │ ├── U1289_mal50_undirected.npy │ │ ├── U1306_mal50.npy │ │ ├── U1306_mal50_undirected.npy │ │ ├── U1450_mal50.npy │ │ ├── U1450_mal50_undirected.npy │ │ ├── U1467_mal50.npy │ │ ├── U1467_mal50_undirected.npy │ │ ├── U1519_mal50.npy │ │ ├── U1519_mal50_undirected.npy │ │ ├── U1569_mal50.npy │ │ ├── U1569_mal50_undirected.npy │ │ ├── U1581_mal50.npy │ │ ├── U1581_mal50_undirected.npy │ │ ├── U1789_mal50.npy │ │ ├── U1789_mal50_undirected.npy │ │ ├── U20_mal50.npy │ │ ├── U20_mal50_undirected.npy │ │ ├── U212_mal50.npy │ │ ├── U212_mal50_undirected.npy │ │ ├── U2231_mal50.npy │ │ ├── U2231_mal50_undirected.npy │ │ ├── U227_mal50.npy │ │ ├── U227_mal50_undirected.npy │ │ ├── U2837_mal50.npy │ │ ├── U2837_mal50_undirected.npy │ │ ├── U288_mal50.npy │ │ ├── U288_mal50_undirected.npy │ │ ├── U3206_mal50.npy │ │ ├── U3206_mal50_undirected.npy │ │ ├── U3277_mal50.npy │ │ ├── U3277_mal50_undirected.npy │ │ ├── U3486_mal50.npy │ │ ├── U3486_mal50_undirected.npy │ │ ├── U3575_mal50.npy │ │ ├── U3575_mal50_undirected.npy │ │ ├── U3718_mal50.npy │ │ ├── U3718_mal50_undirected.npy │ │ ├── U3764_mal50.npy │ │ ├── U3764_mal50_undirected.npy │ │ ├── U3905_mal50.npy │ │ ├── U3905_mal50_undirected.npy │ │ ├── U4112_mal50.npy │ │ ├── U4112_mal50_undirected.npy │ │ ├── U4856_mal50.npy │ │ ├── U4856_mal50_undirected.npy │ │ ├── U4978_mal50.npy │ │ ├── U4978_mal50_undirected.npy │ │ ├── U524_mal50.npy │ │ ├── U524_mal50_undirected.npy │ │ ├── U5254_mal50.npy │ │ ├── U5254_mal50_undirected.npy │ │ ├── U6115_mal50.npy │ │ ├── U6115_mal50_undirected.npy │ │ ├── U620_mal50.npy │ │ ├── U620_mal50_undirected.npy │ │ ├── U642_mal50.npy │ │ ├── U642_mal50_undirected.npy │ │ ├── U6572_mal50.npy │ │ ├── U6572_mal50_undirected.npy │ │ ├── U6764_mal50.npy │ │ ├── U6764_mal50_undirected.npy │ │ ├── U679_mal50.npy │ │ ├── U679_mal50_undirected.npy │ │ ├── U7004_mal50.npy │ │ ├── U7004_mal50_undirected.npy │ │ ├── U7311_mal50.npy │ │ ├── U7311_mal50_undirected.npy │ │ ├── U7394_mal50.npy │ │ ├── U7394_mal50_undirected.npy │ │ ├── U7594_mal50.npy │ │ ├── U7594_mal50_undirected.npy │ │ ├── U7761_mal50.npy │ │ ├── U7761_mal50_undirected.npy │ │ ├── U795_mal50.npy │ │ ├── U795_mal50_undirected.npy │ │ ├── U8168_mal50.npy │ │ ├── U8168_mal50_undirected.npy │ │ ├── U8170_mal50.npy │ │ ├── U8170_mal50_undirected.npy │ │ ├── U825_mal50.npy │ │ ├── U825_mal50_undirected.npy │ │ ├── U882_mal50.npy │ │ ├── U882_mal50_undirected.npy │ │ ├── U8840_mal50.npy │ │ ├── U8840_mal50_undirected.npy │ │ ├── U9407_mal50.npy │ │ ├── U9407_mal50_undirected.npy │ │ ├── U9763_mal50.npy │ │ ├── U9763_mal50_undirected.npy │ │ ├── U995_mal50.npy │ │ └── U995_mal50_undirected.npy └── transform.py ├── evaluation_authentic ├── feature_logic_model.py ├── graph_kernels.py ├── graph_kernels_labeled.py ├── init.py ├── main.py ├── model.py ├── network_feature.py ├── nystrom.py ├── utils.py └── validation.py ├── evaluation_poisson ├── feature_logic_model.py ├── graph_kernels.py ├── graph_kernels_labeled.py ├── init.py ├── main.py ├── model.py ├── network_feature.py ├── nystrom.py ├── utils.py └── validation.py ├── evaluation_scale free ├── 1 ├── feature_logic_model.py ├── graph_kernels.py ├── graph_kernels_labeled.py ├── init.py ├── main.py ├── model.py ├── network_feature.py ├── nystrom.py ├── utils.py └── validation.py ├── figures └── architecture.png ├── main.py ├── model.py ├── ops_.py └── utils_.py /README.md: -------------------------------------------------------------------------------- 1 | # Deep-Graph-Translation- 2 | 3 | This code and dataseta are for the paper "deep graph translation". 4 | 5 | The full dataset can be donwloaded from: https://www.dropbox.com/s/5taptrm07reywb4/dataset.zip?dl=0 6 | 7 | When training the mode 8 | 9 | -------- run main.py 10 | 11 | Before running, modify the args attributes in main.py, and choose command 12 | 13 | ---------model.train() 14 | 15 | ---------model.test() 16 | 17 | Three datasets are included: 'authentication', 'scale free' and 'poisson random' dataset. 18 | -------------------------------------------------------------------------------- /appendix1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/appendix1.png -------------------------------------------------------------------------------- /dataset/authentic/50/name_list50.txt: -------------------------------------------------------------------------------- 1 | U1025 2 | U1025 3 | U1289 4 | U1289 5 | U1289 6 | U1306 7 | U1450 8 | U1450 9 | U1467 10 | U1519 11 | U1569 12 | U1569 13 | U1581 14 | U1581 15 | U1789 16 | U20 17 | U212 18 | U212 19 | U212 20 | U2231 21 | U2231 22 | U227 23 | U2837 24 | U2837 25 | U288 26 | U3206 27 | U3277 28 | U3486 29 | U3486 30 | U3575 31 | U3718 32 | U3718 33 | U3764 34 | U3905 35 | U4112 36 | U4112 37 | U4856 38 | U4978 39 | U4978 40 | U4978 41 | U4978 42 | U524 43 | U5254 44 | U5254 45 | U6115 46 | U620 47 | U642 48 | U6572 49 | U6572 50 | U6764 51 | U679 52 | U679 53 | U7004 54 | U7311 55 | U7394 56 | U7594 57 | U7761 58 | U795 59 | U795 60 | U8168 61 | U8168 62 | U8168 63 | U8170 64 | U8170 65 | U8170 66 | U8170 67 | U825 68 | U825 69 | U882 70 | U8840 71 | U9407 72 | U9763 73 | U9763 74 | U9763 75 | U9763 76 | U995 77 | U995 78 | U995 -------------------------------------------------------------------------------- /dataset/authentic/50/name_list50_1.txt: -------------------------------------------------------------------------------- 1 | U1025 2 | U1289 3 | U1306 4 | U1450 5 | U1467 6 | U1519 7 | U1569 8 | U1581 9 | U1789 10 | U20 11 | U212 12 | U2231 13 | U227 14 | U2837 15 | U288 16 | U3206 17 | U3277 18 | U3486 19 | U3575 20 | U3718 21 | U3764 22 | U3905 23 | U4112 24 | U4856 -------------------------------------------------------------------------------- /dataset/authentic/50/name_list50_2.txt: -------------------------------------------------------------------------------- 1 | U4978 2 | U524 3 | U5254 4 | U6115 5 | U620 6 | U642 7 | U6572 8 | U6764 9 | U679 10 | U7004 11 | U7311 12 | U7394 13 | U7594 14 | U7761 15 | U795 16 | U8168 17 | U8170 18 | U825 19 | U882 20 | U8840 21 | U9407 22 | U9763 23 | U995 -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1025_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1025_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1289_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1289_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1289_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1289_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1306_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1306_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1306_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1306_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1450_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1450_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1450_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1450_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1467_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1467_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1467_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1467_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1519_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1519_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1519_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1519_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1569_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1569_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1569_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1569_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1581_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1581_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1581_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1581_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1789_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1789_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U1789_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U1789_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U20_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U20_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U20_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U20_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U212_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U212_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U212_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U212_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U2231_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U2231_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U2231_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U2231_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U227_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U227_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U227_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U227_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U2837_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U2837_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U2837_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U2837_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U288_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U288_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U288_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U288_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3206_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3206_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3206_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3206_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3277_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3277_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3277_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3277_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3486_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3486_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3486_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3486_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3575_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3575_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3575_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3575_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3718_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3718_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3718_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3718_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3764_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3764_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3764_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3764_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3905_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3905_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U3905_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U3905_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U4112_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U4112_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U4112_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U4112_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U4856_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U4856_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U4856_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U4856_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U4978_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U4978_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U4978_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U4978_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U524_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U524_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U524_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U524_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U5254_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U5254_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U5254_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U5254_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U6115_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U6115_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U6115_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U6115_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U620_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U620_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U620_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U620_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U642_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U642_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U642_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U642_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U6572_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U6572_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U6572_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U6572_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U6764_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U6764_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U6764_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U6764_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U679_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U679_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U679_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U679_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7004_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7004_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7004_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7004_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7311_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7311_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7311_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7311_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7394_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7394_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7394_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7394_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7594_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7594_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7594_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7594_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7761_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7761_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U7761_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U7761_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U795_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U795_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U795_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U795_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U8168_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U8168_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U8168_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U8168_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U8170_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U8170_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U8170_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U8170_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U825_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U825_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U825_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U825_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U882_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U882_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U882_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U882_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U8840_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U8840_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U8840_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U8840_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U9407_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U9407_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U9407_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U9407_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U9763_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U9763_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U9763_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U9763_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U995_mal50.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U995_mal50.npy -------------------------------------------------------------------------------- /dataset/authentic/50/test/U995_mal50_undirected.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/dataset/authentic/50/test/U995_mal50_undirected.npy -------------------------------------------------------------------------------- /dataset/transform.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Apr 26 12:13:10 2018 4 | 5 | @author: gxjco 6 | """ 7 | import numpy as np 8 | 9 | def di_2_undi(filename): 10 | s1=np.load(filename) 11 | n_node=s1.shape[1] 12 | s2=np.zeros((len(s1),2*n_node,2*n_node)) 13 | for i in range(len(s2)): 14 | for m in range(n_node): 15 | for n in range(n_node): 16 | if n=m: s2[i][m][n]=s1[i][m][n] 18 | np.save(filename.split('.')[0]+'_undirected.npy',s2) 19 | return s2 20 | 21 | #di_2_undi('mal_data.npy') 22 | #di_2_undi('reg_data.npy') 23 | 24 | def undi_2_di(filename): 25 | adj=np.load(filename) 26 | n_node=int(adj.shape[1]/2) 27 | adj1=np.zeros((len(adj),n_node,n_node)) 28 | for m in range(len(adj)): 29 | for i in range(n_node): 30 | for j in range(n_node): 31 | if j>=i: adj1[m][i][j]=adj[m][i][j] 32 | if j1: 28 | data=np.concatenate((n_data_,p_data_),axis=0) 29 | #data=np.reshape(data,[data.shape[0],data.shape[1]*data.shape[2]]) 30 | label=np.concatenate((np.zeros(len(n_data_),dtype=np.int32),np.ones(len(p_data_),dtype=np.int32)),axis=0) 31 | sm=SMOTE(k_neighbors=len(p_data_)-1) 32 | data_res, label_res = sm.fit_sample(data, label) 33 | n=data_res[0:int(len(data_res)/2)] 34 | p=data_res[int(len(data_res)/2):] 35 | n_data=n 36 | p_data=p 37 | else: 38 | n_data=n_data_#np.reshape(n_data_,[n_data_.shape[0],n_data_.shape[1]*n_data_.shape[2]]) 39 | p_data_=p_data_#np.reshape(p_data_,[p_data_.shape[0],p_data_.shape[1]*p_data_.shape[2]]) 40 | p_data=n_data 41 | for i in range(len(p_data)): 42 | p_data[i,:]=p_data_[0] 43 | size_n=int(len(n_data)/2) 44 | size_p=int(len(p_data)/2) 45 | n_label=np.zeros(len(n_data)) 46 | p_label=np.ones(len(p_data)) 47 | train_d=np.concatenate((n_data[0:size_n],p_data[0:size_p]), axis=0) 48 | test_d=np.concatenate((n_data[size_n:],p_data[size_p:]), axis=0) 49 | train_l=np.concatenate((n_label[0:size_n],p_label[0:size_p]), axis=0) 50 | test_l=np.concatenate((n_label[size_n:],p_label[size_p:]), axis=0) 51 | return train_d,test_d,train_l,test_l 52 | 53 | def result(predictions, labels,score): 54 | r11=0 55 | r00=0 56 | r10=0 57 | r01=0 58 | for i in range(len(predictions)): 59 | if predictions[i]==1 and labels[i]==1: r11+=1 60 | if predictions[i]==0 and labels[i]==0: r00+=1 61 | if predictions[i]==1 and labels[i]==0: r10+=1 62 | if predictions[i]==0 and labels[i]==1: r01+=1 63 | fpr, tpr, thresholds = metrics.roc_curve(labels,score,pos_label=1) 64 | AUC=metrics.auc(fpr, tpr) 65 | precision=precision_score(labels,predictions) 66 | recall=recall_score(labels,predictions) 67 | return r11,r00,r01,r10,AUC,precision,recall 68 | def writeExcel(row, col, value, file_name,styl=Style.default_style): 69 | rb = xlrd.open_workbook(file_name) 70 | wb = copy(rb) 71 | ws = wb.get_sheet(0) 72 | ws.write(row, col, value, styl) 73 | wb.save(file_name) 74 | def main(user): 75 | train_data,test_data,train_label,test_label=read_data(user,True) 76 | classifier = LogisticRegression() 77 | classifier.fit(train_data, train_label) 78 | prediction = classifier.predict(test_data) 79 | score=classifier.predict_proba(test_data) 80 | result_={} 81 | result_['TP'],result_['TN'],result_['FN'],result_['FP'],result_['AUC'],result_['precision'],result_['recall']=result(prediction, test_label,score[:,1]) 82 | return result_,test_label,score[:,1] 83 | 84 | with open('redteam.txt') as f: 85 | red_team=f.readlines() 86 | red=[] 87 | red_user={} 88 | for i in red_team: 89 | act={} 90 | m=i.split(',') 91 | act['time']=str(m[0]) 92 | act['user']=m[1].split('@')[0] 93 | act['s_c']=m[2] 94 | act['d_c']=m[3].split('\n')[0] 95 | red.append(act) 96 | if act['user'] not in red_user: 97 | red_user[act['user']]=[] 98 | num=0 99 | skip1=[] 100 | lab=[] 101 | sc=[] 102 | for user in red_user: 103 | try: 104 | acc,l,s=main(user) 105 | #writeExcel(int(num)+1,0, user, 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 106 | #writeExcel(int(num)+1,1, acc['TN'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 107 | #writeExcel(int(num)+1,2, acc['TP'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 108 | #writeExcel(int(num)+1,3, acc['FN'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 109 | #writeExcel(int(num)+1,4, acc['FP'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 110 | #writeExcel(int(num)+1,5, acc['AUC'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 111 | #writeExcel(int(num)+1,6, acc['precision'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 112 | #writeExcel(int(num)+1,7, acc['recall'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 113 | print(num) 114 | num+=1 115 | lab.append(l) 116 | sc.append(s) 117 | except: skip1.append(user) 118 | 119 | label_total=lab[0] 120 | score_total=sc[0] 121 | for i in range(97): 122 | label_total=np.concatenate((label_total,lab[i+1]),axis=0) 123 | score_total=np.concatenate((score_total,sc[i+1]),axis=0) 124 | #draw ROC tota 125 | from sklearn.metrics import roc_curve, auc 126 | import matplotlib.pyplot as plt 127 | fpr,tpr,threshold = roc_curve(label_total, score_total) 128 | roc_auc = auc(fpr,tpr) 129 | plt.figure() 130 | lw = 2 131 | plt.figure(figsize=(10,10)) 132 | plt.plot(fpr, tpr, color='darkorange', 133 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) 134 | plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') 135 | plt.xlim([0.0, 1.0]) 136 | plt.ylim([0.0, 1.05]) 137 | plt.xlabel('False Positive Rate') 138 | plt.ylabel('True Positive Rate') 139 | plt.title('Receiver operating characteristic on 10_feature model') 140 | plt.legend(loc="lower right") 141 | plt.show() -------------------------------------------------------------------------------- /evaluation_authentic/graph_kernels.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | from collections import defaultdict 4 | import copy 5 | from scipy.sparse import lil_matrix 6 | 7 | def sp_kernel(g1, g2=None): 8 | if g2 != None: 9 | graphs = [] 10 | for g in g1: 11 | graphs.append(g) 12 | for g in g2: 13 | graphs.append(g) 14 | else: 15 | graphs = g1 16 | 17 | sp_lengths = [] 18 | 19 | for graph in graphs: 20 | sp_lengths.append(nx.shortest_path_length(graph)) 21 | 22 | N = len(graphs) 23 | all_paths = {} 24 | sp_counts = {} 25 | for i in range(N): 26 | sp_counts[i] = {} 27 | nodes = graphs[i].nodes() 28 | for v1 in nodes: 29 | for v2 in nodes: 30 | if v2 in sp_lengths[i][v1]: 31 | label = sp_lengths[i][v1][v2] 32 | if label in sp_counts[i]: 33 | sp_counts[i][label] += 1 34 | else: 35 | sp_counts[i][label] = 1 36 | 37 | if label not in all_paths: 38 | all_paths[label] = len(all_paths) 39 | 40 | phi = lil_matrix((N,len(all_paths))) 41 | 42 | for i in range(N): 43 | for label in sp_counts[i]: 44 | phi[i,all_paths[label]] = sp_counts[i][label] 45 | 46 | if g2 != None: 47 | K = np.dot(phi[:len(g1),:],phi[len(g1):,:].T) 48 | else: 49 | K = np.dot(phi,phi.T) 50 | 51 | K = np.asarray(K.todense()) 52 | 53 | return K 54 | 55 | 56 | def wl_kernel(g1, g2=None, h=6): 57 | if g2 != None: 58 | graphs = [] 59 | for g in g1: 60 | graphs.append(g) 61 | for g in g2: 62 | graphs.append(g) 63 | else: 64 | graphs = g1 65 | 66 | for G in graphs: 67 | for node in G.nodes(): 68 | G.node[node]['label'] = G.degree(node) 69 | 70 | labels = {} 71 | label_lookup = {} 72 | label_counter = 0 73 | 74 | N = len(graphs) 75 | 76 | orig_graph_map = {it: {i: defaultdict(lambda: 0) for i in range(N)} for it in range(-1, h)} 77 | 78 | # initial labeling 79 | ind = 0 80 | for G in graphs: 81 | labels[ind] = np.zeros(G.number_of_nodes(), dtype = np.int32) 82 | node2index = {} 83 | for node in G.nodes(): 84 | node2index[node] = len(node2index) 85 | 86 | for node in G.nodes(): 87 | label = G.node[node]['label'] 88 | if not (label in label_lookup): 89 | label_lookup[label] = len(label_lookup) 90 | 91 | labels[ind][node2index[node]] = label_lookup[label] 92 | orig_graph_map[-1][ind][label] = orig_graph_map[-1][ind].get(label, 0) + 1 93 | 94 | ind += 1 95 | 96 | compressed_labels = copy.deepcopy(labels) 97 | 98 | # WL iterations 99 | for it in range(h): 100 | unique_labels_per_h = set() 101 | label_lookup = {} 102 | ind = 0 103 | for G in graphs: 104 | node2index = {} 105 | for node in G.nodes(): 106 | node2index[node] = len(node2index) 107 | 108 | for node in G.nodes(): 109 | node_label = tuple([labels[ind][node2index[node]]]) 110 | neighbors = list(G.neighbors(node)) 111 | if len(neighbors) > 0: 112 | neighbors_label = tuple([labels[ind][node2index[neigh]] for neigh in neighbors]) 113 | node_label = str(node_label) + "-" + str(sorted(neighbors_label)) 114 | if not (node_label in label_lookup): 115 | label_lookup[node_label] = len(label_lookup) 116 | 117 | compressed_labels[ind][node2index[node]] = label_lookup[node_label] 118 | orig_graph_map[it][ind][node_label] = orig_graph_map[it][ind].get(node_label, 0) + 1 119 | 120 | ind +=1 121 | 122 | labels = copy.deepcopy(compressed_labels) 123 | 124 | if g2 != None: 125 | K = np.zeros((len(g1), len(g2))) 126 | for it in range(-1, h): 127 | for i in range(len(g1)): 128 | for j in range(len(g2)): 129 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][len(g1)+j].keys()) 130 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][len(g1)+j].get(k,0) for k in common_keys]) 131 | else: 132 | K = np.zeros((N, N)) 133 | for it in range(-1, h): 134 | for i in range(N): 135 | for j in range(N): 136 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][j].keys()) 137 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][j].get(k,0) for k in common_keys]) 138 | 139 | return K -------------------------------------------------------------------------------- /evaluation_authentic/graph_kernels_labeled.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | from collections import defaultdict 4 | import copy 5 | 6 | 7 | def sp_kernel(g1, g2=None): 8 | if g2 != None: 9 | graphs = [] 10 | for g in g1: 11 | graphs.append(g) 12 | for g in g2: 13 | graphs.append(g) 14 | else: 15 | graphs = g1 16 | 17 | N = len(graphs) 18 | all_paths = {} 19 | sp_counts = {} 20 | for i in range(N): 21 | sp_lengths = nx.shortest_path_length(graphs[i]) 22 | sp_counts[i] = {} 23 | nodes = graphs[i].nodes() 24 | for v1 in nodes: 25 | for v2 in nodes: 26 | if v2 in sp_lengths[v1]: 27 | label = tuple(sorted([graphs[i].node[v1]['label'], graphs[i].node[v2]['label']]) + [sp_lengths[v1][v2]]) 28 | if label in sp_counts[i]: 29 | sp_counts[i][label] += 1 30 | else: 31 | sp_counts[i][label] = 1 32 | 33 | if label not in all_paths: 34 | all_paths[label] = len(all_paths) 35 | 36 | phi = np.zeros((N,len(all_paths))) 37 | 38 | for i in range(N): 39 | for label in sp_counts[i]: 40 | phi[i,all_paths[label]] = sp_counts[i][label] 41 | 42 | if g2 != None: 43 | K = np.dot(phi[:len(g1),:],phi[len(g1):,:].T) 44 | else: 45 | K = np.dot(phi,phi.T) 46 | 47 | return K 48 | 49 | 50 | def wl_kernel(g1, g2=None, h=6): 51 | if g2 != None: 52 | graphs = [] 53 | for g in g1: 54 | graphs.append(g) 55 | for g in g2: 56 | graphs.append(g) 57 | else: 58 | graphs = g1 59 | 60 | labels = {} 61 | label_lookup = {} 62 | label_counter = 0 63 | 64 | N = len(graphs) 65 | 66 | orig_graph_map = {it: {i: defaultdict(lambda: 0) for i in range(N)} for it in range(-1, h)} 67 | 68 | # initial labeling 69 | ind = 0 70 | for G in graphs: 71 | labels[ind] = np.zeros(G.number_of_nodes(), dtype = np.int32) 72 | node2index = {} 73 | for node in G.nodes(): 74 | node2index[node] = len(node2index) 75 | 76 | for node in G.nodes(): 77 | label = G.node[node]['label'] 78 | if not (label in label_lookup): 79 | label_lookup[label] = len(label_lookup) 80 | 81 | labels[ind][node2index[node]] = label_lookup[label] 82 | orig_graph_map[-1][ind][label] = orig_graph_map[-1][ind].get(label, 0) + 1 83 | 84 | ind += 1 85 | 86 | compressed_labels = copy.deepcopy(labels) 87 | 88 | # WL iterations 89 | for it in range(h): 90 | unique_labels_per_h = set() 91 | label_lookup = {} 92 | ind = 0 93 | for G in graphs: 94 | node2index = {} 95 | for node in G.nodes(): 96 | node2index[node] = len(node2index) 97 | 98 | for node in G.nodes(): 99 | node_label = tuple([labels[ind][node2index[node]]]) 100 | neighbors = G.neighbors(node) 101 | if len(neighbors) > 0: 102 | neighbors_label = tuple([labels[ind][node2index[neigh]] for neigh in neighbors]) 103 | node_label = str(node_label) + "-" + str(sorted(neighbors_label)) 104 | if not (node_label in label_lookup): 105 | label_lookup[node_label] = len(label_lookup) 106 | 107 | compressed_labels[ind][node2index[node]] = label_lookup[node_label] 108 | orig_graph_map[it][ind][node_label] = orig_graph_map[it][ind].get(node_label, 0) + 1 109 | 110 | ind +=1 111 | 112 | labels = copy.deepcopy(compressed_labels) 113 | 114 | if g2 != None: 115 | K = np.zeros((len(g1), len(g2))) 116 | for it in range(-1, h): 117 | for i in range(len(g1)): 118 | for j in range(len(g2)): 119 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][len(g1)+j].keys()) 120 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][len(g1)+j].get(k,0) for k in common_keys]) 121 | else: 122 | K = np.zeros((N, N)) 123 | for it in range(-1, h): 124 | for i in range(N): 125 | for j in range(N): 126 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][j].keys()) 127 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][j].get(k,0) for k in common_keys]) 128 | 129 | return K -------------------------------------------------------------------------------- /evaluation_authentic/init.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torch.autograd import Variable 4 | 5 | def _calculate_fan_in_and_fan_out(tensor): 6 | dimensions = tensor.ndimension() 7 | if dimensions < 2: 8 | raise ValueError("Fan in and fan out can not be computed for tensor with less than 2 dimensions") 9 | 10 | if dimensions == 2: # Linear 11 | fan_in = tensor.size(1) 12 | fan_out = tensor.size(0) 13 | else: 14 | num_input_fmaps = tensor.size(1) 15 | num_output_fmaps = tensor.size(0) 16 | receptive_field_size = 1 17 | if tensor.dim() > 2: 18 | receptive_field_size = tensor[0][0].numel() 19 | fan_in = num_input_fmaps * receptive_field_size 20 | fan_out = num_output_fmaps * receptive_field_size 21 | 22 | return fan_in, fan_out 23 | 24 | 25 | def xavier_uniform(tensor, gain=1): 26 | """Fills the input Tensor or Variable with values according to the method 27 | described in "Understanding the difficulty of training deep feedforward 28 | neural networks" - Glorot, X. & Bengio, Y. (2010), using a uniform 29 | distribution. The resulting tensor will have values sampled from 30 | :math:`U(-a, a)` where 31 | :math:`a = gain \\times \sqrt{2 / (fan\_in + fan\_out)} \\times \sqrt{3}`. 32 | Also known as Glorot initialisation. 33 | 34 | Args: 35 | tensor: an n-dimensional torch.Tensor or autograd.Variable 36 | gain: an optional scaling factor 37 | 38 | Examples: 39 | >>> w = torch.Tensor(3, 5) 40 | >>> nn.init.xavier_uniform(w, gain=nn.init.calculate_gain('relu')) 41 | """ 42 | if isinstance(tensor, Variable): 43 | xavier_uniform(tensor.data, gain=gain) 44 | return tensor 45 | 46 | fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) 47 | std = gain * math.sqrt(2.0 / (fan_in + fan_out)) 48 | a = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation 49 | return tensor.uniform_(-a, a) 50 | 51 | 52 | def xavier_normal(tensor, gain=1): 53 | """Fills the input Tensor or Variable with values according to the method 54 | described in "Understanding the difficulty of training deep feedforward 55 | neural networks" - Glorot, X. & Bengio, Y. (2010), using a normal 56 | distribution. The resulting tensor will have values sampled from 57 | :math:`N(0, std)` where 58 | :math:`std = gain \\times \sqrt{2 / (fan\_in + fan\_out)}`. 59 | Also known as Glorot initialisation. 60 | 61 | Args: 62 | tensor: an n-dimensional torch.Tensor or autograd.Variable 63 | gain: an optional scaling factor 64 | 65 | Examples: 66 | >>> w = torch.Tensor(3, 5) 67 | >>> nn.init.xavier_normal(w) 68 | """ 69 | if isinstance(tensor, Variable): 70 | xavier_normal(tensor.data, gain=gain) 71 | return tensor 72 | 73 | fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) 74 | std = gain * math.sqrt(2.0 / (fan_in + fan_out)) 75 | return tensor.normal_(0, std) 76 | 77 | 78 | def orthogonal(tensor, gain=1): 79 | """Fills the input Tensor or Variable with a (semi) orthogonal matrix, as 80 | described in "Exact solutions to the nonlinear dynamics of learning in deep 81 | linear neural networks" - Saxe, A. et al. (2013). The input tensor must have 82 | at least 2 dimensions, and for tensors with more than 2 dimensions the 83 | trailing dimensions are flattened. 84 | 85 | Args: 86 | tensor: an n-dimensional torch.Tensor or autograd.Variable, where n >= 2 87 | gain: optional scaling factor 88 | 89 | Examples: 90 | >>> w = torch.Tensor(3, 5) 91 | >>> nn.init.orthogonal(w) 92 | """ 93 | if isinstance(tensor, Variable): 94 | orthogonal(tensor.data, gain=gain) 95 | return tensor 96 | 97 | if tensor.ndimension() < 2: 98 | raise ValueError("Only tensors with 2 or more dimensions are supported") 99 | 100 | rows = tensor.size(0) 101 | cols = tensor[0].numel() 102 | flattened = torch.Tensor(rows, cols).normal_(0, 1) 103 | 104 | if rows < cols: 105 | flattened.t_() 106 | 107 | # Compute the qr factorization 108 | q, r = torch.qr(flattened) 109 | # Make Q uniform according to https://arxiv.org/pdf/math-ph/0609050.pdf 110 | d = torch.diag(r, 0) 111 | ph = d.sign() 112 | q *= ph.expand_as(q) 113 | 114 | if rows < cols: 115 | q.t_() 116 | 117 | tensor.view_as(q).copy_(q) 118 | tensor.mul_(gain) 119 | return tensor -------------------------------------------------------------------------------- /evaluation_authentic/main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | #import torchvision.transforms as transforms 4 | import torch.utils.data as utils 5 | from torch.autograd import Variable 6 | import numpy as np 7 | from utils import compute_nystrom,create_train_test_loaders,imbalance 8 | from sklearn.preprocessing import LabelEncoder 9 | from sklearn.model_selection import KFold 10 | from model import CNN 11 | from graph_kernels import sp_kernel, wl_kernel 12 | import xlwt 13 | import xlrd 14 | from xlutils.copy import copy 15 | from xlwt import Style 16 | def writeExcel(row, col, value, file_name,styl=Style.default_style): 17 | rb = xlrd.open_workbook(file_name) 18 | wb = copy(rb) 19 | ws = wb.get_sheet(0) 20 | ws.write(row, col, value, styl) 21 | wb.save(file_name) 22 | 23 | community_detection = "louvain" 24 | 25 | # Hyper Parameters 26 | dim = 100 27 | batch_size = 64 28 | num_epochs = 150 29 | num_filters = 50 30 | hidden_size = 50 31 | learning_rate = 0.005 32 | use_node_labels = False 33 | # Choose kernels 34 | kernels=[wl_kernel] 35 | num_kernels = len(kernels) 36 | 37 | def test_(user): 38 | print("Computing feature maps...") 39 | Q, subgraphs, labels,shapes = compute_nystrom(user, use_node_labels, dim, community_detection, kernels) 40 | M=np.zeros((shapes[0],shapes[1],len(kernels))) 41 | for idx,k in enumerate(kernels): 42 | M[:,:,idx]=Q[idx] 43 | Q=M 44 | # Binarize labels 45 | le = LabelEncoder() 46 | y = le.fit_transform(labels) 47 | # Build vocabulary 48 | max_document_length = max([len(x.split(" ")) for x in subgraphs]) 49 | x = np.zeros((len(subgraphs), max_document_length), dtype=np.int32) 50 | for i in range(len(subgraphs)): 51 | communities = subgraphs[i].split() 52 | for j in range(len(communities)): 53 | x[i,j] = int(communities[j]) 54 | reg=x[0:2400] 55 | gen=x[2400:4800] 56 | mal=x[4800:] 57 | reg_label=y[:2400] 58 | gen_label=y[2400:4800] 59 | mal_label=y[4800:] 60 | 61 | X,Y=imbalance(np.concatenate((reg,mal),axis=0),np.concatenate((reg_label,mal_label),axis=0)) 62 | train_reg=X[0:1200] 63 | test_reg=X[1200:2400] 64 | train_reg_y=Y[0:1200] 65 | test_reg_y=Y[1200:2400] 66 | train_mal=X[2400:3600] 67 | test_mal=X[3600:4800] 68 | train_mal_y=Y[2400:3600] 69 | test_mal_y=Y[3600:4800] 70 | train_gen=gen[0:1200] 71 | train_gen_y=gen_label[0:1200] 72 | 73 | train_fake=np.concatenate((train_reg,train_gen),axis=0) 74 | y_train_fake=np.concatenate((train_reg_y,train_gen_y),axis=0) 75 | train_real=np.concatenate((train_reg,train_mal),axis=0) 76 | y_train_real=np.concatenate((train_reg_y,train_mal_y),axis=0) 77 | test=np.concatenate((test_reg,test_mal),axis=0) 78 | y_test=np.concatenate((test_reg_y,test_mal_y),axis=0) 79 | 80 | 81 | def train_test(Q, x_train, x_test, y_train, y_test, batch_size): 82 | train_loader, test_loader = create_train_test_loaders(Q, x_train, x_test, y_train, y_test, batch_size) 83 | cnn = CNN(input_size=num_filters, hidden_size=hidden_size, num_classes=np.unique(y).size, dim=dim, num_kernels=num_kernels, max_document_length=max_document_length) 84 | if torch.cuda.is_available(): 85 | cnn.cuda() 86 | if torch.cuda.is_available(): 87 | criterion = nn.CrossEntropyLoss().cuda() 88 | else: 89 | criterion = nn.CrossEntropyLoss() 90 | optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate) 91 | for epoch in range(num_epochs): 92 | for i, (graphs, labels) in enumerate(train_loader): 93 | graphs = Variable(graphs) 94 | labels = Variable(labels) 95 | optimizer.zero_grad() 96 | outputs = cnn(graphs) 97 | if torch.cuda.is_available(): 98 | loss = criterion(outputs, labels.cuda()) 99 | else: 100 | loss = criterion(outputs, labels) 101 | loss.backward() 102 | optimizer.step() 103 | # Test the Model 104 | 105 | cnn.eval() 106 | correct = 0 107 | total = 0 108 | TP=0 109 | TN=0 110 | FP=0 111 | FN=0 112 | predict=[] 113 | label=[] 114 | output=[] 115 | for graphs, labels in test_loader: 116 | graphs = Variable(graphs) 117 | outputs = cnn(graphs) 118 | _, predicted = torch.max(outputs.data, 1) 119 | total += labels.size(0) 120 | correct += (predicted == labels.cuda()).sum() 121 | TP += (predicted+labels.cuda()==2).sum() 122 | FP+=(predicted*5+labels.cuda()*1==5).sum() 123 | FN+=(predicted*1+labels.cuda()*5==5).sum() 124 | TN+=(predicted+labels.cuda()==0).sum() 125 | predict.append(predicted) 126 | label.append(labels) 127 | output.append(outputs.data) 128 | if TP+FP==0: precision=0 129 | else: precision=TP/(TP+FP) 130 | if TP+FN==0: recall=0 131 | else: recall=TP/(TP+FN) 132 | l=np.zeros((len(label))) 133 | for i in range(len(label)): 134 | l[i]=int(label[i]) 135 | s=np.zeros((len(output))) 136 | for i in range(len(output)): 137 | s[i]=output[i][0][1] 138 | return TP,TN,FP,FN,precision,recall,l,s 139 | TP_fake,TN_fake,FP_fake,FN_fake,precision_fake,recall_fake,l_fake,s_fake=train_test(Q, train_fake, test, y_train_fake, y_test, batch_size) 140 | TP_real,TN_real,FP_real,FN_real,precision_real,recall_real,l_real,s_real=train_test(Q, train_real, test, y_train_real, y_test, batch_size) 141 | return TP_fake,TN_fake,FP_fake,FN_fake,precision_fake,recall_fake,l_fake,s_fake,TP_real,TN_real,FP_real,FN_real,precision_real,recall_real,l_real,s_real 142 | ''' 143 | num=0 144 | lab=[] 145 | sc=[] 146 | skip=[] 147 | for user in red_user: 148 | try: 149 | TP,TN,FP,FN,precision,recall,l,s=test_(user,red_user) 150 | writeExcel(int(num)+1,0, user, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 151 | writeExcel(int(num)+1,1, TN, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 152 | writeExcel(int(num)+1,2, TP, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 153 | writeExcel(int(num)+1,3, FN, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 154 | writeExcel(int(num)+1,4, FP, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 155 | writeExcel(int(num)+1,5, precision, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 156 | writeExcel(int(num)+1,6, recall, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 157 | print(num) 158 | num+=1 159 | lab.append(l) 160 | sc.append(s) 161 | except: skip.append(user) 162 | 163 | label_total=lab[0] 164 | score_total=sc[0] 165 | for i in range(97): 166 | label_total=np.concatenate((label_total,lab[i+1]),axis=0) 167 | score_total=np.concatenate((score_total,sc[i+1]),axis=0) 168 | #draw ROC tota 169 | from sklearn.metrics import roc_curve, auc 170 | import matplotlib.pyplot as plt 171 | fpr,tpr,threshold = roc_curve(label_total, score_total) 172 | roc_auc = auc(fpr,tpr) 173 | plt.figure() 174 | lw = 2 175 | plt.figure(figsize=(10,10)) 176 | plt.plot(fpr, tpr, color='darkorange', 177 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) 178 | plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') 179 | plt.xlim([0.0, 1.0]) 180 | plt.ylim([0.0, 1.05]) 181 | plt.xlabel('False Positive Rate') 182 | plt.ylabel('True Positive Rate') 183 | plt.title('ROC curve on KCNN model') 184 | plt.legend(loc="lower right") 185 | plt.show() ''' -------------------------------------------------------------------------------- /evaluation_authentic/model.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | from init import xavier_normal,xavier_uniform,orthogonal 4 | 5 | # CNN Model 6 | class CNN(nn.Module): 7 | def __init__(self, input_size, hidden_size, num_classes, dim, num_kernels, max_document_length): 8 | super(CNN, self).__init__() 9 | self.max_document_length = max_document_length 10 | self.conv = nn.Conv3d(1, input_size, (1, 1, dim), padding=0) 11 | self.fc1 = nn.Linear(input_size*num_kernels, hidden_size) 12 | self.fc2 = nn.Linear(hidden_size, num_classes) 13 | self.init_weights() 14 | 15 | def init_weights(self): 16 | xavier_uniform(self.conv.weight.data) 17 | xavier_normal(self.fc1.weight.data) 18 | xavier_normal(self.fc2.weight.data) 19 | 20 | def forward(self, x_in): 21 | out = F.relu(F.max_pool3d(self.conv(x_in), (1, self.max_document_length,1))) 22 | out = out.view(out.size(0), -1) 23 | out = F.relu(self.fc1(out)) 24 | out = F.dropout(out, training=self.training) 25 | out = self.fc2(out) 26 | return F.softmax(out) 27 | -------------------------------------------------------------------------------- /evaluation_authentic/network_feature.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Mar 19 22:52:58 2018 4 | 5 | @author: gxjco 6 | """ 7 | 8 | import networkx as nx 9 | import numpy as np 10 | import os 11 | with open('C:/Users/gxjco/Desktop/redteam.txt/redteam.txt') as f: 12 | red_team=f.readlines() 13 | red=[] 14 | red_user={} 15 | for i in red_team: 16 | act={} 17 | m=i.split(',') 18 | act['time']=m[0] 19 | act['user']=m[1].split('@')[0] 20 | act['s_c']=m[2] 21 | act['d_c']=m[3].split('\n')[0] 22 | red.append(act) 23 | if act['user'] not in red_user: 24 | red_user[act['user']]=[] 25 | for act_ in red: 26 | red_user[act_['user']].append(act_) 27 | def plus(a,b): 28 | for i in b: 29 | if i not in a: 30 | a.append(i) 31 | return a 32 | def feature(node,edge): 33 | G =nx.MultiGraph() 34 | G=G.to_directed() 35 | G.add_nodes_from(node) 36 | G.add_edges_from(edge) 37 | f=np.zeros(10) 38 | f[0]=len(G.nodes) 39 | f[1]=len(G.edges) 40 | f[2]=nx.density(G) 41 | f[3]=nx.degree_pearson_correlation_coefficient(G) 42 | f[4]=nx.algorithms.reciprocity(G) 43 | f[5]=0#nx.transitivity(G) 44 | f[6]=nx.is_weakly_connected(G) 45 | f[7]=nx.number_weakly_connected_components(G) 46 | f[8]=nx.is_strongly_connected(G) 47 | f[9]=nx.number_strongly_connected_components(G) 48 | return f 49 | b=0 50 | path='C:/Users/gxjco/Desktop/redteam.txt/user_new/' 51 | files=os.listdir(path) 52 | for file_ in files: 53 | u=[] 54 | user=file_.split('.')[0] 55 | with open(path+file_) as f: 56 | for line in f: 57 | m=line.split(',') 58 | event={} 59 | event['time']=m[0] 60 | event['user']=m[1].split('@')[0] 61 | event['s_c']=m[3] 62 | event['d_c']=m[4] 63 | u.append(event) 64 | l=red_user[user] 65 | u=plus(u,l) 66 | act={} 67 | for i in range(2600): 68 | act[i]={} 69 | act[i]['node']=[] 70 | act[i]['edge']=[] 71 | for line in u: 72 | if int(line['time'])<4680000: 73 | act[int(int(line['time'])/1800)]['node'].append(line['s_c']) 74 | act[int(int(line['time'])/1800)]['node'].append(line['d_c']) 75 | act[int(int(line['time'])/1800)]['edge'].append((line['s_c'],line['d_c'])) 76 | sample=np.zeros((2600,10)) 77 | for i in range(2600): 78 | try: 79 | sample[i,:]=feature(act[i]['node'],act[i]['edge']) 80 | except:b+=1 81 | 82 | time_red={} 83 | for i in red_user[user]: 84 | time_red[i['time']]=[] 85 | label=[] 86 | for t in range(2600): 87 | o=0 88 | for time in time_red: 89 | if int(time) in range(1800*t,1800*t+1800): 90 | o=1 91 | break 92 | label.append(o) 93 | #seperate the negative and positive 94 | p_data=[] 95 | n_data=[] 96 | for i in range(2600): 97 | if label[i]==1: 98 | p_data.append(sample[i]) 99 | else: 100 | n_data.append(sample[i]) 101 | p_sample=np.zeros((len(p_data),10)) 102 | n_sample=np.zeros((len(n_data),10)) 103 | for i in range(len(p_data)): 104 | p_sample[i]=p_data[i] 105 | for i in range(len(n_data)): 106 | n_sample[i]=n_data[i] 107 | np.save('C:/Users/gxjco/Desktop/redteam.txt/input_new/feature_new_negative_sample__'+user+'.npy',n_sample) 108 | np.save('C:/Users/gxjco/Desktop/redteam.txt/input_new/feature_new_positive_sample__'+user+'.npy',p_sample) 109 | 110 | 111 | -------------------------------------------------------------------------------- /evaluation_authentic/nystrom.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.sparse as sp 3 | from scipy.linalg import svd 4 | from sklearn.utils import check_random_state 5 | import graph_kernels 6 | 7 | class Nystrom(): 8 | 9 | def __init__(self, kernel, kernel_params=None, n_components=100, random_state=None): 10 | self.kernel = kernel 11 | self.kernel_params = kernel_params 12 | self.n_components = n_components 13 | self.random_state = random_state 14 | 15 | def fit(self, graphs, y=None): 16 | rnd = check_random_state(self.random_state) 17 | n_samples = len(graphs) 18 | 19 | # get basis vectors 20 | if self.n_components > n_samples: 21 | n_components = n_samples 22 | else: 23 | n_components = self.n_components 24 | n_components = min(n_samples, n_components) 25 | inds = rnd.permutation(n_samples) 26 | basis_inds = inds[:n_components] 27 | basis = [] 28 | for ind in basis_inds: 29 | basis.append(graphs[ind]) 30 | basis_kernel = self.kernel(basis, basis, **self._get_kernel_params()) 31 | 32 | # sqrt of kernel matrix on basis vectors 33 | U, S, V = svd(basis_kernel) 34 | S = np.maximum(S, 1e-12) 35 | self.normalization_ = np.dot(U * 1. / np.sqrt(S), V) 36 | self.components_ = basis 37 | self.component_indices_ = inds 38 | return self 39 | 40 | def transform(self, graphs): 41 | embedded = self.kernel(graphs, self.components_, **self._get_kernel_params()) 42 | return np.dot(embedded, self.normalization_.T) 43 | 44 | def _get_kernel_params(self): 45 | params = self.kernel_params 46 | if params is None: 47 | params = {} 48 | 49 | return params 50 | 51 | -------------------------------------------------------------------------------- /evaluation_authentic/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | import networkx as nx 3 | import igraph as ig 4 | import numpy as np 5 | from nystrom import Nystrom 6 | import torch 7 | import torch.utils.data as utils 8 | from imblearn.over_sampling import SMOTE 9 | 10 | 11 | def load(user): 12 | def graph(A): 13 | G =nx.MultiGraph() 14 | G=G.to_directed() 15 | for i in range(A.shape[0]): 16 | for j in range(A.shape[0]): 17 | if A[i][j]>0: 18 | G.add_nodes_from([str(i),str(j)]) 19 | for m in range(A[i][j]): 20 | G.add_edges_from([(str(i),str(j))]) 21 | 22 | return G 23 | reg=np.load('./dataset/authentic/50/test/'+user+'_50.npy').astype(int)[:2400] 24 | gen=np.load('./validation_data_auth_50/'+user+'_gen50.npy').astype(int)[:,:,:,0] 25 | mal=np.load('./dataset/authentic/50/test/'+user+'_mal50.npy').astype(int) 26 | Gs_reg=[] 27 | Gs_gen=[] 28 | Gs_mal=[] 29 | for i in range(len(reg)): 30 | Gs_reg.append(graph(reg[i])) 31 | Gs_gen.append(graph(gen[i])) 32 | for i in range(len(mal)): 33 | Gs_mal.append(graph(mal[i])) 34 | label_reg=np.zeros(len(reg)) 35 | label_gen=np.ones(len(gen)) 36 | label_mal=np.ones(len(mal)) 37 | return Gs_reg,label_reg,Gs_gen,label_gen,Gs_mal,label_mal 38 | 39 | 40 | 41 | def networkx_to_igraph(G): 42 | mapping = dict(zip(G.nodes(),range(G.number_of_nodes()))) 43 | reverse_mapping = dict(zip(range(G.number_of_nodes()),G.nodes())) 44 | G = nx.relabel_nodes(G,mapping) 45 | G_ig = ig.Graph(len(G), list(zip(*list(zip(*nx.to_edgelist(G)))[:2]))) 46 | return G_ig, reverse_mapping 47 | 48 | 49 | def community_detection(G_networkx, community_detection_method): 50 | G,reverse_mapping = networkx_to_igraph(G_networkx) 51 | if community_detection_method == "eigenvector": 52 | c = G.community_leading_eigenvector() 53 | elif community_detection_method == "infomap": 54 | c = G.community_infomap() 55 | elif community_detection_method == "fastgreedy": 56 | c = G.community_fastgreedy().as_clustering() 57 | elif community_detection_method == "label_propagation": 58 | c = G.community_label_propagation() 59 | elif community_detection_method == "louvain": 60 | c = G.community_multilevel() 61 | elif community_detection_method == "spinglass": 62 | c = G.community_spinglass() 63 | elif community_detection_method == "walktrap": 64 | c = G.community_walktrap().as_clustering() 65 | else: 66 | c = [] 67 | 68 | communities = [] 69 | for i in range(len(c)): 70 | community = [] 71 | for j in range(len(c[i])): 72 | community.append(reverse_mapping[G.vs[c[i][j]].index]) 73 | 74 | communities.append(community) 75 | 76 | return communities 77 | 78 | def compute_communities(graphs, use_node_labels, community_detection_method): 79 | communities = [] 80 | subgraphs = [] 81 | counter = 0 82 | coms = [] 83 | for G in graphs: 84 | c = community_detection(G, community_detection_method) 85 | coms.append(len(c)) 86 | subgraph = [] 87 | for i in range(len(c)): 88 | communities.append(G.subgraph(c[i])) 89 | subgraph.append(counter) 90 | counter += 1 91 | 92 | subgraphs.append(' '.join(str(s) for s in subgraph)) 93 | 94 | return communities, subgraphs 95 | 96 | 97 | def compute_nystrom(user,use_node_labels, embedding_dim, community_detection_method, kernels): 98 | graphs_reg, labels_reg,graphs_gen, labels_gen,graphs_mal, labels_mal = load(user) 99 | graphs=graphs_reg+graphs_gen+graphs_mal 100 | labels=np.concatenate((labels_reg,labels_gen,labels_mal),axis=0) 101 | communities, subgraphs = compute_communities(graphs, use_node_labels, community_detection_method) 102 | 103 | print("Number of communities: ", len(communities)) 104 | lens = [] 105 | for community in communities: 106 | lens.append(community.number_of_nodes()) 107 | 108 | print("Average size: %.2f" % np.mean(lens)) 109 | Q=[] 110 | for idx, k in enumerate(kernels): 111 | model = Nystrom(k, n_components=embedding_dim) 112 | model.fit(communities) 113 | Q_t = model.transform(communities) 114 | Q_t = np.vstack([np.zeros(embedding_dim), Q_t]) 115 | Q.append(Q_t) 116 | 117 | return Q, subgraphs, labels, Q_t.shape 118 | 119 | 120 | def create_train_test_loaders(Q, x_train, x_test, y_train, y_test, batch_size): 121 | num_kernels = Q.shape[2] 122 | max_document_length = x_train.shape[1] 123 | dim = Q.shape[1] 124 | 125 | my_x = [] 126 | for i in range(x_train.shape[0]): 127 | temp = np.zeros((1, num_kernels, max_document_length, dim)) 128 | for j in range(num_kernels): 129 | for k in range(x_train.shape[1]): 130 | temp[0,j,k,:] = Q[x_train[i,k],:,j].squeeze() 131 | my_x.append(temp) 132 | 133 | if torch.cuda.is_available(): 134 | tensor_x = torch.stack([torch.cuda.FloatTensor(i) for i in my_x]) # transform to torch tensors 135 | tensor_y = torch.cuda.LongTensor(y_train.tolist()) 136 | else: 137 | tensor_x = torch.stack([torch.Tensor(i) for i in my_x]) # transform to torch tensors 138 | tensor_y = torch.from_numpy(np.asarray(y_train,dtype=np.int64)) 139 | 140 | train_dataset = utils.TensorDataset(tensor_x, tensor_y) 141 | train_loader = utils.DataLoader(train_dataset, batch_size=batch_size, shuffle=True) 142 | 143 | my_x = [] 144 | for i in range(x_test.shape[0]): 145 | temp = np.zeros((1, num_kernels, max_document_length, dim)) 146 | for j in range(num_kernels): 147 | for k in range(x_test.shape[1]): 148 | temp[0,j,k,:] = Q[x_test[i,k],:,j].squeeze() 149 | my_x.append(temp) 150 | 151 | if torch.cuda.is_available(): 152 | tensor_x = torch.stack([torch.cuda.FloatTensor(i) for i in my_x]) # transform to torch tensors 153 | tensor_y = torch.cuda.LongTensor(y_test.tolist()) 154 | else: 155 | tensor_x = torch.stack([torch.Tensor(i) for i in my_x]) # transform to torch tensors 156 | tensor_y = torch.from_numpy(np.asarray(y_test,dtype=np.int64)) 157 | 158 | test_dataset = utils.TensorDataset(tensor_x, tensor_y) 159 | test_loader = utils.DataLoader(test_dataset, batch_size=1, shuffle=False) 160 | 161 | return train_loader, test_loader 162 | 163 | def imbalance(x,y): 164 | p_num=sum(y) 165 | if p_num>1: 166 | sm=SMOTE(k_neighbors=p_num-1) 167 | data_res, label_res = sm.fit_sample(x, y) 168 | else: 169 | for i in range(2600): 170 | if y[i]==1: 171 | idx=i 172 | break 173 | new_p=np.zeros((2598,x.shape[1])) 174 | new_p_l=np.ones((2598)) 175 | for i in range(2598): 176 | new_p[i,:]=x[idx] 177 | data_res=np.concatenate((x,new_p),axis=0) 178 | label_res=np.concatenate((y,new_p_l),axis=0) 179 | return data_res.astype(dtype=np.int32),label_res.astype(dtype=np.int32) 180 | 181 | 182 | 183 | 184 | 185 | 186 | -------------------------------------------------------------------------------- /evaluation_authentic/validation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sun Apr 15 20:43:11 2018 4 | 5 | @author: gxjco 6 | """ 7 | import xlwt 8 | import xlrd 9 | from xlutils.copy import copy 10 | from xlwt import Style 11 | from main import* 12 | import numpy as np 13 | from sklearn.metrics import roc_curve, auc 14 | import matplotlib.pyplot as plt 15 | 16 | #find user 17 | with open('./dataset/authentic/50/name_list50.txt') as f: #change the 50 to 300 when size changes 18 | user_name=f.readlines() 19 | name_list=[] 20 | for i in user_name: 21 | if i.split('\n')[0] not in name_list: name_list.append(i.split('\n')[0]) 22 | 23 | def writeExcel(row, col, value, file_name,styl=Style.default_style): 24 | rb = xlrd.open_workbook(file_name) 25 | wb = copy(rb) 26 | ws = wb.get_sheet(0) 27 | ws.write(row, col, value, styl) 28 | wb.save(file_name) 29 | num=0 30 | lab_fake=[] 31 | sc_fake=[] 32 | lab_real=[] 33 | sc_real=[] 34 | skip=[] 35 | for user in name_list: 36 | try: 37 | TP_fake,TN_fake,FP_fake,FN_fake,precision_fake,recall_fake,l_fake,s_fake, TP_real,TN_real,FP_real,FN_real,precision_real,recall_real,l_real,s_real=test_(user) 38 | writeExcel(int(num)+1,0, user, './result_GAN_50.xls') 39 | writeExcel(int(num)+1,1, TN_fake, './result_GAN_50.xls') 40 | writeExcel(int(num)+1,2, TP_fake, './result_GAN_50.xls') 41 | writeExcel(int(num)+1,3, FN_fake, './result_GAN_50.xls') 42 | writeExcel(int(num)+1,4, FP_fake, './result_GAN_50.xls') 43 | writeExcel(int(num)+1,5, precision_fake, './result_GAN_50.xls') 44 | writeExcel(int(num)+1,6, recall_fake, './result_GAN_50.xls') 45 | 46 | writeExcel(int(num)+1,8, user, './result_GAN_50.xls') 47 | writeExcel(int(num)+1,9, TN_real, './result_GAN_50.xls') 48 | writeExcel(int(num)+1,10, TP_real, './result_GAN_50.xls') 49 | writeExcel(int(num)+1,11, FN_real, './result_GAN_50.xls') 50 | writeExcel(int(num)+1,12, FP_real, './result_GAN_50.xls') 51 | writeExcel(int(num)+1,13, precision_real, './result_GAN_50.xls') 52 | writeExcel(int(num)+1,14, recall_real, './result_GAN_50.xls') 53 | print(num) 54 | num+=1 55 | lab_fake.append(l_fake) 56 | sc_fake.append(s_fake) 57 | lab_real.append(l_real) 58 | sc_real.append(s_real) 59 | except: skip.append(user) 60 | label_total_fake=lab_fake[0] 61 | score_total_fake=sc_fake[0] 62 | label_total_real=lab_real[0] 63 | score_total_real=sc_real[0] 64 | for i in range(46): 65 | label_total_fake=np.concatenate((label_total_fake,lab_fake[i+1]),axis=0) 66 | score_total_fake=np.concatenate((score_total_fake,sc_fake[i+1]),axis=0) 67 | label_total_real=np.concatenate((label_total_real,lab_real[i+1]),axis=0) 68 | score_total_real=np.concatenate((score_total_real,sc_real[i+1]),axis=0) 69 | 70 | #draw ROC tota 71 | def draw_roc(label_total_real, score_total_real,label_total_fake, score_total_fake): 72 | 73 | fpr1,tpr1,threshold1 = roc_curve(label_total_fake, score_total_fake) 74 | roc_auc1 = auc(fpr1,tpr1) 75 | fpr2,tpr2,threshold2 = roc_curve(label_total_real, score_total_real) 76 | roc_auc2 = auc(fpr2,tpr2) 77 | plt.figure() 78 | lw = 2 79 | plt.figure(figsize=(10,10)) 80 | plt.plot(fpr1, tpr1, color='darkorange', 81 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc1) 82 | plt.plot(fpr2, tpr2, color='green', 83 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc2) 84 | plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') 85 | plt.xlim([0.0, 1.0]) 86 | plt.ylim([0.0, 1.05]) 87 | plt.xlabel('False Positive Rate') 88 | plt.ylabel('True Positive Rate') 89 | plt.title('ROC curve on KCNN model') 90 | plt.legend(loc="lower right") 91 | plt.show() 92 | draw_roc(label_total_real, score_total_real,label_total_fake, score_total_fake) 93 | roc=[] 94 | ''' 95 | for i in range(49): 96 | j=name_list.index(name_list50[i]) 97 | from sklearn.metrics import roc_curve, auc 98 | fpr1,tpr1,threshold1 = roc_curve(label_total_fake[j*2400:j*2400+2400], score_total_fake[j*2400:j*2400+2400]) 99 | roc.append(auc(fpr1,tpr1))''' -------------------------------------------------------------------------------- /evaluation_poisson/feature_logic_model.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sun Mar 18 19:57:38 2018 4 | 5 | @author: gxjco 6 | """ 7 | import numpy as np 8 | from imblearn.over_sampling import SMOTE 9 | import xlwt 10 | import xlrd 11 | import os 12 | from xlutils.copy import copy 13 | from xlwt import Style 14 | from sklearn.linear_model import LogisticRegression 15 | import math 16 | from sklearn import metrics 17 | from sklearn.metrics import recall_score 18 | from sklearn.metrics import precision_score 19 | from imblearn.over_sampling import SMOTE 20 | def read_data(user,if_smote): 21 | n_data_=np.load('C:/Users/gxjco/Desktop/redteam.txt/input_feature/feature_new_negative_sample__'+user+'.npy') 22 | p_data_=np.load('C:/Users/gxjco/Desktop/redteam.txt/input_feature/feature_new_positive_sample__'+user+'.npy') 23 | for i in range(len(n_data_)): 24 | if math.isnan(n_data_[i,3]): n_data_[i,3]=0 25 | for i in range(len(p_data_)): 26 | if math.isnan(p_data_[i,3]): p_data_[i,3]=0 27 | if len(p_data_)>1: 28 | data=np.concatenate((n_data_,p_data_),axis=0) 29 | #data=np.reshape(data,[data.shape[0],data.shape[1]*data.shape[2]]) 30 | label=np.concatenate((np.zeros(len(n_data_),dtype=np.int32),np.ones(len(p_data_),dtype=np.int32)),axis=0) 31 | sm=SMOTE(k_neighbors=len(p_data_)-1) 32 | data_res, label_res = sm.fit_sample(data, label) 33 | n=data_res[0:int(len(data_res)/2)] 34 | p=data_res[int(len(data_res)/2):] 35 | n_data=n 36 | p_data=p 37 | else: 38 | n_data=n_data_#np.reshape(n_data_,[n_data_.shape[0],n_data_.shape[1]*n_data_.shape[2]]) 39 | p_data_=p_data_#np.reshape(p_data_,[p_data_.shape[0],p_data_.shape[1]*p_data_.shape[2]]) 40 | p_data=n_data 41 | for i in range(len(p_data)): 42 | p_data[i,:]=p_data_[0] 43 | size_n=int(len(n_data)/2) 44 | size_p=int(len(p_data)/2) 45 | n_label=np.zeros(len(n_data)) 46 | p_label=np.ones(len(p_data)) 47 | train_d=np.concatenate((n_data[0:size_n],p_data[0:size_p]), axis=0) 48 | test_d=np.concatenate((n_data[size_n:],p_data[size_p:]), axis=0) 49 | train_l=np.concatenate((n_label[0:size_n],p_label[0:size_p]), axis=0) 50 | test_l=np.concatenate((n_label[size_n:],p_label[size_p:]), axis=0) 51 | return train_d,test_d,train_l,test_l 52 | 53 | def result(predictions, labels,score): 54 | r11=0 55 | r00=0 56 | r10=0 57 | r01=0 58 | for i in range(len(predictions)): 59 | if predictions[i]==1 and labels[i]==1: r11+=1 60 | if predictions[i]==0 and labels[i]==0: r00+=1 61 | if predictions[i]==1 and labels[i]==0: r10+=1 62 | if predictions[i]==0 and labels[i]==1: r01+=1 63 | fpr, tpr, thresholds = metrics.roc_curve(labels,score,pos_label=1) 64 | AUC=metrics.auc(fpr, tpr) 65 | precision=precision_score(labels,predictions) 66 | recall=recall_score(labels,predictions) 67 | return r11,r00,r01,r10,AUC,precision,recall 68 | def writeExcel(row, col, value, file_name,styl=Style.default_style): 69 | rb = xlrd.open_workbook(file_name) 70 | wb = copy(rb) 71 | ws = wb.get_sheet(0) 72 | ws.write(row, col, value, styl) 73 | wb.save(file_name) 74 | def main(user): 75 | train_data,test_data,train_label,test_label=read_data(user,True) 76 | classifier = LogisticRegression() 77 | classifier.fit(train_data, train_label) 78 | prediction = classifier.predict(test_data) 79 | score=classifier.predict_proba(test_data) 80 | result_={} 81 | result_['TP'],result_['TN'],result_['FN'],result_['FP'],result_['AUC'],result_['precision'],result_['recall']=result(prediction, test_label,score[:,1]) 82 | return result_,test_label,score[:,1] 83 | 84 | with open('redteam.txt') as f: 85 | red_team=f.readlines() 86 | red=[] 87 | red_user={} 88 | for i in red_team: 89 | act={} 90 | m=i.split(',') 91 | act['time']=str(m[0]) 92 | act['user']=m[1].split('@')[0] 93 | act['s_c']=m[2] 94 | act['d_c']=m[3].split('\n')[0] 95 | red.append(act) 96 | if act['user'] not in red_user: 97 | red_user[act['user']]=[] 98 | num=0 99 | skip1=[] 100 | lab=[] 101 | sc=[] 102 | for user in red_user: 103 | try: 104 | acc,l,s=main(user) 105 | #writeExcel(int(num)+1,0, user, 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 106 | #writeExcel(int(num)+1,1, acc['TN'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 107 | #writeExcel(int(num)+1,2, acc['TP'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 108 | #writeExcel(int(num)+1,3, acc['FN'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 109 | #writeExcel(int(num)+1,4, acc['FP'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 110 | #writeExcel(int(num)+1,5, acc['AUC'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 111 | #writeExcel(int(num)+1,6, acc['precision'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 112 | #writeExcel(int(num)+1,7, acc['recall'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 113 | print(num) 114 | num+=1 115 | lab.append(l) 116 | sc.append(s) 117 | except: skip1.append(user) 118 | 119 | label_total=lab[0] 120 | score_total=sc[0] 121 | for i in range(97): 122 | label_total=np.concatenate((label_total,lab[i+1]),axis=0) 123 | score_total=np.concatenate((score_total,sc[i+1]),axis=0) 124 | #draw ROC tota 125 | from sklearn.metrics import roc_curve, auc 126 | import matplotlib.pyplot as plt 127 | fpr,tpr,threshold = roc_curve(label_total, score_total) 128 | roc_auc = auc(fpr,tpr) 129 | plt.figure() 130 | lw = 2 131 | plt.figure(figsize=(10,10)) 132 | plt.plot(fpr, tpr, color='darkorange', 133 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) 134 | plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') 135 | plt.xlim([0.0, 1.0]) 136 | plt.ylim([0.0, 1.05]) 137 | plt.xlabel('False Positive Rate') 138 | plt.ylabel('True Positive Rate') 139 | plt.title('Receiver operating characteristic on 10_feature model') 140 | plt.legend(loc="lower right") 141 | plt.show() -------------------------------------------------------------------------------- /evaluation_poisson/graph_kernels.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | from collections import defaultdict 4 | import copy 5 | from scipy.sparse import lil_matrix 6 | 7 | def sp_kernel(g1, g2=None): 8 | if g2 != None: 9 | graphs = [] 10 | for g in g1: 11 | graphs.append(g) 12 | for g in g2: 13 | graphs.append(g) 14 | else: 15 | graphs = g1 16 | 17 | sp_lengths = [] 18 | 19 | for graph in graphs: 20 | sp_lengths.append(nx.shortest_path_length(graph)) 21 | 22 | N = len(graphs) 23 | all_paths = {} 24 | sp_counts = {} 25 | for i in range(N): 26 | sp_counts[i] = {} 27 | nodes = graphs[i].nodes() 28 | for v1 in nodes: 29 | for v2 in nodes: 30 | if v2 in sp_lengths[i][v1]: 31 | label = sp_lengths[i][v1][v2] 32 | if label in sp_counts[i]: 33 | sp_counts[i][label] += 1 34 | else: 35 | sp_counts[i][label] = 1 36 | 37 | if label not in all_paths: 38 | all_paths[label] = len(all_paths) 39 | 40 | phi = lil_matrix((N,len(all_paths))) 41 | 42 | for i in range(N): 43 | for label in sp_counts[i]: 44 | phi[i,all_paths[label]] = sp_counts[i][label] 45 | 46 | if g2 != None: 47 | K = np.dot(phi[:len(g1),:],phi[len(g1):,:].T) 48 | else: 49 | K = np.dot(phi,phi.T) 50 | 51 | K = np.asarray(K.todense()) 52 | 53 | return K 54 | 55 | 56 | def wl_kernel(g1, g2=None, h=6): 57 | if g2 != None: 58 | graphs = [] 59 | for g in g1: 60 | graphs.append(g) 61 | for g in g2: 62 | graphs.append(g) 63 | else: 64 | graphs = g1 65 | 66 | for G in graphs: 67 | for node in G.nodes(): 68 | G.node[node]['label'] = G.degree(node) 69 | 70 | labels = {} 71 | label_lookup = {} 72 | label_counter = 0 73 | 74 | N = len(graphs) 75 | 76 | orig_graph_map = {it: {i: defaultdict(lambda: 0) for i in range(N)} for it in range(-1, h)} 77 | 78 | # initial labeling 79 | ind = 0 80 | for G in graphs: 81 | labels[ind] = np.zeros(G.number_of_nodes(), dtype = np.int32) 82 | node2index = {} 83 | for node in G.nodes(): 84 | node2index[node] = len(node2index) 85 | 86 | for node in G.nodes(): 87 | label = G.node[node]['label'] 88 | if not (label in label_lookup): 89 | label_lookup[label] = len(label_lookup) 90 | 91 | labels[ind][node2index[node]] = label_lookup[label] 92 | orig_graph_map[-1][ind][label] = orig_graph_map[-1][ind].get(label, 0) + 1 93 | 94 | ind += 1 95 | 96 | compressed_labels = copy.deepcopy(labels) 97 | 98 | # WL iterations 99 | for it in range(h): 100 | unique_labels_per_h = set() 101 | label_lookup = {} 102 | ind = 0 103 | for G in graphs: 104 | node2index = {} 105 | for node in G.nodes(): 106 | node2index[node] = len(node2index) 107 | 108 | for node in G.nodes(): 109 | node_label = tuple([labels[ind][node2index[node]]]) 110 | neighbors = list(G.neighbors(node)) 111 | if len(neighbors) > 0: 112 | neighbors_label = tuple([labels[ind][node2index[neigh]] for neigh in neighbors]) 113 | node_label = str(node_label) + "-" + str(sorted(neighbors_label)) 114 | if not (node_label in label_lookup): 115 | label_lookup[node_label] = len(label_lookup) 116 | 117 | compressed_labels[ind][node2index[node]] = label_lookup[node_label] 118 | orig_graph_map[it][ind][node_label] = orig_graph_map[it][ind].get(node_label, 0) + 1 119 | 120 | ind +=1 121 | 122 | labels = copy.deepcopy(compressed_labels) 123 | 124 | if g2 != None: 125 | K = np.zeros((len(g1), len(g2))) 126 | for it in range(-1, h): 127 | for i in range(len(g1)): 128 | for j in range(len(g2)): 129 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][len(g1)+j].keys()) 130 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][len(g1)+j].get(k,0) for k in common_keys]) 131 | else: 132 | K = np.zeros((N, N)) 133 | for it in range(-1, h): 134 | for i in range(N): 135 | for j in range(N): 136 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][j].keys()) 137 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][j].get(k,0) for k in common_keys]) 138 | 139 | return K -------------------------------------------------------------------------------- /evaluation_poisson/graph_kernels_labeled.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | from collections import defaultdict 4 | import copy 5 | 6 | 7 | def sp_kernel(g1, g2=None): 8 | if g2 != None: 9 | graphs = [] 10 | for g in g1: 11 | graphs.append(g) 12 | for g in g2: 13 | graphs.append(g) 14 | else: 15 | graphs = g1 16 | 17 | N = len(graphs) 18 | all_paths = {} 19 | sp_counts = {} 20 | for i in range(N): 21 | sp_lengths = nx.shortest_path_length(graphs[i]) 22 | sp_counts[i] = {} 23 | nodes = graphs[i].nodes() 24 | for v1 in nodes: 25 | for v2 in nodes: 26 | if v2 in sp_lengths[v1]: 27 | label = tuple(sorted([graphs[i].node[v1]['label'], graphs[i].node[v2]['label']]) + [sp_lengths[v1][v2]]) 28 | if label in sp_counts[i]: 29 | sp_counts[i][label] += 1 30 | else: 31 | sp_counts[i][label] = 1 32 | 33 | if label not in all_paths: 34 | all_paths[label] = len(all_paths) 35 | 36 | phi = np.zeros((N,len(all_paths))) 37 | 38 | for i in range(N): 39 | for label in sp_counts[i]: 40 | phi[i,all_paths[label]] = sp_counts[i][label] 41 | 42 | if g2 != None: 43 | K = np.dot(phi[:len(g1),:],phi[len(g1):,:].T) 44 | else: 45 | K = np.dot(phi,phi.T) 46 | 47 | return K 48 | 49 | 50 | def wl_kernel(g1, g2=None, h=6): 51 | if g2 != None: 52 | graphs = [] 53 | for g in g1: 54 | graphs.append(g) 55 | for g in g2: 56 | graphs.append(g) 57 | else: 58 | graphs = g1 59 | 60 | labels = {} 61 | label_lookup = {} 62 | label_counter = 0 63 | 64 | N = len(graphs) 65 | 66 | orig_graph_map = {it: {i: defaultdict(lambda: 0) for i in range(N)} for it in range(-1, h)} 67 | 68 | # initial labeling 69 | ind = 0 70 | for G in graphs: 71 | labels[ind] = np.zeros(G.number_of_nodes(), dtype = np.int32) 72 | node2index = {} 73 | for node in G.nodes(): 74 | node2index[node] = len(node2index) 75 | 76 | for node in G.nodes(): 77 | label = G.node[node]['label'] 78 | if not (label in label_lookup): 79 | label_lookup[label] = len(label_lookup) 80 | 81 | labels[ind][node2index[node]] = label_lookup[label] 82 | orig_graph_map[-1][ind][label] = orig_graph_map[-1][ind].get(label, 0) + 1 83 | 84 | ind += 1 85 | 86 | compressed_labels = copy.deepcopy(labels) 87 | 88 | # WL iterations 89 | for it in range(h): 90 | unique_labels_per_h = set() 91 | label_lookup = {} 92 | ind = 0 93 | for G in graphs: 94 | node2index = {} 95 | for node in G.nodes(): 96 | node2index[node] = len(node2index) 97 | 98 | for node in G.nodes(): 99 | node_label = tuple([labels[ind][node2index[node]]]) 100 | neighbors = G.neighbors(node) 101 | if len(neighbors) > 0: 102 | neighbors_label = tuple([labels[ind][node2index[neigh]] for neigh in neighbors]) 103 | node_label = str(node_label) + "-" + str(sorted(neighbors_label)) 104 | if not (node_label in label_lookup): 105 | label_lookup[node_label] = len(label_lookup) 106 | 107 | compressed_labels[ind][node2index[node]] = label_lookup[node_label] 108 | orig_graph_map[it][ind][node_label] = orig_graph_map[it][ind].get(node_label, 0) + 1 109 | 110 | ind +=1 111 | 112 | labels = copy.deepcopy(compressed_labels) 113 | 114 | if g2 != None: 115 | K = np.zeros((len(g1), len(g2))) 116 | for it in range(-1, h): 117 | for i in range(len(g1)): 118 | for j in range(len(g2)): 119 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][len(g1)+j].keys()) 120 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][len(g1)+j].get(k,0) for k in common_keys]) 121 | else: 122 | K = np.zeros((N, N)) 123 | for it in range(-1, h): 124 | for i in range(N): 125 | for j in range(N): 126 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][j].keys()) 127 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][j].get(k,0) for k in common_keys]) 128 | 129 | return K -------------------------------------------------------------------------------- /evaluation_poisson/init.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torch.autograd import Variable 4 | 5 | def _calculate_fan_in_and_fan_out(tensor): 6 | dimensions = tensor.ndimension() 7 | if dimensions < 2: 8 | raise ValueError("Fan in and fan out can not be computed for tensor with less than 2 dimensions") 9 | 10 | if dimensions == 2: # Linear 11 | fan_in = tensor.size(1) 12 | fan_out = tensor.size(0) 13 | else: 14 | num_input_fmaps = tensor.size(1) 15 | num_output_fmaps = tensor.size(0) 16 | receptive_field_size = 1 17 | if tensor.dim() > 2: 18 | receptive_field_size = tensor[0][0].numel() 19 | fan_in = num_input_fmaps * receptive_field_size 20 | fan_out = num_output_fmaps * receptive_field_size 21 | 22 | return fan_in, fan_out 23 | 24 | 25 | def xavier_uniform(tensor, gain=1): 26 | """Fills the input Tensor or Variable with values according to the method 27 | described in "Understanding the difficulty of training deep feedforward 28 | neural networks" - Glorot, X. & Bengio, Y. (2010), using a uniform 29 | distribution. The resulting tensor will have values sampled from 30 | :math:`U(-a, a)` where 31 | :math:`a = gain \\times \sqrt{2 / (fan\_in + fan\_out)} \\times \sqrt{3}`. 32 | Also known as Glorot initialisation. 33 | 34 | Args: 35 | tensor: an n-dimensional torch.Tensor or autograd.Variable 36 | gain: an optional scaling factor 37 | 38 | Examples: 39 | >>> w = torch.Tensor(3, 5) 40 | >>> nn.init.xavier_uniform(w, gain=nn.init.calculate_gain('relu')) 41 | """ 42 | if isinstance(tensor, Variable): 43 | xavier_uniform(tensor.data, gain=gain) 44 | return tensor 45 | 46 | fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) 47 | std = gain * math.sqrt(2.0 / (fan_in + fan_out)) 48 | a = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation 49 | return tensor.uniform_(-a, a) 50 | 51 | 52 | def xavier_normal(tensor, gain=1): 53 | """Fills the input Tensor or Variable with values according to the method 54 | described in "Understanding the difficulty of training deep feedforward 55 | neural networks" - Glorot, X. & Bengio, Y. (2010), using a normal 56 | distribution. The resulting tensor will have values sampled from 57 | :math:`N(0, std)` where 58 | :math:`std = gain \\times \sqrt{2 / (fan\_in + fan\_out)}`. 59 | Also known as Glorot initialisation. 60 | 61 | Args: 62 | tensor: an n-dimensional torch.Tensor or autograd.Variable 63 | gain: an optional scaling factor 64 | 65 | Examples: 66 | >>> w = torch.Tensor(3, 5) 67 | >>> nn.init.xavier_normal(w) 68 | """ 69 | if isinstance(tensor, Variable): 70 | xavier_normal(tensor.data, gain=gain) 71 | return tensor 72 | 73 | fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) 74 | std = gain * math.sqrt(2.0 / (fan_in + fan_out)) 75 | return tensor.normal_(0, std) 76 | 77 | 78 | def orthogonal(tensor, gain=1): 79 | """Fills the input Tensor or Variable with a (semi) orthogonal matrix, as 80 | described in "Exact solutions to the nonlinear dynamics of learning in deep 81 | linear neural networks" - Saxe, A. et al. (2013). The input tensor must have 82 | at least 2 dimensions, and for tensors with more than 2 dimensions the 83 | trailing dimensions are flattened. 84 | 85 | Args: 86 | tensor: an n-dimensional torch.Tensor or autograd.Variable, where n >= 2 87 | gain: optional scaling factor 88 | 89 | Examples: 90 | >>> w = torch.Tensor(3, 5) 91 | >>> nn.init.orthogonal(w) 92 | """ 93 | if isinstance(tensor, Variable): 94 | orthogonal(tensor.data, gain=gain) 95 | return tensor 96 | 97 | if tensor.ndimension() < 2: 98 | raise ValueError("Only tensors with 2 or more dimensions are supported") 99 | 100 | rows = tensor.size(0) 101 | cols = tensor[0].numel() 102 | flattened = torch.Tensor(rows, cols).normal_(0, 1) 103 | 104 | if rows < cols: 105 | flattened.t_() 106 | 107 | # Compute the qr factorization 108 | q, r = torch.qr(flattened) 109 | # Make Q uniform according to https://arxiv.org/pdf/math-ph/0609050.pdf 110 | d = torch.diag(r, 0) 111 | ph = d.sign() 112 | q *= ph.expand_as(q) 113 | 114 | if rows < cols: 115 | q.t_() 116 | 117 | tensor.view_as(q).copy_(q) 118 | tensor.mul_(gain) 119 | return tensor -------------------------------------------------------------------------------- /evaluation_poisson/main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | #import torchvision.transforms as transforms 4 | import torch.utils.data as utils 5 | from torch.autograd import Variable 6 | import numpy as np 7 | from utils import compute_nystrom,create_train_test_loaders,imbalance 8 | from sklearn.preprocessing import LabelEncoder 9 | from sklearn.model_selection import KFold 10 | from model import CNN 11 | from graph_kernels import sp_kernel, wl_kernel 12 | import xlwt 13 | import xlrd 14 | from xlutils.copy import copy 15 | from xlwt import Style 16 | def writeExcel(row, col, value, file_name,styl=Style.default_style): 17 | rb = xlrd.open_workbook(file_name) 18 | wb = copy(rb) 19 | ws = wb.get_sheet(0) 20 | ws.write(row, col, value, styl) 21 | wb.save(file_name) 22 | 23 | community_detection = "louvain" 24 | 25 | # Hyper Parameters 26 | dim = 100 27 | batch_size = 10 28 | num_epochs = 150 29 | num_filters = 20 30 | hidden_size = 20 31 | learning_rate = 0.005 32 | use_node_labels = False 33 | kernels=[wl_kernel] 34 | num_kernels = len(kernels) 35 | 36 | def test_(): 37 | print("Computing feature maps...") 38 | Q, subgraphs, labels,shapes = compute_nystrom(use_node_labels, dim, community_detection, kernels) 39 | M=np.zeros((shapes[0],shapes[1],len(kernels))) 40 | for idx,k in enumerate(kernels): 41 | M[:,:,idx]=Q[idx] 42 | Q=M 43 | # Binarize labels 44 | le = LabelEncoder() 45 | y = le.fit_transform(labels) 46 | # Build vocabulary 47 | max_document_length = max([len(x.split(" ")) for x in subgraphs]) 48 | x = np.zeros((len(subgraphs), max_document_length), dtype=np.int32) 49 | for i in range(len(subgraphs)): 50 | communities = subgraphs[i].split() 51 | for j in range(len(communities)): 52 | x[i,j] = int(communities[j]) 53 | reg=x[0:2500] 54 | gen=x[2500:5000] 55 | mal=x[5000:] 56 | reg_label=y[:2500] 57 | gen_label=y[2500:5000] 58 | mal_label=y[5000:] 59 | 60 | train_reg=reg[0:1500] 61 | test_reg=reg[1500:] 62 | train_reg_y=reg_label[0:1500] 63 | test_reg_y=reg_label[1500:] 64 | 65 | train_mal=mal[0:1500] 66 | test_mal=mal[1500:] 67 | train_mal_y=mal_label[0:1500] 68 | test_mal_y=mal_label[1500:] 69 | 70 | train_gen=gen[0:1500] 71 | train_gen_y=gen_label[0:1500] 72 | 73 | train_fake=np.concatenate((train_reg,train_gen),axis=0) 74 | y_train_fake=np.concatenate((train_reg_y,train_gen_y),axis=0) 75 | train_real=np.concatenate((train_reg,train_mal),axis=0) 76 | y_train_real=np.concatenate((train_reg_y,train_mal_y),axis=0) 77 | test=np.concatenate((test_reg,test_mal),axis=0) 78 | y_test=np.concatenate((test_reg_y,test_mal_y),axis=0) 79 | 80 | 81 | def train_test(Q, x_train, x_test, y_train, y_test, batch_size): 82 | train_loader, test_loader = create_train_test_loaders(Q, x_train, x_test, y_train, y_test, batch_size) 83 | cnn = CNN(input_size=num_filters, hidden_size=hidden_size, num_classes=np.unique(y).size, dim=dim, num_kernels=num_kernels, max_document_length=max_document_length) 84 | if torch.cuda.is_available(): 85 | cnn.cuda() 86 | if torch.cuda.is_available(): 87 | criterion = nn.CrossEntropyLoss().cuda() 88 | else: 89 | criterion = nn.CrossEntropyLoss() 90 | optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate) 91 | for epoch in range(num_epochs): 92 | for i, (graphs, labels) in enumerate(train_loader): 93 | graphs = Variable(graphs) 94 | labels = Variable(labels) 95 | optimizer.zero_grad() 96 | outputs = cnn(graphs) 97 | if torch.cuda.is_available(): 98 | loss = criterion(outputs, labels.cuda()) 99 | else: 100 | loss = criterion(outputs, labels) 101 | loss.backward() 102 | optimizer.step() 103 | # Test the Model 104 | 105 | cnn.eval() 106 | correct = 0 107 | total = 0 108 | TP=0 109 | TN=0 110 | FP=0 111 | FN=0 112 | predict=[] 113 | label=[] 114 | output=[] 115 | for graphs, labels in test_loader: 116 | graphs = Variable(graphs) 117 | outputs = cnn(graphs) 118 | _, predicted = torch.max(outputs.data, 1) 119 | total += labels.size(0) 120 | correct += (predicted == labels.cuda()).sum() 121 | TP += (predicted+labels.cuda()==2).sum() 122 | FP+=(predicted*5+labels.cuda()*1==5).sum() 123 | FN+=(predicted*1+labels.cuda()*5==5).sum() 124 | TN+=(predicted+labels.cuda()==0).sum() 125 | predict.append(predicted) 126 | label.append(labels) 127 | output.append(outputs.data) 128 | if TP+FP==0: precision=0 129 | else: precision=TP/(TP+FP) 130 | if TP+FN==0: recall=0 131 | else: recall=TP/(TP+FN) 132 | l=np.zeros((len(label))) 133 | for i in range(len(label)): 134 | l[i]=int(label[i]) 135 | s=np.zeros((len(output))) 136 | for i in range(len(output)): 137 | s[i]=output[i][0][1] 138 | return TP,TN,FP,FN,precision,recall,l,s 139 | TP_fake,TN_fake,FP_fake,FN_fake,precision_fake,recall_fake,l_fake,s_fake=train_test(Q, train_fake, test, y_train_fake, y_test, batch_size) 140 | TP_real,TN_real,FP_real,FN_real,precision_real,recall_real,l_real,s_real=train_test(Q, train_real, test, y_train_real, y_test, batch_size) 141 | return TP_fake,TN_fake,FP_fake,FN_fake,precision_fake,recall_fake,l_fake,s_fake,TP_real,TN_real,FP_real,FN_real,precision_real,recall_real,l_real,s_real 142 | ''' 143 | num=0 144 | lab=[] 145 | sc=[] 146 | skip=[] 147 | for user in red_user: 148 | try: 149 | TP,TN,FP,FN,precision,recall,l,s=test_(user,red_user) 150 | writeExcel(int(num)+1,0, user, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 151 | writeExcel(int(num)+1,1, TN, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 152 | writeExcel(int(num)+1,2, TP, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 153 | writeExcel(int(num)+1,3, FN, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 154 | writeExcel(int(num)+1,4, FP, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 155 | writeExcel(int(num)+1,5, precision, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 156 | writeExcel(int(num)+1,6, recall, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 157 | print(num) 158 | num+=1 159 | lab.append(l) 160 | sc.append(s) 161 | except: skip.append(user) 162 | 163 | label_total=lab[0] 164 | score_total=sc[0] 165 | for i in range(97): 166 | label_total=np.concatenate((label_total,lab[i+1]),axis=0) 167 | score_total=np.concatenate((score_total,sc[i+1]),axis=0) 168 | #draw ROC tota 169 | from sklearn.metrics import roc_curve, auc 170 | import matplotlib.pyplot as plt 171 | fpr,tpr,threshold = roc_curve(label_total, score_total) 172 | roc_auc = auc(fpr,tpr) 173 | plt.figure() 174 | lw = 2 175 | plt.figure(figsize=(10,10)) 176 | plt.plot(fpr, tpr, color='darkorange', 177 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) 178 | plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') 179 | plt.xlim([0.0, 1.0]) 180 | plt.ylim([0.0, 1.05]) 181 | plt.xlabel('False Positive Rate') 182 | plt.ylabel('True Positive Rate') 183 | plt.title('ROC curve on KCNN model') 184 | plt.legend(loc="lower right") 185 | plt.show() ''' -------------------------------------------------------------------------------- /evaluation_poisson/model.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | from init import xavier_normal,xavier_uniform,orthogonal 4 | 5 | # CNN Model 6 | class CNN(nn.Module): 7 | def __init__(self, input_size, hidden_size, num_classes, dim, num_kernels, max_document_length): 8 | super(CNN, self).__init__() 9 | self.max_document_length = max_document_length 10 | self.conv = nn.Conv3d(1, input_size, (1, 1, dim), padding=0) 11 | self.fc1 = nn.Linear(input_size*num_kernels, hidden_size) 12 | self.fc2 = nn.Linear(hidden_size, num_classes) 13 | self.init_weights() 14 | 15 | def init_weights(self): 16 | xavier_uniform(self.conv.weight.data) 17 | xavier_normal(self.fc1.weight.data) 18 | xavier_normal(self.fc2.weight.data) 19 | 20 | def forward(self, x_in): 21 | out = F.relu(F.max_pool3d(self.conv(x_in), (1, self.max_document_length,1))) 22 | out = out.view(out.size(0), -1) 23 | out = F.relu(self.fc1(out)) 24 | out = F.dropout(out, training=self.training) 25 | out = self.fc2(out) 26 | return F.softmax(out) 27 | -------------------------------------------------------------------------------- /evaluation_poisson/network_feature.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Mar 19 22:52:58 2018 4 | 5 | @author: gxjco 6 | """ 7 | 8 | import networkx as nx 9 | import numpy as np 10 | import os 11 | with open('C:/Users/gxjco/Desktop/redteam.txt/redteam.txt') as f: 12 | red_team=f.readlines() 13 | red=[] 14 | red_user={} 15 | for i in red_team: 16 | act={} 17 | m=i.split(',') 18 | act['time']=m[0] 19 | act['user']=m[1].split('@')[0] 20 | act['s_c']=m[2] 21 | act['d_c']=m[3].split('\n')[0] 22 | red.append(act) 23 | if act['user'] not in red_user: 24 | red_user[act['user']]=[] 25 | for act_ in red: 26 | red_user[act_['user']].append(act_) 27 | def plus(a,b): 28 | for i in b: 29 | if i not in a: 30 | a.append(i) 31 | return a 32 | def feature(node,edge): 33 | G =nx.MultiGraph() 34 | G=G.to_directed() 35 | G.add_nodes_from(node) 36 | G.add_edges_from(edge) 37 | f=np.zeros(10) 38 | f[0]=len(G.nodes) 39 | f[1]=len(G.edges) 40 | f[2]=nx.density(G) 41 | f[3]=nx.degree_pearson_correlation_coefficient(G) 42 | f[4]=nx.algorithms.reciprocity(G) 43 | f[5]=0#nx.transitivity(G) 44 | f[6]=nx.is_weakly_connected(G) 45 | f[7]=nx.number_weakly_connected_components(G) 46 | f[8]=nx.is_strongly_connected(G) 47 | f[9]=nx.number_strongly_connected_components(G) 48 | return f 49 | b=0 50 | path='C:/Users/gxjco/Desktop/redteam.txt/user_new/' 51 | files=os.listdir(path) 52 | for file_ in files: 53 | u=[] 54 | user=file_.split('.')[0] 55 | with open(path+file_) as f: 56 | for line in f: 57 | m=line.split(',') 58 | event={} 59 | event['time']=m[0] 60 | event['user']=m[1].split('@')[0] 61 | event['s_c']=m[3] 62 | event['d_c']=m[4] 63 | u.append(event) 64 | l=red_user[user] 65 | u=plus(u,l) 66 | act={} 67 | for i in range(2600): 68 | act[i]={} 69 | act[i]['node']=[] 70 | act[i]['edge']=[] 71 | for line in u: 72 | if int(line['time'])<4680000: 73 | act[int(int(line['time'])/1800)]['node'].append(line['s_c']) 74 | act[int(int(line['time'])/1800)]['node'].append(line['d_c']) 75 | act[int(int(line['time'])/1800)]['edge'].append((line['s_c'],line['d_c'])) 76 | sample=np.zeros((2600,10)) 77 | for i in range(2600): 78 | try: 79 | sample[i,:]=feature(act[i]['node'],act[i]['edge']) 80 | except:b+=1 81 | 82 | time_red={} 83 | for i in red_user[user]: 84 | time_red[i['time']]=[] 85 | label=[] 86 | for t in range(2600): 87 | o=0 88 | for time in time_red: 89 | if int(time) in range(1800*t,1800*t+1800): 90 | o=1 91 | break 92 | label.append(o) 93 | #seperate the negative and positive 94 | p_data=[] 95 | n_data=[] 96 | for i in range(2600): 97 | if label[i]==1: 98 | p_data.append(sample[i]) 99 | else: 100 | n_data.append(sample[i]) 101 | p_sample=np.zeros((len(p_data),10)) 102 | n_sample=np.zeros((len(n_data),10)) 103 | for i in range(len(p_data)): 104 | p_sample[i]=p_data[i] 105 | for i in range(len(n_data)): 106 | n_sample[i]=n_data[i] 107 | np.save('C:/Users/gxjco/Desktop/redteam.txt/input_new/feature_new_negative_sample__'+user+'.npy',n_sample) 108 | np.save('C:/Users/gxjco/Desktop/redteam.txt/input_new/feature_new_positive_sample__'+user+'.npy',p_sample) 109 | 110 | 111 | -------------------------------------------------------------------------------- /evaluation_poisson/nystrom.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.sparse as sp 3 | from scipy.linalg import svd 4 | from sklearn.utils import check_random_state 5 | import graph_kernels 6 | 7 | class Nystrom(): 8 | 9 | def __init__(self, kernel, kernel_params=None, n_components=100, random_state=None): 10 | self.kernel = kernel 11 | self.kernel_params = kernel_params 12 | self.n_components = n_components 13 | self.random_state = random_state 14 | 15 | def fit(self, graphs, y=None): 16 | rnd = check_random_state(self.random_state) 17 | n_samples = len(graphs) 18 | 19 | # get basis vectors 20 | if self.n_components > n_samples: 21 | n_components = n_samples 22 | else: 23 | n_components = self.n_components 24 | n_components = min(n_samples, n_components) 25 | inds = rnd.permutation(n_samples) 26 | basis_inds = inds[:n_components] 27 | basis = [] 28 | for ind in basis_inds: 29 | basis.append(graphs[ind]) 30 | basis_kernel = self.kernel(basis, basis, **self._get_kernel_params()) 31 | 32 | # sqrt of kernel matrix on basis vectors 33 | U, S, V = svd(basis_kernel) 34 | S = np.maximum(S, 1e-12) 35 | self.normalization_ = np.dot(U * 1. / np.sqrt(S), V) 36 | self.components_ = basis 37 | self.component_indices_ = inds 38 | return self 39 | 40 | def transform(self, graphs): 41 | embedded = self.kernel(graphs, self.components_, **self._get_kernel_params()) 42 | return np.dot(embedded, self.normalization_.T) 43 | 44 | def _get_kernel_params(self): 45 | params = self.kernel_params 46 | if params is None: 47 | params = {} 48 | 49 | return params 50 | 51 | -------------------------------------------------------------------------------- /evaluation_poisson/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | import networkx as nx 3 | import igraph as ig 4 | import numpy as np 5 | from nystrom import Nystrom 6 | import torch 7 | import torch.utils.data as utils 8 | from imblearn.over_sampling import SMOTE 9 | ''' 10 | def load(user,red_user): 11 | def plus(a,b): 12 | for i in b: 13 | if i not in a: 14 | a.append(i) 15 | return a 16 | path='C:/Users/gxjco/Desktop/New folder (2)/red_user_data_new/user/' 17 | with open(path+user+'.txt') as f: 18 | u=[] 19 | for line in f: 20 | m=line.split(',') 21 | event={} 22 | event['time']=m[0] 23 | event['user']=m[1].split('@')[0] 24 | event['s_c']=m[3] 25 | event['d_c']=m[4] 26 | u.append(event) 27 | l=red_user[user] 28 | u=plus(u,l) 29 | act={} 30 | for i in range(2600): 31 | act[i]={} 32 | act[i]['node']=[] 33 | act[i]['edge']=[] 34 | for line in u: 35 | if int(line['time'])<4680000: 36 | act[int(int(line['time'])/1800)]['node'].append(line['s_c']) 37 | act[int(int(line['time'])/1800)]['node'].append(line['d_c']) 38 | act[int(int(line['time'])/1800)]['edge'].append((line['s_c'],line['d_c'])) 39 | Gs = [] 40 | for i in range(2600): 41 | G =nx.MultiGraph() 42 | G=G.to_directed() 43 | G.add_nodes_from(act[i]['node']) 44 | G.add_edges_from(act[i]['edge']) 45 | Gs.append(G) 46 | time_red={} 47 | for i in red_user[user]: 48 | time_red[i['time']]=[] 49 | label=[] 50 | for t in range(2600): 51 | o=0 52 | for time in time_red: 53 | if int(time) in range(1800*t,1800*t+1800): 54 | o=1 55 | break 56 | label.append(o) 57 | labels = np.array(label, dtype = np.float) 58 | return Gs, labels''' 59 | 60 | def load(): 61 | def graph(A): 62 | G =nx.MultiGraph() 63 | G=G.to_directed() 64 | for i in range(A.shape[0]): 65 | for j in range(A.shape[0]): 66 | if A[i][j]>0: 67 | G.add_nodes_from([str(i),str(j)]) 68 | for m in range(A[i][j]): 69 | G.add_edges_from([(str(i),str(j))]) 70 | 71 | return G 72 | reg=np.load('./dataset/poisson random/10/'+'poisson_reg10.npy').astype(int)[2500:] 73 | gen=np.load('./poisson_gen10.npy').astype(int)[:,:,:,0] #upload the generated graphs 74 | mal=np.load('./dataset/poisson random/10/'+'poisson_mal10.npy').astype(int)[2500:] 75 | Gs_reg=[] 76 | Gs_gen=[] 77 | Gs_mal=[] 78 | for i in range(len(reg)): 79 | Gs_reg.append(graph(reg[i])) 80 | Gs_gen.append(graph(gen[i])) 81 | for i in range(len(mal)): 82 | Gs_mal.append(graph(mal[i])) 83 | label_reg=np.zeros(len(reg)) 84 | label_gen=np.ones(len(gen)) 85 | label_mal=np.ones(len(mal)) 86 | return Gs_reg,label_reg,Gs_gen,label_gen,Gs_mal,label_mal 87 | 88 | 89 | 90 | def networkx_to_igraph(G): 91 | mapping = dict(zip(G.nodes(),range(G.number_of_nodes()))) 92 | reverse_mapping = dict(zip(range(G.number_of_nodes()),G.nodes())) 93 | G = nx.relabel_nodes(G,mapping) 94 | G_ig = ig.Graph(len(G), list(zip(*list(zip(*nx.to_edgelist(G)))[:2]))) 95 | return G_ig, reverse_mapping 96 | 97 | 98 | def community_detection(G_networkx, community_detection_method): 99 | G,reverse_mapping = networkx_to_igraph(G_networkx) 100 | if community_detection_method == "eigenvector": 101 | c = G.community_leading_eigenvector() 102 | elif community_detection_method == "infomap": 103 | c = G.community_infomap() 104 | elif community_detection_method == "fastgreedy": 105 | c = G.community_fastgreedy().as_clustering() 106 | elif community_detection_method == "label_propagation": 107 | c = G.community_label_propagation() 108 | elif community_detection_method == "louvain": 109 | c = G.community_multilevel() 110 | elif community_detection_method == "spinglass": 111 | c = G.community_spinglass() 112 | elif community_detection_method == "walktrap": 113 | c = G.community_walktrap().as_clustering() 114 | else: 115 | c = [] 116 | 117 | communities = [] 118 | for i in range(len(c)): 119 | community = [] 120 | for j in range(len(c[i])): 121 | community.append(reverse_mapping[G.vs[c[i][j]].index]) 122 | 123 | communities.append(community) 124 | 125 | return communities 126 | 127 | def compute_communities(graphs, use_node_labels, community_detection_method): 128 | communities = [] 129 | subgraphs = [] 130 | counter = 0 131 | coms = [] 132 | for G in graphs: 133 | c = community_detection(G, community_detection_method) 134 | coms.append(len(c)) 135 | subgraph = [] 136 | for i in range(len(c)): 137 | communities.append(G.subgraph(c[i])) 138 | subgraph.append(counter) 139 | counter += 1 140 | 141 | subgraphs.append(' '.join(str(s) for s in subgraph)) 142 | 143 | return communities, subgraphs 144 | 145 | 146 | def compute_nystrom(use_node_labels, embedding_dim, community_detection_method, kernels): 147 | graphs_reg, labels_reg,graphs_gen, labels_gen,graphs_mal, labels_mal = load() 148 | graphs=graphs_reg+graphs_gen+graphs_mal 149 | labels=np.concatenate((labels_reg,labels_gen,labels_mal),axis=0) 150 | communities, subgraphs = compute_communities(graphs, use_node_labels, community_detection_method) 151 | 152 | print("Number of communities: ", len(communities)) 153 | lens = [] 154 | for community in communities: 155 | lens.append(community.number_of_nodes()) 156 | 157 | print("Average size: %.2f" % np.mean(lens)) 158 | Q=[] 159 | for idx, k in enumerate(kernels): 160 | model = Nystrom(k, n_components=embedding_dim) 161 | model.fit(communities) 162 | Q_t = model.transform(communities) 163 | Q_t = np.vstack([np.zeros(embedding_dim), Q_t]) 164 | Q.append(Q_t) 165 | 166 | return Q, subgraphs, labels, Q_t.shape 167 | 168 | 169 | def create_train_test_loaders(Q, x_train, x_test, y_train, y_test, batch_size): 170 | num_kernels = Q.shape[2] 171 | max_document_length = x_train.shape[1] 172 | dim = Q.shape[1] 173 | 174 | my_x = [] 175 | for i in range(x_train.shape[0]): 176 | temp = np.zeros((1, num_kernels, max_document_length, dim)) 177 | for j in range(num_kernels): 178 | for k in range(x_train.shape[1]): 179 | temp[0,j,k,:] = Q[x_train[i,k],:,j].squeeze() 180 | my_x.append(temp) 181 | 182 | if torch.cuda.is_available(): 183 | tensor_x = torch.stack([torch.cuda.FloatTensor(i) for i in my_x]) # transform to torch tensors 184 | tensor_y = torch.cuda.LongTensor(y_train.tolist()) 185 | else: 186 | tensor_x = torch.stack([torch.Tensor(i) for i in my_x]) # transform to torch tensors 187 | tensor_y = torch.from_numpy(np.asarray(y_train,dtype=np.int64)) 188 | 189 | train_dataset = utils.TensorDataset(tensor_x, tensor_y) 190 | train_loader = utils.DataLoader(train_dataset, batch_size=batch_size, shuffle=True) 191 | 192 | my_x = [] 193 | for i in range(x_test.shape[0]): 194 | temp = np.zeros((1, num_kernels, max_document_length, dim)) 195 | for j in range(num_kernels): 196 | for k in range(x_test.shape[1]): 197 | temp[0,j,k,:] = Q[x_test[i,k],:,j].squeeze() 198 | my_x.append(temp) 199 | 200 | if torch.cuda.is_available(): 201 | tensor_x = torch.stack([torch.cuda.FloatTensor(i) for i in my_x]) # transform to torch tensors 202 | tensor_y = torch.cuda.LongTensor(y_test.tolist()) 203 | else: 204 | tensor_x = torch.stack([torch.Tensor(i) for i in my_x]) # transform to torch tensors 205 | tensor_y = torch.from_numpy(np.asarray(y_test,dtype=np.int64)) 206 | 207 | test_dataset = utils.TensorDataset(tensor_x, tensor_y) 208 | test_loader = utils.DataLoader(test_dataset, batch_size=1, shuffle=False) 209 | 210 | return train_loader, test_loader 211 | 212 | def imbalance(x,y): 213 | p_num=sum(y) 214 | if p_num>1: 215 | sm=SMOTE(k_neighbors=p_num-1) 216 | data_res, label_res = sm.fit_sample(x, y) 217 | else: 218 | for i in range(2600): 219 | if y[i]==1: 220 | idx=i 221 | break 222 | new_p=np.zeros((2598,x.shape[1])) 223 | new_p_l=np.ones((2598)) 224 | for i in range(2598): 225 | new_p[i,:]=x[idx] 226 | data_res=np.concatenate((x,new_p),axis=0) 227 | label_res=np.concatenate((y,new_p_l),axis=0) 228 | return data_res.astype(dtype=np.int32),label_res.astype(dtype=np.int32) 229 | 230 | 231 | 232 | 233 | 234 | 235 | -------------------------------------------------------------------------------- /evaluation_poisson/validation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sun Apr 15 20:43:11 2018 4 | 5 | @author: gxjco 6 | """ 7 | import xlwt 8 | import xlrd 9 | from xlutils.copy import copy 10 | from xlwt import Style 11 | from main import* 12 | import numpy as np 13 | 14 | 15 | 16 | def writeExcel(row, col, value, file_name,styl=Style.default_style): 17 | rb = xlrd.open_workbook(file_name) 18 | wb = copy(rb) 19 | ws = wb.get_sheet(0) 20 | ws.write(row, col, value, styl) 21 | wb.save(file_name) 22 | 23 | TP_fake,TN_fake,FP_fake,FN_fake,precision_fake,recall_fake,l_fake,s_fake, TP_real,TN_real,FP_real,FN_real,precision_real,recall_real,l_real,s_real=test_() 24 | 25 | 26 | #draw ROC tota 27 | def draw_roc(label_total_real, score_total_real,label_total_fake, score_total_fake): 28 | from sklearn.metrics import roc_curve, auc 29 | import matplotlib.pyplot as plt 30 | fpr1,tpr1,threshold1 = roc_curve(label_total_fake, score_total_fake) 31 | roc_auc1 = auc(fpr1,tpr1) 32 | fpr2,tpr2,threshold2 = roc_curve(label_total_real, score_total_real) 33 | roc_auc2 = auc(fpr2,tpr2) 34 | plt.figure() 35 | lw = 2 36 | plt.figure(figsize=(10,10)) 37 | plt.plot(fpr1, tpr1, color='darkorange', 38 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc1) 39 | plt.plot(fpr2, tpr2, color='green', 40 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc2) 41 | plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') 42 | plt.xlim([0.0, 1.0]) 43 | plt.ylim([0.0, 1.05]) 44 | plt.xlabel('False Positive Rate') 45 | plt.ylabel('True Positive Rate') 46 | plt.title('ROC curve on KCNN model') 47 | plt.legend(loc="lower right") 48 | plt.show() 49 | draw_roc(l_real, s_real,l_fake, s_fake) -------------------------------------------------------------------------------- /evaluation_scale free/1: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /evaluation_scale free/feature_logic_model.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sun Mar 18 19:57:38 2018 4 | 5 | @author: gxjco 6 | """ 7 | import numpy as np 8 | from imblearn.over_sampling import SMOTE 9 | import xlwt 10 | import xlrd 11 | import os 12 | from xlutils.copy import copy 13 | from xlwt import Style 14 | from sklearn.linear_model import LogisticRegression 15 | import math 16 | from sklearn import metrics 17 | from sklearn.metrics import recall_score 18 | from sklearn.metrics import precision_score 19 | from imblearn.over_sampling import SMOTE 20 | def read_data(user,if_smote): 21 | n_data_=np.load('C:/Users/gxjco/Desktop/redteam.txt/input_feature/feature_new_negative_sample__'+user+'.npy') 22 | p_data_=np.load('C:/Users/gxjco/Desktop/redteam.txt/input_feature/feature_new_positive_sample__'+user+'.npy') 23 | for i in range(len(n_data_)): 24 | if math.isnan(n_data_[i,3]): n_data_[i,3]=0 25 | for i in range(len(p_data_)): 26 | if math.isnan(p_data_[i,3]): p_data_[i,3]=0 27 | if len(p_data_)>1: 28 | data=np.concatenate((n_data_,p_data_),axis=0) 29 | #data=np.reshape(data,[data.shape[0],data.shape[1]*data.shape[2]]) 30 | label=np.concatenate((np.zeros(len(n_data_),dtype=np.int32),np.ones(len(p_data_),dtype=np.int32)),axis=0) 31 | sm=SMOTE(k_neighbors=len(p_data_)-1) 32 | data_res, label_res = sm.fit_sample(data, label) 33 | n=data_res[0:int(len(data_res)/2)] 34 | p=data_res[int(len(data_res)/2):] 35 | n_data=n 36 | p_data=p 37 | else: 38 | n_data=n_data_#np.reshape(n_data_,[n_data_.shape[0],n_data_.shape[1]*n_data_.shape[2]]) 39 | p_data_=p_data_#np.reshape(p_data_,[p_data_.shape[0],p_data_.shape[1]*p_data_.shape[2]]) 40 | p_data=n_data 41 | for i in range(len(p_data)): 42 | p_data[i,:]=p_data_[0] 43 | size_n=int(len(n_data)/2) 44 | size_p=int(len(p_data)/2) 45 | n_label=np.zeros(len(n_data)) 46 | p_label=np.ones(len(p_data)) 47 | train_d=np.concatenate((n_data[0:size_n],p_data[0:size_p]), axis=0) 48 | test_d=np.concatenate((n_data[size_n:],p_data[size_p:]), axis=0) 49 | train_l=np.concatenate((n_label[0:size_n],p_label[0:size_p]), axis=0) 50 | test_l=np.concatenate((n_label[size_n:],p_label[size_p:]), axis=0) 51 | return train_d,test_d,train_l,test_l 52 | 53 | def result(predictions, labels,score): 54 | r11=0 55 | r00=0 56 | r10=0 57 | r01=0 58 | for i in range(len(predictions)): 59 | if predictions[i]==1 and labels[i]==1: r11+=1 60 | if predictions[i]==0 and labels[i]==0: r00+=1 61 | if predictions[i]==1 and labels[i]==0: r10+=1 62 | if predictions[i]==0 and labels[i]==1: r01+=1 63 | fpr, tpr, thresholds = metrics.roc_curve(labels,score,pos_label=1) 64 | AUC=metrics.auc(fpr, tpr) 65 | precision=precision_score(labels,predictions) 66 | recall=recall_score(labels,predictions) 67 | return r11,r00,r01,r10,AUC,precision,recall 68 | def writeExcel(row, col, value, file_name,styl=Style.default_style): 69 | rb = xlrd.open_workbook(file_name) 70 | wb = copy(rb) 71 | ws = wb.get_sheet(0) 72 | ws.write(row, col, value, styl) 73 | wb.save(file_name) 74 | def main(user): 75 | train_data,test_data,train_label,test_label=read_data(user,True) 76 | classifier = LogisticRegression() 77 | classifier.fit(train_data, train_label) 78 | prediction = classifier.predict(test_data) 79 | score=classifier.predict_proba(test_data) 80 | result_={} 81 | result_['TP'],result_['TN'],result_['FN'],result_['FP'],result_['AUC'],result_['precision'],result_['recall']=result(prediction, test_label,score[:,1]) 82 | return result_,test_label,score[:,1] 83 | 84 | with open('redteam.txt') as f: 85 | red_team=f.readlines() 86 | red=[] 87 | red_user={} 88 | for i in red_team: 89 | act={} 90 | m=i.split(',') 91 | act['time']=str(m[0]) 92 | act['user']=m[1].split('@')[0] 93 | act['s_c']=m[2] 94 | act['d_c']=m[3].split('\n')[0] 95 | red.append(act) 96 | if act['user'] not in red_user: 97 | red_user[act['user']]=[] 98 | num=0 99 | skip1=[] 100 | lab=[] 101 | sc=[] 102 | for user in red_user: 103 | try: 104 | acc,l,s=main(user) 105 | #writeExcel(int(num)+1,0, user, 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 106 | #writeExcel(int(num)+1,1, acc['TN'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 107 | #writeExcel(int(num)+1,2, acc['TP'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 108 | #writeExcel(int(num)+1,3, acc['FN'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 109 | #writeExcel(int(num)+1,4, acc['FP'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 110 | #writeExcel(int(num)+1,5, acc['AUC'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 111 | #writeExcel(int(num)+1,6, acc['precision'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 112 | #writeExcel(int(num)+1,7, acc['recall'], 'C:/Users/gxjco/Desktop/redteam.txt/feature_result_logic.xls') 113 | print(num) 114 | num+=1 115 | lab.append(l) 116 | sc.append(s) 117 | except: skip1.append(user) 118 | 119 | label_total=lab[0] 120 | score_total=sc[0] 121 | for i in range(97): 122 | label_total=np.concatenate((label_total,lab[i+1]),axis=0) 123 | score_total=np.concatenate((score_total,sc[i+1]),axis=0) 124 | #draw ROC tota 125 | from sklearn.metrics import roc_curve, auc 126 | import matplotlib.pyplot as plt 127 | fpr,tpr,threshold = roc_curve(label_total, score_total) 128 | roc_auc = auc(fpr,tpr) 129 | plt.figure() 130 | lw = 2 131 | plt.figure(figsize=(10,10)) 132 | plt.plot(fpr, tpr, color='darkorange', 133 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) 134 | plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') 135 | plt.xlim([0.0, 1.0]) 136 | plt.ylim([0.0, 1.05]) 137 | plt.xlabel('False Positive Rate') 138 | plt.ylabel('True Positive Rate') 139 | plt.title('Receiver operating characteristic on 10_feature model') 140 | plt.legend(loc="lower right") 141 | plt.show() -------------------------------------------------------------------------------- /evaluation_scale free/graph_kernels.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | from collections import defaultdict 4 | import copy 5 | from scipy.sparse import lil_matrix 6 | 7 | def sp_kernel(g1, g2=None): 8 | if g2 != None: 9 | graphs = [] 10 | for g in g1: 11 | graphs.append(g) 12 | for g in g2: 13 | graphs.append(g) 14 | else: 15 | graphs = g1 16 | 17 | sp_lengths = [] 18 | 19 | for graph in graphs: 20 | sp_lengths.append(nx.shortest_path_length(graph)) 21 | 22 | N = len(graphs) 23 | all_paths = {} 24 | sp_counts = {} 25 | for i in range(N): 26 | sp_counts[i] = {} 27 | nodes = graphs[i].nodes() 28 | for v1 in nodes: 29 | for v2 in nodes: 30 | if v2 in sp_lengths[i][v1]: 31 | label = sp_lengths[i][v1][v2] 32 | if label in sp_counts[i]: 33 | sp_counts[i][label] += 1 34 | else: 35 | sp_counts[i][label] = 1 36 | 37 | if label not in all_paths: 38 | all_paths[label] = len(all_paths) 39 | 40 | phi = lil_matrix((N,len(all_paths))) 41 | 42 | for i in range(N): 43 | for label in sp_counts[i]: 44 | phi[i,all_paths[label]] = sp_counts[i][label] 45 | 46 | if g2 != None: 47 | K = np.dot(phi[:len(g1),:],phi[len(g1):,:].T) 48 | else: 49 | K = np.dot(phi,phi.T) 50 | 51 | K = np.asarray(K.todense()) 52 | 53 | return K 54 | 55 | 56 | def wl_kernel(g1, g2=None, h=6): 57 | if g2 != None: 58 | graphs = [] 59 | for g in g1: 60 | graphs.append(g) 61 | for g in g2: 62 | graphs.append(g) 63 | else: 64 | graphs = g1 65 | 66 | for G in graphs: 67 | for node in G.nodes(): 68 | G.node[node]['label'] = G.degree(node) 69 | 70 | labels = {} 71 | label_lookup = {} 72 | label_counter = 0 73 | 74 | N = len(graphs) 75 | 76 | orig_graph_map = {it: {i: defaultdict(lambda: 0) for i in range(N)} for it in range(-1, h)} 77 | 78 | # initial labeling 79 | ind = 0 80 | for G in graphs: 81 | labels[ind] = np.zeros(G.number_of_nodes(), dtype = np.int32) 82 | node2index = {} 83 | for node in G.nodes(): 84 | node2index[node] = len(node2index) 85 | 86 | for node in G.nodes(): 87 | label = G.node[node]['label'] 88 | if not (label in label_lookup): 89 | label_lookup[label] = len(label_lookup) 90 | 91 | labels[ind][node2index[node]] = label_lookup[label] 92 | orig_graph_map[-1][ind][label] = orig_graph_map[-1][ind].get(label, 0) + 1 93 | 94 | ind += 1 95 | 96 | compressed_labels = copy.deepcopy(labels) 97 | 98 | # WL iterations 99 | for it in range(h): 100 | unique_labels_per_h = set() 101 | label_lookup = {} 102 | ind = 0 103 | for G in graphs: 104 | node2index = {} 105 | for node in G.nodes(): 106 | node2index[node] = len(node2index) 107 | 108 | for node in G.nodes(): 109 | node_label = tuple([labels[ind][node2index[node]]]) 110 | neighbors = list(G.neighbors(node)) 111 | if len(neighbors) > 0: 112 | neighbors_label = tuple([labels[ind][node2index[neigh]] for neigh in neighbors]) 113 | node_label = str(node_label) + "-" + str(sorted(neighbors_label)) 114 | if not (node_label in label_lookup): 115 | label_lookup[node_label] = len(label_lookup) 116 | 117 | compressed_labels[ind][node2index[node]] = label_lookup[node_label] 118 | orig_graph_map[it][ind][node_label] = orig_graph_map[it][ind].get(node_label, 0) + 1 119 | 120 | ind +=1 121 | 122 | labels = copy.deepcopy(compressed_labels) 123 | 124 | if g2 != None: 125 | K = np.zeros((len(g1), len(g2))) 126 | for it in range(-1, h): 127 | for i in range(len(g1)): 128 | for j in range(len(g2)): 129 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][len(g1)+j].keys()) 130 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][len(g1)+j].get(k,0) for k in common_keys]) 131 | else: 132 | K = np.zeros((N, N)) 133 | for it in range(-1, h): 134 | for i in range(N): 135 | for j in range(N): 136 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][j].keys()) 137 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][j].get(k,0) for k in common_keys]) 138 | 139 | return K -------------------------------------------------------------------------------- /evaluation_scale free/graph_kernels_labeled.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | from collections import defaultdict 4 | import copy 5 | 6 | 7 | def sp_kernel(g1, g2=None): 8 | if g2 != None: 9 | graphs = [] 10 | for g in g1: 11 | graphs.append(g) 12 | for g in g2: 13 | graphs.append(g) 14 | else: 15 | graphs = g1 16 | 17 | N = len(graphs) 18 | all_paths = {} 19 | sp_counts = {} 20 | for i in range(N): 21 | sp_lengths = nx.shortest_path_length(graphs[i]) 22 | sp_counts[i] = {} 23 | nodes = graphs[i].nodes() 24 | for v1 in nodes: 25 | for v2 in nodes: 26 | if v2 in sp_lengths[v1]: 27 | label = tuple(sorted([graphs[i].node[v1]['label'], graphs[i].node[v2]['label']]) + [sp_lengths[v1][v2]]) 28 | if label in sp_counts[i]: 29 | sp_counts[i][label] += 1 30 | else: 31 | sp_counts[i][label] = 1 32 | 33 | if label not in all_paths: 34 | all_paths[label] = len(all_paths) 35 | 36 | phi = np.zeros((N,len(all_paths))) 37 | 38 | for i in range(N): 39 | for label in sp_counts[i]: 40 | phi[i,all_paths[label]] = sp_counts[i][label] 41 | 42 | if g2 != None: 43 | K = np.dot(phi[:len(g1),:],phi[len(g1):,:].T) 44 | else: 45 | K = np.dot(phi,phi.T) 46 | 47 | return K 48 | 49 | 50 | def wl_kernel(g1, g2=None, h=6): 51 | if g2 != None: 52 | graphs = [] 53 | for g in g1: 54 | graphs.append(g) 55 | for g in g2: 56 | graphs.append(g) 57 | else: 58 | graphs = g1 59 | 60 | labels = {} 61 | label_lookup = {} 62 | label_counter = 0 63 | 64 | N = len(graphs) 65 | 66 | orig_graph_map = {it: {i: defaultdict(lambda: 0) for i in range(N)} for it in range(-1, h)} 67 | 68 | # initial labeling 69 | ind = 0 70 | for G in graphs: 71 | labels[ind] = np.zeros(G.number_of_nodes(), dtype = np.int32) 72 | node2index = {} 73 | for node in G.nodes(): 74 | node2index[node] = len(node2index) 75 | 76 | for node in G.nodes(): 77 | label = G.node[node]['label'] 78 | if not (label in label_lookup): 79 | label_lookup[label] = len(label_lookup) 80 | 81 | labels[ind][node2index[node]] = label_lookup[label] 82 | orig_graph_map[-1][ind][label] = orig_graph_map[-1][ind].get(label, 0) + 1 83 | 84 | ind += 1 85 | 86 | compressed_labels = copy.deepcopy(labels) 87 | 88 | # WL iterations 89 | for it in range(h): 90 | unique_labels_per_h = set() 91 | label_lookup = {} 92 | ind = 0 93 | for G in graphs: 94 | node2index = {} 95 | for node in G.nodes(): 96 | node2index[node] = len(node2index) 97 | 98 | for node in G.nodes(): 99 | node_label = tuple([labels[ind][node2index[node]]]) 100 | neighbors = G.neighbors(node) 101 | if len(neighbors) > 0: 102 | neighbors_label = tuple([labels[ind][node2index[neigh]] for neigh in neighbors]) 103 | node_label = str(node_label) + "-" + str(sorted(neighbors_label)) 104 | if not (node_label in label_lookup): 105 | label_lookup[node_label] = len(label_lookup) 106 | 107 | compressed_labels[ind][node2index[node]] = label_lookup[node_label] 108 | orig_graph_map[it][ind][node_label] = orig_graph_map[it][ind].get(node_label, 0) + 1 109 | 110 | ind +=1 111 | 112 | labels = copy.deepcopy(compressed_labels) 113 | 114 | if g2 != None: 115 | K = np.zeros((len(g1), len(g2))) 116 | for it in range(-1, h): 117 | for i in range(len(g1)): 118 | for j in range(len(g2)): 119 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][len(g1)+j].keys()) 120 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][len(g1)+j].get(k,0) for k in common_keys]) 121 | else: 122 | K = np.zeros((N, N)) 123 | for it in range(-1, h): 124 | for i in range(N): 125 | for j in range(N): 126 | common_keys = set(orig_graph_map[it][i].keys()) & set(orig_graph_map[it][j].keys()) 127 | K[i][j] += sum([orig_graph_map[it][i].get(k,0)*orig_graph_map[it][j].get(k,0) for k in common_keys]) 128 | 129 | return K -------------------------------------------------------------------------------- /evaluation_scale free/init.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torch.autograd import Variable 4 | 5 | def _calculate_fan_in_and_fan_out(tensor): 6 | dimensions = tensor.ndimension() 7 | if dimensions < 2: 8 | raise ValueError("Fan in and fan out can not be computed for tensor with less than 2 dimensions") 9 | 10 | if dimensions == 2: # Linear 11 | fan_in = tensor.size(1) 12 | fan_out = tensor.size(0) 13 | else: 14 | num_input_fmaps = tensor.size(1) 15 | num_output_fmaps = tensor.size(0) 16 | receptive_field_size = 1 17 | if tensor.dim() > 2: 18 | receptive_field_size = tensor[0][0].numel() 19 | fan_in = num_input_fmaps * receptive_field_size 20 | fan_out = num_output_fmaps * receptive_field_size 21 | 22 | return fan_in, fan_out 23 | 24 | 25 | def xavier_uniform(tensor, gain=1): 26 | """Fills the input Tensor or Variable with values according to the method 27 | described in "Understanding the difficulty of training deep feedforward 28 | neural networks" - Glorot, X. & Bengio, Y. (2010), using a uniform 29 | distribution. The resulting tensor will have values sampled from 30 | :math:`U(-a, a)` where 31 | :math:`a = gain \\times \sqrt{2 / (fan\_in + fan\_out)} \\times \sqrt{3}`. 32 | Also known as Glorot initialisation. 33 | 34 | Args: 35 | tensor: an n-dimensional torch.Tensor or autograd.Variable 36 | gain: an optional scaling factor 37 | 38 | Examples: 39 | >>> w = torch.Tensor(3, 5) 40 | >>> nn.init.xavier_uniform(w, gain=nn.init.calculate_gain('relu')) 41 | """ 42 | if isinstance(tensor, Variable): 43 | xavier_uniform(tensor.data, gain=gain) 44 | return tensor 45 | 46 | fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) 47 | std = gain * math.sqrt(2.0 / (fan_in + fan_out)) 48 | a = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation 49 | return tensor.uniform_(-a, a) 50 | 51 | 52 | def xavier_normal(tensor, gain=1): 53 | """Fills the input Tensor or Variable with values according to the method 54 | described in "Understanding the difficulty of training deep feedforward 55 | neural networks" - Glorot, X. & Bengio, Y. (2010), using a normal 56 | distribution. The resulting tensor will have values sampled from 57 | :math:`N(0, std)` where 58 | :math:`std = gain \\times \sqrt{2 / (fan\_in + fan\_out)}`. 59 | Also known as Glorot initialisation. 60 | 61 | Args: 62 | tensor: an n-dimensional torch.Tensor or autograd.Variable 63 | gain: an optional scaling factor 64 | 65 | Examples: 66 | >>> w = torch.Tensor(3, 5) 67 | >>> nn.init.xavier_normal(w) 68 | """ 69 | if isinstance(tensor, Variable): 70 | xavier_normal(tensor.data, gain=gain) 71 | return tensor 72 | 73 | fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) 74 | std = gain * math.sqrt(2.0 / (fan_in + fan_out)) 75 | return tensor.normal_(0, std) 76 | 77 | 78 | def orthogonal(tensor, gain=1): 79 | """Fills the input Tensor or Variable with a (semi) orthogonal matrix, as 80 | described in "Exact solutions to the nonlinear dynamics of learning in deep 81 | linear neural networks" - Saxe, A. et al. (2013). The input tensor must have 82 | at least 2 dimensions, and for tensors with more than 2 dimensions the 83 | trailing dimensions are flattened. 84 | 85 | Args: 86 | tensor: an n-dimensional torch.Tensor or autograd.Variable, where n >= 2 87 | gain: optional scaling factor 88 | 89 | Examples: 90 | >>> w = torch.Tensor(3, 5) 91 | >>> nn.init.orthogonal(w) 92 | """ 93 | if isinstance(tensor, Variable): 94 | orthogonal(tensor.data, gain=gain) 95 | return tensor 96 | 97 | if tensor.ndimension() < 2: 98 | raise ValueError("Only tensors with 2 or more dimensions are supported") 99 | 100 | rows = tensor.size(0) 101 | cols = tensor[0].numel() 102 | flattened = torch.Tensor(rows, cols).normal_(0, 1) 103 | 104 | if rows < cols: 105 | flattened.t_() 106 | 107 | # Compute the qr factorization 108 | q, r = torch.qr(flattened) 109 | # Make Q uniform according to https://arxiv.org/pdf/math-ph/0609050.pdf 110 | d = torch.diag(r, 0) 111 | ph = d.sign() 112 | q *= ph.expand_as(q) 113 | 114 | if rows < cols: 115 | q.t_() 116 | 117 | tensor.view_as(q).copy_(q) 118 | tensor.mul_(gain) 119 | return tensor -------------------------------------------------------------------------------- /evaluation_scale free/main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | #import torchvision.transforms as transforms 4 | import torch.utils.data as utils 5 | from torch.autograd import Variable 6 | import numpy as np 7 | from utils import compute_nystrom,create_train_test_loaders,imbalance 8 | from sklearn.preprocessing import LabelEncoder 9 | from sklearn.model_selection import KFold 10 | from model import CNN 11 | from graph_kernels import sp_kernel, wl_kernel 12 | import xlwt 13 | import xlrd 14 | from xlutils.copy import copy 15 | from xlwt import Style 16 | def writeExcel(row, col, value, file_name,styl=Style.default_style): 17 | rb = xlrd.open_workbook(file_name) 18 | wb = copy(rb) 19 | ws = wb.get_sheet(0) 20 | ws.write(row, col, value, styl) 21 | wb.save(file_name) 22 | 23 | community_detection = "louvain" 24 | 25 | # Hyper Parameters 26 | dim = 100 27 | batch_size = 10 28 | num_epochs = 250 29 | num_filters = 20 30 | hidden_size = 20 31 | learning_rate = 0.005 32 | use_node_labels = False 33 | kernels=[wl_kernel] 34 | num_kernels = len(kernels) 35 | 36 | def test_(): 37 | print("Computing feature maps...") 38 | Q, subgraphs, labels,shapes = compute_nystrom(use_node_labels, dim, community_detection, kernels) 39 | M=np.zeros((shapes[0],shapes[1],len(kernels))) 40 | for idx,k in enumerate(kernels): 41 | M[:,:,idx]=Q[idx] 42 | Q=M 43 | # Binarize labels 44 | le = LabelEncoder() 45 | y = le.fit_transform(labels) 46 | # Build vocabulary 47 | max_document_length = max([len(x.split(" ")) for x in subgraphs]) 48 | x = np.zeros((len(subgraphs), max_document_length), dtype=np.int32) 49 | for i in range(len(subgraphs)): 50 | communities = subgraphs[i].split() 51 | for j in range(len(communities)): 52 | x[i,j] = int(communities[j]) 53 | reg=x[0:2500] 54 | gen=x[2500:5000] 55 | mal=x[5000:] 56 | reg_label=y[:2500] 57 | gen_label=y[2500:5000] 58 | mal_label=y[5000:] 59 | 60 | train_reg=reg[0:1500] 61 | test_reg=reg[1500:] 62 | train_reg_y=reg_label[0:1500] 63 | test_reg_y=reg_label[1500:] 64 | 65 | train_mal=mal[0:1500] 66 | test_mal=mal[1500:] 67 | train_mal_y=mal_label[0:1500] 68 | test_mal_y=mal_label[1500:] 69 | 70 | train_gen=gen[0:1500] 71 | train_gen_y=gen_label[0:1500] 72 | 73 | train_fake=np.concatenate((train_reg,train_gen),axis=0) 74 | y_train_fake=np.concatenate((train_reg_y,train_gen_y),axis=0) 75 | train_real=np.concatenate((train_reg,train_mal),axis=0) 76 | y_train_real=np.concatenate((train_reg_y,train_mal_y),axis=0) 77 | test=np.concatenate((test_reg,test_mal),axis=0) 78 | y_test=np.concatenate((test_reg_y,test_mal_y),axis=0) 79 | 80 | 81 | def train_test(Q, x_train, x_test, y_train, y_test, batch_size): 82 | train_loader, test_loader = create_train_test_loaders(Q, x_train, x_test, y_train, y_test, batch_size) 83 | cnn = CNN(input_size=num_filters, hidden_size=hidden_size, num_classes=np.unique(y).size, dim=dim, num_kernels=num_kernels, max_document_length=max_document_length) 84 | if torch.cuda.is_available(): 85 | cnn.cuda() 86 | if torch.cuda.is_available(): 87 | criterion = nn.CrossEntropyLoss().cuda() 88 | else: 89 | criterion = nn.CrossEntropyLoss() 90 | optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate) 91 | for epoch in range(num_epochs): 92 | for i, (graphs, labels) in enumerate(train_loader): 93 | graphs = Variable(graphs) 94 | labels = Variable(labels) 95 | optimizer.zero_grad() 96 | outputs = cnn(graphs) 97 | if torch.cuda.is_available(): 98 | loss = criterion(outputs, labels.cuda()) 99 | else: 100 | loss = criterion(outputs, labels) 101 | loss.backward() 102 | optimizer.step() 103 | # Test the Model 104 | 105 | cnn.eval() 106 | correct = 0 107 | total = 0 108 | TP=0 109 | TN=0 110 | FP=0 111 | FN=0 112 | predict=[] 113 | label=[] 114 | output=[] 115 | for graphs, labels in test_loader: 116 | graphs = Variable(graphs) 117 | outputs = cnn(graphs) 118 | _, predicted = torch.max(outputs.data, 1) 119 | total += labels.size(0) 120 | correct += (predicted == labels.cuda()).sum() 121 | TP += (predicted+labels.cuda()==2).sum() 122 | FP+=(predicted*5+labels.cuda()*1==5).sum() 123 | FN+=(predicted*1+labels.cuda()*5==5).sum() 124 | TN+=(predicted+labels.cuda()==0).sum() 125 | predict.append(predicted) 126 | label.append(labels) 127 | output.append(outputs.data) 128 | if TP+FP==0: precision=0 129 | else: precision=TP/(TP+FP) 130 | if TP+FN==0: recall=0 131 | else: recall=TP/(TP+FN) 132 | l=np.zeros((len(label))) 133 | for i in range(len(label)): 134 | l[i]=int(label[i]) 135 | s=np.zeros((len(output))) 136 | for i in range(len(output)): 137 | s[i]=output[i][0][1] 138 | return TP,TN,FP,FN,precision,recall,l,s 139 | TP_fake,TN_fake,FP_fake,FN_fake,precision_fake,recall_fake,l_fake,s_fake=train_test(Q, train_fake, test, y_train_fake, y_test, batch_size) 140 | TP_real,TN_real,FP_real,FN_real,precision_real,recall_real,l_real,s_real=train_test(Q, train_real, test, y_train_real, y_test, batch_size) 141 | return TP_fake,TN_fake,FP_fake,FN_fake,precision_fake,recall_fake,l_fake,s_fake,TP_real,TN_real,FP_real,FN_real,precision_real,recall_real,l_real,s_real 142 | ''' 143 | num=0 144 | lab=[] 145 | sc=[] 146 | skip=[] 147 | for user in red_user: 148 | try: 149 | TP,TN,FP,FN,precision,recall,l,s=test_(user,red_user) 150 | writeExcel(int(num)+1,0, user, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 151 | writeExcel(int(num)+1,1, TN, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 152 | writeExcel(int(num)+1,2, TP, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 153 | writeExcel(int(num)+1,3, FN, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 154 | writeExcel(int(num)+1,4, FP, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 155 | writeExcel(int(num)+1,5, precision, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 156 | writeExcel(int(num)+1,6, recall, 'C:/Users/gxjco/Desktop/New folder (2)/result_CGN.xls') 157 | print(num) 158 | num+=1 159 | lab.append(l) 160 | sc.append(s) 161 | except: skip.append(user) 162 | 163 | label_total=lab[0] 164 | score_total=sc[0] 165 | for i in range(97): 166 | label_total=np.concatenate((label_total,lab[i+1]),axis=0) 167 | score_total=np.concatenate((score_total,sc[i+1]),axis=0) 168 | #draw ROC tota 169 | from sklearn.metrics import roc_curve, auc 170 | import matplotlib.pyplot as plt 171 | fpr,tpr,threshold = roc_curve(label_total, score_total) 172 | roc_auc = auc(fpr,tpr) 173 | plt.figure() 174 | lw = 2 175 | plt.figure(figsize=(10,10)) 176 | plt.plot(fpr, tpr, color='darkorange', 177 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) 178 | plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') 179 | plt.xlim([0.0, 1.0]) 180 | plt.ylim([0.0, 1.05]) 181 | plt.xlabel('False Positive Rate') 182 | plt.ylabel('True Positive Rate') 183 | plt.title('ROC curve on KCNN model') 184 | plt.legend(loc="lower right") 185 | plt.show() ''' -------------------------------------------------------------------------------- /evaluation_scale free/model.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | from init import xavier_normal,xavier_uniform,orthogonal 4 | 5 | # CNN Model 6 | class CNN(nn.Module): 7 | def __init__(self, input_size, hidden_size, num_classes, dim, num_kernels, max_document_length): 8 | super(CNN, self).__init__() 9 | self.max_document_length = max_document_length 10 | self.conv = nn.Conv3d(1, input_size, (1, 1, dim), padding=0) 11 | self.fc1 = nn.Linear(input_size*num_kernels, hidden_size) 12 | self.fc2 = nn.Linear(hidden_size, num_classes) 13 | self.init_weights() 14 | 15 | def init_weights(self): 16 | xavier_uniform(self.conv.weight.data) 17 | xavier_normal(self.fc1.weight.data) 18 | xavier_normal(self.fc2.weight.data) 19 | 20 | def forward(self, x_in): 21 | out = F.relu(F.max_pool3d(self.conv(x_in), (1, self.max_document_length,1))) 22 | out = out.view(out.size(0), -1) 23 | out = F.relu(self.fc1(out)) 24 | out = F.dropout(out, training=self.training) 25 | out = self.fc2(out) 26 | return F.softmax(out) 27 | -------------------------------------------------------------------------------- /evaluation_scale free/network_feature.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Mar 19 22:52:58 2018 4 | 5 | @author: gxjco 6 | """ 7 | 8 | import networkx as nx 9 | import numpy as np 10 | import os 11 | with open('C:/Users/gxjco/Desktop/redteam.txt/redteam.txt') as f: 12 | red_team=f.readlines() 13 | red=[] 14 | red_user={} 15 | for i in red_team: 16 | act={} 17 | m=i.split(',') 18 | act['time']=m[0] 19 | act['user']=m[1].split('@')[0] 20 | act['s_c']=m[2] 21 | act['d_c']=m[3].split('\n')[0] 22 | red.append(act) 23 | if act['user'] not in red_user: 24 | red_user[act['user']]=[] 25 | for act_ in red: 26 | red_user[act_['user']].append(act_) 27 | def plus(a,b): 28 | for i in b: 29 | if i not in a: 30 | a.append(i) 31 | return a 32 | def feature(node,edge): 33 | G =nx.MultiGraph() 34 | G=G.to_directed() 35 | G.add_nodes_from(node) 36 | G.add_edges_from(edge) 37 | f=np.zeros(10) 38 | f[0]=len(G.nodes) 39 | f[1]=len(G.edges) 40 | f[2]=nx.density(G) 41 | f[3]=nx.degree_pearson_correlation_coefficient(G) 42 | f[4]=nx.algorithms.reciprocity(G) 43 | f[5]=0#nx.transitivity(G) 44 | f[6]=nx.is_weakly_connected(G) 45 | f[7]=nx.number_weakly_connected_components(G) 46 | f[8]=nx.is_strongly_connected(G) 47 | f[9]=nx.number_strongly_connected_components(G) 48 | return f 49 | b=0 50 | path='C:/Users/gxjco/Desktop/redteam.txt/user_new/' 51 | files=os.listdir(path) 52 | for file_ in files: 53 | u=[] 54 | user=file_.split('.')[0] 55 | with open(path+file_) as f: 56 | for line in f: 57 | m=line.split(',') 58 | event={} 59 | event['time']=m[0] 60 | event['user']=m[1].split('@')[0] 61 | event['s_c']=m[3] 62 | event['d_c']=m[4] 63 | u.append(event) 64 | l=red_user[user] 65 | u=plus(u,l) 66 | act={} 67 | for i in range(2600): 68 | act[i]={} 69 | act[i]['node']=[] 70 | act[i]['edge']=[] 71 | for line in u: 72 | if int(line['time'])<4680000: 73 | act[int(int(line['time'])/1800)]['node'].append(line['s_c']) 74 | act[int(int(line['time'])/1800)]['node'].append(line['d_c']) 75 | act[int(int(line['time'])/1800)]['edge'].append((line['s_c'],line['d_c'])) 76 | sample=np.zeros((2600,10)) 77 | for i in range(2600): 78 | try: 79 | sample[i,:]=feature(act[i]['node'],act[i]['edge']) 80 | except:b+=1 81 | 82 | time_red={} 83 | for i in red_user[user]: 84 | time_red[i['time']]=[] 85 | label=[] 86 | for t in range(2600): 87 | o=0 88 | for time in time_red: 89 | if int(time) in range(1800*t,1800*t+1800): 90 | o=1 91 | break 92 | label.append(o) 93 | #seperate the negative and positive 94 | p_data=[] 95 | n_data=[] 96 | for i in range(2600): 97 | if label[i]==1: 98 | p_data.append(sample[i]) 99 | else: 100 | n_data.append(sample[i]) 101 | p_sample=np.zeros((len(p_data),10)) 102 | n_sample=np.zeros((len(n_data),10)) 103 | for i in range(len(p_data)): 104 | p_sample[i]=p_data[i] 105 | for i in range(len(n_data)): 106 | n_sample[i]=n_data[i] 107 | np.save('C:/Users/gxjco/Desktop/redteam.txt/input_new/feature_new_negative_sample__'+user+'.npy',n_sample) 108 | np.save('C:/Users/gxjco/Desktop/redteam.txt/input_new/feature_new_positive_sample__'+user+'.npy',p_sample) 109 | 110 | 111 | -------------------------------------------------------------------------------- /evaluation_scale free/nystrom.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.sparse as sp 3 | from scipy.linalg import svd 4 | from sklearn.utils import check_random_state 5 | 6 | class Nystrom(): 7 | 8 | def __init__(self, kernel, kernel_params=None, n_components=100, random_state=None): 9 | self.kernel = kernel 10 | self.kernel_params = kernel_params 11 | self.n_components = n_components 12 | self.random_state = random_state 13 | 14 | def fit(self, graphs, y=None): 15 | rnd = check_random_state(self.random_state) 16 | n_samples = len(graphs) 17 | 18 | # get basis vectors 19 | if self.n_components > n_samples: 20 | n_components = n_samples 21 | else: 22 | n_components = self.n_components 23 | n_components = min(n_samples, n_components) 24 | inds = rnd.permutation(n_samples) 25 | basis_inds = inds[:n_components] 26 | basis = [] 27 | for ind in basis_inds: 28 | basis.append(graphs[ind]) 29 | 30 | basis_kernel = self.kernel(basis, basis, **self._get_kernel_params()) 31 | 32 | # sqrt of kernel matrix on basis vectors 33 | U, S, V = svd(basis_kernel) 34 | S = np.maximum(S, 1e-12) 35 | self.normalization_ = np.dot(U * 1. / np.sqrt(S), V) 36 | self.components_ = basis 37 | self.component_indices_ = inds 38 | return self 39 | 40 | def transform(self, graphs): 41 | embedded = self.kernel(graphs, self.components_, **self._get_kernel_params()) 42 | return np.dot(embedded, self.normalization_.T) 43 | 44 | def _get_kernel_params(self): 45 | params = self.kernel_params 46 | if params is None: 47 | params = {} 48 | 49 | return params 50 | 51 | -------------------------------------------------------------------------------- /evaluation_scale free/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | import networkx as nx 3 | import igraph as ig 4 | import numpy as np 5 | from nystrom import Nystrom 6 | import torch 7 | import torch.utils.data as utils 8 | from imblearn.over_sampling import SMOTE 9 | ''' 10 | def load(user,red_user): 11 | def plus(a,b): 12 | for i in b: 13 | if i not in a: 14 | a.append(i) 15 | return a 16 | path='C:/Users/gxjco/Desktop/New folder (2)/red_user_data_new/user/' 17 | with open(path+user+'.txt') as f: 18 | u=[] 19 | for line in f: 20 | m=line.split(',') 21 | event={} 22 | event['time']=m[0] 23 | event['user']=m[1].split('@')[0] 24 | event['s_c']=m[3] 25 | event['d_c']=m[4] 26 | u.append(event) 27 | l=red_user[user] 28 | u=plus(u,l) 29 | act={} 30 | for i in range(2600): 31 | act[i]={} 32 | act[i]['node']=[] 33 | act[i]['edge']=[] 34 | for line in u: 35 | if int(line['time'])<4680000: 36 | act[int(int(line['time'])/1800)]['node'].append(line['s_c']) 37 | act[int(int(line['time'])/1800)]['node'].append(line['d_c']) 38 | act[int(int(line['time'])/1800)]['edge'].append((line['s_c'],line['d_c'])) 39 | Gs = [] 40 | for i in range(2600): 41 | G =nx.MultiGraph() 42 | G=G.to_directed() 43 | G.add_nodes_from(act[i]['node']) 44 | G.add_edges_from(act[i]['edge']) 45 | Gs.append(G) 46 | time_red={} 47 | for i in red_user[user]: 48 | time_red[i['time']]=[] 49 | label=[] 50 | for t in range(2600): 51 | o=0 52 | for time in time_red: 53 | if int(time) in range(1800*t,1800*t+1800): 54 | o=1 55 | break 56 | label.append(o) 57 | labels = np.array(label, dtype = np.float) 58 | return Gs, labels''' 59 | 60 | def load(): 61 | def graph(A): 62 | G =nx.MultiGraph() 63 | G=G.to_directed() 64 | for i in range(A.shape[0]): 65 | for j in range(A.shape[0]): 66 | if A[i][j]>0: 67 | G.add_nodes_from([str(i),str(j)]) 68 | for m in range(A[i][j]): 69 | G.add_edges_from([(str(i),str(j))]) 70 | 71 | return G 72 | reg=np.load('./dataset/scale free/10/'+'scale_reg10.npy').astype(int)[2500:] 73 | gen=np.load('./scale_gen10.npy').astype(int)[:,:,:,0] 74 | mal=np.load('./dataset/scale free/10/'+'scale_mal10.npy').astype(int)[2500:] 75 | Gs_reg=[] 76 | Gs_gen=[] 77 | Gs_mal=[] 78 | for i in range(len(reg)): 79 | Gs_reg.append(graph(reg[i])) 80 | Gs_gen.append(graph(gen[i])) 81 | for i in range(len(mal)): 82 | Gs_mal.append(graph(mal[i])) 83 | label_reg=np.zeros(len(reg)) 84 | label_gen=np.ones(len(gen)) 85 | label_mal=np.ones(len(mal)) 86 | return Gs_reg,label_reg,Gs_gen,label_gen,Gs_mal,label_mal 87 | 88 | 89 | 90 | def networkx_to_igraph(G): 91 | mapping = dict(zip(G.nodes(),range(G.number_of_nodes()))) 92 | reverse_mapping = dict(zip(range(G.number_of_nodes()),G.nodes())) 93 | G = nx.relabel_nodes(G,mapping) 94 | G_ig = ig.Graph(len(G), list(zip(*list(zip(*nx.to_edgelist(G)))[:2]))) 95 | return G_ig, reverse_mapping 96 | 97 | 98 | def community_detection(G_networkx, community_detection_method): 99 | G,reverse_mapping = networkx_to_igraph(G_networkx) 100 | if community_detection_method == "eigenvector": 101 | c = G.community_leading_eigenvector() 102 | elif community_detection_method == "infomap": 103 | c = G.community_infomap() 104 | elif community_detection_method == "fastgreedy": 105 | c = G.community_fastgreedy().as_clustering() 106 | elif community_detection_method == "label_propagation": 107 | c = G.community_label_propagation() 108 | elif community_detection_method == "louvain": 109 | c = G.community_multilevel() 110 | elif community_detection_method == "spinglass": 111 | c = G.community_spinglass() 112 | elif community_detection_method == "walktrap": 113 | c = G.community_walktrap().as_clustering() 114 | else: 115 | c = [] 116 | 117 | communities = [] 118 | for i in range(len(c)): 119 | community = [] 120 | for j in range(len(c[i])): 121 | community.append(reverse_mapping[G.vs[c[i][j]].index]) 122 | 123 | communities.append(community) 124 | 125 | return communities 126 | 127 | def compute_communities(graphs, use_node_labels, community_detection_method): 128 | communities = [] 129 | subgraphs = [] 130 | counter = 0 131 | coms = [] 132 | for G in graphs: 133 | c = community_detection(G, community_detection_method) 134 | coms.append(len(c)) 135 | subgraph = [] 136 | for i in range(len(c)): 137 | communities.append(G.subgraph(c[i])) 138 | subgraph.append(counter) 139 | counter += 1 140 | 141 | subgraphs.append(' '.join(str(s) for s in subgraph)) 142 | 143 | return communities, subgraphs 144 | 145 | 146 | def compute_nystrom(use_node_labels, embedding_dim, community_detection_method, kernels): 147 | graphs_reg, labels_reg,graphs_gen, labels_gen,graphs_mal, labels_mal = load() 148 | graphs=graphs_reg+graphs_gen+graphs_mal 149 | labels=np.concatenate((labels_reg,labels_gen,labels_mal),axis=0) 150 | communities, subgraphs = compute_communities(graphs, use_node_labels, community_detection_method) 151 | 152 | print("Number of communities: ", len(communities)) 153 | lens = [] 154 | for community in communities: 155 | lens.append(community.number_of_nodes()) 156 | 157 | print("Average size: %.2f" % np.mean(lens)) 158 | Q=[] 159 | for idx, k in enumerate(kernels): 160 | model = Nystrom(k, n_components=embedding_dim) 161 | model.fit(communities) 162 | Q_t = model.transform(communities) 163 | Q_t = np.vstack([np.zeros(embedding_dim), Q_t]) 164 | Q.append(Q_t) 165 | 166 | return Q, subgraphs, labels, Q_t.shape 167 | 168 | 169 | def create_train_test_loaders(Q, x_train, x_test, y_train, y_test, batch_size): 170 | num_kernels = Q.shape[2] 171 | max_document_length = x_train.shape[1] 172 | dim = Q.shape[1] 173 | 174 | my_x = [] 175 | for i in range(x_train.shape[0]): 176 | temp = np.zeros((1, num_kernels, max_document_length, dim)) 177 | for j in range(num_kernels): 178 | for k in range(x_train.shape[1]): 179 | temp[0,j,k,:] = Q[x_train[i,k],:,j].squeeze() 180 | my_x.append(temp) 181 | 182 | if torch.cuda.is_available(): 183 | tensor_x = torch.stack([torch.cuda.FloatTensor(i) for i in my_x]) # transform to torch tensors 184 | tensor_y = torch.cuda.LongTensor(y_train.tolist()) 185 | else: 186 | tensor_x = torch.stack([torch.Tensor(i) for i in my_x]) # transform to torch tensors 187 | tensor_y = torch.from_numpy(np.asarray(y_train,dtype=np.int64)) 188 | 189 | train_dataset = utils.TensorDataset(tensor_x, tensor_y) 190 | train_loader = utils.DataLoader(train_dataset, batch_size=batch_size, shuffle=True) 191 | 192 | my_x = [] 193 | for i in range(x_test.shape[0]): 194 | temp = np.zeros((1, num_kernels, max_document_length, dim)) 195 | for j in range(num_kernels): 196 | for k in range(x_test.shape[1]): 197 | temp[0,j,k,:] = Q[x_test[i,k],:,j].squeeze() 198 | my_x.append(temp) 199 | 200 | if torch.cuda.is_available(): 201 | tensor_x = torch.stack([torch.cuda.FloatTensor(i) for i in my_x]) # transform to torch tensors 202 | tensor_y = torch.cuda.LongTensor(y_test.tolist()) 203 | else: 204 | tensor_x = torch.stack([torch.Tensor(i) for i in my_x]) # transform to torch tensors 205 | tensor_y = torch.from_numpy(np.asarray(y_test,dtype=np.int64)) 206 | 207 | test_dataset = utils.TensorDataset(tensor_x, tensor_y) 208 | test_loader = utils.DataLoader(test_dataset, batch_size=1, shuffle=False) 209 | 210 | return train_loader, test_loader 211 | 212 | def imbalance(x,y): 213 | p_num=sum(y) 214 | if p_num>1: 215 | sm=SMOTE(k_neighbors=p_num-1) 216 | data_res, label_res = sm.fit_sample(x, y) 217 | else: 218 | for i in range(2600): 219 | if y[i]==1: 220 | idx=i 221 | break 222 | new_p=np.zeros((2598,x.shape[1])) 223 | new_p_l=np.ones((2598)) 224 | for i in range(2598): 225 | new_p[i,:]=x[idx] 226 | data_res=np.concatenate((x,new_p),axis=0) 227 | label_res=np.concatenate((y,new_p_l),axis=0) 228 | return data_res.astype(dtype=np.int32),label_res.astype(dtype=np.int32) 229 | 230 | 231 | 232 | 233 | 234 | 235 | -------------------------------------------------------------------------------- /evaluation_scale free/validation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sun Apr 15 20:43:11 2018 4 | 5 | @author: gxjco 6 | """ 7 | import xlwt 8 | import xlrd 9 | from xlutils.copy import copy 10 | from xlwt import Style 11 | from main import* 12 | import numpy as np 13 | 14 | 15 | 16 | def writeExcel(row, col, value, file_name,styl=Style.default_style): 17 | rb = xlrd.open_workbook(file_name) 18 | wb = copy(rb) 19 | ws = wb.get_sheet(0) 20 | ws.write(row, col, value, styl) 21 | wb.save(file_name) 22 | 23 | TP_fake,TN_fake,FP_fake,FN_fake,precision_fake,recall_fake,l_fake,s_fake, TP_real,TN_real,FP_real,FN_real,precision_real,recall_real,l_real,s_real=test_() 24 | 25 | 26 | #draw ROC tota 27 | def draw_roc(label_total_real, score_total_real,label_total_fake, score_total_fake): 28 | from sklearn.metrics import roc_curve, auc 29 | import matplotlib.pyplot as plt 30 | fpr1,tpr1,threshold1 = roc_curve(label_total_fake, score_total_fake) 31 | roc_auc1 = auc(fpr1,tpr1) 32 | fpr2,tpr2,threshold2 = roc_curve(label_total_real, score_total_real) 33 | roc_auc2 = auc(fpr2,tpr2) 34 | plt.figure() 35 | lw = 2 36 | plt.figure(figsize=(10,10)) 37 | plt.plot(fpr1, tpr1, color='darkorange', 38 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc1) 39 | plt.plot(fpr2, tpr2, color='green', 40 | lw=lw, label='ROC curve (area = %0.2f)' % roc_auc2) 41 | plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') 42 | plt.xlim([0.0, 1.0]) 43 | plt.ylim([0.0, 1.05]) 44 | plt.xlabel('False Positive Rate') 45 | plt.ylabel('True Positive Rate') 46 | plt.title('ROC curve on KCNN model') 47 | plt.legend(loc="lower right") 48 | plt.show() 49 | draw_roc(l_real, s_real,l_fake, s_fake) 50 | from sklearn.metrics import precision_score,recall_score 51 | s=s_fake 52 | l=[] 53 | for i in range(len(s)): 54 | if s[i]>1e-20: l.append(1) 55 | else: l.append(0) 56 | print(precision_score(l_real, l)) 57 | print(recall_score(l_real, l)) -------------------------------------------------------------------------------- /figures/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anonymous1025/Deep-Graph-Translation-/fc3ff789281c214a6675ce516dda232baf9bad91/figures/architecture.png -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Mar 26 20:40:35 2018 4 | 5 | @author: gxjco 6 | """ 7 | import argparse 8 | import os 9 | import scipy.misc 10 | import numpy as np 11 | from model import graph2graph 12 | import tensorflow as tf 13 | import datetime 14 | 15 | parser = argparse.ArgumentParser(description='') 16 | parser.add_argument('--epoch', dest='epoch', type=int, default=450, help='# of epoch') 17 | parser.add_argument('--batch_size', dest='batch_size', type=int, default=20, help='# graphs in batch') 18 | parser.add_argument('--train_size', dest='train_size', type=int, default=1e8, help='# graphs used to train') 19 | parser.add_argument('--ngf', dest='ngf', type=int, default=5, help='# of gen filters in first conv layer') 20 | parser.add_argument('--ndf', dest='ndf', type=int, default=5, help='# of discri filters in first conv layer') 21 | parser.add_argument('--input_nc', dest='input_nc', type=int, default=1, help='# of input channels') 22 | parser.add_argument('--output_nc', dest='output_nc', type=int, default=1, help='# of output channels') 23 | parser.add_argument('--niter', dest='niter', type=int, default=100, help='# of iter at starting learning rate') 24 | parser.add_argument('--lr_d', dest='lr_d', type=float, default=0.0001 , help='initial learning rate for adam') 25 | parser.add_argument('--lr_g', dest='lr_g', type=float, default=0.00005 , help='initial learning rate for adam') 26 | parser.add_argument('--beta1', dest='beta1', type=float, default=0.5, help='m omentum term of adam') 27 | parser.add_argument('--flip', dest='flip', type=bool, default=True, help='if flip the graphs for data argumentation') 28 | parser.add_argument('--save_epoch_freq', dest='save_epoch_freq', type=int, default=50, help='save a model every save_epoch_freq epochs (does not overwrite previously saved models)') 29 | parser.add_argument('--save_latest_freq', dest='save_latest_freq', type=int, default=5000, help='save the latest model every latest_freq sgd iterations (overwrites the previous latest model)') 30 | parser.add_argument('--print_freq', dest='print_freq', type=int, default=50, help='print the debug information every print_freq iterations') 31 | parser.add_argument('--continue_train', dest='continue_train', type=bool, default=False, help='if continue training, load the latest model: 1: true, 0: false') 32 | parser.add_argument('--serial_batches', dest='serial_batches', type=bool, default=False, help='f 1, takes graphsin order to make batches, otherwise takes them randomly') 33 | parser.add_argument('--serial_batch_iter', dest='serial_batch_iter', type=bool, default=True, help='iter into serial graph list') 34 | parser.add_argument('--checkpoint_dir', dest='checkpoint_dir', default='./checkpoint_auth_50', help='models are saved here,need to be distinguishable for different dataset') 35 | parser.add_argument('--sample_dir', dest='sample_dir', default='./sample', help='sample are saved here') 36 | parser.add_argument('--test_dir', dest='test_dir', default='./validation_data_auth_50/', help='test sample are saved here, need to be distinguishable for different dataset') 37 | parser.add_argument('--L1_lambda', dest='L1_lambda', type=float, default=100.0, help='weight on L1 term in objective') 38 | parser.add_argument('--train_dir', dest='train_dir', default='./', help='train sample are saved here') 39 | parser.add_argument('--graph_size', dest='graph_size', default=[50,50], help='size of graph') 40 | parser.add_argument('--output_size', dest='output_size', default=[50,50], help='size of graph') 41 | parser.add_argument('--dataset', dest='dataset', default='authentication', help='chose from authentication, scale-free and poisson-random') 42 | args = parser.parse_args() 43 | 44 | 45 | def main_auth(filename): 46 | #start = datetime.datetime.now() 47 | if not os.path.exists(args.checkpoint_dir): 48 | os.makedirs(args.checkpoint_dir) 49 | if not os.path.exists(args.sample_dir): 50 | os.makedirs(args.sample_dir) 51 | if not os.path.exists(args.test_dir): 52 | os.makedirs(args.test_dir) 53 | tf.reset_default_graph() 54 | with tf.Session() as sess: 55 | model = graph2graph(sess, batch_size=args.batch_size, 56 | checkpoint_dir=args.checkpoint_dir, sample_dir=args.sample_dir,test_dir=args.test_dir,train_dir=args.train_dir,graph_size=args.graph_size,output_size=args.output_size) 57 | #model.train(args) 58 | model.test_auth(args,filename) 59 | #end = datetime.datetime.now() 60 | #print (end-start) 61 | 62 | 63 | def main(): 64 | #start = datetime.datetime.now() 65 | if not os.path.exists(args.checkpoint_dir): 66 | os.makedirs(args.checkpoint_dir) 67 | if not os.path.exists(args.sample_dir): 68 | os.makedirs(args.sample_dir) 69 | if not os.path.exists(args.test_dir): 70 | os.makedirs(args.test_dir) 71 | tf.reset_default_graph() 72 | with tf.Session() as sess: 73 | model = graph2graph(sess, batch_size=args.batch_size, 74 | checkpoint_dir=args.checkpoint_dir, sample_dir=args.sample_dir,test_dir=args.test_dir,train_dir=args.train_dir,graph_size=args.graph_size,output_size=args.output_size) 75 | #model.train(args) 76 | model.test(args) 77 | #end = datetime.datetime.now() 78 | #print (end-start) 79 | 80 | if __name__ == '__main__': 81 | 82 | if args.dataset=='authentication': 83 | if args.graph_size[0]==50: 84 | with open('./dataset/authentic/50/name_list50_1.txt') as f: 85 | user_name=f.readlines() 86 | name_list=[] 87 | for i in user_name: 88 | if i.split('\n')[0] not in name_list: name_list.append(i.split('\n')[0]) 89 | 90 | if args.graph_size[0]==300: 91 | with open('./dataset/authentic/300/name_list1.txt') as f: 92 | user_name=f.readlines() 93 | name_list=[] 94 | for i in user_name: 95 | if i.split('\n')[0] not in name_list: name_list.append(i.split('\n')[0]) 96 | 97 | for i in name_list: 98 | main_auth(i) 99 | 100 | if args.dataset=='scale-free' or 'poisson-random': 101 | main() 102 | #tf.app.run() 103 | 104 | #from memory_profiler import memory_usage 105 | # m=memory_usage(main(), interval=20, timeout=240) 106 | # print(max(m))''' 107 | -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Mar 26 20:46:42 2018 4 | 5 | @author: gxjco 6 | """ 7 | 8 | 9 | from __future__ import division 10 | import os 11 | import time 12 | from glob import glob 13 | import tensorflow as tf 14 | import numpy as np 15 | from six.moves import xrange 16 | import csv 17 | from ops_ import * 18 | from utils_ import * 19 | from sklearn.metrics import mean_squared_error 20 | from math import sqrt 21 | 22 | class graph2graph(object): 23 | def __init__(self, sess, test_dir,train_dir,graph_size,output_size,dataset, 24 | batch_size=50, sample_size=1, 25 | gf_dim=10, df_dim=10, L1_lambda=100, 26 | input_c_dim=1, output_c_dim=1, 27 | checkpoint_dir=None, sample_dir=None,g_train_num=6,d_train_num=6): 28 | """ 29 | 30 | Args: 31 | sess: TensorFlow session 32 | batch_size: The size of batch. Should be specified before training. 33 | output_size: (optional) The resolution in pixels of the graphs. [256] 34 | gf_dim: (optional) Dimension of gen filters in first conv layer. [64] 35 | df_dim: (optional) Dimension of discrim filters in first conv layer. [64] 36 | input_c_dim: (optional) Dimension of input graph channel. For grayscale input, set to 1. [3] 37 | output_c_dim: (optional) Dimension of output graph channel. For grayscale input, set to 1. [3] 38 | """ 39 | self.sess = sess 40 | self.is_grayscale = (input_c_dim == 1) 41 | self.batch_size = batch_size 42 | self.graph_size = graph_size 43 | self.sample_size = sample_size 44 | self.output_size = output_size 45 | self.g_train_num=g_train_num 46 | self.d_train_num=d_train_num 47 | self.test_dir=test_dir 48 | self.gf_dim = gf_dim 49 | self.df_dim = df_dim 50 | 51 | self.input_c_dim = input_c_dim 52 | self.output_c_dim = output_c_dim 53 | self.dataset=dataset 54 | self.L1_lambda = L1_lambda 55 | 56 | # batch normalization : deals with poor initialization helps gradient flow 57 | self.d_bn1 = batch_norm(name='d_bn1') 58 | self.d_bn2 = batch_norm(name='d_bn2') 59 | self.d_bn3 = batch_norm(name='d_bn3') 60 | 61 | 62 | self.g_bn_e1 = batch_norm(name='g_bn_e1') 63 | self.g_bn_e2 = batch_norm(name='g_bn_e2') 64 | self.g_bn_e3 = batch_norm(name='g_bn_e3') 65 | self.g_bn_e4 = batch_norm(name='g_bn_e4') 66 | 67 | self.g_bn_d1 = batch_norm(name='g_bn_d1') 68 | self.g_bn_d2 = batch_norm(name='g_bn_d2') 69 | self.g_bn_d3 = batch_norm(name='g_bn_d3') 70 | 71 | self.checkpoint_dir = checkpoint_dir 72 | self.build_model() 73 | 74 | def build_model(self): 75 | self.real_data = tf.placeholder(tf.float32, 76 | [self.batch_size, self.graph_size[0], self.graph_size[1], 77 | self.input_c_dim + self.output_c_dim], 78 | name='real_A_and_B_graphs') 79 | 80 | self.real_A = self.real_data[:, :, :, :self.input_c_dim] 81 | self.real_B = self.real_data[:, :, :, self.input_c_dim:self.input_c_dim + self.output_c_dim] 82 | 83 | self.fake_B = self.generator(self.real_A) 84 | 85 | self.real_AB = tf.concat([self.real_A, self.real_B], 3) 86 | self.fake_AB = tf.concat([self.real_A, self.fake_B], 3) 87 | self.D, self.D_logits = self.discriminator(self.real_AB, reuse=False) #define the input from graph 88 | self.D_, self.D_logits_ = self.discriminator(self.fake_AB, reuse=True) 89 | 90 | self.d_sum = tf.summary.histogram("d", self.D) 91 | self.d__sum = tf.summary.histogram("d_", self.D_) 92 | self.fake_B_sum = tf.summary.graph("fake_B", self.fake_B) 93 | 94 | self.d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.D_logits, labels=tf.ones_like(self.D))) 95 | self.d_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.D_logits_, labels=tf.zeros_like(self.D_))) 96 | self.g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.D_logits_, labels=tf.ones_like(self.D_))) \ 97 | + self.L1_lambda * tf.reduce_mean(tf.abs(self.real_AB - self.fake_AB)) 98 | 99 | self.d_loss_real_sum = tf.summary.scalar("d_loss_real", self.d_loss_real) 100 | self.d_loss_fake_sum = tf.summary.scalar("d_loss_fake", self.d_loss_fake) 101 | 102 | self.d_loss = self.d_loss_real + self.d_loss_fake 103 | 104 | self.g_loss_sum = tf.summary.scalar("g_loss", self.g_loss) 105 | self.d_loss_sum = tf.summary.scalar("d_loss", self.d_loss) 106 | 107 | t_vars = tf.trainable_variables() 108 | 109 | self.d_vars = [var for var in t_vars if 'd_' in var.name] 110 | self.g_vars = [var for var in t_vars if 'g_' in var.name] 111 | 112 | self.saver = tf.train.Saver() 113 | 114 | 115 | def load_random_samples(self,sample_dir): 116 | sample_data=load_data(sample_dir) 117 | sample = np.random.choice(sample_data, self.batch_size) 118 | sample_graphs = np.array(sample).astype(np.float32) 119 | return sample_graphs 120 | 121 | def sample_model(self, sample_dir, epoch, idx): 122 | sample_graphs = self.load_random_samples(sample_dir) 123 | samples, d_loss, g_loss = self.sess.run( 124 | [self.fake_B_sample, self.d_loss, self.g_loss], 125 | feed_dict={self.real_data: sample_graphs} 126 | ) 127 | # save_graphs(samples, [self.batch_size, 1], 128 | # './{}/train_{:02d}_{:04d}.png'.format(sample_dir, epoch, idx)) 129 | print("[Sample] d_loss: {:.8f}, g_loss: {:.8f}".format(d_loss, g_loss)) 130 | 131 | def train(self, args): 132 | """Train pix2pix""" 133 | #d_optim = tf.train.GradientDescentOptimizer(args.lr) \ 134 | # .minimize(self.d_loss, var_list=self.d_vars) 135 | d_optim = tf.train.AdamOptimizer(args.lr_d, beta1=args.beta1) \ 136 | .minimize(self.d_loss, var_list=self.d_vars) 137 | #g_optim = tf.train.GradientDescentOptimizer(args.lr) \ 138 | # .minimize(self.g_loss, var_list=self.g_vars) 139 | g_optim = tf.train.AdamOptimizer(args.lr_g, beta1=args.beta1) \ 140 | .minimize(self.g_loss, var_list=self.g_vars) 141 | 142 | init_op = tf.global_variables_initializer() 143 | self.sess.run(init_op) 144 | 145 | self.g_sum = tf.summary.merge([self.d__sum, 146 | self.fake_B_sum, self.d_loss_fake_sum, self.g_loss_sum]) 147 | self.d_sum = tf.summary.merge([self.d_sum, self.d_loss_real_sum, self.d_loss_sum]) 148 | self.writer = tf.summary.FileWriter("./logs", self.sess.graph) 149 | 150 | counter = 1 151 | start_time = time.time() 152 | data = load_data(args.train_dir,'train',self.graph_size[0],self.dataset) 153 | errD_fake = 0 154 | errD_real = 0 155 | errG = 0 156 | best=5 157 | for epoch in xrange(args.epoch): 158 | #np.random.shuffle(data) 159 | batch_idxs = min(len(data), args.train_size) // self.batch_size 160 | for idx in xrange(0, batch_idxs): 161 | batch = data[idx*self.batch_size:(idx+1)*self.batch_size] 162 | batch_graphs = np.array(batch).astype(np.float32) 163 | if errD_fake+errD_real>0.5: 164 | for i in range(self.d_train_num): 165 | # Update D network 166 | _, summary_str = self.sess.run([d_optim, self.d_sum], 167 | feed_dict={ self.real_data: batch_graphs }) 168 | self.writer.add_summary(summary_str, counter) 169 | 170 | for i in range(self.g_train_num): 171 | # Update G network 172 | _, summary_str = self.sess.run([g_optim, self.g_sum], 173 | feed_dict={ self.real_data: batch_graphs }) 174 | self.writer.add_summary(summary_str, counter) 175 | 176 | # Run g_optim twice to make sure that d_loss does not go to zero (different from paper) 177 | 178 | 179 | errD_fake = self.d_loss_fake.eval({self.real_data: batch_graphs}) 180 | errD_real = self.d_loss_real.eval({self.real_data: batch_graphs}) 181 | errG = self.g_loss.eval({self.real_data: batch_graphs}) 182 | 183 | counter += 1 184 | print("Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f" \ 185 | % (epoch, idx, batch_idxs, 186 | time.time() - start_time, errD_fake+errD_real, errG)) 187 | 188 | # if np.mod(counter, 100) == 1: 189 | # self.sample_model(args.sample_dir, epoch, idx) 190 | 191 | if errG0: gen_data=np.concatenate((gen_data,samples),axis=0) 315 | 316 | np.save(self.test_dir+filename.split('.')[0]+'_gen50.npy',gen_data) 317 | 318 | 319 | def test(self, args): 320 | score=[] 321 | gen_data=[] 322 | 323 | init_op = tf.global_variables_initializer() 324 | self.sess.run(init_op) 325 | 326 | # load testing input 327 | print("Loading testing graphs ...") 328 | sample_graphs_all =load_data_test(self.graph_size[0],self.dataset) 329 | sample_graphs = [sample_graphs_all[i:i+self.batch_size] 330 | for i in xrange(0, len(sample_graphs_all), self.batch_size)] 331 | sample_graphs = np.array(sample_graphs) 332 | if self.load(self.checkpoint_dir): 333 | print(" [*] Load SUCCESS") 334 | else: 335 | print(" [!] Load failed...") 336 | for i, sample_graph in enumerate(sample_graphs): 337 | idx = i+1 338 | print("sampling graph ", idx) 339 | samples = self.sess.run( 340 | self.fake_B, 341 | feed_dict={self.real_data: sample_graphs[i]} 342 | ) 343 | 344 | label = self.sess.run( 345 | self.real_B, 346 | feed_dict={self.real_data: sample_graphs[i]} 347 | ) 348 | if i==0: gen_data=samples 349 | if i>0: gen_data=np.concatenate((gen_data,samples),axis=0) 350 | for i in range(gen_data.shape[0]): 351 | for j in range(gen_data.shape[1]): 352 | for k in range(gen_data.shape[2]): 353 | gen_data[i,j,k,0]=round(gen_data[i,j,k,0]) 354 | 355 | if self.dataset=='scale-free': 356 | np.save(self.test_dir+'scale_gen'+str(self.graph_size[0])+'.npy',gen_data) 357 | if self.dataset=='poisson-random': 358 | np.save(self.test_dir+'poisson_gen'+str(self.graph_size[0])+'.npy',gen_data) -------------------------------------------------------------------------------- /ops_.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import tensorflow as tf 4 | from tensorflow.python.framework import ops 5 | from utils_ import * 6 | 7 | class batch_norm(object): 8 | # h1 = lrelu(tf.contrib.layers.batch_norm(conv2d(h0, self.df_dim*2, name='d_h1_conv'),decay=0.9,updates_collections=None,epsilon=0.00001,scale=True,scope="d_h1_conv")) 9 | def __init__(self, epsilon=1e-5, momentum = 0.9, name="batch_norm"): 10 | with tf.variable_scope(name): 11 | self.epsilon = epsilon 12 | self.momentum = momentum 13 | self.name = name 14 | 15 | def __call__(self, x, train=True): 16 | return tf.contrib.layers.batch_norm(x, decay=self.momentum, updates_collections=None, epsilon=self.epsilon, scale=True, scope=self.name) 17 | 18 | def binary_cross_entropy(preds, targets, name=None): 19 | """Computes binary cross entropy given `preds`. 20 | 21 | For brevity, let `x = `, `z = targets`. The logistic loss is 22 | 23 | loss(x, z) = - sum_i (x[i] * log(z[i]) + (1 - x[i]) * log(1 - z[i])) 24 | 25 | Args: 26 | preds: A `Tensor` of type `float32` or `float64`. 27 | targets: A `Tensor` of the same type and shape as `preds`. 28 | """ 29 | eps = 1e-12 30 | with ops.op_scope([preds, targets], name, "bce_loss") as name: 31 | preds = ops.convert_to_tensor(preds, name="preds") 32 | targets = ops.convert_to_tensor(targets, name="targets") 33 | return tf.reduce_mean(-(targets * tf.log(preds + eps) + 34 | (1. - targets) * tf.log(1. - preds + eps))) 35 | 36 | def conv_cond_concat(x, y): 37 | """Concatenate conditioning vector on feature map axis.""" 38 | x_shapes = x.get_shape() 39 | y_shapes = y.get_shape() 40 | return tf.concat([x, y*tf.ones([x_shapes[0], x_shapes[1], x_shapes[2], y_shapes[3]])], 3) 41 | 42 | 43 | 44 | def conv2d(input_, output_dim, 45 | k_h=5, k_w=5, d_h=3, d_w=3, stddev=0.02, 46 | name="conv2d"): 47 | with tf.variable_scope(name): 48 | w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim], 49 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 50 | conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME') 51 | 52 | biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) 53 | conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) 54 | 55 | return conv 56 | def e2e(input_,output_dim,k_h=50, d_h=1, d_w=1, stddev=0.02, 57 | name="e2e"): 58 | with tf.variable_scope(name): 59 | w1 = tf.get_variable('w1', [k_h, k_h, input_.get_shape()[-1], output_dim], 60 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 61 | conv1 = tf.nn.conv2d(input_, w1[0:1,:,:,:], strides=[1, d_h, d_w, 1], padding='VALID') 62 | biases1 = tf.get_variable('biases1', [output_dim], initializer=tf.constant_initializer(0.0)) 63 | conv1 = tf.reshape(tf.nn.bias_add(conv1, biases1), conv1.get_shape()) 64 | w2 = tf.get_variable('w2', [k_h,k_h, input_.get_shape()[-1], output_dim], 65 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 66 | conv2 = tf.nn.conv2d(input_, w2[:,0:1,:,:], strides=[1, d_h, d_w, 1], padding='VALID') 67 | biases2 = tf.get_variable('biases2', [output_dim], initializer=tf.constant_initializer(0.0)) 68 | conv2 = tf.reshape(tf.nn.bias_add(conv2, biases2), conv2.get_shape()) 69 | m1 = tf.tile(conv1,[1,1,k_h,1]) 70 | m2 = tf.tile(conv2,[1,k_h,1,1]) 71 | conv = tf.add(m1, m2) 72 | return conv 73 | 74 | def e2n(input_,output_dim,k_h=50, d_h=1, d_w=1, stddev=0.02, 75 | name="e2n"): 76 | with tf.variable_scope(name): 77 | w = tf.get_variable('w', [1, k_h, input_.get_shape()[-1], output_dim], 78 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 79 | conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='VALID') 80 | 81 | biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) 82 | conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) 83 | return conv 84 | 85 | def n2g(input_,output_dim,k_h=50, d_h=1, d_w=1, stddev=0.02, 86 | name="e2n"): 87 | with tf.variable_scope(name): 88 | w = tf.get_variable('w', [k_h,1, input_.get_shape()[-1], output_dim], 89 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 90 | conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='VALID') 91 | biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) 92 | conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) 93 | return conv 94 | 95 | def de_n2g(input_, output_shape, 96 | k_h=50, d_h=1, d_w=1, stddev=0.02, 97 | name="de_n2g", with_w=False): 98 | with tf.variable_scope(name): 99 | # filter : [height, width, output_channels, in_channels] 100 | w = tf.get_variable('w', [k_h,1, output_shape[-1], input_.get_shape()[-1]], 101 | initializer=tf.random_normal_initializer(stddev=stddev)) 102 | 103 | deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, 104 | strides=[1, d_h, d_w, 1],padding='VALID') 105 | biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) 106 | deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape()) 107 | if with_w: 108 | return deconv, w, biases 109 | else: 110 | return deconv 111 | 112 | def de_e2n(input_, output_shape, 113 | k_h=50, d_h=1, d_w=1, stddev=0.02, 114 | name="de_n2g", with_w=False): 115 | with tf.variable_scope(name): 116 | # filter : [height, width, output_channels, in_channels] 117 | w = tf.get_variable('w', [1,k_h, output_shape[-1], input_.get_shape()[-1]], 118 | initializer=tf.random_normal_initializer(stddev=stddev)) 119 | 120 | deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, 121 | strides=[1, d_h, d_w, 1],padding='VALID') 122 | biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) 123 | deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape()) 124 | 125 | if with_w: 126 | return deconv, w, biases 127 | else: 128 | return deconv 129 | 130 | def de_e2e(input_, output_shape, 131 | k_h=50, d_h=1, d_w=1, stddev=0.02, 132 | name="de_n2g", with_w=False): 133 | with tf.variable_scope(name): 134 | # filter : [height, width, output_channels, in_channels] 135 | input_1=tf.reshape(tf.reduce_sum(input_,axis=1),(int(input_.shape[0]),k_h,1,int(input_.shape[3]))) 136 | input_2=tf.reshape(tf.reduce_sum(input_,axis=2),(int(input_.shape[0]),1,k_h,int(input_.shape[3]))) 137 | 138 | w1 = tf.get_variable('w1', [1,k_h, output_shape[-1], input_.get_shape()[-1]], 139 | initializer=tf.random_normal_initializer(stddev=stddev)) 140 | deconv1 = tf.nn.conv2d_transpose(input_1, w1, output_shape=output_shape, 141 | strides=[1, d_h, d_w, 1],padding='VALID') 142 | biases1 = tf.get_variable('biases1', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) 143 | deconv1 = tf.reshape(tf.nn.bias_add(deconv1, biases1), deconv1.get_shape()) 144 | 145 | w2 = tf.get_variable('w2', [k_h,1, output_shape[-1], input_.get_shape()[-1]], 146 | initializer=tf.random_normal_initializer(stddev=stddev)) 147 | deconv2 = tf.nn.conv2d_transpose(input_2, w2, output_shape=output_shape, 148 | strides=[1, d_h, d_w, 1],padding='VALID') 149 | biases2 = tf.get_variable('biases2', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) 150 | deconv2 = tf.reshape(tf.nn.bias_add(deconv2, biases2), deconv2.get_shape()) 151 | 152 | deconv=tf.add(deconv1,deconv2)/2 153 | if with_w: 154 | return deconv, w1, biases1 155 | else: 156 | return deconv 157 | def deconv2d(input_, output_shape, 158 | k_h=5, k_w=5, d_h=3, d_w=3, stddev=0.02, 159 | name="deconv2d", with_w=False): 160 | with tf.variable_scope(name): 161 | # filter : [height, width, output_channels, in_channels] 162 | w = tf.get_variable('w', [k_h, k_w, output_shape[-1], input_.get_shape()[-1]], 163 | initializer=tf.random_normal_initializer(stddev=stddev)) 164 | 165 | try: 166 | deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, 167 | strides=[1, d_h, d_w, 1]) 168 | 169 | # Support for verisons of TensorFlow before 0.7.0 170 | except AttributeError: 171 | deconv = tf.nn.deconv2d(input_, w, output_shape=output_shape, 172 | strides=[1, d_h, d_w, 1]) 173 | 174 | biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) 175 | deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape()) 176 | 177 | if with_w: 178 | return deconv, w, biases 179 | else: 180 | return deconv 181 | 182 | 183 | 184 | def lrelu(x, leak=0.2, name="lrelu"): 185 | return tf.maximum(x, leak*x) 186 | 187 | def linear(input_, output_size, scope=None, stddev=0.02, bias_start=0.0, with_w=False): 188 | shape = input_.get_shape().as_list() 189 | with tf.variable_scope(scope or "Linear"): 190 | matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32, 191 | tf.random_normal_initializer(stddev=stddev)) 192 | bias = tf.get_variable("bias", [output_size], 193 | initializer=tf.constant_initializer(bias_start)) 194 | if with_w: 195 | return tf.matmul(input_, matrix) + bias, matrix, bias 196 | else: 197 | return tf.matmul(input_, matrix) + bias 198 | def linear_mask(input_, output_size, scope=None, stddev=0.02, bias_start=0.0, with_w=False): 199 | shape = input_.get_shape().as_list() 200 | with tf.variable_scope(scope or "Linear_mask"): 201 | matrix = np.ones((shape[1], output_size[1])).astype('float32') 202 | mask_ = np.loadtxt('mask.csv',delimiter=',') 203 | mask = mask_.astype('float32') 204 | output=tf.matmul(input_, matrix*mask) 205 | return tf.reshape(output,[shape[0],54,1,1]) 206 | -------------------------------------------------------------------------------- /utils_.py: -------------------------------------------------------------------------------- 1 | """ 2 | Some codes from https://github.com/Newmu/dcgan_code 3 | """ 4 | from __future__ import division 5 | import math 6 | import json 7 | import random 8 | import pprint 9 | import scipy.misc 10 | import numpy as np 11 | from time import gmtime, strftime 12 | import os 13 | import csv 14 | import numpy 15 | from sklearn import preprocessing 16 | import urllib 17 | 18 | pp = pprint.PrettyPrinter() 19 | 20 | get_stddev = lambda x, k_h, k_w: 1/math.sqrt(k_w*k_h*x.get_shape()[-1]) 21 | 22 | 23 | def load_data(path,type_,size,dataset): 24 | 25 | if dataset=='authentication': 26 | 27 | if size==50: 28 | reg=np.load('./dataset/authentic/'+str(size)+'/train/reg_data'+str(size)+'.npy') 29 | mal=np.load('./dataset/authentic/'+str(size)+'/train/mal_data'+str(size)+'.npy') 30 | if size==300: 31 | reg=np.load('./dataset/authentic/'+str(size)+'/train/reg_data.npy') 32 | mal=np.load('./dataset/authentic/'+str(size)+'/train/mal_data.npy') 33 | 34 | data=np.zeros((reg.shape[0],reg.shape[1],reg.shape[2],2)) 35 | for i in range(reg.shape[0]): 36 | data[i,:,:,0]=reg[i] 37 | data[i,:,:,1]=mal[i] 38 | if size==50: 39 | return data[:37] 40 | #return data[37:] ----cross validation 41 | if size==300: 42 | return data[:249] 43 | #return np.concatenate((data[:136],data[249:]),axis=0) 44 | #return data[136:] --cross validation 45 | 46 | 47 | if dataset=='scale-free': 48 | reg=np.load('./dataset/scale free/'+str(size)+'/scale_reg'+str(size)+'.npy') 49 | mal=np.load('./dataset/scale free/'+str(size)+'/scale_mal'+str(size)+'.npy') 50 | data=np.zeros((reg.shape[0],reg.shape[1],reg.shape[2],2)) 51 | for i in range(reg.shape[0]): 52 | data[i,:,:,0]=reg[i] 53 | data[i,:,:,1]=mal[i] 54 | return data[:2500] 55 | 56 | if dataset=='poisson-random': 57 | reg=np.load('./dataset/poisson random/'+str(size)+'/poisson_reg'+str(size)+'.npy') 58 | mal=np.load('./dataset/poisson random/'+str(size)+'/poisson_mal'+str(size)+'.npy') 59 | data=np.zeros((reg.shape[0],reg.shape[1],reg.shape[2],2)) 60 | for i in range(reg.shape[0]): 61 | data[i,:,:,0]=reg[i] 62 | data[i,:,:,1]=mal[i] 63 | return data[:2500] 64 | 65 | def load_data_test_auth(path,filename,size): 66 | 67 | if size==50: 68 | reg=np.load('./dataset/authentic/'+str(size)+'/test/'+filename+'_50.npy')[:2400] 69 | if size==300: 70 | reg=np.load('./dataset/authentic/'+str(size)+'/test/'+filename+'.npy')[:2400] 71 | 72 | data=np.zeros((reg.shape[0],reg.shape[1],reg.shape[2],2)) 73 | for i in range(reg.shape[0]): 74 | data[i,:,:,0]=reg[i] 75 | data[i,:,:,1]=reg[i] 76 | return data 77 | 78 | def load_data_test(size,dataset): 79 | 80 | if dataset=='scale-free': 81 | reg=np.load('./dataset/scale free/'+str(size)+'/scale_reg'+str(size)+'.npy')[2500:] 82 | if dataset=='poisson-random': 83 | reg=np.load('./dataset/poisson random/'+str(size)+'/poisson_reg'+str(size)+'.npy')[2500:] 84 | data=np.zeros((reg.shape[0],reg.shape[1],reg.shape[2],2)) 85 | for i in range(reg.shape[0]): 86 | data[i,:,:,0]=reg[i] 87 | data[i,:,:,1]=reg[i] 88 | return data 89 | 90 | 91 | 92 | --------------------------------------------------------------------------------