├── README.md └── deep_and_cross_tf.py /README.md: -------------------------------------------------------------------------------- 1 | # Deep-Cross-Tensorflow 2 | Ref: https://github.com/Nirvanada/Deep-and-Cross-Keras 3 | Simple tensorflow implementation for [Deep and Cross Network](https://arxiv.org/pdf/1708.05123.pdf), But the model need to be tuned by yourself and the model create by this project only achieves 76.66%. 4 | Data:download from https://www.kaggle.com/uciml/forest-cover-type-dataset/data 5 | -------------------------------------------------------------------------------- /deep_and_cross_tf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Sat Mar 17 21:29:21 2018 5 | 6 | @author: flyaway 7 | """ 8 | from sklearn.preprocessing import OneHotEncoder,StandardScaler 9 | import numpy as np 10 | from sklearn.metrics import accuracy_score 11 | from sklearn.cross_validation import train_test_split 12 | import tensorflow as tf 13 | import pandas as pd 14 | import keras.backend as K 15 | 16 | def preprocessing(data): 17 | # inverse transform one-hot to continuous column 18 | df_onehot = data[[col for col in data.columns.tolist() if "Soil_Type" in col]] 19 | #for i in df_onehot.columns.tolist(): 20 | # if df_onehot[i].sum() == 0: 21 | # del df_onehot[i] 22 | data["Soil"] = df_onehot.dot(np.array(range(df_onehot.columns.size))).astype(int) 23 | data.drop([col for col in data.columns.tolist() if "Soil_Type" in col], axis = 1, inplace = True) 24 | label = np.array(OneHotEncoder().fit_transform(data["Cover_Type"].values.reshape(-1, 1)).todense()) 25 | del data["Cover_Type"] 26 | cate_columns = ["Soil"] 27 | cont_columns = [col for col in data.columns if col != "Soil"] 28 | # Feature normilization 29 | scaler = StandardScaler() 30 | data_cont = pd.DataFrame(scaler.fit_transform(data[cont_columns]), columns = cont_columns) 31 | data_cate = data[cate_columns] 32 | data = pd.concat([data_cate, data_cont], axis = 1) 33 | cate_values = data[cate_columns].values 34 | cont_values = data[cont_columns].values 35 | return data, label, cate_values, cont_values #cate_columns, cont_columns 36 | 37 | 38 | 39 | def embedding_input(inp,name, n_in, n_out): 40 | #inp = Input(shape = (1, ), dtype = 'int64', name = name) 41 | #inp = tf.placeholder(tf.int32,shape = (1,)) 42 | print type(inp) 43 | with tf.variable_scope(name) as scope: 44 | embeddings = tf.Variable(tf.random_uniform([n_in,n_out],-1.0,1.0)) 45 | #embeddings = tf.Variable(tf.random_uniform([voc_size, embedding_size], -1.0, 1.0)) 46 | print embeddings.shape 47 | return inp,tf.nn.embedding_lookup(embeddings, inp,name = scope.name) 48 | 49 | 50 | def embedding_feature_generate(cate_values,num_cate): 51 | #data, label, cate_columns, cont_columns = preprocessing(data) 52 | embeddings_tensors = [] 53 | 54 | col = cate_values.shape[1] 55 | for i in range(col): 56 | layer_name = 'inp_' + str(i) 57 | 58 | #nunique = np.unique(cate_values[:,i]).shape[0] 59 | nunique = num_cate[i] 60 | embed_dim = nunique if int(6 * np.power(nunique, 1/4)) > nunique \ 61 | else int(6 * np.power(nunique, 1/4)) 62 | t_inp, t_build = embedding_input(cate_values[:,i],layer_name, nunique, embed_dim) 63 | embeddings_tensors.append((t_inp, t_build)) 64 | del(t_inp, t_build) 65 | inp_embed = [et[1] for et in embeddings_tensors] 66 | return inp_embed 67 | 68 | def fclayer(x,output_dim,reluFlag,name): 69 | with tf.variable_scope(name) as scope: 70 | input_dim = x.get_shape().as_list()[1] 71 | W = tf.Variable(tf.random_normal([input_dim,output_dim], stddev=0.01)) 72 | b = tf.Variable(tf.random_normal([output_dim], stddev=0.01)) 73 | out = tf.nn.xw_plus_b(x,W,b,name = scope.name) 74 | if reluFlag: 75 | return tf.nn.relu(out) 76 | else: 77 | return out 78 | 79 | def crosslayer(x,inp_embed,name): 80 | with tf.variable_scope(name) as scope: 81 | 82 | input_dim = x.get_shape().as_list()[1] 83 | 84 | w = tf.Variable(tf.random_normal([1, input_dim], stddev=0.01)) 85 | b = tf.Variable(tf.random_normal([1,input_dim], stddev=0.01)) 86 | 87 | 88 | tmp1 = K.batch_dot(K.reshape(x, (-1, input_dim, 1)), tf.reshape(inp_embed,(-1,1,input_dim))) 89 | 90 | 91 | tmp = K.sum(w * tmp1, 1, keepdims = True) 92 | 93 | tmp = tf.reshape(tmp,shape=(-1,input_dim)) 94 | 95 | #one = tf.ones_like(tmp) 96 | 97 | #bb = tf.tensordot(one ,b,1) 98 | output = tf.add(tmp,b) 99 | output = tf.add(output,inp_embed) 100 | return output 101 | 102 | 103 | def build_model(X_cate,X_cont,num_cate): 104 | 105 | inp_embed = embedding_feature_generate(X_cate,num_cate) 106 | inp_embed = tf.concat([inp_embed],axis = 1,name = "concat") 107 | input_dim = inp_embed.get_shape().as_list() 108 | inp_embed = tf.reshape(inp_embed,input_dim[1:3]) 109 | inp_embed = tf.concat([inp_embed,X_cont],axis = 1,name = "concat") 110 | 111 | fc1 = fclayer(inp_embed,272,reluFlag=False,name = 'fc_1') 112 | fc2 = fclayer(fc1,272,reluFlag=False,name = 'fc_2') 113 | fc3 = fclayer(fc2,272,reluFlag=False,name = 'fc_3') 114 | fc4 = fclayer(fc3,272,reluFlag=False,name = 'fc_4') 115 | fc5 = fclayer(fc4,272,reluFlag=False,name = 'fc_5') 116 | fc6 = fclayer(fc5,272,reluFlag=False,name = 'fc_5') 117 | #return fc6 118 | #print inp_embed.shape 119 | cross1 = crosslayer(inp_embed,inp_embed,name = 'cross1') 120 | #print cross1.shape 121 | cross2 = crosslayer(cross1,inp_embed,name = 'cross2') 122 | cross3 = crosslayer(cross2,inp_embed,name = 'cross3') 123 | cross4 = crosslayer(cross3,inp_embed,name = 'cross4') 124 | cross5 = crosslayer(cross4,inp_embed,name = 'cross5') 125 | cross6 = crosslayer(cross5,inp_embed,name = 'cross6') 126 | cross7 = crosslayer(cross6,inp_embed,name = 'cross7') 127 | cross8 = crosslayer(cross7,inp_embed,name = 'cross8') 128 | 129 | output = tf.concat([fc6, cross8], axis = 1,name = 'concat') 130 | #print output.shape 131 | out = fclayer(output,7,reluFlag=False,name = 'out') 132 | return out 133 | 134 | if __name__ == "__main__": 135 | data = pd.read_csv("./covtype.csv") 136 | data, label, cate_values, cont_values = preprocessing(data) 137 | #获取每种类别特征的最大值，这里仅有一种类别 138 | num_cate = [] 139 | for i in range(cate_values.shape[1]): 140 | nunique = np.unique(cate_values[:,i]).shape[0] 141 | num_cate.append(nunique) 142 | 143 | X_cate = tf.placeholder(tf.int32, [None, cate_values.shape[1]]) 144 | X_cont = tf.placeholder("float",[None,cont_values.shape[1]]) 145 | Y = tf.placeholder("float", [None, 7]) 146 | 147 | 148 | 149 | #print(inp_embed.get_shape().as_list()) 150 | #input_dim = inp_embed.get_shape().as_list() 151 | #X_train, X_test, y_train, y_test = train_test_split(inp_embed, label, test_size=0.1,random_state=1024) 152 | output = build_model(X_cate,X_cont,num_cate) 153 | 154 | entropy=tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=Y) 155 | cost = tf.reduce_sum(entropy) 156 | train_op = tf.train.AdamOptimizer(0.001).minimize(cost) 157 | predict_op = tf.arg_max(output,1) 158 | 159 | ## prepare for data 160 | X_train_cate, X_test_cate,X_train_cont,X_test_cont,\ 161 | y_train, y_test = train_test_split(cate_values,cont_values,label, test_size=0.1,random_state=1024) 162 | 163 | with tf.Session() as sess: 164 | sess.run(tf.global_variables_initializer()) 165 | for i in range(100): 166 | lost = [] 167 | for start, end in zip(range(0, len(X_train_cate), 256), range(256, len(X_train_cate)+1, 256)): 168 | sess.run(train_op, feed_dict={X_cate: X_train_cate[start:end], X_cont: X_train_cont[start:end],Y:y_train[start:end]}) 169 | lost.append(sess.run(cost,feed_dict={X_cate: X_train_cate[start:end], X_cont: X_train_cont[start:end],Y:y_train[start:end]})) 170 | print(i,np.mean(lost)) 171 | print(i,np.mean(np.argmax(y_train, axis=1) == sess.run(predict_op, feed_dict={X_cate: X_train_cate[start:end], X_cont: X_train_cont[start:end]}))) 172 | print(i, np.mean(np.argmax(y_test, axis=1) == sess.run(predict_op, feed_dict={X_cate: X_test_cate[start:end], X_cont: X_test_cont[start:end]}))) --------------------------------------------------------------------------------