2050:
169 | var_df.loc[var_df['index']==col,'id_col'] = 1
170 | else:
171 | try:
172 | pd.to_datetime(train[col],infer_datetime_format=True)
173 | var_df.loc[var_df['index']==col,'date_time'] = 1
174 | except:
175 | var_df.loc[var_df['index']==col,'id_col'] = 1
176 | else:
177 | if train[col].min() < 1900 or train[col].max() > 2050:
178 | if col not in num_bool_vars:
179 | var_df.loc[var_df['index']==col,'int'] = 1
180 | else:
181 | try:
182 | pd.to_datetime(train[col],infer_datetime_format=True)
183 | var_df.loc[var_df['index']==col,'date_time'] = 1
184 | except:
185 | if col not in num_bool_vars:
186 | var_df.loc[var_df['index']==col,'int'] = 1
187 | else:
188 | pass
189 | int_vars = list(var_df[(var_df['int'] ==1)]['index'])
190 | date_vars = list(var_df[(var_df['date_time'] == 1)]['index'])
191 | id_vars = list(var_df[(var_df['id_col'] == 1)]['index'])
192 | sum_all_cols['int_vars'] = int_vars
193 | copy_date_vars = copy.deepcopy(date_vars)
194 | for date_var in copy_date_vars:
195 | #### This test is to make sure sure date vars are actually date vars
196 | try:
197 | pd.to_datetime(train[date_var],infer_datetime_format=True)
198 | except:
199 | ##### if not a date var, then just add it to delete it from processing
200 | cols_delete.append(date_var)
201 | date_vars.remove(date_var)
202 | sum_all_cols['date_vars'] = date_vars
203 | sum_all_cols['id_vars'] = id_vars
204 | sum_all_cols['cols_delete'] = cols_delete
205 | ## This is an EXTREMELY complicated logic for cat vars. Don't change it unless you test it many times!
206 | var_df['numeric'] = 0
207 | float_or_cat = var_df.apply(lambda x: 1 if x['type_of_column'] in ['float16',
208 | 'float32','float64'] else 0,
209 | axis=1)
210 | ####### We need to make sure there are no categorical vars in float #######
211 | if len(var_df.loc[float_or_cat == 1]) > 0:
212 | for col in var_df.loc[float_or_cat == 1]['index'].values.tolist():
213 | if len(train[col].value_counts()) > 2 and len(train[col].value_counts()
214 | ) <= float_limit and len(train[col].value_counts()) <= len(train):
215 | var_df.loc[var_df['index']==col,'cat'] = 1
216 | else:
217 | if col not in (num_bool_vars + factor_vars):
218 | var_df.loc[var_df['index']==col,'numeric'] = 1
219 | cat_vars = list(var_df[(var_df['cat'] ==1)]['index'])
220 | continuous_vars = list(var_df[(var_df['numeric'] ==1)]['index'])
221 |
222 | ######## V E R Y I M P O R T A N T ###################################################
223 | cat_vars_copy = copy.deepcopy(factor_vars)
224 | for cat in cat_vars_copy:
225 | if df_preds[cat].dtype==float:
226 | continuous_vars.append(cat)
227 | factor_vars.remove(cat)
228 | var_df.loc[var_df['index']==cat,'dcat'] = 0
229 | var_df.loc[var_df['index']==cat,'numeric'] = 1
230 | elif len(df_preds[cat].value_counts()) == df_preds.shape[0]:
231 | id_vars.append(cat)
232 | factor_vars.remove(cat)
233 | var_df.loc[var_df['index']==cat,'dcat'] = 0
234 | var_df.loc[var_df['index']==cat,'id_col'] = 1
235 |
236 | sum_all_cols['factor_vars'] = factor_vars
237 | ##### There are a couple of extra tests you need to do to remove abberations in cat_vars ###
238 | cat_vars_copy = copy.deepcopy(cat_vars)
239 | for cat in cat_vars_copy:
240 | if df_preds[cat].dtype==float:
241 | continuous_vars.append(cat)
242 | cat_vars.remove(cat)
243 | var_df.loc[var_df['index']==cat,'cat'] = 0
244 | var_df.loc[var_df['index']==cat,'numeric'] = 1
245 | elif len(df_preds[cat].value_counts()) == df_preds.shape[0]:
246 | id_vars.append(cat)
247 | cat_vars.remove(cat)
248 | var_df.loc[var_df['index']==cat,'cat'] = 0
249 | var_df.loc[var_df['index']==cat,'id_col'] = 1
250 | sum_all_cols['cat_vars'] = cat_vars
251 | sum_all_cols['continuous_vars'] = continuous_vars
252 | sum_all_cols['id_vars'] = id_vars
253 | ###### This is where you consoldate the numbers ###########
254 | var_dict_sum = dict(zip(var_df.values[:,0], var_df.values[:,2:].sum(1)))
255 | for col, sumval in var_dict_sum.items():
256 | if sumval == 0:
257 | print('%s of type=%s is not classified' %(col,train[col].dtype))
258 | elif sumval > 1:
259 | print('%s of type=%s is classified into more then one type' %(col,train[col].dtype))
260 | else:
261 | pass
262 | ##### If there are more than 1000 unique values, then add it to NLP vars ###
263 | copy_discretes = copy.deepcopy(discrete_string_vars)
264 | for each_discrete in copy_discretes:
265 | if train[each_discrete].nunique() >= 1000:
266 | nlp_vars.append(each_discrete)
267 | discrete_string_vars.remove(each_discrete)
268 | elif train[each_discrete].nunique() > 100 and train[each_discrete].nunique() < 1000:
269 | pass
270 | else:
271 | ### If it is less than 100 unique values, then make it categorical var
272 | cat_vars.append(each_discrete)
273 | discrete_string_vars.remove(each_discrete)
274 | sum_all_cols['discrete_string_vars'] = discrete_string_vars
275 | sum_all_cols['cat_vars'] = cat_vars
276 | sum_all_cols['nlp_vars'] = nlp_vars
277 | ############### This is where you print all the types of variables ##############
278 | ####### Returns 8 vars in the following order: continuous_vars,int_vars,cat_vars,
279 | ### string_bool_vars,discrete_string_vars,nlp_vars,date_or_id_vars,cols_delete
280 | if verbose == 1:
281 | print(" Number of Numeric Columns = ", len(continuous_vars))
282 | print(" Number of Integer-Categorical Columns = ", len(int_vars))
283 | print(" Number of String-Categorical Columns = ", len(cat_vars))
284 | print(" Number of Factor-Categorical Columns = ", len(factor_vars))
285 | print(" Number of String-Boolean Columns = ", len(string_bool_vars))
286 | print(" Number of Numeric-Boolean Columns = ", len(num_bool_vars))
287 | print(" Number of Discrete String Columns = ", len(discrete_string_vars))
288 | print(" Number of NLP String Columns = ", len(nlp_vars))
289 | print(" Number of Date Time Columns = ", len(date_vars))
290 | print(" Number of ID Columns = ", len(id_vars))
291 | print(" Number of Columns to Delete = ", len(cols_delete))
292 | if verbose == 2:
293 | print(' Printing upto %d columns max in each category:' %max_cols_to_print)
294 | print(" Numeric Columns : %s" %continuous_vars[:max_cols_to_print])
295 | print(" Integer-Categorical Columns: %s" %int_vars[:max_cols_to_print])
296 | print(" String-Categorical Columns: %s" %cat_vars[:max_cols_to_print])
297 | print(" Factor-Categorical Columns: %s" %factor_vars[:max_cols_to_print])
298 | print(" String-Boolean Columns: %s" %string_bool_vars[:max_cols_to_print])
299 | print(" Numeric-Boolean Columns: %s" %num_bool_vars[:max_cols_to_print])
300 | print(" Discrete String Columns: %s" %discrete_string_vars[:max_cols_to_print])
301 | print(" NLP text Columns: %s" %nlp_vars[:max_cols_to_print])
302 | print(" Date Time Columns: %s" %date_vars[:max_cols_to_print])
303 | print(" ID Columns: %s" %id_vars[:max_cols_to_print])
304 | print(" Columns that will not be considered in modeling: %s" %cols_delete[:max_cols_to_print])
305 | ##### now collect all the column types and column names into a single dictionary to return!
306 |
307 | len_sum_all_cols = reduce(add,[len(v) for v in sum_all_cols.values()])
308 | if len_sum_all_cols == orig_cols_total:
309 | if verbose:
310 | print(' %d Predictors classified...' %orig_cols_total)
311 | #print(' This does not include the Target column(s)')
312 | else:
313 | print('No of columns classified %d does not match %d total cols. Continuing...' %(
314 | len_sum_all_cols, orig_cols_total))
315 | ls = sum_all_cols.values()
316 | flat_list = [item for sublist in ls for item in sublist]
317 | if len(left_subtract(list(train),flat_list)) > 0:
318 | print(' Error: some columns missing from classification are: %s' %left_subtract(list(train),flat_list))
319 | return sum_all_cols
320 | ####################################################################################
321 |
--------------------------------------------------------------------------------
/featurewiz/databunch.py:
--------------------------------------------------------------------------------
1 | ###############################################################################
2 | # MIT License
3 | #
4 | # Copyright (c) 2020 Alex Lekov
5 | #
6 | # Permission is hereby granted, free of charge, to any person obtaining a copy
7 | # of this software and associated documentation files (the "Software"), to deal
8 | # in the Software without restriction, including without limitation the rights
9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 | # copies of the Software, and to permit persons to whom the Software is
11 | # furnished to do so, subject to the following conditions:
12 | #
13 | # The above copyright notice and this permission notice shall be included in all
14 | # copies or substantial portions of the Software.
15 | #
16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 | # SOFTWARE.
23 | ###############################################################################
24 | ##### This amazing Library was created by Alex Lekov: Many Thanks to Alex! ###
25 | ##### https://github.com/Alex-Lekov/AutoML_Alex ###
26 | ###############################################################################
27 | import pandas as pd
28 | import numpy as np
29 | from itertools import combinations
30 | from sklearn.preprocessing import StandardScaler
31 | from category_encoders import HashingEncoder, SumEncoder, PolynomialEncoder, BackwardDifferenceEncoder
32 | from category_encoders import OneHotEncoder, HelmertEncoder, OrdinalEncoder, CountEncoder, BaseNEncoder
33 | from category_encoders import TargetEncoder, CatBoostEncoder, WOEEncoder, JamesSteinEncoder
34 | from category_encoders.glmm import GLMMEncoder
35 | from sklearn.preprocessing import LabelEncoder
36 | from category_encoders.wrapper import PolynomialWrapper
37 | from .encoders import FrequencyEncoder
38 | from . import settings
39 |
40 | import pdb
41 | # disable chained assignments
42 | pd.options.mode.chained_assignment = None
43 | import copy
44 | import dask
45 | import dask.dataframe as dd
46 |
47 | class DataBunch(object):
48 | """
49 | Сlass for storing, cleaning and processing your dataset
50 | """
51 | def __init__(self,
52 | X_train=None,
53 | y_train=None,
54 | X_test=None,
55 | y_test=None,
56 | cat_features=None,
57 | clean_and_encod_data=True,
58 | cat_encoder_names=None,
59 | clean_nan=True,
60 | num_generator_features=True,
61 | group_generator_features=True,
62 | target_enc_cat_features=True,
63 | normalization=True,
64 | random_state=42,
65 | verbose=1):
66 | """
67 | Description of __init__
68 |
69 | Args:
70 | X_train=None (undefined): dataset
71 | y_train=None (undefined): y
72 | X_test=None (undefined): dataset
73 | y_test=None (undefined): y
74 | cat_features=None (list or None):
75 | clean_and_encod_data=True (undefined):
76 | cat_encoder_names=None (list or None):
77 | clean_nan=True (undefined):
78 | num_generator_features=True (undefined):
79 | group_generator_features=True (undefined):
80 | target_enc_cat_features=True (undefined)
81 | random_state=42 (undefined):
82 | verbose = 1 (undefined)
83 | """
84 | self.random_state = random_state
85 |
86 | self.X_train = None
87 | self.y_train = None
88 | self.X_test = None
89 | self.y_test = None
90 | self.X_train_predicts = None
91 | self.X_test_predicts = None
92 | self.cat_features = None
93 |
94 | # Encoders
95 | self.cat_encoders_names = settings.cat_encoders_names
96 | self.target_encoders_names = settings.target_encoders_names
97 |
98 |
99 | self.cat_encoder_names = cat_encoder_names
100 | self.cat_encoder_names_list = list(self.cat_encoders_names.keys()) + list(self.target_encoders_names.keys())
101 | self.target_encoders_names_list = list(self.target_encoders_names.keys())
102 |
103 | # check X_train, y_train, X_test
104 | if self.check_data_format(X_train):
105 | if type(X_train) == dask.dataframe.core.DataFrame:
106 | self.X_train_source = X_train.compute()
107 | else:
108 | self.X_train_source = pd.DataFrame(X_train)
109 | self.X_train_source = remove_duplicate_cols_in_dataset(self.X_train_source)
110 | if X_test is not None:
111 | if self.check_data_format(X_test):
112 | if type(X_test) == dask.dataframe.core.DataFrame:
113 | self.X_test_source = X_test.compute()
114 | else:
115 | self.X_test_source = pd.DataFrame(X_test)
116 | self.X_test_source = remove_duplicate_cols_in_dataset(self.X_test_source)
117 |
118 |
119 | ### There is a chance for an error in this - so worth watching!
120 | if y_train is not None:
121 | le = LabelEncoder()
122 | if self.check_data_format(y_train):
123 | if settings.multi_label:
124 | ### if the model is mult-Label, don't transform it since it won't work
125 | self.y_train_source = y_train
126 | else:
127 | if not isinstance(y_train, pd.DataFrame):
128 | if y_train.dtype == 'object' or str(y_train.dtype) == 'category':
129 | self.y_train_source = le.fit_transform(y_train)
130 | else:
131 | if settings.modeltype == 'Multi_Classification':
132 | rare_class = find_rare_class(y_train)
133 | if rare_class != 0:
134 | ### if the rare class is not zero, then transform it using Label Encoder
135 | y_train = le.fit_transform(y_train)
136 | self.y_train_source = copy.deepcopy(y_train)
137 | else:
138 | print('Error: y_train should be a series. Skipping target encoding for dataset...')
139 | target_enc_cat_features = False
140 | else:
141 | if settings.multi_label:
142 | self.y_train_source = pd.DataFrame(y_train)
143 | else:
144 | if y_train.dtype == 'object' or str(y_train.dtype) == 'category':
145 | self.y_train_source = le.fit_transform(pd.DataFrame(y_train))
146 | else:
147 | self.y_train_source = copy.deepcopy(y_train)
148 | else:
149 | print("No target data found!")
150 | return
151 |
152 | if y_test is not None:
153 | self.y_test = y_test
154 |
155 | if verbose > 0:
156 | print('Source X_train shape: ', self.X_train_source.shape)
157 | if not X_test is None:
158 | print('| Source X_test shape: ', self.X_test_source.shape)
159 | print('#'*50)
160 |
161 | # add categorical features in DataBunch
162 | if cat_features is None:
163 | self.cat_features = self.auto_detect_cat_features(self.X_train_source)
164 | if verbose > 0:
165 | print('Auto detect cat features: ', len(self.cat_features))
166 |
167 | else:
168 | self.cat_features = list(cat_features)
169 |
170 | # preproc_data in DataBunch
171 | if clean_and_encod_data:
172 | if verbose > 0:
173 | print('> Start preprocessing with %d variables' %self.X_train_source.shape[1])
174 | self.X_train, self.X_test = self.preproc_data(self.X_train_source,
175 | self.X_test_source,
176 | self.y_train_source,
177 | cat_features=self.cat_features,
178 | cat_encoder_names=cat_encoder_names,
179 | clean_nan=clean_nan,
180 | num_generator_features=num_generator_features,
181 | group_generator_features=group_generator_features,
182 | target_enc_cat_features=target_enc_cat_features,
183 | normalization=normalization,
184 | verbose=verbose,)
185 | else:
186 | self.X_train, self.X_test = X_train, X_test
187 |
188 |
189 | def check_data_format(self, data):
190 | """
191 | Description of check_data_format:
192 | Check that data is not pd.DataFrame or empty
193 |
194 | Args:
195 | data (undefined): dataset
196 | Return:
197 | True or Exception
198 | """
199 | data_tmp = pd.DataFrame(data)
200 | if data_tmp is None or data_tmp.empty:
201 | raise Exception("data is not pd.DataFrame or empty")
202 | else:
203 | if isinstance(data, pd.Series) or isinstance(data, pd.DataFrame):
204 | return True
205 | elif isinstance(data, np.ndarray):
206 | return True
207 | elif type(data) == dask.dataframe.core.DataFrame:
208 | return True
209 | else:
210 | False
211 |
212 | def clean_nans(self, data, cols=None):
213 | """
214 | Fill Nans and add column, that there were nans in this column
215 |
216 | Args:
217 | data (pd.DataFrame, shape (n_samples, n_features)): the input data
218 | cols list() features: the input data
219 | Return:
220 | Clean data (pd.DataFrame, shape (n_samples, n_features))
221 |
222 | """
223 | if cols is not None:
224 | nan_columns = list(data[cols].columns[data[cols].isnull().sum() > 0])
225 | if nan_columns:
226 | for nan_column in nan_columns:
227 | data[nan_column+'_isNAN'] = pd.isna(data[nan_column]).astype('uint8')
228 | data.fillna(data.median(), inplace=True)
229 | return(data)
230 |
231 |
232 | def auto_detect_cat_features(self, data):
233 | """
234 | Description of _auto_detect_cat_features:
235 | Auto-detection categorical_features by simple rule:
236 | categorical feature == if feature nunique low 1% of data
237 |
238 | Args:
239 | data (pd.DataFrame): dataset
240 |
241 | Returns:
242 | cat_features (list): columns names cat features
243 |
244 | """
245 | #object_features = list(data.columns[data.dtypes == 'object'])
246 | cat_features = data.columns[(data.nunique(dropna=False) < len(data)//100) & \
247 | (data.nunique(dropna=False) >2)]
248 | #cat_features = list(set([*object_features, *cat_features]))
249 | return (cat_features)
250 |
251 |
252 | def gen_cat_encodet_features(self, data, cat_encoder_name):
253 | """
254 | Description of _encode_features:
255 | Encode car features
256 |
257 | Args:
258 | data (pd.DataFrame):
259 | cat_encoder_name (str): cat Encoder name
260 |
261 | Returns:
262 | pd.DataFrame
263 |
264 | """
265 |
266 | if isinstance(cat_encoder_name, str):
267 | if cat_encoder_name in self.cat_encoder_names_list and cat_encoder_name not in self.target_encoders_names_list:
268 | if cat_encoder_name == 'HashingEncoder':
269 | encoder = self.cat_encoders_names[cat_encoder_name][0](cols=self.cat_features, n_components=int(np.log(len(data.columns))*1000),
270 | drop_invariant=True)
271 | else:
272 | encoder = self.cat_encoders_names[cat_encoder_name][0](cols=self.cat_features, drop_invariant=True)
273 | data_encodet = encoder.fit_transform(data)
274 | data_encodet = data_encodet.add_prefix(cat_encoder_name + '_')
275 | else:
276 | print(f"{cat_encoder_name} is not supported!")
277 | return ('', '')
278 | else:
279 | encoder = copy.deepcopy(cat_encoder_name)
280 | data_encodet = encoder.transform(data)
281 | data_encodet = data_encodet.add_prefix(str(cat_encoder_name).split("(")[0] + '_')
282 |
283 |
284 | return (data_encodet, encoder)
285 |
286 |
287 | def gen_target_encodet_features(self, x_data, y_data=None, cat_encoder_name=''):
288 | """
289 | Description of _encode_features:
290 | Encode car features
291 |
292 | Args:
293 | data (pd.DataFrame):
294 | cat_encoder_name (str): cat Encoder name
295 |
296 | Returns:
297 | pd.DataFrame
298 |
299 | """
300 |
301 |
302 | if isinstance(cat_encoder_name, str):
303 | ### If it is the first time, it will perform fit_transform !
304 | if cat_encoder_name in self.target_encoders_names_list:
305 | encoder = self.target_encoders_names[cat_encoder_name][0](cols=self.cat_features, drop_invariant=True)
306 | if settings.modeltype == 'Multi_Classification':
307 | ### you must put a Polynomial Wrapper on the cat_encoder in case the model is multi-class
308 | if cat_encoder_name in ['WOEEncoder']:
309 | encoder = PolynomialWrapper(encoder)
310 | ### All other encoders TargetEncoder CatBoostEncoder GLMMEncoder don't need
311 | ### Polynomial Wrappers since they handle multi-class (label encoded) very well!
312 | cols = encoder.cols
313 | for each_col in cols:
314 | x_data[each_col] = encoder.fit_transform(x_data[each_col], y_data).values
315 | data_encodet = encoder.fit_transform(x_data, y_data)
316 | data_encodet = data_encodet.add_prefix(cat_encoder_name + '_')
317 | else:
318 | print(f"{cat_encoder_name} is not supported!")
319 | return ('', '')
320 | else:
321 | ### if it is already fit, then it will only do transform here !
322 | encoder = copy.deepcopy(cat_encoder_name)
323 | data_encodet = encoder.transform(x_data)
324 | data_encodet = data_encodet.add_prefix(str(cat_encoder_name).split("(")[0] + '_')
325 |
326 |
327 | return (data_encodet, encoder)
328 |
329 | def gen_numeric_interaction_features(self,
330 | df,
331 | columns,
332 | operations=['/','*','-','+'],):
333 | """
334 | Description of numeric_interaction_terms:
335 | Numerical interaction generator features: A/B, A*B, A-B,
336 |
337 | Args:
338 | df (pd.DataFrame):
339 | columns (list): num columns names
340 | operations (list): operations type
341 |
342 | Returns:
343 | pd.DataFrame
344 |
345 | """
346 | copy_columns = copy.deepcopy(columns)
347 | fe_df = pd.DataFrame()
348 | for combo_col in combinations(columns,2):
349 | if '/' in operations:
350 | fe_df['{}_div_by_{}'.format(combo_col[0], combo_col[1]) ] = (df[combo_col[0]]*1.) / df[combo_col[1]]
351 | if '*' in operations:
352 | fe_df['{}_mult_by_{}'.format(combo_col[0], combo_col[1]) ] = df[combo_col[0]] * df[combo_col[1]]
353 | if '-' in operations:
354 | fe_df['{}_minus_{}'.format(combo_col[0], combo_col[1]) ] = df[combo_col[0]] - df[combo_col[1]]
355 | if '+' in operations:
356 | fe_df['{}_plus_{}'.format(combo_col[0], combo_col[1]) ] = df[combo_col[0]] + df[combo_col[1]]
357 |
358 | for each_col in copy_columns:
359 | fe_df['{}_squared'.format(each_col) ] = df[each_col].pow(2)
360 | return (fe_df)
361 |
362 |
363 | def gen_groupby_cat_encode_features(self, data, cat_columns, num_column,
364 | cat_encoder_name='JamesSteinEncoder'):
365 | """
366 | Description of group_encoder
367 |
368 | Args:
369 | data (pd.DataFrame): dataset
370 | cat_columns (list): cat columns names
371 | num_column (str): num column name
372 |
373 | Returns:
374 | pd.DataFrame
375 |
376 | """
377 |
378 | if isinstance(cat_encoder_name, str):
379 | if cat_encoder_name in self.cat_encoder_names_list:
380 | encoder = JamesSteinEncoder(cols=self.cat_features, model='beta', return_df = True, drop_invariant=True)
381 | encoder.fit(X=data[cat_columns], y=data[num_column].values)
382 | else:
383 | print(f"{cat_encoder_name} is not supported!")
384 | return ('', '')
385 | else:
386 | encoder = copy.deepcopy(cat_encoder_name)
387 |
388 | data_encodet = encoder.transform(X=data[cat_columns], y=data[num_column].values)
389 | data_encodet = data_encodet.add_prefix('GroupEncoded_' + num_column + '_')
390 |
391 | return (data_encodet, encoder)
392 |
393 | def preproc_data(self, X_train=None,
394 | X_test=None,
395 | y_train=None,
396 | cat_features=None,
397 | cat_encoder_names=None,
398 | clean_nan=True,
399 | num_generator_features=True,
400 | group_generator_features=True,
401 | target_enc_cat_features=True,
402 | normalization=True,
403 | verbose=1,):
404 | """
405 | Description of preproc_data:
406 | dataset preprocessing function
407 |
408 | Args:
409 | X_train=None (pd.DataFrame):
410 | X_test=None (pd.DataFrame):
411 | y_train=None (pd.DataFrame):
412 | cat_features=None (list):
413 | cat_encoder_names=None (list):
414 | clean_nan=True (Bool):
415 | num_generator_features=True (Bool):
416 | group_generator_features=True (Bool):
417 |
418 | Returns:
419 | X_train (pd.DataFrame)
420 | X_test (pd.DataFrame)
421 |
422 | """
423 |
424 | #### Sometimes there are duplicates in column names. You must remove them here. ###
425 | cat_features = find_remove_duplicates(cat_features)
426 |
427 | # concat datasets for correct processing.
428 | df_train = X_train.copy()
429 |
430 | if X_test is None:
431 | data = df_train
432 | test_data = None ### Set test_data to None if X_test is None
433 | else:
434 | test_data = X_test.copy()
435 | test_data = remove_duplicate_cols_in_dataset(test_data)
436 | data = copy.deepcopy(df_train)
437 |
438 | data = remove_duplicate_cols_in_dataset(data)
439 |
440 | # object & num features
441 | object_features = list(data.columns[(data.dtypes == 'object') | (data.dtypes == 'category')])
442 | num_features = list(set(data.columns) - set(cat_features) - set(object_features) - {'test'})
443 | encodet_features_names = list(set(object_features + list(cat_features)))
444 |
445 | original_number_features = len(encodet_features_names)
446 | count_number_features = df_train.shape[1]
447 |
448 | self.encodet_features_names = encodet_features_names
449 | self.num_features_names = num_features
450 | self.binary_features_names = []
451 |
452 | # LabelEncode all Binary Features - leave the rest alone
453 | cols = data.columns.tolist()
454 | #### This sometimes errors because there are duplicate columns in a dataset ###
455 | print('LabelEncode all Boolean Features. Leave the rest alone')
456 | for feature in cols:
457 | if data[feature].dtype == bool :
458 | print(' boolean feature = ',feature)
459 |
460 | for feature in cols:
461 | if (data[feature].dtype == bool):
462 | data[feature] = data[feature].astype('category').cat.codes
463 | if test_data is not None:
464 | test_data[feature] = test_data[feature].astype('category').cat.codes
465 | self.binary_features_names.append(feature)
466 |
467 | # Convert all Category features "Category" type variables if no encoding is specified
468 | cat_only_encoders = [x for x in self.cat_encoder_names if x in self.cat_encoders_names]
469 | if len(cat_only_encoders) > 0:
470 | ### Just skip if this encoder is not in the list of category encoders ##
471 | if encodet_features_names:
472 | if cat_encoder_names is None:
473 | for feature in encodet_features_names:
474 | data[feature] = data[feature].fillna('missing')
475 | data[feature] = data[feature].astype('category').cat.codes
476 | if test_data is not None:
477 | test_data[feature] = test_data[feature].fillna('missing')
478 | test_data[feature] = test_data[feature].astype('category').cat.codes
479 | else:
480 | #### If an encoder is specified, then use that encoder to transform categorical variables
481 | if verbose > 0:
482 | print('> Generate Categorical Encoded features')
483 |
484 | copy_cat_encoder_names = copy.deepcopy(cat_encoder_names)
485 | for encoder_name in copy_cat_encoder_names:
486 | if verbose > 0:
487 | print(' + To know more, click: %s' %self.cat_encoders_names[encoder_name][1])
488 | data_encodet, train_encoder = self.gen_cat_encodet_features(data[encodet_features_names],
489 | encoder_name)
490 | if not isinstance(data_encodet, str):
491 | data = pd.concat([data, data_encodet], axis=1)
492 | if test_data is not None:
493 | test_encodet, _ = self.gen_cat_encodet_features(test_data[encodet_features_names],
494 | train_encoder)
495 | if not isinstance(test_encodet, str):
496 | test_data = pd.concat([test_data, test_encodet], axis=1)
497 |
498 | if verbose > 0:
499 | if not isinstance(data_encodet, str):
500 | addl_features = data_encodet.shape[1] - original_number_features
501 | count_number_features += addl_features
502 | print(' + added ', addl_features, ' additional Features using',encoder_name)
503 |
504 | # Generate Target related Encoder features for cat variables:
505 |
506 |
507 | target_encoders = [x for x in self.cat_encoder_names if x in self.target_encoders_names_list]
508 | if len(target_encoders) > 0:
509 | target_enc_cat_features = True
510 | if target_enc_cat_features:
511 | if encodet_features_names:
512 | if verbose > 0:
513 | print('> Generate Target Encoded categorical features')
514 |
515 | if len(target_encoders) == 0:
516 | target_encoders = ['TargetEncoder'] ### set the default as TargetEncoder if nothing is specified
517 | copy_target_encoders = copy.deepcopy(target_encoders)
518 | for encoder_name in copy_target_encoders:
519 | if verbose > 0:
520 | print(' + To know more, click: %s' %self.target_encoders_names[encoder_name][1])
521 | data_encodet, train_encoder = self.gen_target_encodet_features(data[encodet_features_names],
522 | self.y_train_source, encoder_name)
523 | if not isinstance(data_encodet, str):
524 | data = pd.concat([data, data_encodet], axis=1)
525 |
526 | if test_data is not None:
527 | test_encodet, _ = self.gen_target_encodet_features(test_data[encodet_features_names],'',
528 | train_encoder)
529 | if not isinstance(test_encodet, str):
530 | test_data = pd.concat([test_data, test_encodet], axis=1)
531 |
532 |
533 | if verbose > 0:
534 | if not isinstance(data_encodet, str):
535 | addl_features = data_encodet.shape[1] - original_number_features
536 | count_number_features += addl_features
537 | print(' + added ', len(encodet_features_names) , ' additional Features using ', encoder_name)
538 |
539 | # Clean NaNs in Numeric variables only
540 | if clean_nan:
541 | if verbose > 0:
542 | print('> Cleaned NaNs in numeric features')
543 | data = self.clean_nans(data, cols=num_features)
544 | if test_data is not None:
545 | test_data = self.clean_nans(test_data, cols=num_features)
546 | ### Sometimes, train has nulls while test doesn't and vice versa
547 | if test_data is not None:
548 | rem_cols = left_subtract(list(data),list(test_data))
549 | if len(rem_cols) > 0:
550 | for rem_col in rem_cols:
551 | test_data[rem_col] = 0
552 | elif len(left_subtract(list(test_data),list(data))) > 0:
553 | rem_cols = left_subtract(list(test_data),list(data))
554 | for rem_col in rem_cols:
555 | data[rem_col] = 0
556 | else:
557 | print(' + test and train have similar NaN columns')
558 |
559 | # Generate interaction features for Numeric variables
560 | if num_generator_features:
561 | if len(num_features) > 1:
562 | if verbose > 0:
563 | print('> Generate Interactions features among Numeric variables')
564 | fe_df = self.gen_numeric_interaction_features(data[num_features],
565 | num_features,
566 | operations=['/','*','-','+'],)
567 |
568 | if not isinstance(fe_df, str):
569 | data = pd.concat([data,fe_df],axis=1)
570 | if test_data is not None:
571 | fe_test = self.gen_numeric_interaction_features(test_data[num_features],
572 | num_features,
573 | operations=['/','*','-','+'],)
574 | if not isinstance(fe_test, str):
575 | test_data = pd.concat([test_data, fe_test], axis=1)
576 |
577 | if verbose > 0:
578 | if not isinstance(fe_df, str):
579 | addl_features = fe_df.shape[1]
580 | count_number_features += addl_features
581 | print(' + added ', addl_features, ' Interaction Features ',)
582 |
583 | # Generate Group Encoded Features for Numeric variables only using all Categorical variables
584 | if group_generator_features:
585 | if encodet_features_names and num_features:
586 | if verbose > 0:
587 | print('> Generate Group-by Encoded Features')
588 | print(' + To know more, click: %s' %self.target_encoders_names['JamesSteinEncoder'][1])
589 |
590 | for num_col in num_features:
591 | data_encodet, train_group_encoder = self.gen_groupby_cat_encode_features(
592 | data,
593 | encodet_features_names,
594 | num_col,)
595 | if not isinstance(data_encodet, str):
596 | data = pd.concat([data, data_encodet],axis=1)
597 | if test_data is not None:
598 | test_encodet, _ = self.gen_groupby_cat_encode_features(
599 | data,
600 | encodet_features_names,
601 | num_col,train_group_encoder)
602 | if not isinstance(test_encodet, str):
603 | test_data = pd.concat([test_data, test_encodet], axis=1)
604 |
605 | if verbose > 0:
606 | addl_features = data_encodet.shape[1]*len(num_features)
607 | count_number_features += addl_features
608 | print(' + added ', addl_features, ' Group-by Encoded Features using JamesSteinEncoder')
609 |
610 |
611 | # Drop source cat features
612 | if not len(cat_encoder_names) == 0:
613 | ### if there is no categorical encoding, then let the categorical_vars pass through.
614 | ### If they have been transformed into Cat Encoded variables, then you can drop them!
615 | data.drop(columns=encodet_features_names, inplace=True)
616 | # In this case, there may be some inf values, replace them ######
617 | data.replace([np.inf, -np.inf], np.nan, inplace=True)
618 | #data.fillna(0, inplace=True)
619 | if test_data is not None:
620 | if not len(cat_encoder_names) == 0:
621 | ### if there is no categorical encoding, then let the categorical_vars pass through.
622 | test_data.drop(columns=encodet_features_names, inplace=True)
623 | test_data.replace([np.inf, -np.inf], np.nan, inplace=True)
624 | #test_data.fillna(0, inplace=True)
625 |
626 | X_train = copy.deepcopy(data)
627 | X_test = copy.deepcopy(test_data)
628 |
629 | # Normalization Data
630 | if normalization:
631 | if verbose > 0:
632 | print('> Normalization Features')
633 | columns_name = X_train.columns.values
634 | scaler = StandardScaler().fit(X_train)
635 | X_train = scaler.transform(X_train)
636 | X_test = scaler.transform(X_test)
637 | X_train = pd.DataFrame(X_train, columns=columns_name)
638 | X_test = pd.DataFrame(X_test, columns=columns_name)
639 |
640 | if verbose > 0:
641 | print('#'*50)
642 | print('> Final Number of Features: ', (X_train.shape[1]))
643 | print('#'*50)
644 | print('New X_train rows: %s, X_test rows: %s' %(X_train.shape[0], X_test.shape[0]))
645 | print('New X_train columns: %s, X_test columns: %s' %(X_train.shape[1], X_test.shape[1]))
646 | if len(left_subtract(X_test.columns, X_train.columns)) > 0:
647 | print("""There are more columns in test than train
648 | due to missing columns being more in test than train. Continuing...""")
649 |
650 | return X_train, X_test
651 | ################################################################################
652 | def find_rare_class(series, verbose=0):
653 | ######### Print the % count of each class in a Target variable #####
654 | """
655 | Works on Multi Class too. Prints class percentages count of target variable.
656 | It returns the name of the Rare class (the one with the minimum class member count).
657 | This can also be helpful in using it as pos_label in Binary and Multi Class problems.
658 | """
659 | return series.value_counts().index[-1]
660 | #################################################################################
661 | def left_subtract(l1,l2):
662 | lst = []
663 | for i in l1:
664 | if i not in l2:
665 | lst.append(i)
666 | return lst
667 | #################################################################################
668 | def remove_duplicate_cols_in_dataset(df):
669 | df = copy.deepcopy(df)
670 | cols = df.columns.tolist()
671 | number_duplicates = df.columns.duplicated().astype(int).sum()
672 | if number_duplicates > 0:
673 | print('Detected %d duplicate columns in dataset. Removing duplicates...' %number_duplicates)
674 | df = df.loc[:,~df.columns.duplicated()]
675 | return df
676 | ###########################################################################
677 | # Removes duplicates from a list to return unique values - USED ONLYONCE
678 | def find_remove_duplicates(values):
679 | output = []
680 | seen = set()
681 | for value in values:
682 | if value not in seen:
683 | output.append(value)
684 | seen.add(value)
685 | return output
686 | #################################################################################
687 |
--------------------------------------------------------------------------------
/featurewiz/encoders.py:
--------------------------------------------------------------------------------
1 | ###############################################################################
2 | # MIT License
3 | #
4 | # Copyright (c) 2020 Alex Lekov
5 | #
6 | # Permission is hereby granted, free of charge, to any person obtaining a copy
7 | # of this software and associated documentation files (the "Software"), to deal
8 | # in the Software without restriction, including without limitation the rights
9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 | # copies of the Software, and to permit persons to whom the Software is
11 | # furnished to do so, subject to the following conditions:
12 | #
13 | # The above copyright notice and this permission notice shall be included in all
14 | # copies or substantial portions of the Software.
15 | #
16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 | # SOFTWARE.
23 | ###############################################################################
24 | ##### This amazing Library was created by Alex Lekov: Many Thanks to Alex! ###
25 | ##### https://github.com/Alex-Lekov/AutoML_Alex ###
26 | ###############################################################################
27 | import pandas as pd
28 | import numpy as np
29 |
30 | ################################################################
31 | # Simple Encoders
32 | # (do not use information about target)
33 | ################################################################
34 |
35 | class FrequencyEncoder():
36 | """
37 | FrequencyEncoder
38 | Conversion of category into frequencies.
39 | Parameters
40 | ----------
41 | cols : list of categorical features.
42 | drop_invariant : not used
43 | """
44 | def __init__(self, cols=None, drop_invariant=None):
45 | """
46 | Description of __init__
47 |
48 | Args:
49 | cols=None (undefined): columns in dataset
50 | drop_invariant=None (undefined): not used
51 |
52 | """
53 | self.cols = cols
54 | self.counts_dict = None
55 |
56 | def fit(self, X: pd.DataFrame, y=None) -> pd.DataFrame:
57 | """
58 | Description of fit
59 |
60 | Args:
61 | X (pd.DataFrame): dataset
62 | y=None (not used): not used
63 |
64 | Returns:
65 | pd.DataFrame
66 |
67 | """
68 | counts_dict = {}
69 | if self.cols is None:
70 | self.cols = X.columns
71 | for col in self.cols:
72 | values = X[col].value_counts(dropna=False).index
73 | n_obs = np.float(len(X))
74 | counts = list(X[col].value_counts(dropna=False) / n_obs)
75 | counts_dict[col] = dict(zip(values, counts))
76 | self.counts_dict = counts_dict
77 |
78 | def transform(self, X: pd.DataFrame) -> pd.DataFrame:
79 | """
80 | Description of transform
81 |
82 | Args:
83 | X (pd.DataFrame): dataset
84 |
85 | Returns:
86 | pd.DataFrame
87 |
88 | """
89 | counts_dict_test = {}
90 | res = []
91 | for col in self.cols:
92 | values = X[col].value_counts(1,dropna=False).index.tolist()
93 | counts = X[col].value_counts(1,dropna=False).values.tolist()
94 | counts_dict_test[col] = dict(zip(values, counts))
95 |
96 | # if value is in "train" keys - replace "test" counts with "train" counts
97 | for k in [
98 | key
99 | for key in counts_dict_test[col].keys()
100 | if key in self.counts_dict[col].keys()
101 | ]:
102 | counts_dict_test[col][k] = self.counts_dict[col][k]
103 | res.append(X[col].map(counts_dict_test[col]).values.reshape(-1, 1))
104 | try:
105 | res = np.hstack(res)
106 | except:
107 | pdb.set_trace()
108 | X[self.cols] = res
109 | return X
110 |
111 | def fit_transform(self, X: pd.DataFrame, y=None) -> pd.DataFrame:
112 | """
113 | Description of fit_transform
114 |
115 | Args:
116 | X (pd.DataFrame): dataset
117 | y=None (undefined): not used
118 |
119 | Returns:
120 | pd.DataFrame
121 |
122 | """
123 | self.fit(X, y)
124 | X = self.transform(X)
125 | return X
126 |
--------------------------------------------------------------------------------
/featurewiz/settings.py:
--------------------------------------------------------------------------------
1 | ### this defines some of the global settings for encoder names in one place ####
2 | from category_encoders import HashingEncoder, SumEncoder, PolynomialEncoder, BackwardDifferenceEncoder
3 | from category_encoders import OneHotEncoder, HelmertEncoder, OrdinalEncoder, CountEncoder, BaseNEncoder
4 | from category_encoders import TargetEncoder, CatBoostEncoder, WOEEncoder, JamesSteinEncoder
5 | from category_encoders.glmm import GLMMEncoder
6 | from sklearn.preprocessing import LabelEncoder
7 | from category_encoders.wrapper import PolynomialWrapper
8 | from .encoders import FrequencyEncoder
9 | #################################################################################
10 | def init():
11 | global cat_encoders_names
12 | cat_encoders_names = {
13 | 'HashingEncoder': [HashingEncoder,'https://contrib.scikit-learn.org/category_encoders/hashing.html'],
14 | 'SumEncoder': [SumEncoder,'https://contrib.scikit-learn.org/category_encoders/sum.html'],
15 | 'PolynomialEncoder': [PolynomialEncoder,'https://contrib.scikit-learn.org/category_encoders/polynomial.html'],
16 | 'BackwardDifferenceEncoder': [BackwardDifferenceEncoder,'https://contrib.scikit-learn.org/category_encoders/backward_difference.html'],
17 | 'OneHotEncoder': [OneHotEncoder,'https://contrib.scikit-learn.org/category_encoders/onehot.html'],
18 | 'HelmertEncoder': [HelmertEncoder,'https://contrib.scikit-learn.org/category_encoders/helmert.html'],
19 | 'OrdinalEncoder': [OrdinalEncoder,'https://contrib.scikit-learn.org/category_encoders/ordinal.html'],
20 | 'BaseNEncoder': [BaseNEncoder,'https://contrib.scikit-learn.org/category_encoders/basen.html'],
21 | 'FrequencyEncoder': [FrequencyEncoder,'https://github.com/Alex-Lekov/AutoML_Alex/blob/master/automl_alex/encoders.py'],
22 | }
23 |
24 | global target_encoders_names
25 | target_encoders_names = {
26 | 'TargetEncoder': [TargetEncoder,'https://contrib.scikit-learn.org/category_encoders/targetencoder.html'],
27 | 'CatBoostEncoder': [CatBoostEncoder,'https://contrib.scikit-learn.org/category_encoders/catboost.html'],
28 | 'WOEEncoder': [WOEEncoder,'https://contrib.scikit-learn.org/category_encoders/woe.html'],
29 | 'JamesSteinEncoder': [JamesSteinEncoder,'https://contrib.scikit-learn.org/category_encoders/jamesstein.html'],
30 | 'GLMMEncoder': [GLMMEncoder,'https://contrib.scikit-learn.org/category_encoders/glmm.html'],
31 | }
32 |
33 | global modeltpe
34 | modeltpe = ''
35 | global multi_label
36 | multi_label = False
37 | #################################################################################
38 |
--------------------------------------------------------------------------------
/featurewiz/sulov_method.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | import random
4 | np.random.seed(99)
5 | random.seed(42)
6 | from . import settings
7 | settings.init()
8 | ################################################################################
9 | #### The warnings from Sklearn are so annoying that I have to shut it off #######
10 | import warnings
11 | warnings.filterwarnings("ignore")
12 | from sklearn.exceptions import DataConversionWarning
13 | warnings.filterwarnings(action='ignore', category=DataConversionWarning)
14 | def warn(*args, **kwargs):
15 | pass
16 | warnings.warn = warn
17 | import logging
18 | ####################################################################################
19 | import pdb
20 | import copy
21 | import time
22 | from sklearn.feature_selection import chi2, mutual_info_regression, mutual_info_classif
23 | from sklearn.feature_selection import SelectKBest
24 | from itertools import combinations
25 | import matplotlib.patches as mpatches
26 | import matplotlib.pyplot as plt
27 | #################################################################################################
28 | from collections import defaultdict
29 | from collections import OrderedDict
30 | import time
31 | import networkx as nx # Import networkx for groupwise method
32 | #################################################################################
33 | def left_subtract(l1,l2):
34 | lst = []
35 | for i in l1:
36 | if i not in l2:
37 | lst.append(i)
38 | return lst
39 | #################################################################################
40 | def return_dictionary_list(lst_of_tuples):
41 | """ Returns a dictionary of lists if you send in a list of Tuples"""
42 | orDict = defaultdict(list)
43 | # iterating over list of tuples
44 | for key, val in lst_of_tuples:
45 | orDict[key].append(val)
46 | return orDict
47 | ################################################################################
48 | def find_remove_duplicates(list_of_values):
49 | """
50 | # Removes duplicates from a list to return unique values - USED ONLY ONCE
51 | """
52 | output = []
53 | seen = set()
54 | for value in list_of_values:
55 | if value not in seen:
56 | output.append(value)
57 | seen.add(value)
58 | return output
59 | ##################################################################################
60 | def remove_highly_correlated_vars_fast(df, corr_limit): # Keeping original function for fallback
61 | """Fast method to remove highly correlated vars using just linear correlation."""
62 | corr_matrix = df.corr().abs()
63 | upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))
64 | to_drop = [column for column in upper.columns if any(upper[column] > corr_limit)]
65 | return to_drop
66 |
67 | def FE_remove_variables_using_SULOV_method(df, numvars, modeltype, target,
68 | corr_limit = 0.70, verbose=0, dask_xgboost_flag=False,
69 | correlation_types = ['pearson'], # New parameter for correlation types
70 | adaptive_threshold = False, # New parameter for adaptive threshold
71 | sulov_mode = 'pairwise'): # New parameter for SULOV mode (pairwise/groupwise)
72 | """
73 | FE stands for Feature Engineering - it means this function performs feature engineering
74 | ###########################################################################################
75 | ##### SULOV stands for Searching Uncorrelated List Of Variables #############
76 | ###########################################################################################
77 | SULOV method was created by Ram Seshadri in 2018. This highly efficient method removes
78 | variables that are highly correlated using a series of pair-wise correlation knockout
79 | rounds. It is extremely fast and hence can work on thousands of variables in less than
80 | a minute, even on a laptop. You need to send in a list of numeric variables and that's
81 | all! The method defines high Correlation as anything over 0.70 (absolute) but this can
82 | be changed. If two variables have absolute correlation higher than this, they will be
83 | marked, and using a process of elimination, one of them will get knocked out:
84 | To decide order of variables to keep, we use mutuail information score to select. MIS returns
85 | a ranked list of these correlated variables: when we select one, we knock out others that
86 | are highly correlated to it. Then we select next variable to inspect. This continues until
87 | we knock out all highly correlated variables in each set of variables. Finally we are
88 | left with only uncorrelated variables that are also highly important in mutual score.
89 | ###########################################################################################
90 | ######## YOU MUST INCLUDE THE ABOVE MESSAGE IF YOU COPY SULOV method IN YOUR LIBRARY ##########
91 | ###########################################################################################
92 | """
93 | df = copy.deepcopy(df)
94 | df_target = df[target]
95 | df = df[numvars]
96 | ### for some reason, doing a mass fillna of vars doesn't work! Hence doing it individually!
97 | null_vars = np.array(numvars)[df.isnull().sum()>0]
98 | for each_num in null_vars:
99 | df[each_num] = df[each_num].fillna(0)
100 | target = copy.deepcopy(target)
101 | if verbose:
102 | print('#######################################################################################')
103 | print('##### Searching for Uncorrelated List Of Variables (SULOV) in %s features ############' %len(numvars))
104 | print('#######################################################################################')
105 | print('Starting SULOV with %d features...' %len(numvars))
106 |
107 | # 1. Calculate Correlation Matrices based on correlation_types parameter
108 | correlation_matrices = {}
109 | for corr_type in correlation_types:
110 | correlation_matrices[corr_type] = df.corr(method=corr_type).abs()
111 |
112 | # 2. Adaptive Threshold (if enabled)
113 | current_corr_threshold = corr_limit
114 | if adaptive_threshold:
115 | combined_corr_matrix = pd.concat(correlation_matrices.values()).max(level=0) # Max across all corr types
116 | upper_triangle_corrs = combined_corr_matrix.where(np.triu(np.ones(combined_corr_matrix.shape),k=1).astype(bool)).stack().sort_values(ascending=False)
117 | correlation_values = upper_triangle_corrs.values
118 | current_corr_threshold = np.percentile(correlation_values, 75) # Example: 75th percentile
119 | print(f"Adaptive Correlation Threshold: {current_corr_threshold:.3f}")
120 |
121 | # 3. Find Correlated Pairs based on all selected correlation types
122 | correlated_pairs = []
123 | for i in range(len(df.columns)):
124 | for j in range(i + 1, len(df.columns)):
125 | col1 = df.columns[i]
126 | col2 = df.columns[j]
127 | is_correlated = False
128 | for corr_type, corr_matrix in correlation_matrices.items():
129 | if corr_matrix.loc[col1, col2] >= current_corr_threshold:
130 | is_correlated = True
131 | break # If correlated by any type, consider them correlated
132 | if is_correlated:
133 | correlated_pairs.append((col1, col2))
134 |
135 | # Deterministic sorting of correlated pairs (always applied)
136 | correlated_pairs.sort()
137 |
138 | if modeltype == 'Regression':
139 | sel_function = mutual_info_regression
140 | else:
141 | sel_function = mutual_info_classif
142 |
143 | if correlated_pairs: # Proceed only if correlated pairs are found
144 | if isinstance(target, list):
145 | target = target[0]
146 | max_feats = len(numvars) # Changed from len(corr_list) to numvars to be more robust
147 |
148 | ##### you must ensure there are no infinite nor null values in corr_list df ##
149 | df_fit_cols = find_remove_duplicates(sum(correlated_pairs,())) # Unique cols from correlated pairs
150 | df_fit = df[df_fit_cols]
151 |
152 | ### Now check if there are any NaN values in the dataset #####
153 | if df_fit.isnull().sum().sum() > 0:
154 | df_fit = df_fit.dropna()
155 | else:
156 | print(' there are no null values in dataset...')
157 |
158 | if df_target.isnull().sum().sum() > 0:
159 | print(' there are null values in target. Returning with all vars...')
160 | return numvars
161 | else:
162 | print(' there are no null values in target column...')
163 |
164 | ##### Ready to perform fit and find mutual information score ####
165 |
166 | try:
167 | if modeltype == 'Regression':
168 | fs = mutual_info_regression(df_fit, df_target, n_neighbors=5, discrete_features=False, random_state=42)
169 | else:
170 | fs = mutual_info_classif(df_fit, df_target, n_neighbors=5, discrete_features=False, random_state=42)
171 | except Exception as e:
172 | print(f' SelectKBest() function is erroring with: {e}. Returning with all {len(numvars)} variables...')
173 | return numvars
174 |
175 | try:
176 | #################################################################################
177 | ####### This is the main section where we use mutual info score to select vars
178 | #################################################################################
179 | mutual_info = dict(zip(df_fit_cols,fs)) # Use df_fit_cols as keys
180 | #### The first variable in list has the highest correlation to the target variable ###
181 | sorted_by_mutual_info =[key for (key,val) in sorted(mutual_info.items(), key=lambda kv: kv[1],reverse=True)]
182 |
183 | if sulov_mode == 'pairwise':
184 | ##### Now we select the final list of correlated variables (Pairwise SULOV) ###########
185 | selected_corr_list = []
186 | copy_sorted = copy.deepcopy(sorted_by_mutual_info)
187 | analyzed_pairs = set() # Track analyzed pairs to avoid redundancy
188 |
189 | for col1_sorted in copy_sorted:
190 | selected_corr_list.append(col1_sorted)
191 | for col2_tup in correlated_pairs:
192 | col1_corr, col2_corr = col2_tup
193 | pair = tuple(sorted(col2_tup)) # Ensure consistent pair order
194 | if col1_sorted == col1_corr and pair not in analyzed_pairs: # Check if current sorted col is part of a correlated pair
195 | analyzed_pairs.add(pair)
196 | if col2_corr in copy_sorted:
197 | copy_sorted.remove(col2_corr)
198 | elif col1_sorted == col2_corr and pair not in analyzed_pairs: # Check if current sorted col is part of a correlated pair
199 | analyzed_pairs.add(pair)
200 | if col1_corr in copy_sorted:
201 | copy_sorted.remove(col1_corr)
202 |
203 | elif sulov_mode == 'groupwise':
204 | ##### Groupwise SULOV ###########
205 | G = nx.Graph()
206 | for col in df_fit_cols: # Use df_fit_cols for graph nodes
207 | G.add_node(col)
208 | for col1_g, col2_g in correlated_pairs:
209 | G.add_edge(col1_g, col2_g)
210 | correlated_feature_groups = list(nx.connected_components(G))
211 |
212 | selected_corr_list = []
213 | features_to_drop_in_group = set()
214 | for group in correlated_feature_groups:
215 | if len(group) > 1:
216 | group_mis_scores = {feature: mutual_info.get(feature, 0) for feature in group} # Get MIS for group, default 0 if not found
217 | best_feature = max(group_mis_scores, key=group_mis_scores.get) # Feature with max MIS
218 | selected_corr_list.append(best_feature) # Keep best feature
219 | for feature in group:
220 | if feature != best_feature:
221 | features_to_drop_in_group.add(feature)
222 | selected_corr_list = list(set(selected_corr_list)) # Ensure unique selected features
223 | removed_cols_sulov = list(features_to_drop_in_group) # Features removed in groupwise mode
224 | final_list_corr_part = selected_corr_list # Renamed for clarity
225 |
226 | else: # Default to original pairwise logic if mode is not recognized
227 | print(f"Warning: Unknown SULOV mode '{sulov_mode}'. Defaulting to pairwise mode.")
228 | ##### Original Pairwise SULOV logic (as fallback) ######
229 | selected_corr_list = []
230 | copy_sorted = copy.deepcopy(sorted_by_mutual_info)
231 | copy_pair = dict(return_dictionary_list(correlated_pairs)) # Recreate pair dict if needed
232 | for each_corr_name in copy_sorted:
233 | selected_corr_list.append(each_corr_name)
234 | if each_corr_name in copy_pair: # Check if key exists before accessing
235 | for each_remove in copy_pair[each_corr_name]:
236 | if each_remove in copy_sorted:
237 | copy_sorted.remove(each_remove)
238 | final_list_corr_part = selected_corr_list # Renamed for clarity
239 |
240 |
241 | if sulov_mode != 'groupwise': # For pairwise and default modes
242 | final_list_corr_part = selected_corr_list # Renamed for consistency
243 | removed_cols_sulov = left_subtract(df_fit_cols, final_list_corr_part) # Calculate removed cols
244 |
245 | ##### Now we combine the uncorrelated list to the selected correlated list above
246 | rem_col_list = left_subtract(numvars, df_fit_cols) # Uncorrelated columns are those not in df_fit_cols
247 | final_list = rem_col_list + final_list_corr_part
248 | removed_cols = left_subtract(numvars, final_list) + removed_cols_sulov # Combine all removed cols
249 |
250 | except Exception as e:
251 | print(f' SULOV Method crashing due to: {e}')
252 | #### Dropping highly correlated Features fast using simple linear correlation ###
253 | removed_cols = remove_highly_correlated_vars_fast(df,corr_limit)
254 | final_list = left_subtract(numvars, removed_cols)
255 |
256 | if len(removed_cols) > 0:
257 | if verbose:
258 | print(f' Removing ({len(removed_cols)}) highly correlated variables:')
259 | if len(removed_cols) <= 30:
260 | print(f' {removed_cols}')
261 | if len(final_list) <= 30:
262 | print(f' Following ({len(final_list)}) vars selected: {final_list}')
263 |
264 | ############## D R A W C O R R E L A T I O N N E T W O R K ##################
265 | selected = copy.deepcopy(final_list)
266 | if verbose and len(selected) <= 1000 and correlated_pairs: # Draw only if correlated pairs exist
267 | try:
268 | #### Now start building the graph ###################
269 | gf = nx.Graph()
270 | ### the mutual info score gives the size of the bubble ###
271 | multiplier = 2100
272 | for each in sorted_by_mutual_info: # Use sorted_by_mutual_info for node order
273 | if each in mutual_info: # Check if mutual_info exists for the node
274 | gf.add_node(each, size=int(max(1,mutual_info[each]*multiplier)))
275 |
276 | ######### This is where you calculate the size of each node to draw
277 | sizes = [mutual_info.get(x,0)*multiplier for x in list(gf.nodes())] # Use .get with default 0 for robustness
278 |
279 | corr = df[df_fit_cols].corr() # Use df_fit_cols for correlation calculation
280 | high_corr = corr[abs(corr)>current_corr_threshold] # Use adaptive/original threshold
281 | combos = combinations(df_fit_cols,2) # Use df_fit_cols for combinations
282 |
283 | ### this gives the strength of correlation between 2 nodes ##
284 | multiplier_edge = 20 # Renamed to avoid confusion
285 | for (var1, var2) in combos:
286 | if np.isnan(high_corr.loc[var1,var2]):
287 | pass
288 | else:
289 | gf.add_edge(var1, var2,weight=multiplier_edge*high_corr.loc[var1,var2])
290 |
291 | ######## Now start building the networkx graph ##########################
292 | widths = nx.get_edge_attributes(gf, 'weight')
293 | nodelist = gf.nodes()
294 | cols = 5
295 | height_size = 5
296 | width_size = 15
297 | rows = int(len(df_fit_cols)/cols) # Use df_fit_cols length
298 | if rows < 1:
299 | rows = 1
300 | plt.figure(figsize=(width_size,min(20,height_size*rows)))
301 | pos = nx.shell_layout(gf)
302 | nx.draw_networkx_nodes(gf,pos,
303 | nodelist=nodelist,
304 | node_size=sizes,
305 | node_color='blue',
306 | alpha=0.5)
307 | nx.draw_networkx_edges(gf,pos,
308 | edgelist = widths.keys(),
309 | width=list(widths.values()),
310 | edge_color='lightblue',
311 | alpha=0.6)
312 | pos_higher = {}
313 | x_off = 0.04 # offset on the x axis
314 | y_off = 0.04 # offset on the y axis
315 | for k, v in pos.items():
316 | pos_higher[k] = (v[0]+x_off, v[1]+y_off)
317 |
318 | labels_dict = {} # Create labels dictionary
319 | for x in nodelist:
320 | if x in selected:
321 | labels_dict[x] = x+' (selected)'
322 | else:
323 | labels_dict[x] = x+' (removed)'
324 |
325 | nx.draw_networkx_labels(gf, pos=pos_higher,
326 | labels = labels_dict,
327 | font_color='black')
328 |
329 | plt.box(True)
330 | plt.title("""In SULOV, we repeatedly remove features with lower mutual info scores among highly correlated pairs (see figure),
331 | SULOV selects the feature with higher mutual info score related to target when choosing between a pair. """, fontsize=10)
332 | plt.suptitle('How SULOV Method Works by Removing Highly Correlated Features', fontsize=20,y=1.03)
333 | red_patch = mpatches.Patch(color='blue', label='Bigger circle denotes higher mutual info score with target')
334 | blue_patch = mpatches.Patch(color='lightblue', label='Thicker line denotes higher correlation between two variables')
335 | plt.legend(handles=[red_patch, blue_patch],loc='best')
336 | plt.show();
337 | ##### N E T W O R K D I A G R A M C O M P L E T E #################
338 | return final_list
339 | except Exception as e:
340 | print(f' Networkx library visualization crashing due to {e}')
341 | print(f'Completed SULOV. {len(final_list)} features selected')
342 | return final_list
343 | else:
344 | print(f'Completed SULOV. {len(final_list)} features selected')
345 | return final_list
346 | print(f'Completed SULOV. All {len(numvars)} features selected')
347 | return numvars
348 | ###################################################################################
--------------------------------------------------------------------------------
/images/MRMR.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/MRMR.png
--------------------------------------------------------------------------------
/images/SULOV.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/SULOV.jpg
--------------------------------------------------------------------------------
/images/feather_example.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/feather_example.jpg
--------------------------------------------------------------------------------
/images/feature_engg.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/feature_engg.png
--------------------------------------------------------------------------------
/images/feature_engg_old.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/feature_engg_old.jpg
--------------------------------------------------------------------------------
/images/featurewiz_background.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/featurewiz_background.jpg
--------------------------------------------------------------------------------
/images/featurewiz_logo.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/featurewiz_logo.jpg
--------------------------------------------------------------------------------
/images/featurewiz_logos.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/featurewiz_logos.png
--------------------------------------------------------------------------------
/images/featurewiz_logos_old.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/featurewiz_logos_old.png
--------------------------------------------------------------------------------
/images/featurewiz_mrmr.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/featurewiz_mrmr.png
--------------------------------------------------------------------------------
/images/xgboost.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AutoViML/featurewiz/1fe1341a4272957a32d42f63a2446f5f8c4011c2/images/xgboost.jpg
--------------------------------------------------------------------------------
/old_README.md:
--------------------------------------------------------------------------------
1 | # featurewiz
2 | `featurewiz` is the best feature selection library for boosting your machine learning performance with minimal effort and maximum relevance using the famous MRMR algorithm.
3 |
4 | 
5 |
6 | # Table of Contents
7 |
23 |
24 | ## Latest
25 | `featurewiz` 5.0 version is out! It contains brand new Deep Learning Auto Encoders to enrich your data for the toughest imbalanced and multi-class datasets. If you are looking for the latest and greatest updates about our library, check out our updates page.
26 |
27 |
28 | ## Citation
29 | If you use featurewiz in your research project or paper, please use the following format for citations:
30 |
31 | "Seshadri, Ram (2020). GitHub - AutoViML/featurewiz: Use advanced feature engineering strategies and select the best features from your data set fast with a single line of code. source code: https://github.com/AutoViML/featurewiz "
32 |
33 | Current citations for featurewiz
34 |
35 | [Google Scholar](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C31&q=featurewiz&btnG=)
36 |
37 | ## Highlights
38 | `featurewiz` stands out as a versatile and powerful tool for feature selection and engineering, capable of significantly enhancing model performance through intelligent feature transformation and selection techniques. Its unique methods like SULOV and recursive XGBoost, combined with advanced feature engineering options, make it a valuable addition to any data scientist's toolkit:
39 | ### Best Feature Selection Algorithm
40 |
It provides one of the best automatic feature selection algorithms (Minimum Redundancy Maximum Relevance (MRMR) algorithm) as described by wikipedia in this page: "The MRMR selection has been found to be more powerful than the maximum relevance feature selection" such as Boruta.
41 |
42 | ### Advanced Feature Engineering Options
43 | featurewiz extends beyond traditional feature selection by including powerful feature engineering capabilities such as:
44 | Auto Encoders, including Denoising Auto Encoders (DAEs) and Variational Auto Encoders (VAEs), for improved model performance, especially on imbalanced datasets.
45 | A variety of category encoders like HashingEncoder, SumEncoder, PolynomialEncoder, BackwardDifferenceEncoder, OneHotEncoder, HelmertEncoder, OrdinalEncoder, and BaseNEncoder.
46 | The ability to add interaction features (e.g., x1x2, x2x3, x1^2), group by features, and target encoding.
47 |
48 | ### SULOV Method for Feature Selection
49 | SULOV stands for "Searching for Uncorrelated List Of Variables". It selects features that are uncorrelated with each other but have high correlation with the target variable, based on the Minimum Redundancy Maximum Relevance (mRMR) principle. This method effectively reduces redundancy in features while retaining those with high relevance to the target.
50 |
51 | ### Recursive XGBoost Method
52 | After applying the SULOV method, featurewiz employs a recursive approach using XGBoost's feature importance. This process is repeated multiple times on subsets of data, combining and deduplicating selected features to identify the most impactful ones.
53 |
54 | ### Comprehensive Encoding and Transformation
55 | featurewiz allows for extensive customization in how features are encoded and transformed, making it highly adaptable to various types of data.
56 | The ability to combine multiple encoding and transformation methods enhances its flexibility and effectiveness in feature engineering.
57 |
58 | ### Used by PhD's and Researchers and actively maintained
59 | featurewiz is used by researchers and PhD data scientists around the world: there are 64 citations for featurewiz since its release:
60 |
61 | [Google Scholar](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C31&q=featurewiz&btnG=)
62 | It's efficient in handling large datasets, making it suitable for a wide range of applications from small to big data scenarios.
63 | It is well-documented, and it comes with a number of examples.
64 | It is actively maintained, and it is regularly updated with new features and bug fixes.
65 |
66 | ## Internals
67 | `featurewiz` has two major internal modules. They are explained below.
68 | ### 1. Feature Engineering module
69 | The first step is not absolutely necessary but it can be used to create new features that may or may not be helpful (be careful with automated feature engineering tools!).
70 | One of the gaps in open-source AutoML tools and especially Auto_ViML has been the lack of feature engineering capabilities that high-powered competitions such as Kaggle required. The ability to create "interaction" variables or adding "group-by" features or "target-encoding" categorical variables was difficult and sifting through those hundreds of new features to find the best features was difficult and left only to "experts" or "professionals". featurewiz was created to help you in this endeavor.
71 |
featurewiz now enables you to add hundreds of such features with a single line of code. Set the "feature_engg" flag to "interactions", "groupby" or "target" and featurewiz will select the best encoders for each of those options and create hundreds (perhaps thousands) of features in one go. Not only that, using the next step, featurewiz will sift through numerous such variables and find only the least correlated and most relevant features to your model. All in one step!.
72 |
73 | 
74 |
75 | ### 2. Feature Selection module
76 |
The second step is Feature Selection. `featurewiz` uses the MRMR (Minimum Redundancy Maximum Relevance) algorithm as the basis for its feature selection.
77 | Why perform Feature Selection? Once you have created 100's of new features, you still have three questions left to answer:
78 | 1. How do we interpret those newly created features?
79 | 2. Which of these features is important and which is useless? How many of them are highly correlated to each other causing redundancy?
80 | 3. Does the model overfit now on these new features and perform better or worse than before?
81 |
82 | All are very important questions and featurewiz answers them by using the SULOV method and Recursive XGBoost to reduce features in your dataset to the best "minimum optimal" features for the model.
83 |
SULOV: SULOV stands for `Searching for Uncorrelated List of Variables`. The SULOV algorithm is based on the Minimum-Redundancy-Maximum-Relevance (MRMR) algorithm explained in this article as one of the best feature selection methods. To understand how MRMR works and how it is different from `Boruta` and other feature selection methods, see the chart below. Here "Minimal Optimal" refers to MRMR (featurewiz) while "all-relevant" refers to Boruta.
84 |
85 | 
86 |
87 | ## Working
88 | `featurewiz` performs feature selection in 2 steps. Each step is explained below.
89 | The working of the `SULOV` algorithm is as follows:
90 |
91 | - Find all the pairs of highly correlated variables exceeding a correlation threshold (say absolute(0.7)).
92 | - Then find their MIS score (Mutual Information Score) to the target variable. MIS is a non-parametric scoring method. So its suitable for all kinds of variables and target.
93 | - Now take each pair of correlated variables, then knock off the one with the lower MIS score.
94 | - What’s left is the ones with the highest Information scores and least correlation with each other.
95 |
96 |
97 | 
98 |
99 | The working of the Recursive XGBoost is as follows:
100 | Once SULOV has selected variables that have high mutual information scores with the least correlation among them, featurewiz uses XGBoost to repeatedly find the best features among the remaining variables after SULOV.
101 |
102 | - Select all variables in the data set and the full data split into train and valid sets.
103 | - Find top X features (could be 10) on train using valid for early stopping (to prevent over-fitting)
104 | - Then take the next set of vars and find top X
105 | - Do this 5 times. Combine all selected features and de-duplicate them.
106 |
107 |
108 | 
109 |
110 | ## Tips
111 | Here are some additional tips for ML engineers and data scientists when using featurewiz:
112 |
113 | - How to cross-validate your results: When you use featurewiz, we automatically perform multiple rounds of feature selection using permutations on the number of columns. However, you can perform feature selection using permutations of rows as follows in cross_validate using featurewiz.
114 |
- Use multiple feature selection tools: It is a good idea to use multiple feature selection tools and compare the results. This will help you to get a better understanding of which features are most important for your data.
115 | - Don't forget to engineer new features: Feature selection is only one part of the process of building a good machine learning model. You should also spend time engineering your features to make them as informative as possible. This can involve things like creating new features, transforming existing features, and removing irrelevant features.
116 | - Don't overfit your model: It is important to avoid overfitting your model to the training data. Overfitting occurs when your model learns the noise in the training data, rather than the underlying signal. To avoid overfitting, you can use regularization techniques, such as lasso or elasticnet.
117 | - Start with a small number of features: When you are first starting out, it is a good idea to start with a small number of features. This will help you to avoid overfitting your model. As you become more experienced, you can experiment with adding more features.
118 |
119 |
120 | ## Install
121 |
122 | **Prerequisites:**
123 |
124 | - featurewiz is built using xgboost, dask, numpy, pandas and matplotlib. It should run on most Python 3 Anaconda installations. You won't have to import any special libraries other than "dask", "XGBoost" and "networkx" library. Optionally, it uses LightGBM for fast modeling, which it installs automatically.
125 | - We use "networkx" library for charts and interpretability.
But if you don't have these libraries, featurewiz will install those for you automatically.
126 |
127 | To install from source:
128 |
129 | ```
130 | cd
131 | git clone git@github.com:AutoViML/featurewiz.git
132 | # or download and unzip https://github.com/AutoViML/featurewiz/archive/master.zip
133 | conda create -n python=3.7 anaconda
134 | conda activate # ON WINDOWS: `source activate `
135 | cd featurewiz
136 | pip install -r requirements.txt
137 | ```
138 |
139 | ## Good News: You can install featurewiz on Colab and Kaggle easily in 2 steps!
140 | As of June 2022, thanks to [arturdaraujo](https://github.com/arturdaraujo), featurewiz is now available on conda-forge. You can try:
141 |
142 | ```
143 | conda install -c conda-forge featurewiz
144 | ```
145 |
146 | ### If the above conda install fails, you can try installing featurewiz this way:
147 | #### Install featurewiz using git+
148 |
149 | ```
150 | !pip install git+https://github.com/AutoViML/featurewiz.git
151 | ```
152 |
153 | ## Usage
154 |
155 | There are two ways to use featurewiz.
156 |
157 | - The first way is the new way where you use scikit-learn's `fit and predict` syntax. It also includes the `lazytransformer` library that I created to transform datetime, NLP and categorical variables into numeric variables automatically. We recommend that you use it as the main syntax for all your future needs.
158 |
159 | ```
160 | from featurewiz import FeatureWiz
161 | fwiz = FeatureWiz(feature_engg = '', nrows=None, transform_target=True, scalers="std",
162 | category_encoders="auto", add_missing=False, verbose=0)
163 | X_train_selected, y_train = fwiz.fit_transform(X_train, y_train)
164 | X_test_selected = fwiz.transform(X_test)
165 | ### get list of selected features ###
166 | fwiz.features
167 | ```
168 |
169 | - The second way is the old way and this was the original syntax of featurewiz. It is still being used by thousands of researchers in the field. Hence it will continue to be maintained. However, it can be discontinued any time without notice. You can use it if you like it.
170 |
171 | ```
172 | import featurewiz as fwiz
173 | outputs = fwiz.featurewiz(dataname=train, target=target, corr_limit=0.70, verbose=2, sep=',',
174 | header=0, test_data='',feature_engg='', category_encoders='',
175 | dask_xgboost_flag=False, nrows=None, skip_sulov=False, skip_xgboost=False)
176 | ```
177 |
178 | `outputs` is a tuple: There will always be two objects in output. It can vary:
179 | - In the first case, it can be `features` and `trainm`: features is a list (of selected features) and trainm is the transformed dataframe (if you sent in train only)
180 | - In the second case, it can be `trainm` and `testm`: It can be two transformed dataframes when you send in both test and train but with selected features.
181 |
182 | In both cases, the features and dataframes are ready for you to do further modeling.
183 |
184 | Featurewiz works on any multi-class, multi-label data Set. So you can have as many target labels as you want.
185 | You don't have to tell Featurewiz whether it is a Regression or Classification problem. It will decide that automatically.
186 |
187 | ## API
188 |
189 | **Input Arguments for NEW syntax**
190 |
191 | Parameters
192 | ----------
193 | corr_limit : float, default=0.90
194 | The correlation limit to consider for feature selection. Features with correlations
195 | above this limit may be excluded.
196 |
197 | verbose : int, default=0
198 | Level of verbosity in output messages.
199 |
200 | feature_engg : str or list, default=''
201 | Specifies the feature engineering methods to apply, such as 'interactions', 'groupby',
202 | and 'target'.
203 |
204 | category_encoders : str or list, default=''
205 | Encoders for handling categorical variables. Supported encoders include 'onehot',
206 | 'ordinal', 'hashing', 'count', 'catboost', 'target', 'glm', 'sum', 'woe', 'bdc',
207 | 'loo', 'base', 'james', 'helmert', 'label', 'auto', etc.
208 |
209 | add_missing : bool, default=False
210 | If True, adds indicators for missing values in the dataset.
211 |
212 | dask_xgboost_flag : bool, default=False
213 | If set to True, enables the use of Dask for parallel computing with XGBoost.
214 |
215 | nrows : int or None, default=None
216 | Limits the number of rows to process.
217 |
218 | skip_sulov : bool, default=False
219 | If True, skips the application of the Super Learning Optimized (SULO) method in
220 | feature selection.
221 |
222 | skip_xgboost : bool, default=False
223 | If True, bypasses the recursive XGBoost feature selection.
224 |
225 | transform_target : bool, default=False
226 | When True, transforms the target variable(s) into numeric format if they are not
227 | already.
228 |
229 | scalers : str or None, default=None
230 | Specifies the scaler to use for feature scaling. Available options include
231 | 'std', 'standard', 'minmax', 'max', 'robust', 'maxabs'.
232 |
233 | **Input Arguments for old syntax**
234 |
235 | - `dataname`: could be a datapath+filename or a dataframe. It will detect whether your input is a filename or a dataframe and load it automatically.
236 | - `target`: name of the target variable in the data set.
237 | - `corr_limit`: if you want to set your own threshold for removing variables as highly correlated, then give it here. The default is 0.9 which means variables less than -0.9 and greater than 0.9 in pearson's correlation will be candidates for removal.
238 | - `verbose`: This has 3 possible states:
239 | - `0` - limited output. Great for running this silently and getting fast results.
240 | - `1` - verbose. Great for knowing how results were and making changes to flags in input.
241 | - `2` - more charts such as SULOV and output. Great for finding out what happens under the hood for SULOV method.
242 | - `test_data`: This is only applicable to the old syntax if you want to transform both train and test data at the same time in the same way. `test_data` could be the name of a datapath+filename or a dataframe. featurewiz will detect whether your input is a filename or a dataframe and load it automatically. Default is empty string.
243 | - `dask_xgboost_flag`: default False. If you want to use dask with your data, then set this to True.
244 | - `feature_engg`: You can let featurewiz select its best encoders for your data set by setting this flag
245 | for adding feature engineering. There are three choices. You can choose one, two, or all three.
246 | - `interactions`: This will add interaction features to your data such as x1*x2, x2*x3, x1**2, x2**2, etc.
247 | - `groupby`: This will generate Group By features to your numeric vars by grouping all categorical vars.
248 | - `target`: This will encode and transform all your categorical features using certain target encoders.
249 | Default is empty string (which means no additional features)
250 | - `add_missing`: default is False. This is a new flag: the `add_missing` flag will add a new column for missing values for all your variables in your dataset. This will help you catch missing values as an added signal.
251 | - `category_encoders`: default is "auto". Instead, you can choose your own category encoders from the list below.
252 | We recommend you do not use more than two of these. Featurewiz will automatically select only two if you have more than two in your list. You can set "auto" for our own choice or the empty string "" (which means no encoding of your categorical features)
These descriptions are derived from the excellent category_encoders python library. Please check it out!
253 | - `HashingEncoder`: HashingEncoder is a multivariate hashing implementation with configurable dimensionality/precision. The advantage of this encoder is that it does not maintain a dictionary of observed categories. Consequently, the encoder does not grow in size and accepts new values during data scoring by design.
254 | - `SumEncoder`: SumEncoder is a Sum contrast coding for the encoding of categorical features.
255 | - `PolynomialEncoder`: PolynomialEncoder is a Polynomial contrast coding for the encoding of categorical features.
256 | - `BackwardDifferenceEncoder`: BackwardDifferenceEncoder is a Backward difference contrast coding for encoding categorical variables.
257 | - `OneHotEncoder`: OneHotEncoder is the traditional Onehot (or dummy) coding for categorical features. It produces one feature per category, each being a binary.
258 | - `HelmertEncoder`: HelmertEncoder uses the Helmert contrast coding for encoding categorical features.
259 | - `OrdinalEncoder`: OrdinalEncoder uses Ordinal encoding to designate a single column of integers to represent the categories in your data. Integers however start in the same order in which the categories are found in your dataset. If you want to change the order, just sort the column and send it in for encoding.
260 | - `FrequencyEncoder`: FrequencyEncoder is a count encoding technique for categorical features. For a given categorical feature, it replaces the names of the categories with the group counts of each category.
261 | - `BaseNEncoder`: BaseNEncoder encodes the categories into arrays of their base-N representation. A base of 1 is equivalent to one-hot encoding (not really base-1, but useful), and a base of 2 is equivalent to binary encoding. N=number of actual categories is equivalent to vanilla ordinal encoding.
262 | - `TargetEncoder`: TargetEncoder performs Target encoding for categorical features. It supports the following kinds of targets: binary and continuous. For multi-class targets, it uses a PolynomialWrapper.
263 | - `CatBoostEncoder`: CatBoostEncoder performs CatBoost coding for categorical features. It supports the following kinds of targets: binary and continuous. For polynomial target support, it uses a PolynomialWrapper. This is very similar to leave-one-out encoding, but calculates the values “on-the-fly”. Consequently, the values naturally vary during the training phase and it is not necessary to add random noise.
264 | - `WOEEncoder`: WOEEncoder uses the Weight of Evidence technique for categorical features. It supports only one kind of target: binary. For polynomial target support, it uses a PolynomialWrapper. It cannot be used for Regression.
265 | - `JamesSteinEncoder`: JamesSteinEncoder uses the James-Stein estimator. It supports 2 kinds of targets: binary and continuous. For polynomial target support, it uses PolynomialWrapper.
266 | For feature value i, James-Stein estimator returns a weighted average of:
267 | The mean target value for the observed feature value i.
268 | The mean target value (regardless of the feature value).
269 | - `nrows`: default `None`. You can set the number of rows to read from your datafile if it is too large to fit into either dask or pandas. But you won't have to if you use dask.
270 | - `skip_sulov`: default `False`. You can set the flag to skip the SULOV method if you want.
271 | - `skip_xgboost`: default `False`. You can set the flag to skip the Recursive XGBoost method if you want.
272 |
273 | **Output values for old syntax** This applies only to the old syntax.
274 | - `outputs`: Output is always a tuple. We can call our outputs in that tuple as `out1` and `out2` below.
275 | - `out1` and `out2`: If you sent in just one dataframe or filename as input, you will get:
276 | - 1. `features`: It will be a list (of selected features) and
277 | - 2. `trainm`: It will be a dataframe (if you sent in a file or dataname as input)
278 | - `out1` and `out2`: If you sent in two files or dataframes (train and test), you will get:
279 | - 1. `trainm`: a modified train dataframe with engineered and selected features from dataname and
280 | - 2. `testm`: a modified test dataframe with engineered and selected features from test_data.
281 |
282 | ## Additional
283 | To learn more about how featurewiz works under the hood, watch this [video](https://www.youtube.com/embed/ZiNutwPcAU0)
284 |
285 | 
286 |
287 | featurewiz was designed for selecting High Performance variables with the fewest steps.
288 | In most cases, featurewiz builds models with 20%-99% fewer features than your original data set with nearly the same or slightly lower performance (this is based on my trials. Your experience may vary).
289 |
290 | featurewiz is every Data Scientist's feature wizard that will:
291 | - Automatically pre-process data: you can send in your entire dataframe "as is" and featurewiz will classify and change/label encode categorical variables changes to help XGBoost processing. It classifies variables as numeric or categorical or NLP or date-time variables automatically so it can use them correctly to model.
292 | - Perform feature engineering automatically: The ability to create "interaction" variables or adding "group-by" features or "target-encoding" categorical variables is difficult and sifting through those hundreds of new features is painstaking and left only to "experts". Now, with featurewiz you can create hundreds or even thousands of new features with the click of a mouse. This is very helpful when you have a small number of features to start with. However, be careful with this option. You can very easily create a monster with this option.
293 |
- Perform feature reduction automatically. When you have small data sets and you know your domain well, it is easy to perhaps do EDA and identify which variables are important. But when you have a very large data set with hundreds if not thousands of variables, selecting the best features from your model can mean the difference between a bloated and highly complex model or a simple model with the fewest and most information-rich features. featurewiz uses XGBoost repeatedly to perform feature selection. You must try it on your large data sets and compare!
294 | - Explain SULOV method graphically using networkx library so you can see which variables are highly correlated to which ones and which of those have high or low mutual information scores automatically. Just set verbose = 2 to see the graph.
295 | - Build a fast XGBoost or LightGBM model using the features selected by featurewiz. There is a function called "simple_lightgbm_model" which you can use to build a fast model. It is a new module, so check it out.
296 |
297 |
298 | *** Special thanks to fellow open source Contributors ***:
299 |
300 | - Alex Lekov (https://github.com/Alex-Lekov/AutoML_Alex/tree/master/automl_alex) for his DataBunch and encoders modules which are used by the tool (although with some modifications).
301 | - Category Encoders library in Python : This is an amazing library. Make sure you read all about the encoders that featurewiz uses here: https://contrib.scikit-learn.org/category_encoders/index.html
302 |
303 |
304 | ## Maintainers
305 |
306 | * [@AutoViML](https://github.com/AutoViML)
307 |
308 | ## Contributing
309 |
310 | See [the contributing file](CONTRIBUTING.md)!
311 |
312 | PRs accepted.
313 |
314 | ## License
315 |
316 | Apache License 2.0 © 2020 Ram Seshadri
317 |
318 | ## DISCLAIMER
319 | This project is not an official Google project. It is not supported by Google and Google specifically disclaims all warranties as to its quality, merchantability, or fitness for a particular purpose.
320 |
321 |
322 | [page]: examples/cross_validate.py
323 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy<2.0
2 | ipython
3 | jupyter
4 | xgboost>=1.6.2,<=1.7.6
5 | pandas>=2.0
6 | matplotlib
7 | seaborn
8 | scipy
9 | scikit-learn>=1.2.2,<=1.5.2
10 | networkx
11 | category_encoders==2.6.3
12 | xlrd>=2.0.0
13 | dask>=2021.11.0
14 | lightgbm>=3.2.1
15 | distributed>=2021.11.0
16 | feather-format>=0.4.1
17 | pyarrow>=7.0.0
18 | fsspec>=0.3.3
19 | Pillow>=9.0.0
20 | tqdm>=4.61.1
21 | numexpr>=2.7.3
22 | tensorflow>=2.5.2
23 | lazytransform>=1.17
24 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | import setuptools
4 |
5 | with open("README.md", "r", encoding="utf-8") as fh:
6 | long_description = fh.read()
7 |
8 | setuptools.setup(
9 | name="featurewiz",
10 | version="0.6.1",
11 | author="Ram Seshadri",
12 | author_email="rsesha2001@yahoo.com",
13 | description="Select Best Features from your data set - any size - now with XGBoost!",
14 | long_description=long_description,
15 | long_description_content_type="text/markdown",
16 | license='Apache License 2.0',
17 | url="https://github.com/AutoViML/featurewiz",
18 | packages=setuptools.find_packages(exclude=("tests",)),
19 | install_requires=[
20 | "numpy<2.0",
21 | "ipython",
22 | "jupyter",
23 | "xgboost>=1.6.2,<=1.7.6",
24 | "pandas>=2.0",
25 | "matplotlib",
26 | "seaborn",
27 | "scipy",
28 | "scikit-learn>=1.2.2,<=1.5.2",
29 | "networkx",
30 | "category_encoders==2.6.3",
31 | "xlrd>=2.0.0",
32 | "dask>=2021.11.0",
33 | "lightgbm>=3.2.1",
34 | "distributed>=2021.11.0",
35 | "feather-format>=0.4.1",
36 | "pyarrow>=7.0.0",
37 | "fsspec>=0.3.3",
38 | "Pillow>=9.0.0",
39 | "tqdm>=4.61.1",
40 | "numexpr>=2.7.3",
41 | "tensorflow>=2.5.2",
42 | "lazytransform>=1.17",
43 | ],
44 | classifiers=[
45 | "Programming Language :: Python :: 3",
46 | "Operating System :: OS Independent",
47 | ],
48 | )
49 |
--------------------------------------------------------------------------------
/updates.md:
--------------------------------------------------------------------------------
1 | # featurewiz latest updates page
2 | This is the main page where we will post the latest updates to the featurewiz library. Make sure you bookmark this page and upgrade your featurewiz library before you run it. There are new updates almost every week to featurewiz!
3 |
4 | ### Update (Jan 2024): Introducing BlaggingClassifier, built to handle imbalanced distributions!
5 | Featurewiz now introduces The BlaggingClassifier one of the best classifiers ever, built by # Original Author: Gilles Louppe and Licensed in BSD 3 clause with Adaptations by Tom E Fawcett. This is an amazing classifier that everyone must try for their best imbalanced and multi-class problems. Don't just take our word for it, try it out yourself and see!
6 |
7 | ## Jan 2024 update
8 | `featurewiz` 5.0 version is out! It contains brand new Deep Learning Auto Encoders to enrich your data for the toughest imbalanced and multi-class datasets. In addition, it has multiple brand-new Classifiers built for imbalanced and multi-class problems such as the `IterativeDoubleClassifier` and the `BlaggingClassifier`. If you are looking for the latest and greatest updates about our library, check out our updates page.
9 |
10 |
11 | 
12 |
13 | #### Update (December 2023): FeatureWiz 0.5 is here! Includes powerful deep learning autoencoders!
14 | The FeatureWiz transformer now includes powerful deep learning auto encoders in the new `auto_encoders` argument. They will transform your features into a lower dimension space but capturing important patterns inherent in your dataset. This is known as "feature extraction" and is a powerful tool for tackling very difficult problems in classification. You can set the `auto_encoders` option to `VAE` for Variational Auto Encoder, `DAE` for Denoising Auto Encoder, `CNN` for CNN's and `GAN` for GAN data augmentation for generating synthetic data. These options will completely replace your existing features. Suppose you want to add them as additional features? You can do that by setting the `auto_encoders` option to `VAE_ADD`, `DAE_ADD`, and `CNN_ADD` and featurewiz will automatically add these features to your existing dataset. In addition, it will do feature selection among your old and new features. Isn't that awesome? I have uploaded a sample notebook for you to test and improve your Classifier performance using these options for Imbalanced datasets. Please send me comments via email which is displayed on my Github main page.
15 |
16 | 
17 |
18 | #### Update (November 2023): The FeatureWiz transformer (version 0.4.3 on) includes an "add_missing" flag
19 | The FeatureWiz transformer now includes an `add_missing` flag which will add a new column for missing values for all your variables in your dataset. This will help you catch missing values as an added signal when you use FeatureWiz library. Try it out and let us know in your comments via email.
20 |
21 |
22 | #### Update (October 2023): FeatureWiz transformer (version 0.4.0 on) now has lazytransformer library
23 | The new FeatureWiz transformer includes a categorical encoder + date-time + NLP transfomer that transforms all your string, text, date-time columns into numeric variables in one step. You will see a fully transformed `(all-numeric)` dataset when you use FeatureWiz transformer. Try it out and let us know in your comments via email.
24 |
25 | #### Update (June 2023): featurewiz now has skip_sulov and skip_xgboost flags
26 |
27 | There are two flags that are available to skip the recursive xgboost and/or SULOV methods. They are the `skip_xgboost` and `skip_sulov` flags. They are by default set to `False`. But you can change them to `True` if you want to skip them.
28 |
29 | #### Update (May 2023): featurewiz 3.0 is here with better accuracy and speed
30 |
31 | The latest version of featurewiz is here! The new 3.0 version of featurewiz provides slightly better performance by about 1-2% in diverse datasets (your experience may vary). Install it and check it out!
32 |
33 |
34 | #### Update (March 2023): XGBoost 1.7 and higher versions have issues with featurewiz
35 |
36 | The latest version of XGBoost 1.7+ does not work with featurewiz. They have made massive changes to their API. So please switch to xgboost 1.5 if you want to run featurewiz.
37 |
38 | #### Update (October 2022): FeatureWiz 2.0 is here.
39 | featurewiz 2.0 is here. You have two small performance improvements:
40 | 1. SULOV method now has a higher correlation limit of 0.90 as default. This means fewer variables are removed and hence more vars are selected. You can always set it back to the old limit by setting `corr_limit`=0.70 if you want.
41 |
42 | 2. Recursive XGBoost algorithm is tighter in that it selects fewer features in each iteration. To see how many it selects, set `verbose` flag to 1.
43 | The net effect is that the same number of features are selected but they are better at producing more accurate models. Try it out and let us know.
44 |
45 | #### Update (September 2022): You can now skip SULOV method using skip_sulov flag
46 | featurewiz now has a new input: `skip_sulov` flag is here. You can set it to `True` to skip the SULOV method if needed.
47 |
48 | #### Update (August 2022): Silent mode with verbose=0
49 | featurewiz now has a "silent" mode which you can set using the "verbose=0" option. It will run silently with no charts or graphs and very minimal verbose output. Hope this helps!
50 |
51 | #### Update (May 2022): New high performance modules based on XGBoost and LightGBM
52 | featurewiz as of version 0.1.50 or higher has multiple high performance models that you can use to build highly performant models once you have completed feature selection. These models are based on LightGBM and XGBoost and have even Stacking and Blending ensembles. You can find them as functions starting with "simple_" and "complex_" under featurewiz. All the best!
53 |
54 | #### Update (March 2022): Ability to read feather format files
55 | featurewiz as of version 0.1.04 or higher can read `feather-format` files at blazing speeds. See example below on how to convert your CSV files to feather. Then you can feed those '.ftr' files to featurewiz and it will read it 10-100X faster!
56 |
57 |
58 | 
59 |
60 | featurewiz now runs at blazing speeds thanks to using GPU's by default. So if you are running a large data set on Colab and/or Kaggle, make sure you turn on the GPU kernels. featurewiz will automatically detect that GPU is turned on and will utilize XGBoost using GPU-hist. That will ensure it will crunch your datasets even faster. I have tested it with a very large data set and it reduced the running time from 52 mins to 1 minute! That's a 98% reduction in running time using GPU compared to CPU!
61 |
62 | ### Update (Jan 2022): FeatureWiz is now a sklearn-compatible transformer that you can use in data pipelines
63 | FeatureWiz as of version 0.0.90 or higher is a scikit-learn compatible feature selection transformer. You can perform fit and predict as follows. You will get a Transformer that can select the top variables from your dataset. You can also use it in sklearn pipelines as a Transformer.
64 |
65 | ```
66 | from featurewiz import FeatureWiz
67 | features = FeatureWiz(corr_limit=0.70, feature_engg='', category_encoders='',
68 | dask_xgboost_flag=False, nrows=None, verbose=2)
69 | X_train_selected, y_train = features.fit_transform(X_train, y_train)
70 | X_test_selected = features.transform(X_test)
71 | features.features ### provides the list of selected features ###
72 | ```
73 |
74 | ### Featurewiz is now upgraded with XGBOOST 1.5.1 for DASK for blazing fast performance even for very large data sets! Set `dask_xgboost_flag = True` to run dask + xgboost.
75 | featurewiz now runs with a default setting of `nrows=None`. This means it will run using all rows. But if you want it to run faster, then you can change `nrows` to 1000 or whatever, so it will sample that many rows and run.
76 |
77 | ### Featurewiz has lots of new fast model training functions that you can use to train highly performant models with the features selected by featurewiz. They are:
78 | 1. simple_LightGBM_model() - simple regression and classification with one target label.
79 | 2. simple_XGBoost_model() - simple regression and classification with one target label.
80 | 3. complex_LightGBM_model() - more complex multi-label and multi-class models.
81 | 4. complex_XGBoost_model() - more complex multi-label and multi-class models.
82 | 5. Stacking_Classifier(): Stacking model that can handle multi-label, multi-class problems.
83 | 6. Stacking_Regressor(): Stacking model that can handle multi-label, regression problems.
84 | 7. Blending_Regressor(): Blending model that can handle multi-label, regression problems.
85 |
86 |
--------------------------------------------------------------------------------
