├── README-image ├── TensorBoard模型结构.png ├── image-20221221105305280.png ├── image-20221221120034751.png ├── image-20221221120327989.png ├── image-20221221154708292.png ├── image-20221221193916976.png ├── image-20221221201443669.png ├── image-20221221201647481.png ├── image-20221221202231406.png ├── image-20221221232108413.png ├── image-20221221232728887.png ├── image-20221222103639371.png ├── image-20221222155058742.png ├── image-20221222160928336.png ├── image-20221222190109549.png ├── image-20221222190543853.png ├── image-20221222192146000.png ├── image-20221222192511533.png ├── image-20221222192830751.png ├── image-20221222192849349.png ├── image-20221222193631521.png ├── image-20221222193750323.png ├── image-20221222194347192.png ├── image-20221222214439733.png ├── image-20221223104907559.png ├── image-20221223104915746.png ├── image-20221223120215460.png ├── image-20221223120440331.png ├── image-20221223120549749.png └── image-20221223121130542.png ├── README.md ├── data ├── X_test.npy ├── X_train.npy ├── get_data.ipynb ├── get_data.py ├── y_test.npy └── y_train.npy └── model ├── alphanet_v1_pool.ipynb ├── alphanet_v1_pool.pth ├── alphanet_v3_gru.ipynb ├── alphanet_v3_gru.pdf ├── alphanet_v3_gru.pth ├── alphanet_v3_gru_classification.ipynb ├── alphanet_v3_gru_classification.pth ├── alphanet_v3_gru_classification_excess_return.ipynb ├── alphanet_v3_gru_classification_excess_return.pth └── baseline_random_forest.ipynb /README-image/TensorBoard模型结构.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/TensorBoard模型结构.png -------------------------------------------------------------------------------- /README-image/image-20221221105305280.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221105305280.png -------------------------------------------------------------------------------- /README-image/image-20221221120034751.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221120034751.png -------------------------------------------------------------------------------- /README-image/image-20221221120327989.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221120327989.png -------------------------------------------------------------------------------- /README-image/image-20221221154708292.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221154708292.png -------------------------------------------------------------------------------- /README-image/image-20221221193916976.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221193916976.png -------------------------------------------------------------------------------- /README-image/image-20221221201443669.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221201443669.png -------------------------------------------------------------------------------- /README-image/image-20221221201647481.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221201647481.png -------------------------------------------------------------------------------- /README-image/image-20221221202231406.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221202231406.png -------------------------------------------------------------------------------- /README-image/image-20221221232108413.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221232108413.png -------------------------------------------------------------------------------- /README-image/image-20221221232728887.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221221232728887.png -------------------------------------------------------------------------------- /README-image/image-20221222103639371.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222103639371.png -------------------------------------------------------------------------------- /README-image/image-20221222155058742.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222155058742.png -------------------------------------------------------------------------------- /README-image/image-20221222160928336.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222160928336.png -------------------------------------------------------------------------------- /README-image/image-20221222190109549.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222190109549.png -------------------------------------------------------------------------------- /README-image/image-20221222190543853.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222190543853.png -------------------------------------------------------------------------------- /README-image/image-20221222192146000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222192146000.png -------------------------------------------------------------------------------- /README-image/image-20221222192511533.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222192511533.png -------------------------------------------------------------------------------- /README-image/image-20221222192830751.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222192830751.png -------------------------------------------------------------------------------- /README-image/image-20221222192849349.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222192849349.png -------------------------------------------------------------------------------- /README-image/image-20221222193631521.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222193631521.png -------------------------------------------------------------------------------- /README-image/image-20221222193750323.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222193750323.png -------------------------------------------------------------------------------- /README-image/image-20221222194347192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222194347192.png -------------------------------------------------------------------------------- /README-image/image-20221222214439733.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221222214439733.png -------------------------------------------------------------------------------- /README-image/image-20221223104907559.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221223104907559.png -------------------------------------------------------------------------------- /README-image/image-20221223104915746.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221223104915746.png -------------------------------------------------------------------------------- /README-image/image-20221223120215460.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221223120215460.png -------------------------------------------------------------------------------- /README-image/image-20221223120440331.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221223120440331.png -------------------------------------------------------------------------------- /README-image/image-20221223120549749.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221223120549749.png -------------------------------------------------------------------------------- /README-image/image-20221223121130542.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/README-image/image-20221223121130542.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## 思路框架 2 | 3 | ### 问题背景 4 | 5 | 传统的因子挖掘过程通常是由人工构造因子表达式，对多个单因子进行加权合成。面对大量的原始数据，人工基于投资经验，手动构造因子表达式生成单因子的过程是极其繁琐的。在因子合成阶段，通常是用ICIR加权平均等手段进行合成，这种简单的线性加权方式也限制了因子合成的多种可能性。 6 | 7 | ### 将卷积思想应用于因子挖掘 8 | 9 | 在卷积神经网络中，最关键的特征提取组件是卷积核。在图像识别领域，卷积核通过一个带有可优化的权重和偏置项的矩阵，对原始数据进行互相关操作。 10 | 11 | ![image-20221222103639371](README-image/image-20221222103639371.png) 12 | 13 | 14 | 15 | 我们可以将原始量价数据整理成一个二维矩阵，尝试使用卷积核对数据进行特征提取。 16 | 17 | 但是，如果完全采用传统的卷积操作，提取的特征就是：一定感受野范围内的特征的加权组合。这样的操作会有两个问题： 18 | 19 | 1. 提取的特征只是某些特征数据的固定的加权组合，这极大地限制了因子表达式的可能性。 20 | 2. 传统的卷积核只能感受局部范围内的数据，因此，我们输入的特征变量的上下顺序会影响提取出的特征。因此输入变量的顺序还需要人工干预。 21 | 22 | 由此看来，简单地套用卷积操作并不合适，但我们可以借鉴卷积核的“遍历操作”的思想，自定义运算符函数，实现类似“卷积层”的特征提取层。 23 | 24 | ![image-20221222214439733](README-image/image-20221222214439733.png) 25 | 26 | 具体的特征提取层将在后文介绍。经过特征提取后，可再添加批标准化层、池化层、全连接层，将原始数据转换为收益率的预测。 27 | 28 | ### 优化模型 29 | 30 | 对于上述将卷积思想应用于因子挖掘的方法，可以尝试对两个方向进行优化。 31 | 32 | 1. 调整网络结构。添加更丰富的特征提取层，将池化层转换为可以记忆时序信息的循环神经网络。 33 | 2. 调整标签值。将收益率值得预测转换为涨跌方向的预测和超额收益率方向的预测。 34 | 35 | ## 准备数据集 36 | 37 | 我们需要的特征均为量价数据，即open, high, low, close, vwap, volume, return1, turn, free_turn这9个量价指标在$t-29$到$t$时间段的$9\times 30$个特征。 38 | 39 | ![image-20221221105305280](README-image/image-20221221105305280.png) 40 | 41 | Tushare提供了免费的量价数据接口，在程序中导入token，即可使用`pro.daily()`下载数据。 42 | 43 | 下面具体介绍获取数据的细节。 44 | 45 | ### 训练集和测试集包含的时间段 46 | 47 | 由于通过Tushare的免费接口获取数据的速度较慢（逐股票、逐日获取后再合并，而不是批量一次性获取，因此耗时较久），本文只截取了`20220101`至`20220630`这半年的数据作为训练集，`20220930`至`20221231`这一季度的数据作为测试集。没有用`20220630`至`20221231`的数据作为测试集，是因为希望训练集和验证集之间能够暂停一段时间，否则训练集的标签可能会包含未来信息，进而夸大测试集上的预测效果。 48 | 49 | > 本项目下载数据的时间为2022年12月初，因此实际所用的验证集并不是完整的一季度。 50 | 51 | ### 采样的日期 52 | 53 | 如果对训练集和验证集包含的时间段中的**每一个交易日**均进行采样，会造成两个问题： 54 | 55 | 1. 采样过于频繁，导致相邻日期的数据基本相近。 56 | 2. 采样天数过多，下载数据的时间会非常久。 57 | 58 | 因此，本文使用间隔采样的方法，**每间隔10个交易日进行一次采样**。具体判断哪一天为采样日的函数为： 59 | 60 | ```python 61 | # 给定日期区间的端点，输出期间的定长采样交易日列表 62 | def get_datelist(start: str, end: str, interval: int): 63 | 64 | df = pro.index_daily(ts_code='399300.SZ', start_date=start, end_date=end) 65 | date_list = list(df.iloc[::-1]['trade_date']) 66 | sample_list = [] 67 | for i in range(len(date_list)): 68 | if i % interval == 0: 69 | sample_list.append(date_list[i]) 70 | 71 | return sample_list 72 | ``` 73 | 74 | 其原理是基于沪深300指数（399300.SZ）的交易数据进行间隔采样。沪深300指数有数据的日期一定是交易日。 75 | 76 | ### 采样的股票 77 | 78 | A股市场的股票数量近5000只，若对每一只股票均进行采样也将耗费大量时间。本文对每个采样日，**获取前1000只股票的数据**。具体判断对哪些股票进行采样的函数为： 79 | 80 | ```python 81 | # 给定一个交易日，返回该日满足条件的A股股票列表 82 | def get_stocklist(date: str, num: int): 83 | 84 | start = str(pd.to_datetime(date)-timedelta(30)) 85 | start = start[0:4]+start[5:7]+start[8:10] 86 | df1 = pro.index_weight(index_code='000002.SH', 87 | start_date=start, end_date=date) # 交易日当天的股票列表 88 | codes = list(df1['con_code']) 89 | codes = codes[0:1000] # 在每个截面期只选取1000只股票 90 | 91 | return codes 92 | ``` 93 | 94 | 其原理是基于A股指数（000002.SH）的前1000只成分股进行采样。 95 | 96 | ### 获取单个股票在单个交易日的数据 97 | 98 | `get_x_y()`函数返回两个值，一个是前30个交易日的9个指标面板（9*30），一个是未来10天的收益率。 99 | 100 | ```python 101 | def get_x_y(code: str, date: str, pass_day: int, future_day: int, len1: int, len2: int): 102 | 103 | start = str(pd.to_datetime(date)-timedelta(pass_day*2)) 104 | start = start[0:4]+start[5:7]+start[8:10] 105 | end = str(pd.to_datetime(date)+timedelta(future_day*2)) 106 | end = end[0:4]+end[5:7]+end[8:10] 107 | df_price = pro.daily(ts_code=code, # OHLC,pct_change,volume 108 | start_date=start, end_date=date) 109 | df_basic = pro.daily_basic(ts_code=code, 110 | start_date=start, end_date=date) 111 | df_return = pro.daily(ts_code=code, 112 | start_date=date, end_date=end).iloc[::-1]['close'] 113 | if (df_price.shape[0] == df_basic.shape[0]) & (df_price.shape[0] == len1) & (df_return.shape[0] == len2): # 判断数据的完整性 114 | df_price = df_price.iloc[0:pass_day, [2, 3, 4, 5, 8, 9]].fillna(0.1) 115 | df_basic = df_basic.iloc[0:pass_day, [3, 4, 5]].fillna(0.1) 116 | data = np.array(pd.merge(df_price, df_basic, 117 | left_index=True, right_index=True).iloc[::-1].T) 118 | # print(data.shape) 119 | # 未来十个交易日的收益率 120 | dfr = df_return.iloc[0:future_day] 121 | ret = dfr.iloc[-1]/dfr.iloc[0]-1 # 后十个交易日的收益率 122 | return data, ret 123 | else: 124 | return None, None # 数据缺失的预处理 125 | ``` 126 | 127 | ### 舍弃缺失值 128 | 129 | 在获取单个股票在单个交易日的数据时，若某只股票的数据有缺失，则需舍弃它，否则在输入到神经网络时会带有缺失值。 130 | 131 | 基于沪深300指数，判断某日应有的数据长度的函数： 132 | 133 | ```python 134 | def get_length(date: str, pass_day: int, future_day: int): 135 | start = str(pd.to_datetime(date)-timedelta(pass_day*2)) 136 | start = start[0:4]+start[5:7]+start[8:10] 137 | end = str(pd.to_datetime(date)+timedelta(future_day*2)) 138 | end = end[0:4]+end[5:7]+end[8:10] 139 | len_1 = pro.index_daily(ts_code='399300.SZ', 140 | start_date=start, end_date=date).shape[0] 141 | len_2 = pro.index_daily(ts_code='399300.SZ', 142 | start_date=date, end_date=end).shape[0] 143 | return len_1, len_2 144 | ``` 145 | 146 | 在`get_x_y()`函数中，基于`len_1`和`len_2`判断了数据的完整性。若有缺失值则返回空值，不会计入数据集中。 147 | 148 | ### 获取数据集 149 | 150 | 筛选出哪一天、哪一只股票需要进行采样后，我们就可以获取数据了。 151 | 152 | 对每一个采样日、每一只股票进行循环。配合`rich.progress`可以展示下载数据的进度条。 153 | 154 | > `rich.progress`的使用示例可以参考[这里](https://fengchao.pro/pin/frequently-used-python-data-processing-code/#richprogress%E5%B1%95%E7%A4%BA%E8%BF%9B%E5%BA%A6%E6%9D%A1)。 155 | 156 | ```python 157 | def get_dataset(num: int, start: str, end: str, interval: int, pass_day: int, future_day: int): 158 | X_train = [] 159 | y_train = [] 160 | trade_date_list = get_datelist(start, end, interval) 161 | # 添加进度条 162 | with Progress() as progress: 163 | task_date = progress.add_task( 164 | "[red]Date...", total=len(trade_date_list)) 165 | for date in trade_date_list: 166 | # 更新进度条 167 | progress.update(task_date, advance=1) 168 | stock_list = get_stocklist(date, num) 169 | len1, len2 = get_length(date, pass_day, future_day) 170 | task_stock = progress.add_task( 171 | "[green]Stock...", total=len(range(len(stock_list)))) 172 | for i in range(len(stock_list)): 173 | # 更新进度条 174 | progress.update(task_stock, advance=1) 175 | code = stock_list[i] 176 | x, y = get_x_y(code, date, pass_day, future_day, len1, len2) 177 | try: 178 | if (x.shape[0] == 9) & (x.shape[1] == pass_day): 179 | X_train.append(x) 180 | y_train.append(y) 181 | except Exception: 182 | continue 183 | return X_train, y_train 184 | ``` 185 | 186 | 数据示例： 187 | 188 | ![image-20221223104907559](README-image/image-20221223104907559.png) 189 | 190 | ![image-20221223104915746](README-image/image-20221223104915746.png) 191 | 192 | ### 保存`.npy`数据到本地 193 | 194 | 为了方便训练模型，可以将数据以`.npy`格式存储到本地。在训练模型时可以直接使用`np.load('../data/X_train.npy')`载入数据。 195 | 196 | ```python 197 | # 参数设定：使用过去30天的数据预测未来10天的收益率，回归问题 198 | X_train, y_train = get_dataset( 199 | num=1000, start='20220101', end='20220630', interval=10, pass_day=30, future_day=10) 200 | X_test, y_test = get_dataset(num=1000, start='20220931', 201 | end='20221231', interval=10, pass_day=30, future_day=10) 202 | print("there are in total", len(X_train), "training samples") 203 | print("there are in total", len(X_test), "testing samples") 204 | # 将数据保存到本地供离线训练 205 | Xa = np.array(X_train) 206 | ya = np.array(y_train) 207 | Xe = np.array(X_test) 208 | ye = np.array(y_test) 209 | np.save('./X_train.npy', Xa) 210 | np.save('./y_train.npy', ya) 211 | np.save('./X_test.npy', Xe) 212 | np.save('./y_test.npy', ye) 213 | ``` 214 | 215 | 整个获取数据的时间约为3个小时，共获取到11825条训练数据和4943条测试数据（数据量不为1000的整数倍，是因为舍弃了部分缺失值）。 216 | 217 | - 特征数据为$9\times 30$的个股量价数据构成的矩阵。9行代表9个量价特征，30代表$t-29$至$t$这30天的数据。 218 | 219 | - 标签数据为个股在某个交易日往后10个交易日的收益率。 220 | 221 | ![image-20221221120034751](README-image/image-20221221120034751.png) 222 | 223 | ![image-20221221120327989](README-image/image-20221221120327989.png) 224 | 225 | ## 搭建AlphaNet-V1 226 | 227 | ### AlphaNet-V1的整体网络结构 228 | 229 | 下图展示了AlphaNet-V1的整体网络结构。它由7个平行的特征提取层、3个平行的池化层和1个全连接层组成。其中，特征提取层和池化层后都有一个批标准化层（Batch Normalization）。 230 | 231 | 输入数据是一个$9\times30$的个股量价“数据图片”，预测目标为个股从当日到10个交易日后的收益率数值。 232 | 233 | ![image-20221221154708292](README-image/image-20221221154708292.png) 234 | 235 | ### 特征提取层（类似卷积层） 236 | 237 | AlphaNet的输入数据是一个$9\times30$的个股量价“数据图片”。如果简单地套用卷积神经网络处理图片像素数据的操作，则**卷积操作只能在感受野内将若干日期的若干量价数据进行加权平均**，经过卷积层得到的特征将变得很难解释，也不符合传统构造量价因子的方式。 238 | 239 | 因此，借鉴卷积神经网络CNN的思想，我们可以将**多种运算符函数作为自定义网络层**进行特征提取。本文实现了7种运算符，分别是`ts_corr`, `ts_cov`, `ts_stddev`, `ts_zscore`, `ts_return`, `ts_decaylinear`, `ts_mean`，它们的含义如下： 240 | 241 | | 名称 | 定义 | 242 | | ---------------- | ------------------------------------------------------------ | 243 | | `ts_corr` | 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的相关系数。 | 244 | | `ts_cov` | 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的协方差。 | 245 | | `ts_stddev` | 过去 d 天 X 值构成的时序数列的标准差。 | 246 | | `ts_zscore` | 过去 d 天 X 值构成的时序数列的平均值除以标准差。 | 247 | | `ts_return` | (X - delay(X, d))/delay(X, d)-1, delay(X, d)为 X 在 d 天前的取值。 | 248 | | `ts_decaylinear` | 过去 d 天 X 值构成的时序数列的加权平均值，权数为 d, d – 1, …, 1(权数之和应为 1，需进行归一化处理)，其中离现在越近的日子权数越大。 | 249 | | `ts_mean` | 过去 d 天 X 值构成的时序数列的平均值。 | 250 | 251 | 这7个运算符函数中，`ts_corr`和`ts_cov`需要从9行数据中提取2行数据，并计算相关系数和协方差。其他5个运算符函数仅需针对某一行数据计算标准差、变化率等。下面针对这两种情况分别举例说明。 252 | 253 | #### 基于双变量的特征提取层——以`ts_corr`为例 254 | 255 | 我们的输入数据是$9\times30$的矩阵，每一行是某个量价指标在最近30个交易日的值。基于双变量进行特征提取的步骤为： 256 | 257 | 1. 取出两行数据。 258 | 2. 对于取出的两行数据，给定步长`stride`，在时间维度上对两行数据进行遍历，计算两行数据的相关系数。例如，当$stride=3$时，下一次计算将在时间维度上往右步进3步，我们将进行$\frac{30}{3}=10$次运算。 259 | 3. 将运算结果整理到新的矩阵，得到新的“特征图片”，作为后续池化层的输入。 260 | 261 | ![image-20221221193916976](README-image/image-20221221193916976.png) 262 | 263 | > 从9行数据中任取2行，有$\tbinom{9}{2}=36$种取法。假设我们设定步长为10，则得到的新的“特征图片”的维数是$36\times3$。 264 | 265 | #### 基于双变量的特征提取层——代码实现 266 | 267 | 需要给定原始矩阵`Matrix`、两两组合的列表`combination`、反转的两两组合的列表`combination_rev`以及每次遍历运算的起始索引列表`index_list`。 268 | 269 | - 生成`combination`和`combination_rev`的代码为： 270 | 271 | ```python 272 | # 生成卷积操作时需要的两列数据的组合的列表 273 | def generate_combination(N): 274 | """ 275 | args: 276 | N: int, the number of rows of the matrix 277 | 278 | return: 279 | combination: list, the combination of two columns of the matrix 280 | combination_rev: list, the combination of two rows of the matrix, which is the reverse of combination 281 | """ 282 | col = [] 283 | col_rev = [] 284 | for i in range(1,N): 285 | for j in range(0,i): 286 | col.append([i,j]) 287 | col_rev.append([j,i]) 288 | return col, col_rev 289 | # 生成卷积操作时需要的两列数据的组合的列表 290 | combination, combination_rev = generate_combination(9) 291 | ``` 292 | 293 | ![image-20221221201443669](README-image/image-20221221201443669.png) 294 | 295 | - 生成`index_list`的代码为： 296 | 297 | ```python 298 | # 根据输入的矩阵和卷积操作的步长, 计算卷积操作的索引 299 | def get_index_list(matrix, stride): 300 | """ 301 | args: 302 | matrix: torch.tensor, the input matrix 303 | stride: int, the stride of the convolution operation 304 | 305 | return: 306 | index_list: list, the index of the convolution operation 307 | 308 | """ 309 | W = matrix.shape[3] 310 | if W % stride == 0: 311 | index_list = list(np.arange(0, W+stride, stride)) 312 | else: 313 | mod = W % stride 314 | index_list = list(np.arange(0, W+stride-mod, stride)) + [W] 315 | return index_list 316 | # 根据输入的矩阵和卷积操作的步长, 计算卷积操作的索引 317 | # Inception模块使用的卷积操作的步长为10 318 | index_list = get_index_list(np.zeros((1,1,9,30)), 10) 319 | ``` 320 | 321 | ![image-20221221201647481](README-image/image-20221221201647481.png) 322 | 323 | - 基于双变量的特征提取代码为： 324 | 325 | ```python 326 | # 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的相关系数 327 | def ts_corr4d(self, Matrix, combination, combination_rev): 328 | new_H = len(combination) 329 | index_list = self.index_list 330 | list = [] # 存放长度为len(index_list)-1的相关系数 331 | for i in range(len(index_list)-1): 332 | start_index = index_list[i] 333 | end_index = index_list[i+1] 334 | data = Matrix[:, :, combination, start_index:end_index] # N*1*new_H*2*d 335 | data2 = Matrix[:, :, combination_rev, 336 | start_index:end_index] # N*1*new_H*2*d 337 | std1 = data.std(axis=4, keepdims=True) # N*1*new_H*2*1, 在时序上求标准差 338 | std2 = data2.std(axis=4, keepdims=True) # N*1*new_H*2*1, 在时序上求标准差 339 | std = (std1*std2).mean(axis=3, keepdims=True) # N*1*new_H*1*1 340 | list.append(std) 341 | std = np.squeeze(np.array(list)).transpose(1, 2, 0).reshape(-1, 1, new_H, len(index_list)-1)+0.01 # N*1*new_H*len(index_list)-1 # 加上0.01, 防止除0 342 | # N*1*new_H*len(index_list)-1 343 | cov = self.ts_cov4d(Matrix, combination, combination_rev) 344 | corr = cov/std # N*1*new_H*len(index_list)-1 345 | return corr 346 | ``` 347 | 348 | ```python 349 | # 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的协方差 350 | def ts_cov4d(self, Matrix, combination, combination_rev): 351 | new_H = len(combination) 352 | index_list = self.index_list 353 | list = [] # 存放长度为len(index_list)-1的协方差 354 | for i in range(len(index_list)-1): 355 | start_index = index_list[i] 356 | end_index = index_list[i+1] 357 | data = Matrix[:, :, combination, start_index:end_index] # N*1*new_H*2*d 358 | data2 = Matrix[:, :, combination_rev, 359 | start_index:end_index] # N*1*new_H*2*d 360 | mean1 = data.mean(axis=4, keepdims=True) # N*1*new_H*2*1, 在时序上求均值 361 | mean2 = data2.mean(axis=4, keepdims=True) # N*1*new_H*2*1, 在时序上求均值 362 | spread1 = data - mean1 # N*1*new_H*2*d, 在时序上求偏差 363 | spread2 = data2 - mean2 # N*1*new_H*2*d, 在时序上求偏差 364 | cov = ((spread1 * spread2).sum(axis=4, keepdims=True) / 365 | (data.shape[4]-1)).mean(axis=3, keepdims=True) # N*1*new_H*1*1 366 | list.append(cov) 367 | cov = np.squeeze(np.array(list)).transpose( 368 | 1, 2, 0).reshape(-1, 1, new_H, len(index_list)-1) # N*1*new_H*len(index_list)-1 369 | return torch.from_numpy(cov) 370 | ``` 371 | 372 | 经过上述特征提取后，得到的新的“特征图片”的维数是$36\times3$。我们后续会将其进行池化和展平。注意到，这36行数据的上下位置不影响池化和展平操作得到的结果（展平后每个量的地位都一样），因此原始输入数据的$9\times30$的矩阵内部的上下**可以任意排列**。这也避免了卷积神经网络处理图片像素数据时**只能感知局部数据**的问题。 373 | 374 | #### 基于单变量的特征提取层——以`ts_stddev`为例 375 | 376 | 基于单变量进行特征提取的步骤为： 377 | 378 | 1. 取出一行数据。 379 | 2. 对于取出的一行数据，给定步长`stride`，在时间维度上对这一行数据进行遍历，计算这一行数据的标准差。例如，当$stride=3$时，下一次计算将在时间维度上往右步进3步，我们将进行$\frac{30}{3}=10$次运算。 380 | 3. 将运算结果整理到新的矩阵，得到新的“特征图片”，作为后续池化层的输入。 381 | 382 | ![image-20221221202231406](README-image/image-20221221202231406.png) 383 | 384 | > 从9行数据中任取1行，有$\tbinom{9}{1}=9$种取法。假设我们设定步长为10，则得到的新的“特征图片”的维数是9*3。 385 | 386 | #### 基于单变量的特征提取层——代码实现 387 | 388 | 单变量的特征提取只需给定原始数据`Matrix`和每次遍历运算的起始索引列表`index_list`。 389 | 390 | ```python 391 | # 过去 d 天 X 值构成的时序数列的标准差 392 | def ts_stddev4d(self, Matrix): 393 | # 只需要对单变量做卷积操作, 不需要将变量两两组合。因此输出的 H 可以保持和输入的 H 一致 394 | new_H = Matrix.shape[2] 395 | index_list = self.index_list 396 | list = [] # 存放长度为len(index_list)-1的标准差 397 | for i in range(len(index_list)-1): 398 | start_index = index_list[i] 399 | end_index = index_list[i+1] 400 | data = Matrix[:, :, :, start_index:end_index] # N*1*H*d 401 | std = data.std(axis=3, keepdims=True) # N*1*H*1 402 | list.append(std) 403 | std4d = np.squeeze(np.array(list)).transpose( 404 | 1, 2, 0).reshape(-1, 1, new_H, len(index_list)-1) # N*1*new_H*len(index_list)-1 405 | return torch.from_numpy(std4d) 406 | 407 | # 过去 d 天 X 值构成的时序数列的平均值除以标准差 408 | def ts_zcore4d(self, Matrix): 409 | # 只需要对单变量做卷积操作, 不需要将变量两两组合。因此输出的 H 可以保持和输入的 H 一致 410 | new_H = Matrix.shape[2] 411 | index_list = self.index_list 412 | list = [] # 存放长度为len(index_list)-1的zcore 413 | for i in range(len(index_list)-1): 414 | start_index = index_list[i] 415 | end_index = index_list[i+1] 416 | data = Matrix[:, :, :, start_index:end_index] # N*1*H*d 417 | mean = data.mean(axis=3, keepdims=True) # N*1*H*1 418 | std = data.std(axis=3, keepdims=True) + \ 419 | 0.01 # N*1*H*1, 加上0.01, 防止除以0 420 | list.append(mean/std) 421 | zscore = np.squeeze(np.array(list)).transpose( 422 | 1, 2, 0).reshape(-1, 1, new_H, len(index_list)-1) # N*1*new_H*len(index_list)-1 423 | return torch.from_numpy(zscore) 424 | 425 | # (X - delay(X, d))/delay(X, d)-1, 其中 delay(X, d)为 X 在 d 天前的取值 426 | def ts_return4d(self, Matrix): 427 | # 只需要对单变量做卷积操作, 不需要将变量两两组合。因此输出的 H 可以保持和输入的 H 一致 428 | new_H = Matrix.shape[2] 429 | index_list = self.index_list 430 | list = [] # 存放长度为len(index_list)-1的return 431 | for i in range(len(index_list)-1): 432 | start_index = index_list[i] 433 | end_index = index_list[i+1] 434 | data = Matrix[:, :, :, start_index:end_index] # N*1*H*d 435 | # N*1*H*1, 在分母加上0.01, 防止除以0 436 | return_ = data[:, :, :, -1]/(data[:, :, :, 0]+0.01)-1 437 | list.append(return_) 438 | ts_return = np.squeeze(np.array(list)).transpose( 439 | 1, 2, 0).reshape(-1, 1, new_H, len(index_list)-1) # N*1*new_H*len(index_list)-1 440 | return torch.from_numpy(ts_return) 441 | 442 | # 过去 d 天 X 值构成的时序数列的加权平均值, 权数为 d, d – 1, …, 1(权数之和应为 1, 需进行归一化处理), 其中离现在越近的日子权数越大 443 | def ts_decaylinear4d(self, Matrix): 444 | new_H = Matrix.shape[2] 445 | index_list = self.index_list 446 | list = [] # 存放长度为len(index_list)-1的加权平均值 447 | for i in range(len(index_list)-1): 448 | start_index = index_list[i] 449 | end_index = index_list[i+1] 450 | range_ = end_index-start_index 451 | weight = np.arange(1, range_+1) 452 | weight = weight/weight.sum() # 权重向量 453 | data = Matrix[:, :, :, start_index:end_index] # N*1*H*d 454 | wd = (data*weight).sum(axis=3, keepdims=True) # N*1*H*1 455 | list.append(wd) 456 | ts_decaylinear = np.squeeze(np.array(list)).transpose( 457 | 1, 2, 0).reshape(-1, 1, new_H, len(index_list)-1) # N*1*new_H*len(index_list)-1 458 | return torch.from_numpy(ts_decaylinear) 459 | 460 | # 过去 d 天 X 值构成的时序数列的平均值 461 | def ts_mean4d(self, Matrix): 462 | new_H = Matrix.shape[2] 463 | index_list = self.index_list 464 | list = [] # 存放长度为len(index_list)-1的平均值 465 | for i in range(len(index_list)-1): 466 | start_index = index_list[i] 467 | end_index = index_list[i+1] 468 | data = Matrix[:, :, :, start_index:end_index] # N*1*H*d 469 | mean_ = data.mean(axis=3, keepdims=True) # N*1*H*1 470 | list.append(mean_) 471 | ts_mean = np.squeeze(np.array(list)).transpose( 472 | 1, 2, 0).reshape(-1, 1, new_H, len(index_list)-1) # N*1*new_H*len(index_list)-1 473 | return torch.from_numpy(ts_mean) 474 | ``` 475 | 476 | 上述7个函数就相当于定义好了卷积操作的“卷积核”，并且这些操作中**没有需要优化的参数**，只需要按照给定的运算符进行前向传播。 477 | 478 | 图像识别领域中的卷积操作是需要优化卷积核的，这也是和图像识别领域中的卷积操作有区别的地方。 479 | 480 | ### 批标准化层（Batch Normalization） 481 | 482 | Batch Normalization通过将每一层的原始输出进行标准化（减去均值，除以标准差），还可以乘以$\gamma$（Scale），再加上$\beta$（Offset）。$\gamma$和$\beta$都是超参数，可以用神经网络训练它们。 483 | 484 | 具体的数学公式如下。 485 | 486 | 求第$l$层的批均值： 487 | $$ 488 | \mu=\frac{1}{m} \sum_{i=1}^m Z^{l(i)} 489 | $$ 490 | 求第$l$层的批方差： 491 | $$ 492 | \sigma^2=\frac{1}{m} \sum_{i=1}^m\left(Z^{l(i)}-\mu\right)^2 493 | $$ 494 | 批标准化的结果： 495 | $$ 496 | \hat{Z}^l=\gamma * \frac{Z^l-\mu}{\sqrt{\sigma^2+\varepsilon}}+\beta 497 | $$ 498 | 经过上述操作，即可将$Z^l$转换为$\hat{Z}^l$。 499 | 500 | 在批标准化中，可优化的参数是$\gamma$和$\beta$。如果没有$\gamma$和$\beta$，则批标准化的运算就为常规的 z-score 标准化。 501 | 502 | > 关于Batch Normalization的笔记可以参考[https://fengchao.pro/post/batch-normalization/](https://fengchao.pro/post/batch-normalization/) 503 | 504 | ### 池化层 505 | 506 | 在AlphaNet-V1中，池化层与传统的图像识别中的池化层一致，都是对某个区域的数据提取最大值、平均值和最小值。 507 | 508 | 由于PyTorch中没有内置最小池化层，我们可以将数据取相反数后，进行最大池化，再将最大池化的结果取相反数，就可以实现最小池化。 509 | 510 | ```python 511 | # 池化层, 尺度为1*d 512 | self.max_pool = nn.MaxPool2d(kernel_size=(1, self.d)) 513 | self.avg_pool = nn.AvgPool2d(kernel_size=(1, self.d)) 514 | # 最小池化等价于相反数的最大池化, 后续会对结果取反 515 | self.min_pool = nn.MaxPool2d(kernel_size=(1, self.d)) 516 | ``` 517 | 518 | ### 特征提取层和池化层的特征维数变化分析 519 | 520 | 在经过特征提取层、池化层后（批标准化层不改变特征维数，因此不考虑），将特征提取层和池化层的输出均展平后再拼接，得到$702\times1$的特征。 521 | 522 | 下面解释维数为$702\times 1$是如何得到的。 523 | 524 | 1. 特征提取层展平后得到$351\times1$的特征。 525 | 526 | - 有2个特征提取层是基于双变量的，它们的输出维数是$36\times3$。其他5个特征提取层都是基于单变量的，它们的输出维数是$9\times3$。 527 | 528 | - 因此，特征提取层展平后得到$(2\times36+5\times9)\times3=117\times3=351$维特征。 529 | 530 | 2. 池化层展平后也会得到$351\times1$的特征。 531 | 532 | - 池化层的输入就是特征提取层的输出，因此池化层的输入是$117\times3$。 533 | - 池化操作的步长`stride`是`len(index_list)-1`$=3$，即最大池化、平均池化和最小池化都将$117\times3$转换为$117\times1$的矩阵。 534 | - 将3个$117\times1$的矩阵展平后，得到$3*117=351$维特征。 535 | 536 | 3. 将特征提取层和池化层的输出均展平后再拼接，得到$351+351=702$维特征。 537 | 538 | ### 全连接层 539 | 540 | 全连接层包含30个神经元，接受输入为$702\times 1$的矩阵，经过一个隐藏层转换为$30\times1$的矩阵，经过RELU激活函数、Dropout Rate为0.5，再输出到$1*1$的神经元作为预测结果。 541 | 542 | ```python 543 | data = self.fc1(data) # N*30 544 | data = self.relu(data) 545 | data = self.dropout(data) 546 | data = self.fc2(data) # N*1 547 | # 线性激活函数, 无需再进行激活 548 | data = data.to(torch.float) 549 | ``` 550 | 551 | ### 继承`nn.module`类，改写前向传播`forward`函数 552 | 553 | 为了让我们自定义的函数在神经网络中能够运行，我们需要继承`nn.module`类并改写前向传播`forward`函数。 554 | 555 | #### 自定义特征提取层、池化层的代码实现 556 | 557 | 自定义的`Inception`类实现了特征提取层（以及随后的批标准化）、池化层（以及随后的批标准化），并将输出结果展平成$702\times 1$的矩阵。 558 | 559 | 具体的运算符函数在前面已经定义过了，这里不再详细展开。 560 | 561 | ```python 562 | class Inception(nn.Module): 563 | """ 564 | Inception, 用于提取时间序列的特征, 具体操作包括： 565 | 566 | 1. kernel_size和stride均为d=10的特征提取层, 类似于卷积层，用于提取时间序列的特征. 具体包括: 567 | 568 | 1. ts_corr4d: 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的相关系数 569 | 2. ts_cov4d: 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的协方差 570 | 3. ts_stddev4d: 过去 d 天 X 值构成的时序数列的标准差 571 | 4. ts_zscore4d: 过去 d 天 X 值构成的时序数列的平均值除以标准差 572 | 5. ts_return4d: (X - delay(X, d))/delay(X, d)-1, 其中delay(X, d)为 X 在 d 天前的取值 573 | 6. ts_decaylinear4d: 过去 d 天 X 值构成的时序数列的加权平均值，权数为 d, d – 1, …, 1(权数之和应为 1，需进行归一化处理)，其中离现在越近的日子权数越大 574 | 7. ts_mean4d: 过去 d 天 X 值构成的时序数列的平均值 575 | 576 | 各操作得到的张量维数： 577 | 1. 由于涉及两个变量的协方差, 因此ts_corr4d和ts_cov4d的输出为 N*1*36*3 578 | 2. 其余操作均只涉及单变量的时序计算, 因此输出为 N*1*9*3 579 | 580 | 2. 对第1步的输出进行Batch Normalization操作, 输出维数仍为 N*1*36*3 或 N*1*9*3 581 | 582 | 3. 对于第2步得到的张量, kernel_size为3的池化层. 具体包括: 583 | 1. max_pool: 过去 d 天 X 值构成的时序数列的最大值 584 | 2. avg_pool: 过去 d 天 X 值构成的时序数列的平均值 585 | 3. min_pool: 过去 d 天 X 值构成的时序数列的最小值 586 | 587 | 以上三个操作的输出均为 N*1*117*1 588 | 589 | 4. 对第3步的输出进行Batch Normalization操作, 输出维数仍为 N*1*117*1 590 | 591 | 5. 将第2步和第4步的输出展平后进行拼接, 得到的张量维数为 N*(2*36*3+5*9*3+3*117) = N*702 592 | 593 | """ 594 | 595 | def __init__(self, combination, combination_rev, index_list): 596 | """ 597 | combination: 卷积操作时需要的两列数据的组合 598 | combination_rev: 卷积操作时需要的两列数据的组合, 与combination相反 599 | index_list: 卷积操作时需要的时间索引 600 | 601 | """ 602 | 603 | super(Inception, self).__init__() 604 | # 卷积操作时需要的两列数据的组合 605 | self.combination = combination 606 | self.combination_rev = combination_rev 607 | 608 | # 卷积操作时需要的时间索引 609 | self.index_list = index_list 610 | self.d = len(index_list)-1 611 | 612 | # 卷积操作后的Batch Normalization层 613 | self.bc1 = nn.BatchNorm2d(1) 614 | self.bc2 = nn.BatchNorm2d(1) 615 | self.bc3 = nn.BatchNorm2d(1) 616 | self.bc4 = nn.BatchNorm2d(1) 617 | self.bc5 = nn.BatchNorm2d(1) 618 | self.bc6 = nn.BatchNorm2d(1) 619 | self.bc7 = nn.BatchNorm2d(1) 620 | 621 | # 池化层, 尺度为1*d 622 | self.max_pool = nn.MaxPool2d(kernel_size=(1, self.d)) 623 | self.avg_pool = nn.AvgPool2d(kernel_size=(1, self.d)) 624 | # 最小池化等价于相反数的最大池化, 后续会对结果取反 625 | self.min_pool = nn.MaxPool2d(kernel_size=(1, self.d)) 626 | 627 | # 池化操作后的Batch Normalization层 628 | self.bc_pool1 = nn.BatchNorm2d(1) 629 | self.bc_pool2 = nn.BatchNorm2d(1) 630 | self.bc_pool3 = nn.BatchNorm2d(1) 631 | 632 | def forward(self, data): 633 | """ 634 | data: 输入的数据, 维度为batch_size*1*9*30 635 | 636 | """ 637 | # 本层的输入为batch_size*1*9*30, 在训练时不需要反向传播, 因此可以使用detach()函数 638 | data = data.detach().cpu().numpy() 639 | combination = self.combination 640 | combination_rev = self.combination_rev 641 | 642 | # 卷积操作 643 | conv1 = self.ts_corr4d(data, combination, combination_rev).to(torch.float) 644 | conv2 = self.ts_cov4d(data, combination, combination_rev).to(torch.float) 645 | conv3 = self.ts_stddev4d(data).to(torch.float) 646 | conv4 = self.ts_zcore4d(data).to(torch.float) 647 | conv5 = self.ts_return4d(data).to(torch.float) 648 | conv6 = self.ts_decaylinear4d(data).to(torch.float) 649 | conv7 = self.ts_mean4d(data).to(torch.float) 650 | 651 | # 卷积操作后的Batch Normalization 652 | batch1 = self.bc1(conv1) 653 | batch2 = self.bc2(conv2) 654 | batch3 = self.bc3(conv3) 655 | batch4 = self.bc4(conv4) 656 | batch5 = self.bc5(conv5) 657 | batch6 = self.bc6(conv6) 658 | batch7 = self.bc7(conv7) 659 | 660 | # 在 H 维度上进行特征拼接 661 | feature = torch.cat( 662 | [batch1, batch2, batch3, batch4, batch5, batch6, batch7], axis=2) # N*1*(2*36+5*9)*3 = N*1*117*3 663 | 664 | # 同时将特征展平, 准备输入到全连接层 665 | feature_flatten = feature.flatten(start_dim=1) # N*(117*3) = N*351 666 | 667 | # 对多通道特征进行池化操作, 每层池化后面都有Batch Normalization 668 | # 最大池化 669 | maxpool = self.max_pool(feature) # N*1*117*1 670 | maxpool = self.bc_pool1(maxpool) 671 | # 平均池化 672 | avgpool = self.avg_pool(feature) # N*1*117*1 673 | avgpool = self.bc_pool2(avgpool) 674 | # 最小池化 675 | # N*1*117*1, 最小池化等价于相反数的最大池化, 并对结果取反 676 | minpool = -self.min_pool(-1*feature) 677 | minpool = self.bc_pool3(minpool) 678 | # 特征拼接 679 | pool_cat = torch.cat([maxpool, avgpool, minpool], 680 | axis=2) # N*1*(3*117)*1 = N*1*351*1 681 | # 将池化层的特征展平 682 | pool_cat_flatten = pool_cat.flatten(start_dim=1) # N*351 683 | 684 | # 拼接展平后的特征 685 | feature_final = torch.cat( 686 | [feature_flatten, pool_cat_flatten], axis=1) # N*(351+351) = N*702 687 | return feature_final 688 | 689 | # 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的相关系数 690 | def ts_corr4d(self, Matrix, combination, combination_rev): 691 | ... 692 | 693 | # 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的协方差 694 | def ts_cov4d(self, Matrix, combination, combination_rev): 695 | ... 696 | 697 | # 过去 d 天 X 值构成的时序数列的标准差 698 | def ts_stddev4d(self, Matrix): 699 | ... 700 | 701 | # 过去 d 天 X 值构成的时序数列的平均值除以标准差 702 | def ts_zcore4d(self, Matrix): 703 | ... 704 | 705 | # (X - delay(X, d))/delay(X, d)-1, 其中 delay(X, d)为 X 在 d 天前的取值 706 | def ts_return4d(self, Matrix): 707 | ... 708 | 709 | # 过去 d 天 X 值构成的时序数列的加权平均值, 权数为 d, d – 1, …, 1(权数之和应为 1, 需进行归一化处理), 其中离现在越近的日子权数越大 710 | def ts_decaylinear4d(self, Matrix): 711 | ... 712 | 713 | # 过去 d 天 X 值构成的时序数列的平均值 714 | def ts_mean4d(self, Matrix): 715 | ... 716 | ``` 717 | 718 | #### 自定义AlphaNet的代码实现 719 | 720 | 整合前面的各层，封装到AlphaNet中。 721 | 722 | ```python 723 | class AlphaNet(nn.Module): 724 | 725 | def __init__(self, combination, combination_rev, index_list, fc1_num, fc2_num, dropout_rate): 726 | super(AlphaNet, self).__init__() 727 | self.combination = combination 728 | self.combination_rev = combination_rev 729 | self.fc1_num = fc1_num 730 | self.fc2_num = fc2_num 731 | # 自定义的Inception模块 732 | self.Inception = Inception(combination, combination_rev, index_list) 733 | # 两个全连接层 734 | self.fc1 = nn.Linear(fc1_num, fc2_num) # 702 -> 30 735 | self.fc2 = nn.Linear(fc2_num, 1) # 30 -> 1 736 | # 激活函数 737 | self.relu = nn.ReLU() 738 | # dropout 739 | self.dropout = nn.Dropout(dropout_rate) 740 | # 初始化权重 741 | self._init_weights() 742 | 743 | def _init_weights(self): 744 | # 使用xavier的均匀分布对weights进行初始化 745 | nn.init.xavier_uniform_(self.fc1.weight) 746 | nn.init.xavier_uniform_(self.fc2.weight) 747 | # 使用正态分布对bias进行初始化 748 | nn.init.normal_(self.fc1.bias, std=1e-6) 749 | nn.init.normal_(self.fc2.bias, std=1e-6) 750 | 751 | def forward(self, data): 752 | data = self.Inception(data) # N*702 753 | data = self.fc1(data) # N*30 754 | data = self.relu(data) 755 | data = self.dropout(data) 756 | data = self.fc2(data) # N*1 757 | # 线性激活函数, 无需再进行激活 758 | data = data.to(torch.float) 759 | 760 | return data 761 | ``` 762 | 763 | ### 使用`torchsummary`包查看网络结构 764 | 765 | ```python 766 | from torchsummary import summary 767 | 768 | test = AlphaNet(combination, combination_rev, index_list, fc1_num=702, fc2_num=30, dropout_rate=0.5) 769 | 770 | summary(test, input_size=(1, 9, 30)) 771 | ``` 772 | 773 | ``` 774 | ---------------------------------------------------------------- 775 | Layer (type) Output Shape Param # 776 | ================================================================ 777 | BatchNorm2d-1 [-1, 1, 36, 3] 2 778 | BatchNorm2d-2 [-1, 1, 36, 3] 2 779 | BatchNorm2d-3 [-1, 1, 9, 3] 2 780 | BatchNorm2d-4 [-1, 1, 9, 3] 2 781 | BatchNorm2d-5 [-1, 1, 9, 3] 2 782 | BatchNorm2d-6 [-1, 1, 9, 3] 2 783 | BatchNorm2d-7 [-1, 1, 9, 3] 2 784 | MaxPool2d-8 [-1, 1, 117, 1] 0 785 | BatchNorm2d-9 [-1, 1, 117, 1] 2 786 | AvgPool2d-10 [-1, 1, 117, 1] 0 787 | BatchNorm2d-11 [-1, 1, 117, 1] 2 788 | MaxPool2d-12 [-1, 1, 117, 1] 0 789 | BatchNorm2d-13 [-1, 1, 117, 1] 2 790 | Inception-14 [-1, 702] 0 791 | Linear-15 [-1, 30] 21,090 792 | ReLU-16 [-1, 30] 0 793 | Dropout-17 [-1, 30] 0 794 | Linear-18 [-1, 1] 31 795 | ================================================================ 796 | Total params: 21,141 797 | Trainable params: 21,141 798 | Non-trainable params: 0 799 | ---------------------------------------------------------------- 800 | Input size (MB): 0.00 801 | Forward/backward pass size (MB): 0.01 802 | Params size (MB): 0.08 803 | Estimated Total Size (MB): 0.10 804 | ---------------------------------------------------------------- 805 | ``` 806 | 807 | 自定义的特征提取层不需要训练参数，只需要按照规定的运算函数完成前向传播即可。同样地，池化层也不需要训练参数。 808 | 809 | 需要训练的参数来自：批标准化的$\gamma$和$\beta$，全连接层的weight和bias。 810 | 811 | ## 载入数据 812 | 813 | ### 继承`Dataset`类，改写`__getitem__` 814 | 815 | ```python 816 | class FactorData(Dataset): 817 | 818 | def __init__(self, train_x, train_y): 819 | self.len = len(train_x) 820 | self.x_data = train_x 821 | self.y_data = train_y 822 | 823 | def __getitem__(self, index): 824 | """ 825 | 指定读取数据的方式: 根据索引index返回dataset[index] 826 | 827 | """ 828 | return self.x_data[index], self.y_data[index] 829 | 830 | def __len__(self): 831 | return self.len 832 | 833 | ``` 834 | 835 | ### 设定Batch Size 836 | 837 | ```python 838 | batch_size = 1000 839 | ``` 840 | 841 | ### 将数据载入到DataLoader中 842 | 843 | ```python 844 | # 将数据载入到DataLoader中 845 | train_data = FactorData(trainx, trainy) 846 | train_loader = DataLoader(dataset=train_data, 847 | batch_size=batch_size, 848 | shuffle=False) # 不打乱数据集 849 | test_data = FactorData(testx, testy) 850 | test_loader = DataLoader(dataset=test_data, 851 | batch_size=batch_size, 852 | shuffle=False) # 不打乱数据集 853 | ``` 854 | 855 | ## 训练和测试模型 856 | 857 | ### 将AlphaNet实例化 858 | 859 | ```python 860 | # 构建模型 861 | alphanet = AlphaNet(combination=combination, combination_rev=combination_rev, 862 | index_list=index_list, fc1_num=702, fc2_num=30, dropout_rate=0.5) 863 | ``` 864 | 865 | ### 设定优化器 866 | 867 | ```python 868 | weight_list, bias_list = [], [] 869 | for name, p in alphanet.named_parameters(): 870 | # 将所有的bias参数放入bias_list中 871 | if 'bias' in name: 872 | bias_list += [p] 873 | # 将所有的weight参数放入weight_list中 874 | else: 875 | weight_list += [p] 876 | 877 | # weight decay: 对所有weight参数进行L2正则化 878 | optimizer = optim.RMSprop([{'params': weight_list, 'weight_decay': 1e-5}, 879 | {'params': bias_list, 'weight_decay': 0}], 880 | lr=1e-4, 881 | momentum=0.9) 882 | ``` 883 | 884 | ### 设定损失函数为均方误差MSE 885 | 886 | ```python 887 | # 损失函数为均方误差 MSE 888 | criterion = nn.MSELoss() 889 | ``` 890 | 891 | ### 训练和测试模型 892 | 893 | 由于AlphaNet-V1的网络结构较为简单，模型在5到10个Epoch的训练后即收敛，因此我们设定训练轮次`epoch_num`为5。 894 | 895 | 训练和测试模型的代码如下： 896 | 897 | ```python 898 | epoch_num = 5 899 | train_loss_list = [] 900 | test_loss_list = [] 901 | best_test_epoch, best_test_loss = 0, np.inf 902 | seed = 0 903 | for epoch in range(1, epoch_num+1): 904 | train_loss, test_loss = 0, 0 905 | # 在训练集中训练模型 906 | alphanet.train() # 关于.train()的作用，可以参考https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch 907 | train_batch_num = 0 908 | for data, label in tqdm(train_loader, f'Epoch {epoch}-train', leave=False): 909 | train_batch_num += 1 910 | # 准备数据 911 | data, label = data.to(torch.float), label.to(torch.float) 912 | # 得到训练集的预测值 913 | out_put = alphanet(data) 914 | # 计算损失 915 | loss = criterion(out_put, label) 916 | # 将损失值加入到本轮训练的损失中 917 | train_loss += loss.item() 918 | # 梯度清零 919 | optimizer.zero_grad() # 关于.zero_grad()的作用，可以参考https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch 920 | # 反向传播求解梯度 921 | loss.backward() 922 | # 更新权重参数 923 | optimizer.step() 924 | 925 | # 测试模式 926 | alphanet.eval() 927 | test_batch_num = 0 928 | with torch.no_grad(): 929 | for data, label in tqdm(test_loader, f'Epoch {epoch}-test ', leave=False): 930 | test_batch_num += 1 931 | data, label = data.to(torch.float), label.to(torch.float) 932 | # 得到测试集的预测值 933 | y_pred = alphanet(data) 934 | # 计算损失 935 | loss = criterion(y_pred, label) 936 | # 将损失值加入到本轮测试的损失中 937 | test_loss += loss.item() 938 | 939 | train_loss_list.append(train_loss/train_batch_num) 940 | test_loss_list.append(test_loss/test_batch_num) 941 | ``` 942 | 943 | ### 绘制训练集和测试集上损失的变化 944 | 945 | ```python 946 | # 画出损失函数的变化 947 | fig = plt.figure(figsize=(9, 6)) 948 | # 字号 949 | plt.rcParams['font.size'] = 16 950 | ax = fig.add_subplot(111) 951 | ax.plot(range(1, epoch_num+1), train_loss_list, 'r', label='train loss') 952 | ax.plot(range(1, epoch_num+1), test_loss_list, 'b', label='test loss') 953 | # 设置x轴刻度为整数 954 | ax.xaxis.set_major_locator(ticker.MultipleLocator(1)) 955 | # 设置y轴范围 956 | ax.set_ylim(bottom=0, top=0.1) 957 | ax.legend() 958 | ax.set_xlabel('Epoch') 959 | ax.set_ylabel('MSE Loss') 960 | plt.show() 961 | ``` 962 | 963 | ![image-20221221232108413](README-image/image-20221221232108413.png) 964 | 965 | 经过测试发现，训练集和验证集在训练5轮左右就已经收敛，损失已不再下降。为了探究模型给出了什么样的预测值，我们接下来比较部分样本标签的预测值和真实值。 966 | 967 | ### 预测值和真实值的比较 968 | 969 | 为方便查看，我们截取最后一批训练样本的前200个样本，对比模型的预测标签值和真实标签值。 970 | 971 | ```python 972 | # 绘制部分预测值和真实值 973 | y_pred = y_pred.detach().numpy() 974 | label = label.detach().numpy() 975 | 976 | # 截取部分数据 977 | part = range(0, 200) 978 | plt.plot(y_pred[part], label='pred') 979 | plt.plot(label[part], label='true') 980 | plt.legend() 981 | plt.show() 982 | ``` 983 | 984 | ![image-20221221232728887](README-image/image-20221221232728887.png) 985 | 986 | AlphaNet-V1给出的标签预测值几乎都为常数，只在少部分样本点有突出值，这并不是一个理想的收益率预测值。对于标签预测值几乎都为常数的情况，我推测原因可能是： 987 | 988 | 1. AlphaNet-V1的网络结构较简单，只使用了7个平行的特征提取层、3个平行的池化层和一个全连接层。简单的网络结构限制了参数的选择范围，为了减少损失，模型可能会倾向于选择常数作为标签预测值。 989 | 2. AlphaNet-V1用的池化层通过取最大、平均和最小的操作，将3个在时序值转换为1个值。而这三个时序值仍然具有时序信息，但简单的池化层会丧失这一时序信息。 990 | 991 | ### 保存模型 992 | 993 | 将模型的结构和参数均保存到本地，方便后续使用。 994 | 995 | ```python 996 | # 保存模型 997 | torch.save(alphanet, 'alphanet_v1_pool.pth') 998 | ``` 999 | 1000 | ## 改进方向 1001 | 1002 | 在实证检验后，我们发现AlphaNet-V1给出的标签预测值几乎都为常数，只在少部分样本点有突出值，这并不是一个理想的收益率预测值。因此，本文尝试对AlphaNet-V1进行两方面的改进： 1003 | 1004 | 1. 调整网络结构。AlphaNet-V1的网络结构较简单，我们将在AlphaNet-V3版本中加入不同步长`stride`的特征提取层，并将现有的池化层转换为可以记忆时序信息的门控循环单元（GRU）。 1005 | 2. 调整标签值。AlphaNet-V1预测的标签值是个股在未来10个交易日的收益率。我们将在AlphaNet-V3版本中将预测标签转换为涨跌方向（1代表上涨，即收益率大于0；0代表不变或下跌，即收益率小于或等于0）和超额收益方向（1代表收益率大于当批的平均收益率，0代表收益率小于或等于当批的平均收益率）。 1006 | 1007 | ## 搭建AlphaNet-V3 1008 | 1009 | ### 加入多步长的特征提取层 1010 | 1011 | AlphaNet-V1的特征提取层中，固定步长`stride=10`，我们可以设定其他的步长，拓宽因子挖掘的可能性。 1012 | 1013 | 下面我们加入步长`stride=3`的特征提取层，仍使用AlphaNet-V1中的7个运算符函数。 1014 | 1015 | 由于步长发生变化，因此卷积操作的索引列表也发生了变化。我们需要生成两个不同的`index_list`，供两个步长不同的特征提取层使用。 1016 | 1017 | ```python 1018 | # 根据输入的矩阵和卷积操作的步长, 计算卷积操作的索引 1019 | def get_index_list(matrix, stride): 1020 | """ 1021 | args: 1022 | matrix: torch.tensor, the input matrix 1023 | stride: int, the stride of the convolution operation 1024 | 1025 | return: 1026 | index_list: list, the index of the convolution operation 1027 | 1028 | """ 1029 | W = matrix.shape[3] 1030 | if W % stride == 0: 1031 | index_list = list(np.arange(0, W+stride, stride)) 1032 | else: 1033 | mod = W % stride 1034 | index_list = list(np.arange(0, W+stride-mod, stride)) + [W] 1035 | return index_list 1036 | # 根据输入的矩阵和卷积操作的步长, 计算卷积操作的索引 1037 | # inception1模块使用的卷积操作的步长为10 1038 | index_list_1 = get_index_list(np.zeros((1,1,9,30)), 10) 1039 | # inception2模块使用的卷积操作的步长为3 1040 | index_list_2 = get_index_list(np.zeros((1,1,9,30)), 3) 1041 | ``` 1042 | 1043 | ![image-20221222155058742](README-image/image-20221222155058742.png) 1044 | 1045 | 在AlphaNet-V1中，我们已经编写了`Inception`类，它可以接收不同的`index_list`作为参数，因此我们只需要传入`index_list_2`即可实现步长为`stride=3`的特征提取层。 1046 | 1047 | ```python 1048 | class AlphaNet(nn.Module): 1049 | 1050 | def __init__(self, combination, combination_rev, index_list_1, index_list_2, fc_num): 1051 | super(AlphaNet, self).__init__() 1052 | self.combination = combination 1053 | self.combination_rev = combination_rev 1054 | # 自定义的Inception1模块 1055 | self.Inception_1 = Inception(combination, combination_rev, index_list_1) 1056 | # 自定义的Inception2模块 1057 | self.Inception_2 = Inception(combination, combination_rev, index_list_2) 1058 | # 输出层 1059 | self.fc = nn.Linear(fc_num, 1) # 30 -> 1 1060 | # 初始化权重 1061 | self._init_weights() 1062 | ``` 1063 | 1064 | 上面的代码完成了两个平行的特征提取层。 1065 | 1066 | ### 将池化层替换为门控循环单元（GRU） 1067 | 1068 | AlphaNet-V1中的池化层仅是将特征提取层的输出在时序上取最大、平均和最小池化。若特征提取层中的步长`stride=10`，则池化的作用是将时序上的3个值取最大、平均和最小值，转换为1个值。这样的操作丧失了特征提取层得到的3个值本身的时序信息。 1069 | 1070 | ![image-20221222160928336](README-image/image-20221222160928336.png) 1071 | 1072 | 为了保留特征提取层得到的输出本身的时序信息，我们将池化层转换为门控循环单元，它是循环神经网络的一种，可以接受时序输入，并输出带有时序记忆的隐藏状态。 1073 | 1074 | 我们自定义的GRU层的代码如下。由于特征提取层将得到维数为$117\times3$或$117\times10$的矩阵，因此我们设定`input_size=117`。`num_layers`设为2。我们需要的输出是带有时序记忆的隐藏状态。最终将得到维数为$30$的矩阵。 1075 | 1076 | ```python 1077 | class GRU(nn.Module): 1078 | def __init__(self): 1079 | super(GRU, self).__init__() 1080 | self.gru = nn.GRU(input_size=117, hidden_size=30, 1081 | num_layers=2, batch_first=True, bidirectional=False) 1082 | 1083 | def forward(self, data): 1084 | # N*time_step*117 -> output: torch.Size([1000, time_step, 30]), hn: torch.Size([2, 1000, 30]), 对于Inception1, time_step=3, 对于Inception2, time_step=10 1085 | output, hn = self.gru(data) 1086 | h = hn[-1:] # 使用最后一层hidden state的输出, h: torch.Size([1, 1000, 30]) 1087 | data = h.squeeze(0) # torch.Size([1000, 30]) 1088 | return data # torch.Size([1000, 30]) 1089 | 1090 | ``` 1091 | 1092 | 将GRU层嵌入到自定义的`Inception`类中： 1093 | 1094 | ```python 1095 | class Inception(nn.Module): 1096 | """ 1097 | Inception, 用于提取时间序列的特征, 具体操作包括： 1098 | 1099 | 1. kernel_size和stride为d=10或3的特征提取层, 类似于卷积层，用于提取时间序列的特征. 具体包括:(下面以d=10为例) 1100 | 1101 | 1. ts_corr4d: 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的相关系数 1102 | 2. ts_cov4d: 过去 d 天 X 值构成的时序数列和 Y 值构成的时序数列的协方差 1103 | 3. ts_stddev4d: 过去 d 天 X 值构成的时序数列的标准差 1104 | 4. ts_zscore4d: 过去 d 天 X 值构成的时序数列的平均值除以标准差 1105 | 5. ts_return4d: (X - delay(X, d))/delay(X, d)-1, 其中delay(X, d)为 X 在 d 天前的取值 1106 | 6. ts_decaylinear4d: 过去 d 天 X 值构成的时序数列的加权平均值，权数为 d, d – 1, …, 1(权数之和应为 1，需进行归一化处理)，其中离现在越近的日子权数越大 1107 | 7. ts_mean4d: 过去 d 天 X 值构成的时序数列的平均值 1108 | 1109 | 各操作得到的张量维数： 1110 | 1. 由于涉及两个变量的协方差, 因此ts_corr4d和ts_cov4d的输出为 N*1*36*3 1111 | 2. 其余操作均只涉及单变量的时序计算, 因此输出为 N*1*9*3 1112 | 1113 | 2. 对第1步的输出进行Batch Normalization操作, 输出维数仍为 N*1*36*3 或 N*1*9*3 1114 | 1115 | 3. 对第2步的输出, 在 H 维度上拼接, 输出维数为 N*1*(2*36+5*9)*3 = N*1*117*3 1116 | 1117 | 4. 对第3步的输出, 使用GRU进行特征提取, 输出维数为 N*30 1118 | 1119 | """ 1120 | 1121 | def __init__(self, combination, combination_rev, index_list): 1122 | """ 1123 | combination: 卷积操作时需要的两列数据的组合 1124 | combination_rev: 卷积操作时需要的两列数据的组合, 与combination相反 1125 | index_list: 卷积操作时需要的时间索引 1126 | 1127 | """ 1128 | 1129 | super(Inception, self).__init__() 1130 | # 卷积操作时需要的两列数据的组合 1131 | self.combination = combination 1132 | self.combination_rev = combination_rev 1133 | 1134 | # 卷积操作时需要的时间索引 1135 | self.index_list = index_list 1136 | 1137 | # 卷积操作后的Batch Normalization层 1138 | self.bc1 = nn.BatchNorm2d(1) 1139 | self.bc2 = nn.BatchNorm2d(1) 1140 | self.bc3 = nn.BatchNorm2d(1) 1141 | self.bc4 = nn.BatchNorm2d(1) 1142 | self.bc5 = nn.BatchNorm2d(1) 1143 | self.bc6 = nn.BatchNorm2d(1) 1144 | self.bc7 = nn.BatchNorm2d(1) 1145 | 1146 | # GRU层 1147 | self.GRU = GRU() 1148 | 1149 | def forward(self, data): 1150 | """ 1151 | data: 输入的数据, 维度为batch_size*1*9*30 1152 | 1153 | """ 1154 | # 本层的输入为batch_size*1*9*30, 在训练时不需要反向传播, 因此可以使用detach()函数 1155 | data = data.detach().cpu().numpy() 1156 | combination = self.combination 1157 | combination_rev = self.combination_rev 1158 | 1159 | # 卷积操作 1160 | conv1 = self.ts_corr4d(data, combination, combination_rev).to(torch.float) 1161 | conv2 = self.ts_cov4d(data, combination, combination_rev).to(torch.float) 1162 | conv3 = self.ts_stddev4d(data).to(torch.float) 1163 | conv4 = self.ts_zcore4d(data).to(torch.float) 1164 | conv5 = self.ts_return4d(data).to(torch.float) 1165 | conv6 = self.ts_decaylinear4d(data).to(torch.float) 1166 | conv7 = self.ts_mean4d(data).to(torch.float) 1167 | 1168 | # 卷积操作后的Batch Normalization 1169 | batch1 = self.bc1(conv1) 1170 | batch2 = self.bc2(conv2) 1171 | batch3 = self.bc3(conv3) 1172 | batch4 = self.bc4(conv4) 1173 | batch5 = self.bc5(conv5) 1174 | batch6 = self.bc6(conv6) 1175 | batch7 = self.bc7(conv7) 1176 | 1177 | # 在 H 维度上进行特征拼接 1178 | feature = torch.cat( 1179 | [batch1, batch2, batch3, batch4, batch5, batch6, batch7], axis=2) # N*1*(2*36+5*9)*3 = N*1*117*3 1180 | 1181 | # GRU层 1182 | feature = feature.squeeze(1) # N*1*117*3 -> N*117*3 1183 | feature = feature.permute(0, 2, 1) # N*117*3 -> N*3*117 1184 | feature = self.GRU(feature) # N*3*117 -> N*30 1185 | 1186 | return feature 1187 | 1188 | ``` 1189 | 1190 | ### 使用`torchsummary`包查看网络结构 1191 | 1192 | ```python 1193 | test = AlphaNet(combination, combination_rev, index_list_1, index_list_2, fc_num=60) 1194 | 1195 | summary(test, input_size=(1, 9, 30)) 1196 | ``` 1197 | 1198 | ``` 1199 | ---------------------------------------------------------------- 1200 | Layer (type) Output Shape Param # 1201 | ================================================================ 1202 | BatchNorm2d-1 [-1, 1, 36, 3] 2 1203 | BatchNorm2d-2 [-1, 1, 36, 3] 2 1204 | BatchNorm2d-3 [-1, 1, 9, 3] 2 1205 | BatchNorm2d-4 [-1, 1, 9, 3] 2 1206 | BatchNorm2d-5 [-1, 1, 9, 3] 2 1207 | BatchNorm2d-6 [-1, 1, 9, 3] 2 1208 | BatchNorm2d-7 [-1, 1, 9, 3] 2 1209 | GRU-8 [[-1, 3, 30], [-1, 2, 30]] 0 1210 | GRU-9 [-1, 30] 0 1211 | Inception-10 [-1, 30] 0 1212 | BatchNorm2d-11 [-1, 1, 36, 10] 2 1213 | BatchNorm2d-12 [-1, 1, 36, 10] 2 1214 | BatchNorm2d-13 [-1, 1, 9, 10] 2 1215 | BatchNorm2d-14 [-1, 1, 9, 10] 2 1216 | BatchNorm2d-15 [-1, 1, 9, 10] 2 1217 | BatchNorm2d-16 [-1, 1, 9, 10] 2 1218 | BatchNorm2d-17 [-1, 1, 9, 10] 2 1219 | GRU-18 [[-1, 10, 30], [-1, 2, 30]] 0 1220 | GRU-19 [-1, 30] 0 1221 | Inception-20 [-1, 30] 0 1222 | Linear-21 [-1, 1] 61 1223 | ================================================================ 1224 | Total params: 89 1225 | Trainable params: 89 1226 | Non-trainable params: 0 1227 | ---------------------------------------------------------------- 1228 | Input size (MB): 0.00 1229 | Forward/backward pass size (MB): 0.17 1230 | Params size (MB): 0.00 1231 | Estimated Total Size (MB): 0.17 1232 | ---------------------------------------------------------------- 1233 | 1234 | ``` 1235 | 1236 | > 不知道为什么，这里的GRU层并没有显示待估计的参数，实际上GRU层也是需要估计非常多的参数的。 1237 | 1238 | 在2个特征提取层及GRU层后，将输出展平后拼接，得到$60\times1$的矩阵，直接连接到预测目标即可输出预测值，这样就构建了AlphaNet-V3。 1239 | 1240 | ### 使用`TensorBoard`查看网络结构 1241 | 1242 | 我们可以使用`TensorBoard`对模型进行可视化，主要代码如下。 1243 | 1244 | ```python 1245 | from tensorboardX import SummaryWriter # 用于进行可视化 1246 | 1247 | from torchviz import make_dot 1248 | 1249 | sample_data = torch.rand(1000, 1, 9, 30) 1250 | # 1. 来用tensorflow进行可视化 1251 | with SummaryWriter("./log", comment="sample_model_visualization") as sw: 1252 | sw.add_graph(alphanet, sample_data) 1253 | 1254 | # 2. 保存成pt文件后进行可视化 1255 | torch.save(alphanet, "./log/alphanet_v3_gru.pt") 1256 | 1257 | # 3. 使用graphviz进行可视化 1258 | out = alphanet(sample_data) 1259 | g = make_dot(out) 1260 | g.render('alphanet_v3_gru', view=False) # 这种方式会生成一个pdf文件 1261 | ``` 1262 | 1263 | 在`TensorBoard`中，查看网络结构如下。由于特征提取层的运算符函数太多，且有两个步长不同的的平行的特征提取层，若全部展开则宽度太大，因此这里只展开其中一个。 1264 | 1265 | ![TensorBoard模型结构](README-image/TensorBoard模型结构.png) 1266 | 1267 | ## 训练和测试模型 1268 | 1269 | 相比AlphaNet-V1，AlphaNet-V3加入了步长`stride=3`的特征提取层、将池化层转换为保留时序信息的GRU层，模型变得更复杂。在训练模型时，训练20轮以上才显现出收敛的趋势。因此我们将`epoch_num`设为30。 1270 | 1271 | ```python 1272 | epoch_num = 30 1273 | train_loss_list = [] 1274 | test_loss_list = [] 1275 | best_test_epoch, best_test_loss = 0, np.inf 1276 | for epoch in range(1, epoch_num+1): 1277 | train_loss, test_loss = 0, 0 1278 | # 在训练集中训练模型 1279 | alphanet.train() # 关于.train()的作用，可以参考https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch 1280 | train_batch_num = 0 1281 | for data, label in tqdm(train_loader, f'Epoch {epoch}-train', leave=False): 1282 | train_batch_num += 1 1283 | # 准备数据 1284 | data, label = data.to(torch.float), label.to(torch.float) 1285 | # 得到训练集的预测值 1286 | out_put = alphanet(data) 1287 | # 计算损失 1288 | loss = criterion(out_put, label) 1289 | # 将损失值加入到本轮训练的损失中 1290 | train_loss += loss.item() 1291 | # 梯度清零 1292 | optimizer.zero_grad() # 关于.zero_grad()的作用，可以参考https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch 1293 | # 反向传播求解梯度 1294 | loss.backward() 1295 | # 更新权重参数 1296 | optimizer.step() 1297 | 1298 | # 测试模式 1299 | alphanet.eval() 1300 | test_batch_num = 0 1301 | with torch.no_grad(): 1302 | for data, label in tqdm(test_loader, f'Epoch {epoch}-test ', leave=False): 1303 | test_batch_num += 1 1304 | data, label = data.to(torch.float), label.to(torch.float) 1305 | # 得到测试集的预测值 1306 | y_pred = alphanet(data) 1307 | # 计算损失 1308 | loss = criterion(y_pred, label) 1309 | # 将损失值加入到本轮测试的损失中 1310 | test_loss += loss.item() 1311 | 1312 | train_loss_list.append(train_loss/train_batch_num) 1313 | test_loss_list.append(test_loss/test_batch_num) 1314 | ``` 1315 | 1316 | ### 绘制训练集和测试集上损失的变化 1317 | 1318 | ```python 1319 | # 画出损失函数的变化 1320 | fig = plt.figure(figsize=(9, 6)) 1321 | # 字号 1322 | plt.rcParams['font.size'] = 16 1323 | ax = fig.add_subplot(111) 1324 | ax.plot(train_loss_list, 'r', label='train loss') 1325 | ax.plot(test_loss_list, 'b', label='test loss') 1326 | # 设置y轴范围 1327 | ax.set_ylim(bottom=0, top=0.1) 1328 | ax.legend() 1329 | ax.set(xlabel='Epoch') 1330 | ax.set(ylabel='MSE Loss') 1331 | plt.show() 1332 | ``` 1333 | 1334 | ![image-20221222190109549](README-image/image-20221222190109549.png) 1335 | 1336 | 经过测试发现，训练集和验证集在训练20轮左右就已经收敛，损失已不再下降。为了探究模型给出了什么样的预测值，我们接下来比较部分样本标签的预测值和真实值。 1337 | 1338 | ### 预测值和真实值的比较 1339 | 1340 | 为方便查看，我们截取最后一批训练样本的前200个样本，对比模型的预测标签值和真实标签值。 1341 | 1342 | ```python 1343 | # 绘制部分预测值和真实值 1344 | y_pred = y_pred.detach().numpy() 1345 | label = label.detach().numpy() 1346 | # 截取部分数据 1347 | part = range(0, 200) 1348 | plt.plot(y_pred[part], label='pred') 1349 | plt.plot(label[part], label='true') 1350 | plt.legend() 1351 | plt.show() 1352 | ``` 1353 | 1354 | ![image-20221222190543853](README-image/image-20221222190543853.png) 1355 | 1356 | AlphaNet-V3的模型结构更复杂，输出的预测值并不全是常数，可以作为对收益率的预测值。 1357 | 1358 | 但我们也可以注意到AlphaNet-V3给出的收益率预测值的波动明显比真实值的波动更大，从这一点上看，预测效果并不是很理想。这可能是因为预测收益率的具体数值的难度较大，我们后续考虑将预测目标转换为收益率的方向（即个股在未来10个交易日的价格涨跌），以及超额收益的方向（即个股在未来10个交易日的收益率相对截面平均收益率的大小）。 1359 | 1360 | ## 调整预测目标：收益率的方向 1361 | 1362 | 为了调整预测目标，将回归问题转换为二分类问题，我们需要在代码中作如下的调整。 1363 | 1364 | ### 将预测标签转换为0和1 1365 | 1366 | ```python 1367 | # 由于是分类问题, 因此将y大于0的标签设置为1，小于0的标签设置为0 1368 | trainy[trainy > 0] = 1 1369 | trainy[trainy < 0] = 0 1370 | testy[testy > 0] = 1 1371 | testy[testy < 0] = 0 1372 | ``` 1373 | 1374 | ### 全连接层最后的激活函数设为`sigmoid`，使得最终输出在0到1之间 1375 | 1376 | ```python 1377 | def forward(self, data): 1378 | data_1 = self.Inception_1(data) # N*30 1379 | data_2 = self.Inception_2(data) # N*30 1380 | pool_cat = torch.cat([data_1, data_2], axis=1) # N*60 1381 | # 输出层 1382 | data = self.fc(pool_cat) 1383 | # 激活函数, 使输出值在0到1之间 1384 | data = torch.sigmoid(data) 1385 | data = data.to(torch.float) 1386 | 1387 | return data 1388 | ``` 1389 | 1390 | ### 损失函数设为`Binary Cross Entropy` 1391 | 1392 | 在回归问题中，我们使用的损失函数为均方误差MSE。在二分类问题中，适合使用`BCELoss()`，即Binary Cross Entropy Loss二元交叉熵。 1393 | 1394 | ```python 1395 | # 由于是分类问题, 因此使用Binary Cross Entropy损失函数 1396 | criterion = nn.BCELoss() 1397 | ``` 1398 | 1399 | ### 绘制训练集和测试集上损失的变化 1400 | 1401 | ![image-20221222192146000](README-image/image-20221222192146000.png) 1402 | 1403 | 训练集和测试集的损失值较为接近，均在0.69左右。训练轮次在5轮左右时效果较好。 1404 | 1405 | ### 预测值和真实值的比较 1406 | 1407 | #### ROC曲线 1408 | 1409 | 对于二分类问题，我们可以绘制ROC曲线来评价预测效果。 1410 | 1411 | ```python 1412 | # 绘制ROC曲线 1413 | fpr, tpr, thresholds = metrics.roc_curve(label, y_pred) 1414 | plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve') 1415 | plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--') 1416 | plt.xlabel('False Positive Rate') 1417 | plt.ylabel('True Positive Rate') 1418 | plt.title('ROC Curve') 1419 | plt.legend(loc="lower right") 1420 | plt.show() 1421 | ``` 1422 | 1423 | ![image-20221222192511533](README-image/image-20221222192511533.png) 1424 | 1425 | 可以看到，ROC曲线仅略高于次对角线，说明预测效果仅略高于基于样本正负比率的随机猜测。 1426 | 1427 | #### 混淆矩阵 1428 | 1429 | 我们可以绘制混淆矩阵，查看正负例样本的预测情况。 1430 | 1431 | ```python 1432 | # 绘制混淆矩阵 1433 | y_pred = np.where(y_pred > 0.5, 1, 0) 1434 | cm = metrics.confusion_matrix(label, y_pred) 1435 | ``` 1436 | 1437 | ![image-20221222192830751](README-image/image-20221222192830751.png) 1438 | 1439 | ```python 1440 | import seaborn as sn 1441 | plt.figure(figsize = (10,8)) 1442 | sn.heatmap(cm, annot=True, cmap='Blues', fmt='d') 1443 | plt.xlabel('y_pred') 1444 | plt.ylabel('y_true') 1445 | ``` 1446 | 1447 | ![image-20221222192849349](README-image/image-20221222192849349.png) 1448 | 1449 | 从混淆矩阵中可以看出，预测结果确实和真实数据中的正负比例很接近，说明我们的预测结果确实近似于基于样本数据的“随机猜测”。 1450 | 1451 | 这在信噪比极低的金融领域是可以理解的，我们的模型只用到了简单的量价数据，在生成涨跌信号的准确度上并不应奢求能获得非常好的预测效果。 1452 | 1453 | ## 调整预测目标：超额收益率的方向 1454 | 1455 | 由于市场存在不可分散的系统性风险，个股的收益率数值通常受市场环境的影响很大。在积极组合管理领域，投资者更希望找到能够打败市场、具有超额收益的个股。因此我们再次调整预测目标为个股在10个交易日后的超额收益的正负值。 1456 | 1457 | ### 将预测标签转换为超额收益率的正负 1458 | 1459 | 需要先计算截面的收益率均值，再根据个股收益率相对于截面收益率的大小，确定超额收益率的正负，并以此作为预测目标。 1460 | 1461 | 由于我们的样本并没有包含全市场的所有个股，因此这样的计算会有偏差。 1462 | 1463 | ```python 1464 | # 由于是超额收益的分类问题, 因此将y大于trainy.mean()的标签设置为1，小于trainy.mean()的标签设置为0 1465 | train_mean_y = trainy.mean().item() 1466 | trainy[trainy > train_mean_y] = 1 1467 | trainy[trainy < train_mean_y] = 0 1468 | test_mean_y = testy.mean().item() 1469 | testy[testy > test_mean_y] = 1 1470 | testy[testy < test_mean_y] = 0 1471 | ``` 1472 | 1473 | 其他代码基本与“预测收益率的方向”一致，在此不再赘述。 1474 | 1475 | ### 绘制训练集和测试集上损失的变化 1476 | 1477 | ![image-20221222193631521](README-image/image-20221222193631521.png) 1478 | 1479 | 损失值得结果与“预测收益率的方向”相比略有下降，或许说明模型在预测超额收益率的方向上表现更好。 1480 | 1481 | ### 预测值和真实值的比较 1482 | 1483 | #### ROC曲线 1484 | 1485 | ![image-20221222193750323](README-image/image-20221222193750323.png) 1486 | 1487 | #### 混淆矩阵 1488 | 1489 | ![image-20221222194347192](README-image/image-20221222194347192.png) 1490 | 1491 | 尽管我们做了多种尝试，但发现ROC曲线和混淆矩阵的表现依旧一般，模型在分类准确率上效果并不理想。 1492 | 1493 | ## 将随机森林模型作为baseline进行比较 1494 | 1495 | 我们的特征均为量价数据，个股在每一天有$9\times30=270$个特征。随机森林模型在训练时并不会用到全部特征，而是会随机抽取样本和特征。作为基准模型，我们不进行额外的特征工程，也不刻意调整模型参数，仅将基准结果与前面构造的AlphaNet-V3进行比较。 1496 | 1497 | 我们以“超额收益率的方向”作为预测目标，前文已经介绍过了AlphaNet-V3的预测效果，这里我们介绍随机森林模型进行分类预测的过程。 1498 | 1499 | 在前面的神经网络中，我们的输入数据是$9\times30$的矩阵，在随机森林模型中，我们需要将二维矩阵展平，再输入到模型中。 1500 | 1501 | ```python 1502 | # 读取数据 1503 | X_train = np.load('../data/X_train.npy') 1504 | # # 将数据转换为二维 1505 | X_train = X_train.reshape(X_train.shape[0], -1) 1506 | y_train = np.load('../data/y_train.npy') 1507 | X_test = np.load('../data/X_test.npy') 1508 | # 将数据转换为二维 1509 | X_test = X_test.reshape(X_test.shape[0], -1) 1510 | y_test = np.load('../data/y_test.npy') 1511 | # 查看数据的大小 1512 | print("训练集特征维数: ", X_train.shape) 1513 | print("训练集标签维数: ", y_train.shape) 1514 | print("测试集特征维数: ", X_test.shape) 1515 | print("测试集标签维数: ", y_test.shape) 1516 | 1517 | # 由于是超额收益的分类问题, 因此将y大于y_train.mean()的标签设置为1，小于y_train.mean()的标签设置为0 1518 | train_mean_y = y_train.mean() 1519 | y_train[y_train > train_mean_y] = 1 1520 | y_train[y_train < train_mean_y] = 0 1521 | test_mean_y = y_test.mean() 1522 | y_test[y_test > test_mean_y] = 1 1523 | y_test[y_test < test_mean_y] = 0 1524 | ``` 1525 | 1526 | ### 训练和测试模型 1527 | 1528 | ```python 1529 | # 构建随机森林模型 1530 | clf = RandomForestClassifier(random_state=0, n_jobs=-1) 1531 | clf.fit(X_train, y_train) 1532 | # 在测试集上预测 1533 | y_pred = clf.predict(X_test) 1534 | y_pred_proba = clf.predict_proba(X_test)[:,1] 1535 | ``` 1536 | 1537 | #### 输出模型评价指标的函数 1538 | 1539 | ```python 1540 | # 在测试集上给出模型分类的效果 1541 | def evaluate(true, pred): 1542 | print('accuracy:{:.2%}'.format(metrics.accuracy_score(true, pred))) 1543 | print('precision:{:.2%}'.format(metrics.precision_score(true, pred))) 1544 | print('recall:{:.2%}'.format(metrics.recall_score(true, pred))) 1545 | print('f1-score:{:.2%}'.format(metrics.f1_score(true, pred))) 1546 | ``` 1547 | 1548 | ```python 1549 | evaluate(y_test, y_pred) 1550 | ``` 1551 | 1552 | ![image-20221223120215460](README-image/image-20221223120215460.png) 1553 | 1554 | 精确率、召回率均较低，基准模型的表现也不好。 1555 | 1556 | #### 特征重要性排序 1557 | 1558 | 随机森林模型可以基于信息增益对特征重要性进行排序。 1559 | 1560 | 在代码中，需要为展平后的270个特征设定名称。 1561 | 1562 | ```python 1563 | # 为270个特征设定名字 1564 | feature_names = ['open', 'high', 'low', 'close', 'vwap', 'volume', 'return1', 'turn', 'free_turn'] 1565 | time = range(29, -1, -1) 1566 | all_feature_names = [] 1567 | for i in feature_names: 1568 | for j in time: 1569 | all_feature_names.append(i + '_' + str(j)) 1570 | feat_importances = pd.Series(clf.feature_importances_, index=all_feature_names) 1571 | ``` 1572 | 1573 | ```python 1574 | # 将特征重要性排序后绘图 1575 | ax = feat_importances.sort_values()[250:].plot(kind='barh', figsize=(12, 6)) 1576 | # 设置横坐标格式 1577 | ax.xaxis.set_major_formatter(plt.FuncFormatter(lambda x, pos: "{:.2%}".format(x))) 1578 | # 设置标题 1579 | ax.set_title('Importance of Features') 1580 | # 如果不需要显示特征重要性的大小数值，可以使用下面2行代码 1581 | # for container in ax.containers: 1582 | # ax.bar_label(container) 1583 | # 如果需要显示特征重要性的大小数值，可以使用下面的代码 1584 | x_offset = 0 1585 | y_offset = 0 1586 | for p in ax.patches: 1587 | b = p.get_bbox() 1588 | val = "{:.2%}".format(b.x1) 1589 | ax.annotate(val, (b.x1 + 0.0002, b.y1 - 0.4)) 1590 | ``` 1591 | 1592 | ![image-20221223120440331](README-image/image-20221223120440331.png) 1593 | 1594 | 最重要的特征几乎都为`vwap`，说明基于成交量的加权价格可能是预测超额收益的重要因素。 1595 | 1596 | #### ROC曲线 1597 | 1598 | ```python 1599 | # 绘制ROC曲线 1600 | fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred_proba) 1601 | plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve') 1602 | plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--') 1603 | plt.xlabel('False Positive Rate') 1604 | plt.ylabel('True Positive Rate') 1605 | plt.title('ROC Curve') 1606 | plt.legend(loc="lower right") 1607 | plt.show() 1608 | ``` 1609 | 1610 | ![image-20221223120549749](README-image/image-20221223120549749.png) 1611 | 1612 | #### 混淆矩阵 1613 | 1614 | ![image-20221223121130542](README-image/image-20221223121130542.png) 1615 | 1616 | ROC曲线几乎和次对角线重合，混淆矩阵的分类准确率、召回率都不高，说明在随机森林模型上的预测效果同样不佳。 1617 | 1618 | ## 总结和结论 1619 | 1620 | 本文主要实现了： 1621 | 1622 | 1. 基于个股在过去30个交易日的9个量价数据，构造“图片”矩阵，作为AlphaNet的输入数据。 1623 | 2. 构造自定义的特征提取层，实现2种基于双变量的特征提取层和5种基于单变量的特征提取层。 1624 | 3. 以10为步长，将自定义的特征提取层与批标准化层、最大（平均、最小）池化层、全连接层结合，以均方误差MSE为损失函数，输出收益率值的预测目标。 1625 | 4. 在训练集和验证集上做检验，发现可能是由于模型过于简单，损失值很快收敛，且预测值几乎都为常数。为此做了一些改进。 1626 | 5. 改进1：将池化层替换为门控循环单元（GRU）。改进后的预测结果能给出不同的收益率值，但波动率明显比真实值更大。 1627 | 6. 改进2：调整预测目标为收益率和方向和超额收益的方向。将全连接层的激活函数设为`sigmoid`，损失函数设为`Binary Cross Entropy`，对测试集上的预测效果进行评估，发现ROC曲线和混淆矩阵都接近随机猜测。 1628 | 7. 与基Baseline随机森林模型进行比较。我们将随机森林模型应用于对个股在未来10个交易日的超额收益的预测，发现`vwap`是最重要的特征。但模型评价指标也告诉我们很难准确地预测超额收益率的方向。 1629 | 1630 | ## 未来研究方向 1631 | 1632 | 虽然本文测试的几个模型均算不上理想，但这或许就是金融市场的特点：低信噪比、难以预测。在开始进行这个项目前，我也并不期待能搭建出预测准确率高得惊人的模型。在回顾模型是如何构建的同时，我也想到了未来可以继续拓展的方向： 1633 | 1634 | 1. 扩充特征数据。本文只用到了9个量价数据，且都是日频的数据。在高频领域可以构造出更多的分钟频甚至更高频的特征数据。对于数据结构与本文相似的特征，AlphaNet都可以接受作为输入。更多的特征数据也意味着更多解释收益率的可能性，也许能提高模型的预测效果。 1635 | 2. 进一步调整模型结构和参数。由于前期搭建的模型表现并不理想，且研究时间有限，我并没有花费大量时间进行参数调优，只是参考了原始研究报告中的参数。模型中的步长、GRU层数、全连接层的设计、Dropout rate等，都可以进行参数寻优，或许能取得更好的效果。 1636 | 3. 使用更大量的数据进行训练。由于数据的可得性，本文下载15000余个样本数据耗时3个多小时。在有充足的数据来源的情况下，可以考虑对更大量的样本进行训练和测试。 1637 | 1638 | ## 参考资料 1639 | 1640 | 1. 本文的研究思路来自华泰证券研究所金融工程组的两篇研究报告：[《华泰人工智能系列之三十二-AlphaNet：因子挖掘神经网络》](https://crm.htsc.com.cn/doc/2020/10750101/74856806-a2e3-41cb-be4c-695dc6cc1341.pdf)和[《华泰人工智能系列之三十四-再探AlphaNet：结构和特征优化》](https://crm.htsc.com.cn/doc/2020/10750101/74619658-f648-4001-a255-5b78174b073a.pdf)。这两篇报告介绍了如何借鉴卷积神经网络CNN和门控循环单元GRU的思想，搭建基于量价数据进行因子挖掘的神经网络。我使用的模型参数也是基于研究报告中给出的。 1641 | 1642 | 2. 本文获取数据和构造运算符函数的方法参考了知乎的回答：[如何实现用遗传算法或神经网络进行因子挖掘？](https://www.zhihu.com/question/336207872/answer/2592739064) -------------------------------------------------------------------------------- /data/X_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/data/X_test.npy -------------------------------------------------------------------------------- /data/X_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/data/X_train.npy -------------------------------------------------------------------------------- /data/get_data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from rich.progress import Progress\n", 10 | "from datetime import timedelta \n", 11 | "import pandas as pd\n", 12 | "import numpy as np\n", 13 | "import tushare as ts\n", 14 | "pro = ts.pro_api()" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "# 给定一个交易日，返回该日满足条件的A股股票列表\n", 24 | "def get_stocklist(date: str, num: int):\n", 25 | "\n", 26 | " start = str(pd.to_datetime(date)-timedelta(30))\n", 27 | " start = start[0:4]+start[5:7]+start[8:10]\n", 28 | " df1 = pro.index_weight(index_code='000002.SH',\n", 29 | " start_date=start, end_date=date) # 交易日当天的股票列表\n", 30 | " codes = list(df1['con_code'])\n", 31 | " codes = codes[0:1000] # 在每个截面期只选取1000只股票\n", 32 | "\n", 33 | " return codes\n", 34 | "\n", 35 | "# 给定日期区间的端点，输出期间的定长采样交易日列表\n", 36 | "\n", 37 | "\n", 38 | "def get_datelist(start: str, end: str, interval: int):\n", 39 | "\n", 40 | " df = pro.index_daily(ts_code='399300.SZ', start_date=start, end_date=end)\n", 41 | " date_list = list(df.iloc[::-1]['trade_date'])\n", 42 | " sample_list = []\n", 43 | " for i in range(len(date_list)):\n", 44 | " if i % interval == 0:\n", 45 | " sample_list.append(date_list[i])\n", 46 | "\n", 47 | " return sample_list\n", 48 | "\n", 49 | "# 返回两个，一个是前30个交易日的9个指标面板（9*30），一个是未来10天的收益率\n", 50 | "\n", 51 | "\n", 52 | "def get_x_y(code: str, date: str, pass_day: int, future_day: int, len1: int, len2: int):\n", 53 | "\n", 54 | " start = str(pd.to_datetime(date)-timedelta(pass_day*2))\n", 55 | " start = start[0:4]+start[5:7]+start[8:10]\n", 56 | " end = str(pd.to_datetime(date)+timedelta(future_day*2))\n", 57 | " end = end[0:4]+end[5:7]+end[8:10]\n", 58 | " df_price = pro.daily(ts_code=code, # OHLC,pct_change,volume\n", 59 | " start_date=start, end_date=date)\n", 60 | " df_basic = pro.daily_basic(ts_code=code,\n", 61 | " start_date=start, end_date=date)\n", 62 | " df_return = pro.daily(ts_code=code,\n", 63 | " start_date=date, end_date=end).iloc[::-1]['close']\n", 64 | " if (df_price.shape[0] == df_basic.shape[0]) & (df_price.shape[0] == len1) & (df_return.shape[0] == len2): # 判断数据的完整性\n", 65 | " df_price = df_price.iloc[0:pass_day, [2, 3, 4, 5, 8, 9]].fillna(0.1)\n", 66 | " df_basic = df_basic.iloc[0:pass_day, [3, 4, 5]].fillna(0.1)\n", 67 | " data = np.array(pd.merge(df_price, df_basic,\n", 68 | " left_index=True, right_index=True).iloc[::-1].T)\n", 69 | " # print(data.shape)\n", 70 | " # 未来十个交易日的收益率\n", 71 | " dfr = df_return.iloc[0:future_day]\n", 72 | " ret = dfr.iloc[-1]/dfr.iloc[0]-1 # 后十个交易日的收益率\n", 73 | " return data, ret\n", 74 | " else:\n", 75 | " return None, None # 数据缺失的预处理\n", 76 | "\n", 77 | "\n", 78 | "def get_length(date: str, pass_day: int, future_day: int):\n", 79 | " start = str(pd.to_datetime(date)-timedelta(pass_day*2))\n", 80 | " start = start[0:4]+start[5:7]+start[8:10]\n", 81 | " end = str(pd.to_datetime(date)+timedelta(future_day*2))\n", 82 | " end = end[0:4]+end[5:7]+end[8:10]\n", 83 | " len_1 = pro.index_daily(ts_code='399300.SZ',\n", 84 | " start_date=start, end_date=date).shape[0]\n", 85 | " len_2 = pro.index_daily(ts_code='399300.SZ',\n", 86 | " start_date=date, end_date=end).shape[0]\n", 87 | " return len_1, len_2\n", 88 | "\n", 89 | "# 构造数据集的函数：输入一个时间区间的端点，得到该区间内采样交易日期的所有数据\n", 90 | "\n", 91 | "\n", 92 | "def get_dataset(num: int, start: str, end: str, interval: int, pass_day: int, future_day: int):\n", 93 | " X_train = []\n", 94 | " y_train = []\n", 95 | " trade_date_list = get_datelist(start, end, interval)\n", 96 | " # 添加进度条\n", 97 | " with Progress() as progress:\n", 98 | " task_date = progress.add_task(\n", 99 | " \"[red]Date...\", total=len(trade_date_list))\n", 100 | " for date in trade_date_list:\n", 101 | " # 更新进度条\n", 102 | " progress.update(task_date, advance=1)\n", 103 | " stock_list = get_stocklist(date, num)\n", 104 | " len1, len2 = get_length(date, pass_day, future_day)\n", 105 | " task_stock = progress.add_task(\n", 106 | " \"[green]Stock...\", total=len(range(len(stock_list))))\n", 107 | " for i in range(len(stock_list)):\n", 108 | " # 更新进度条\n", 109 | " progress.update(task_stock, advance=1)\n", 110 | " code = stock_list[i]\n", 111 | " x, y = get_x_y(code, date, pass_day, future_day, len1, len2)\n", 112 | " try:\n", 113 | " if (x.shape[0] == 9) & (x.shape[1] == pass_day):\n", 114 | " X_train.append(x)\n", 115 | " y_train.append(y)\n", 116 | " except Exception:\n", 117 | " continue\n", 118 | " return X_train, y_train" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "# 参数设定：使用过去30天的数据预测未来10天的收益率，回归问题\n", 128 | "X_train, y_train = get_dataset(\n", 129 | " num=1000, start='20220101', end='20220630', interval=10, pass_day=30, future_day=10)\n", 130 | "X_test, y_test = get_dataset(num=1000, start='20220931',\n", 131 | " end='20221231', interval=10, pass_day=30, future_day=10)\n", 132 | "print(\"there are in total\", len(X_train), \"training samples\")\n", 133 | "print(\"there are in total\", len(X_test), \"testing samples\")\n", 134 | "# 将数据保存到本地供离线训练\n", 135 | "Xa = np.array(X_train)\n", 136 | "ya = np.array(y_train)\n", 137 | "Xe = np.array(X_test)\n", 138 | "ye = np.array(y_test)\n", 139 | "np.save('./X_train.npy', Xa)\n", 140 | "np.save('./y_train.npy', ya)\n", 141 | "np.save('./X_test.npy', Xe)\n", 142 | "np.save('./y_test.npy', ye)\n" 143 | ] 144 | } 145 | ], 146 | "metadata": { 147 | "kernelspec": { 148 | "display_name": "Python 3.8.0 64-bit", 149 | "language": "python", 150 | "name": "python3" 151 | }, 152 | "language_info": { 153 | "codemirror_mode": { 154 | "name": "ipython", 155 | "version": 3 156 | }, 157 | "file_extension": ".py", 158 | "mimetype": "text/x-python", 159 | "name": "python", 160 | "nbconvert_exporter": "python", 161 | "pygments_lexer": "ipython3", 162 | "version": "3.8.0" 163 | }, 164 | "orig_nbformat": 4, 165 | "vscode": { 166 | "interpreter": { 167 | "hash": "767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90" 168 | } 169 | } 170 | }, 171 | "nbformat": 4, 172 | "nbformat_minor": 2 173 | } 174 | -------------------------------------------------------------------------------- /data/get_data.py: -------------------------------------------------------------------------------- 1 | from rich.progress import Progress 2 | from datetime import timedelta 3 | import pandas as pd 4 | import numpy as np 5 | import tushare as ts 6 | pro = ts.pro_api() 7 | 8 | 9 | # 给定一个交易日，返回该日满足条件的A股股票列表 10 | def get_stocklist(date: str, num: int): 11 | 12 | start = str(pd.to_datetime(date)-timedelta(30)) 13 | start = start[0:4]+start[5:7]+start[8:10] 14 | df1 = pro.index_weight(index_code='000002.SH', 15 | start_date=start, end_date=date) # 交易日当天的股票列表 16 | codes = list(df1['con_code']) 17 | codes = codes[0:1000] # 在每个截面期只选取1000只股票 18 | 19 | return codes 20 | 21 | # 给定日期区间的端点，输出期间的定长采样交易日列表 22 | 23 | 24 | def get_datelist(start: str, end: str, interval: int): 25 | 26 | df = pro.index_daily(ts_code='399300.SZ', start_date=start, end_date=end) 27 | date_list = list(df.iloc[::-1]['trade_date']) 28 | sample_list = [] 29 | for i in range(len(date_list)): 30 | if i % interval == 0: 31 | sample_list.append(date_list[i]) 32 | 33 | return sample_list 34 | 35 | # 返回两个，一个是前30个交易日的9个指标面板（9*30），一个是未来10天的收益率 36 | 37 | 38 | def get_x_y(code: str, date: str, pass_day: int, future_day: int, len1: int, len2: int): 39 | 40 | start = str(pd.to_datetime(date)-timedelta(pass_day*2)) 41 | start = start[0:4]+start[5:7]+start[8:10] 42 | end = str(pd.to_datetime(date)+timedelta(future_day*2)) 43 | end = end[0:4]+end[5:7]+end[8:10] 44 | df_price = pro.daily(ts_code=code, # OHLC,pct_change,volume 45 | start_date=start, end_date=date) 46 | df_basic = pro.daily_basic(ts_code=code, 47 | start_date=start, end_date=date) 48 | df_return = pro.daily(ts_code=code, 49 | start_date=date, end_date=end).iloc[::-1]['close'] 50 | if (df_price.shape[0] == df_basic.shape[0]) & (df_price.shape[0] == len1) & (df_return.shape[0] == len2): # 判断数据的完整性 51 | df_price = df_price.iloc[0:pass_day, [2, 3, 4, 5, 8, 9]].fillna(0.1) 52 | df_basic = df_basic.iloc[0:pass_day, [3, 4, 5]].fillna(0.1) 53 | data = np.array(pd.merge(df_price, df_basic, 54 | left_index=True, right_index=True).iloc[::-1].T) 55 | # print(data.shape) 56 | # 未来十个交易日的收益率 57 | dfr = df_return.iloc[0:future_day] 58 | ret = dfr.iloc[-1]/dfr.iloc[0]-1 # 后十个交易日的收益率 59 | return data, ret 60 | else: 61 | return None, None # 数据缺失的预处理 62 | 63 | 64 | def get_length(date: str, pass_day: int, future_day: int): 65 | start = str(pd.to_datetime(date)-timedelta(pass_day*2)) 66 | start = start[0:4]+start[5:7]+start[8:10] 67 | end = str(pd.to_datetime(date)+timedelta(future_day*2)) 68 | end = end[0:4]+end[5:7]+end[8:10] 69 | len_1 = pro.index_daily(ts_code='399300.SZ', 70 | start_date=start, end_date=date).shape[0] 71 | len_2 = pro.index_daily(ts_code='399300.SZ', 72 | start_date=date, end_date=end).shape[0] 73 | return len_1, len_2 74 | 75 | # 构造数据集的函数：输入一个时间区间的端点，得到该区间内采样交易日期的所有数据 76 | 77 | 78 | def get_dataset(num: int, start: str, end: str, interval: int, pass_day: int, future_day: int): 79 | X_train = [] 80 | y_train = [] 81 | trade_date_list = get_datelist(start, end, interval) 82 | # 添加进度条 83 | with Progress() as progress: 84 | task_date = progress.add_task( 85 | "[red]Date...", total=len(trade_date_list)) 86 | for date in trade_date_list: 87 | # 更新进度条 88 | progress.update(task_date, advance=1) 89 | stock_list = get_stocklist(date, num) 90 | len1, len2 = get_length(date, pass_day, future_day) 91 | task_stock = progress.add_task( 92 | "[green]Stock...", total=len(range(len(stock_list)))) 93 | for i in range(len(stock_list)): 94 | # 更新进度条 95 | progress.update(task_stock, advance=1) 96 | code = stock_list[i] 97 | x, y = get_x_y(code, date, pass_day, future_day, len1, len2) 98 | try: 99 | if (x.shape[0] == 9) & (x.shape[1] == pass_day): 100 | X_train.append(x) 101 | y_train.append(y) 102 | except Exception: 103 | continue 104 | return X_train, y_train 105 | 106 | 107 | # 参数设定：使用过去30天的数据预测未来10天的收益率，回归问题 108 | X_train, y_train = get_dataset( 109 | num=1000, start='20220101', end='20220630', interval=10, pass_day=30, future_day=10) 110 | X_test, y_test = get_dataset(num=1000, start='20220931', 111 | end='20221231', interval=10, pass_day=30, future_day=10) 112 | print("there are in total", len(X_train), "training samples") 113 | print("there are in total", len(X_test), "testing samples") 114 | # 将数据保存到本地供离线训练 115 | Xa = np.array(X_train) 116 | ya = np.array(y_train) 117 | Xe = np.array(X_test) 118 | ye = np.array(y_test) 119 | np.save('./X_train.npy', Xa) 120 | np.save('./y_train.npy', ya) 121 | np.save('./X_test.npy', Xe) 122 | np.save('./y_test.npy', ye) 123 | -------------------------------------------------------------------------------- /data/y_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/data/y_test.npy -------------------------------------------------------------------------------- /data/y_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/data/y_train.npy -------------------------------------------------------------------------------- /model/alphanet_v1_pool.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/model/alphanet_v1_pool.pth -------------------------------------------------------------------------------- /model/alphanet_v3_gru.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/model/alphanet_v3_gru.pdf -------------------------------------------------------------------------------- /model/alphanet_v3_gru.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/model/alphanet_v3_gru.pth -------------------------------------------------------------------------------- /model/alphanet_v3_gru_classification.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/model/alphanet_v3_gru_classification.pth -------------------------------------------------------------------------------- /model/alphanet_v3_gru_classification_excess_return.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremy-feng/AlphaNet/40df993a1e57a7b65336b23bc0a15602d38131f6/model/alphanet_v3_gru_classification_excess_return.pth -------------------------------------------------------------------------------- /model/baseline_random_forest.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 56, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import warnings\n", 10 | "warnings.filterwarnings('ignore')\n", 11 | "import numpy as np\n", 12 | "from matplotlib import pyplot as plt\n", 13 | "# 导入数据处理和绘图的包\n", 14 | "import pandas as pd\n", 15 | "import matplotlib.pyplot as plt\n", 16 | "# 随机森林相关的包\n", 17 | "from sklearn import preprocessing\n", 18 | "from sklearn.ensemble import RandomForestClassifier\n", 19 | "from sklearn import metrics\n", 20 | "from tqdm import tqdm\n", 21 | "import itertools" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 46, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stdout", 31 | "output_type": "stream", 32 | "text": [ 33 | "训练集特征维数: (11825, 270)\n", 34 | "训练集标签维数: (11825,)\n", 35 | "测试集特征维数: (4943, 270)\n", 36 | "测试集标签维数: (4943,)\n" 37 | ] 38 | } 39 | ], 40 | "source": [ 41 | "# 读取数据\n", 42 | "X_train = np.load('../data/X_train.npy')\n", 43 | "# # 将数据转换为二维\n", 44 | "X_train = X_train.reshape(X_train.shape[0], -1)\n", 45 | "y_train = np.load('../data/y_train.npy')\n", 46 | "X_test = np.load('../data/X_test.npy')\n", 47 | "# 将数据转换为二维\n", 48 | "X_test = X_test.reshape(X_test.shape[0], -1)\n", 49 | "y_test = np.load('../data/y_test.npy')\n", 50 | "# 查看数据的大小\n", 51 | "print(\"训练集特征维数: \", X_train.shape)\n", 52 | "print(\"训练集标签维数: \", y_train.shape)\n", 53 | "print(\"测试集特征维数: \", X_test.shape)\n", 54 | "print(\"测试集标签维数: \", y_test.shape)\n", 55 | "\n", 56 | "# 由于是超额收益的分类问题, 因此将y大于y_train.mean()的标签设置为1，小于y_train.mean()的标签设置为0\n", 57 | "train_mean_y = y_train.mean()\n", 58 | "y_train[y_train > train_mean_y] = 1\n", 59 | "y_train[y_train < train_mean_y] = 0\n", 60 | "test_mean_y = y_test.mean()\n", 61 | "y_test[y_test > test_mean_y] = 1\n", 62 | "y_test[y_test < test_mean_y] = 0" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 47, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# 构建随机森林模型\n", 72 | "clf = RandomForestClassifier(random_state=0, n_jobs=-1)\n", 73 | "clf.fit(X_train, y_train)\n", 74 | "# 在测试集上预测\n", 75 | "y_pred = clf.predict(X_test)\n", 76 | "y_pred_proba = clf.predict_proba(X_test)[:,1]" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 48, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "# 在测试集上给出模型分类的效果\n", 86 | "def evaluate(true, pred):\n", 87 | " print('accuracy:{:.2%}'.format(metrics.accuracy_score(true, pred)))\n", 88 | " print('precision:{:.2%}'.format(metrics.precision_score(true, pred)))\n", 89 | " print('recall:{:.2%}'.format(metrics.recall_score(true, pred)))\n", 90 | " print('f1-score:{:.2%}'.format(metrics.f1_score(true, pred)))" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 49, 96 | "metadata": {}, 97 | "outputs": [ 98 | { 99 | "name": "stdout", 100 | "output_type": "stream", 101 | "text": [ 102 | "accuracy:53.17%\n", 103 | "precision:41.54%\n", 104 | "recall:24.88%\n", 105 | "f1-score:31.12%\n" 106 | ] 107 | } 108 | ], 109 | "source": [ 110 | "evaluate(y_test, y_pred)" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | "为270个特征设定名字" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 50, 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "# 为270个特征设定名字\n", 127 | "feature_names = ['open', 'high', 'low', 'close', 'vwap', 'volume', 'return1', 'turn', 'free_turn']\n", 128 | "time = range(29, -1, -1)\n", 129 | "all_feature_names = []\n", 130 | "for i in feature_names:\n", 131 | " for j in time:\n", 132 | " all_feature_names.append(i + '_' + str(j))" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 51, 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "feat_importances = pd.Series(clf.feature_importances_, index=all_feature_names)" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 52, 147 | "metadata": {}, 148 | "outputs": [ 149 | { 150 | "data": { 151 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAu8AAAF1CAYAAABYlgTpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAAB1R0lEQVR4nOzde3SV1b3v//c3BFDEhoJoSYLb0ghCSAghsbABFS2EYonFjSm0h8jg1GwUN23FbS0IRZRderZsUSHH6kGtUkgsFhIvxfKDglihNNyiXCwoEZJSFItoCJeQfH9/rIc0gYSbufN5jbEGz7w887LWAL9rOtd8zN0REREREZHGL6yhByAiIiIiIudGwbuIiIiISBOh4F1EREREpIlQ8C4iIiIi0kQoeBcRERERaSIUvIuIiIiINBEK3kVEpMGYWTcz22xmX5jZxIYej4hIY6fgXUTkSzCzAjP7VkOPA8DMVpnZDxt6HOfpAeCP7n65uz95amEwp6NmVlzp1e/LdNhE3ycREUDBu4hIk2chTfXf838Btp6lzr3u3rbSa219DKwmZhbekP2LyMWtqf5jLyLS6JjZWDP7k5k9bmafmdmHZvavQf5eM/vYzO6sVP8FM3vazJYH20ZWm9m/VCr/VzP7i5kdCv7810plq8xsppn9CSgBXgIGAnOD1em5Qb0ngr4/N7MNZjawUhvTzexlM3sx6H+rmSVVKu9sZr8zs0/M7NOTbQZl48xsu5kdNLM3K4+7mvclNWj7s2Dc3YP8lcCgSmPueh7vdWsze8zM9pjZ/uB9vDQo+6qZvRaM+2BwHR2UzTz1fTKza8zMKwfllVfnT/lcPwWmn6X/K4I+PzOzf5jZmib85UpEGhn9YyIiUru+CeQDHYCFQBaQDMQA/4tQ0Ni2Uv0fAI8AVwCbgd8AmFl74HXgyaCt/wFeN7MOle4dA2QAlwNjgTX8c5X63qDOX4AEoH0wnt+a2SWV2kgNxtgOyAVOBv0tgNeAj4BrgKigHmZ2GzAZuB3oGPS7qLo3IwjIFwE/Duq+AbxqZq3c/eZTxvzXat/R6s0CugZziwnGNy0oCwOeJ7SqfzVw5OS83H1KDe/T2XwT+BC4Cph5lv4nAYXBfK8i9F75ecxNRKRGCt5FRGrXbnd/3t3LgGygMzDD3Y+5+x+A44SCvZNed/e33P0YMAXoZ2adgVuBne7+krufcPdFwA5geKV7X3D3rUF5aXWDcfcF7v5pUGc20BroVqnK2+7+RjDel4BeQf71QCTwn+5+2N2PuvvbQdl44Bfuvt3dTwD/BSTUsPr+vWCOy4MxPgZcCvxrNXVr8mSwiv2ZmW00MyP0peUn7v4Pd/8iGMOoYM6fuvsr7l4SlM0EbjyP/qrzN3d/Kpjv0TP1D5QCnYB/cfdSd1/j7greRaRWKHgXEald+ytdHwFw91PzKq+87z154e7FwD8IBc2RhFa9K/uI0ArvaffWxMzuD7a3HDKzz4AIQqv8J/290nUJcEmwfaQz8FEQrJ7qX4AnTgbUwZjtlLGdVGUe7l4ejLu6ujWZ6O7tglcioRXtNsCGSmNYFuRjZm3M7Fdm9pGZfQ68BbQL/m/Char8Xp+xf+C/gV3AH4KtUw9+iX5FRKpQ8C4i0rA6n7wIttO0B/4WvE5dyb4aKKqUPnU1t0o62N/+AJAGfNXd2wGHCAXaZ7MXuNqq/3HmXuDfKwXU7dz9Und/p5q6VeYRrJp3PmUe5+sAoS9BsZX6j3D3k1+KJhH6vwvfdPevADec7D7489T37XDwZ5tKeV87pU7le87Yv7t/4e6T3L0LoW1J95nZLRc4VxGRKhS8i4g0rGFmNsDMWhHa+77O3fcS2hve1cy+b2bhZvY9oAehfeg12Q90qZS+HDgBfAKEm9k04CvnOK71wD5glpldZmaXmFn/oOxp4GdmFgtgZhFmdkcN7bwM3Gpmt5hZS0KB9TGgukD/nASr988Cj5vZlcEYoswsJahyOaHg+rPgtwM/P6WJKu+Tu39C6MvE/zKzFmY2DvjGhfZvZt8xs5jgi8ohoAwov9D5iohUpuBdRKRhLSQUXP4D6EPoR624+6fAdwgFu58SWkH/jrsfOENbTwAjgxNWngTeJLSd46+Etq4c5Ry22gT9lxHaXx8D7CH0A8zvBWVLgF8CWcG2lPeAb9fQzvvBnJ4itGI9HBju7sfPZRxn8FNCW1PWBWP4//jnXv45hPbVHwDWEXoPKjv1fQK4C/hPQu91LGf/cnGm/q8N0sXAWiDT3f94AXMUETmN6Tc0IiINw8xeAArd/aGGHouIiDQNWnkXEREREWkiFLyLiIiIiDQR2jYjIiIiItJEaOVdRERERKSJUPAuIiIiItJEVPfwDQlcccUVfs011zT0MERERESkGduwYcMBd+949poK3s/ommuuIS8vr6GHISIiIiLNmJl9dK51tW1GRERERKSJUPAuIiIiIlKNvXv3MmjQIHr06EFsbCxPPPHEaXUOHTrE8OHD6dWrF7GxsTz//PNVyj///HOio6O59957ATh27BhDhw6lZ8+eZGZmVtQzs2fMLPFsY9JRkWfQutO13unOOQ09DBERERGpYwWzbj0tb9++fezbt4/ExES++OIL+vTpw9KlS+nRo0dFnf/6r//i0KFD/PKXv+STTz6hW7du/P3vf6dVq1YA/OhHP+KTTz6hffv2zJ07l9zcXPLz85k8eTL9+/dn7dq1mNk2YJ27/++zjbNZrbyb2R1mttXMys0sqVJ+SzP7tZm9a2bbzexnDTlOEREREWn8OnXqRGJiaDH88ssvp3v37hQVFVWpY2Z88cUXuDvFxcW0b9+e8PDQz0o3bNjA/v37GTJkSEX9li1bUlJSQmlpKZUW0aOAqecypmYVvAPvAbcDb52SfwfQ2t3jgD7Av5vZNfU8NhERERFpogoKCti0aRPf/OY3q+Tfe++9bN++ncjISOLi4njiiScICwujvLycSZMm8dhjj1WpP3jwYAoKCujbty8TJ04kNzcXoMTd/3Yu46j102bMbBaw193nBenpwH8Co90918yWAAfdfZyZjQO+4e5TzGwp0Bm4BHjC3Z8J7i8GngWGAH8HRrn7J9X17e7bg3tOKwIuM7Nw4FLgOPB57c1aRERERJqr4uJi/u3f/o05c+bwla98pUrZm2++SUJCAitXruSDDz5g8ODBDBw4kBdffJFhw4YRHR1dpX54eDgLFy4EoLS0lJSUFIC/m9n/AFcDL7p7bk1jqYuV92wgrVI6DRgPDAzSUcDJjUID+ecq+Th37wMkARPNrEOQfxmQ5+6xwGrg5xcwpsXAYWAfsAd4zN3/UV1FM8swszwzyysrOXQBXYmIiIhIc1FaWsq//du/8YMf/IDbb7/9tPLnn3+e22+/HTMjJiaGr3/96+zYsYO1a9cyd+5crrnmGu6//35efPFFHnzwwSr3ZmZmkp6eDtAWOAR8D5h0pvHUevDu7puAK80s0sx6AQeBlcBAM+sBbAP2m1knoB/wTnDrRDPbAqwjtAJ/bZBfTugLAcACYMAFDOt6oAyIBL4OTDKzLjWM/xl3T3L3pBZtIi6gKxERERFpDtyd//2//zfdu3fnvvvuq7bO1VdfzYoVKwDYv38/77//Pl26dOE3v/kNe/bsoaCggMcee4z09HRmzZpVcd/Bgwd57bXXTgbvYYRiXie0S6RGdfWQpt8CI4GvAdnuXmRm7YChhFba2xNakS929y/M7CbgW0A/dy8xs1WEts9U50KOx/k+sMzdS4GPzexPhFb4P7yAtkRERETkIvCnP/2Jl156ibi4OBISEoDQ6TJ79uwBYPz48UydOpWxY8cSFxeHu/PLX/6SK6644qxtz5gxgylTphAWFgahVfeBwLvA02e6r66C92xC+9SvAG4M8tYBPwZuBjoQ2sqyOCiLILQPvsTMrgP6VmorjNAXgSxCQfjbFzCePUG/L5nZZUH7cy6gHRERERG5SAwYMICzHaseGRnJH/7whzPWGTt2LGPHjq2S9/jjj1dOursP4RzUSfDu7lvN7HKgyN33BdlrgCHuvit4BGz7IA9gGTDezLYD7xMK9E86DFxvZg8BHxPaC1QtMxsBPAV0BF43s83ungLMA543s62AAc+7e/7Z5hEXFUFeNWd+ioiIiIg0hEb/kCYzK3b3tg3Rd1JSkufl5TVE1yIiIiJykTCzDe6edPaaze+cdxERERGRZquu9rzXmupW3c1sHtD/lOwn3P35+hmViIiIiEj9a/TBe3XcfUJDj0FEREREpL5p24yIiIiISBOh4F1EREREpIlQ8C4iIowbN44rr7ySnj17Vluek5NDfHw8CQkJJCUl8fbb/3zkxgMPPEBsbCzdu3dn4sSJuDvHjh1j6NCh9OzZk8zMzIq6GRkZbNy4sc7nIyLSXDXJPe/15d2iQ1zz4OsNPQwRkVpTUMOzK8aOHcu999578jHdp7nllltITU3FzMjPzyctLY0dO3bwzjvv8Kc//Yn8/NCjMwYMGMDq1av5/PPPGTBgAJMnT6Z///7cc889bNmyhbKyMhITE+tsfiIizZ2CdxER4YYbbqCgoKDG8rZt/3nw1+HDhzEzAMyMo0ePcvz4cdyd0tJSrrrqKo4cOUJJSQmlpaUVTyecOnUqTz99xqd+i4jIWVwU22bMLMHM1pnZZjPLM7PrG3pMIiJNzZIlS7juuuu49dZbee655wDo168fgwYNolOnTnTq1ImUlBS6d+/O4MGDKSgooG/fvkycOJHc3FwSExOJjIxs4FmIiDRtF8vK+/8BHnb335vZsCB9U8MOSUSkaRkxYgQjRozgrbfeYurUqfx//9//x65du9i+fTuFhYUADB48mDVr1jBw4EAWLlwIQGlpKSkpKeTk5HDfffexZ88e0tPTSU1NbcjpiIg0SfW28m5ms8xsQqX0dDM7bGapQXqJmT0XXI8zs5nB9VIz22BmW80so9L9xWb2eJC/wsw6nqF7B74SXEcAfzvDODOC1fm8spJDFz5hEZFm6oYbbuDDDz/kwIEDLFmyhL59+9K2bVvatm3Lt7/9bdauXVulfmZmJunp6axbt46IiAiys7OZPXt2A41eRKRpq89tM9lAWqV0GjAeGBiko4AewfVA4K3gepy79wGSgIlm1iHIvwzIc/dYYDXw8zP0/WPgv81sL/AY8LOaKrr7M+6e5O5JLdpEnOvcRESatV27dlXsXd+4cSPHjh2jQ4cOXH311axevZoTJ05QWlrK6tWr6d69e8V9Bw8e5LXXXiM9PZ2SkhLCwsIwM44cOdJQUxERadLqbduMu28ysyvNLBLoCBwEVgITzKwHsA34qpl1AvoBE4NbJ5rZiOC6M3At8ClQTugLAcAC4Hdn6P5u4Cfu/oqZpQHzgW/V3uxERJq20aNHs2rVKg4cOEB0dDQPP/wwpaWlAIwfP55XXnmFF198kZYtW3LppZeSnZ2NmTFy5EhWrlxJXFwcZsbQoUMZPnx4RbszZsxgypQphIWFkZKSwrx584iLi2P8+PENNVURkSbNTq6k1EtnZjOAA8DXgL+7+5NmtgN4BvgMaA+UAmPcPcnMbgIeBYa4e4mZrQKmu/sqMysDWrv7CTPrArzi7r1r6PcQ0M7d3UJHJBxy969UV7ey1p2u9U53zvlykxYRaURqOipSREQajpltcPekc6lb3z9YzQaeBa4Abgzy1hHa1nIz0AFYHLwgtD/9YBC4Xwf0rdRWGDASyAK+D7xNzf4W9Lcq6GfnuQw2LiqCPP2HTkREREQaiXoN3t19q5ldDhS5+74gew2hlfVdZvYRodX3NUHZMmC8mW0H3icU6J90GLjezB4CPga+d4au7wKeMLNw4CiQcYa6IiIiIiKNUr1um6lNZlbs7m3PXvPCJSUleV5eXl12ISIiIiIXufPZNnNRPKRJRERERKQ5aLIPaapu1d3M5gH9T8l+wt2fr59RiYiIiIjUnSYbvFfH3SecvZaIiIiISNOkbTMiIiIiIk2EgncRkYvYuHHjuPLKK+nZs2e15Tk5OcTHx5OQkEBSUhJvv/3PU3kfeOABYmNj6d69OxMnTsTdOXbsGEOHDqVnz55kZmZW1M3IyGDjxo11Ph8RkeauWW2bqW3vFh3imgdfb+hhiIjUiuoe0DR27Fjuvfde0tPTq73nlltuITU1FTMjPz+ftLQ0duzYwTvvvMOf/vQn8vPzARgwYACrV6/m888/Z8CAAUyePJn+/ftzzz33sGXLFsrKykhMTKzT+YmIXAwUvIuIXMRuuOEGCgoKaixv2/afZwMcPnyY0EOqwcw4evQox48fx90pLS3lqquu4siRI5SUlFBaWsrJo4inTp3K008/XafzEBG5WDSrbTNm9t9mtsPM8s1siZm1q1T2MzPbZWbvm1lKAw5TRKRJWbJkCddddx233norzz33HAD9+vVj0KBBdOrUiU6dOpGSkkL37t0ZPHgwBQUF9O3bl4kTJ5Kbm0tiYiKRkZENPAsRkeahWQXvwHKgp7vHA38FfgZgZj2AUUAsMBTINLMWDTZKEZEmZMSIEezYsYOlS5cydepUAHbt2sX27dspLCykqKiIlStXsmbNGsLDw1m4cCGbNm3ijjvuYM6cOUyaNIn77ruPkSNHkpub28CzERFp2mo9eDezWWY2oVJ6upkdNrPUIL3EzJ4LrseZ2czgeqmZbTCzrWaWUen+YjN7PMhfYWYda+rb3f/g7ieC5DogOri+Dchy92PuvhvYBVxfw/gzzCzPzPLKSg5d+BshItLM3HDDDXz44YccOHCAJUuW0LdvX9q2bUvbtm359re/zdq1a6vUz8zMJD09nXXr1hEREUF2djazZ89uoNGLiDQPdbHyng2kVUqnAeOBgUE6CugRXA8E3gqux7l7HyAJmGhmHYL8y4A8d48FVgM/P8dxjAN+X6nPvZXKCoO807j7M+6e5O5JLdpEnGNXIiLN065duyr2rm/cuJFjx47RoUMHrr76alavXs2JEycoLS1l9erVdO/eveK+gwcP8tprr5Genk5JSQlhYWGYGUeOHGmoqYiINAu1/oNVd99kZleaWSTQETgIrAQmBNtXtgFfNbNOQD9gYnDrRDMbEVx3Bq4FPgXKCX0hAFgA/O5sYzCzKcAJ4De1MysRkeZp9OjRrFq1igMHDhAdHc3DDz9MaWkpAOPHj+eVV17hxRdfpGXLllx66aVkZ2djZowcOZKVK1cSFxeHmTF06FCGDx9e0e6MGTOYMmUKYWFhpKSkMG/ePOLi4hg/fnxDTVVEpFmwkysqtdqo2QzgAPA14O/u/qSZ7QCeAT4D2gOlwBh3TzKzm4BHgSHuXmJmq4Dp7r7KzMqA1u5+wsy6AK+4e+8z9D0W+HfgFncvCfJ+BuDuvwjSbwbtr62pHYDWna71TnfOubA3QUSkkanuqEgREWl4ZrbB3ZPOpW5dHRWZDTwLXAHcGOStA34M3Ax0ABYHL4AI4GAQuF8H9K3UVhgwEsgCvg+8TQ3MbCjwAHDjycA9kAssNLP/ASIJreqvP9sk4qIiyNN/7ERERESkkaiT02bcfStwOVDk7vuC7DVAuLvvAjYSWn1fE5QtA8LNbDswi1Cgf9Jh4Hoze49Q4D/jDF3PDfpdbmabzezpSuN5mdCWnWXABHcv+/IzFRERERGpP3WybaY2mVmxu7c9e83al5SU5Hl5eQ3RtYiIiIhcJM5n20xzO+ddRERERKTZqqs977WmulV3M5sH9D8l+wl3f75+RiUiIiIiUv8affBeHXefcPZaIiIiIiLNi7bNiIiIiIg0EQreRURERESaCAXvIiIXkXHjxnHllVfSs2fPastzcnKIj48nISGBpKQk3n77n4/W2LNnD0OGDKF79+706NGDgoICAH7wgx8QHx/P5MmTK+o++uijLF26tC6nIiJyUWqSe97ry7tFh7jmwdcbehgiIuetpqepjh07lnvvvZf09PRqy2+55RZSU1MxM/Lz80lLS2PHjh0ApKenM2XKFAYPHkxxcTFhYWHk5+dz6aWXkp+fz+DBgzl06BAlJSX8+c9/5qGHHqqz+YmIXKya1cq7md1hZlvNrNzMkirl/yB4aNPJV7mZJTTgUEVEGsQNN9xA+/btayxv27YtZgbA4cOHK663bdvGiRMnGDx4cEW9Nm3a0LJlS44cOUJ5eTmlpaW0aNGCadOm8fDDD9f9ZERELkLNKngH3gNuB96qnOnuv3H3BHdPAMYAu919c/0PT0Sk8VuyZAnXXXcdt956K8899xwAf/3rX2nXrh233347vXv35j//8z8pKyuje/fudOzYkcTERIYPH86uXbsoLy8nMTGxgWchItI81XrwbmazzGxCpfR0MztsZqlBeomZPRdcjzOzmcH1UjPbEKycZ1S6v9jMHg/yV5hZx5r6dvft7v7+WYY4Gsj6MnMUEWnORowYwY4dO1i6dClTp04F4MSJE6xZs4bHHnuMv/zlL3z44Ye88MILAMyZM4fNmzczadIkpk6dyiOPPMLMmTNJS0vj2WefbcCZiIg0P3Wx8p4NpFVKpwHjgYFBOgroEVwP5J+r5OPcvQ+QBEw0sw5B/mVAnrvHAquBn3/J8X0PWFRToZllmFmemeWVlRz6kl2JiDRdN9xwAx9++CEHDhwgOjqahIQEunTpQnh4ON/97nfZuHFjlfo5OTn06dOH4uJiPvjgA15++WUWL15MSUlJA81ARKT5qfXg3d03AVeaWaSZ9QIOAiuBgWbWA9gG7DezTkA/4J3g1olmtgVYB3QGrg3yywl9IQBYAAy40LGZ2TeBEnd/7wzjf8bdk9w9qUWbiAvtSkSkSdq1axfuDsDGjRs5duwYHTp0IDk5mc8++4xPPvkEgJUrV9KjR4+K+0pLS5kzZw4PPPAAR44cqdgrX1ZWxvHjx+t/IiIizVRdnTbzW2Ak8DUg292LzKwdMJTQSnt7Qivyxe7+hZndBHwL6OfuJWa2Crikhrb9S4xrFGdYdRcRae5Gjx7NqlWrKlbTH374YUpLSwEYP348r7zyCi+++CItW7bk0ksvJTs7GzOjRYsWPPbYY9xyyy24O3369OGuu+6qaHfevHnceeedtGnThvj4eEpKSoiLi2PYsGG0a9eugWYrItL82MkVllpt1CwWeBa4ArjR3feZ2QvAzcGrA7AYWOzuPzGz24AfuvtwM7sO2AwMdfdVZubAaHfPMrOHgKvc/T/O0v8q4H53z6uUFwbsBQa6+4fnMo/Wna71TnfOOZ+pi4g0CjUdFSkiIo2PmW1w96Sz16yjlXd332pmlwNF7r4vyF4DDHH3XWb2EaHV9zVB2TJgvJltB94ntHXmpMPA9UHg/jGhPevVMrMRwFNAR+B1M9vs7ilB8Q3A3nMN3AHioiLI038ARURERKSRqJOV99pkZsXu3rYh+k5KSvK8vLyzVxQRERERuUDns/Le3M55FxERERFpturqB6u1prpVdzObB/Q/JfsJd3++fkYlIiIiIlL/Gn3wXh13n3D2WiIiIiIizYu2zYiIiIiINBEK3kVEREREmggF7yIizdy4ceO48sor6dmzZ7XlOTk5xMfHk5CQQFJSEm+//XaV8s8//5zo6GjuvfdeAI4dO8bQoUPp2bMnmZmZFfUyMjLYuHFj3U1EREQa/1GRDUkPaRKRpqSmBzO99dZbtG3blvT0dN57773TyouLi7nsssswM/Lz80lLS2PHjh0V5T/60Y/45JNPaN++PXPnziU3N5f8/HwmT55M//79Wbt2LVu2bOHJJ59k/vz5dTY/EZHm6qI9KtLM7jCzrWZWbmZJp5TFm9naoPxdM7ukocYpIlKfbrjhBtq3b19jedu2bTEzAA4fPlxxDbBhwwb279/PkCFDKvJatmxJSUkJpaWlnFwAmjp1Ko888kgdzUBERE5qVsE78B5wO/BW5UwzCwcWAOPdPRa4CSit99GJiDRSS5Ys4brrruPWW2/lueeeA6C8vJxJkybx2GOPVak7ePBgCgoK6Nu3LxMnTiQ3N5fExEQiIyMbYugiIheVWg/ezWyWmU2olJ5uZofNLDVILzGz54LrcWY2M7heamYbgpXxjEr3F5vZ40H+CjPrWFPf7r7d3d+vpmgIkO/uW4J6n7p7We3MWESk6RsxYgQ7duxg6dKlTJ06FYDMzEyGDRtGdHR0lbrh4eEsXLiQTZs2cccddzBnzhwmTZrEfffdx8iRI8nNzW2IKYiIXBTqYuU9G0irlE4DxgMDg3QU0CO4Hsg/V8nHuXsfIAmYaGYdgvzLgLxgxXw18PMLGFNXwM3sTTPbaGYP1FTRzDLMLM/M8spKDl1AVyIiTdcNN9zAhx9+yIEDB1i7di1z587lmmuu4f777+fFF1/kwQcfrFI/MzOT9PR01q1bR0REBNnZ2cyePbuBRi8i0vzV+kOa3H2TmV1pZpFAR+AgsBKYYGY9gG3AV82sE9APmBjcOtHMRgTXnYFrgU+BckJfCCC09eV3FzCscGAAkAyUACuCHwasqGb8zwDPQOgHqxfQl4hIk7Jr1y6+8Y1vYGZs3LiRY8eO0aFDB37zm99U1HnhhRfIy8tj1qxZFXkHDx7ktdde48033+TVV18lLCwMM+PIkSMNMQ0RkYtCXT1h9bfASOBrQLa7F5lZO2AooZX29oRW5Ivd/Qszuwn4FtDP3UvMbBVQ0w9KLySgLgTecvcDAGb2BpAInBa8i4g0N6NHj2bVqlUcOHCA6OhoHn74YUpLQz/7GT9+PK+88govvvgiLVu25NJLLyU7O7vKj1ZrMmPGDKZMmUJYWBgpKSnMmzePuLg4xo8fX9dTEhG5aNXJUZFmFgs8C1wB3Oju+8zsBeDm4NUBWAwsdvefmNltwA/dfbiZXQdsBoa6+yozc2C0u2eZ2UPAVe7+H2fpfxVwv7vnBemvEgrUBwDHgWXA4+7++pna0VGRItKU1HRUpIiING7nc1Rknay8u/tWM7scKHL3fUH2GmCIu+8ys48Irb6vCcqWAePNbDvwPrCuUnOHgeuDwP1j4Hs19Rtsu3mK0Had181ss7unuPtBM/sf4C+EVu7fOFvgDhAXFUGe/mMoIiIiIo1Eo39Ik5kVu3vbhug7KSnJ8/LyGqJrEREREblIXLQPaRIRERERac7q6gertaa6VXczmwf0PyX7CXd/vn5GJSIiIiJS/xp98F4dd59w9loiIiIiIs2Lts2IiIiIiDQRCt5FRERERJoIBe8iIs3QuHHjuPLKK+nZs2e15Tk5OcTHx5OQkEBSUhJvv/02AB999BGJiYkkJCQQGxvL008/DcCxY8cYOnQoPXv2JDMzs6KdjIwMNm7cWPcTEhERoAkcFdmQ9JAmEWnsanow01tvvUXbtm1JT0/nvffeO628uLiYyy67DDMjPz+ftLQ0duzYwfHjx3F3WrduTXFxMT179uSdd94hLy+P/Px8Jk+eTP/+/Vm7di1btmzhySefZP78+XU9TRGRZq3BH9IkIiIN64YbbqCgoKDG8rZt/3mQ1+HDhzEzAFq1alWRf+zYMcrLywFo2bIlJSUllJaWcnLRZ+rUqRUr8yIiUj8uim0zZtbezJab2c7gz6829JhERBrakiVLuO6667j11lt57rnnKvL37t1LfHw8nTt35qc//SmRkZEMHjyYgoIC+vbty8SJE8nNzSUxMZHIyMgGnIGIyMXnogjegQeBFe5+LbAiSIuIXNRGjBjBjh07WLp0KVOnTq3I79y5M/n5+ezatYtf//rX7N+/n/DwcBYuXMimTZu44447mDNnDpMmTeK+++5j5MiR5ObmNuBMREQuHvUWvJvZLDObUCk93cwOm1lqkF5iZs8F1+PMbGZwvdTMNpjZVjPLqHR/sZk9HuSvMLOOZ+j+NuDXwfWvge+eYZwZZpZnZnllJYcueL4iIk3FDTfcwIcffsiBAweq5EdGRtKzZ0/WrFlTJT8zM5P09HTWrVtHREQE2dnZzJ49uz6HLCJy0arPlfdsIK1SOg0YDwwM0lFAj+B6IPBWcD3O3fsAScBEM+sQ5F8G5Ll7LLAa+PkZ+r7K3fcF138Hrqqpors/4+5J7p7Uok3Euc1MRKSJ2bVrV8Xe9Y0bN3Ls2DE6dOhAYWEhR44cAeDgwYO8/fbbdOvWreK+gwcP8tprr5Genk5JSQlhYWGYWcU9IiJSt+rtB6vuvsnMrjSzSKAjcBBYCUwwsx7ANuCrZtYJ6AdMDG6daGYjguvOwLXAp0A5oS8EAAuA353jONzMdMSOiDRro0ePZtWqVRw4cIDo6GgefvhhSktLARg/fjyvvPIKL774Ii1btuTSSy8lOzsbM2P79u1MmjQJM8Pduf/++4mLi6tod8aMGUyZMoWwsDBSUlKYN28ecXFxjB8/vqGmKiJyUanXoyLNbAZwAPga8Hd3f9LMdgDPAJ8B7YFSYIy7J5nZTcCjwBB3LzGzVcB0d19lZmVAa3c/YWZdgFfcvXcN/b4P3OTu+4IvB6vcvVt1dSvTUZEi0tjVdFSkiIg0HY35qMhs4FngCuDGIG8d8GPgZqADsDh4AUQAB4PA/Tqgb6W2woCRQBbwfeDtM/SbC9wJzAr+zDmXwcZFRZCn/zCKiIiISCNRr6fNuPtW4HKgqNIe9DVAuLvvAjYSWn0/+euoZUC4mW0nFHivq9TcYeB6M3uPUOA/4wxdzwIGm9lO4FtBWkRERESkSWmyT1g1s2J3b3v2mhcuKSnJ8/Ly6rILEREREbnInc+2mYvlnHcRERERkSavvve815rqVt3NbB7Q/5TsJ9z9+foZlYiIiIhI3WmywXt13H3C2WuJiIiIiDRN2jYjIiIiItJEKHgXEREREWkiFLyLSKOzbNkyunXrRkxMDLNmnX6y60cffcQtt9xCfHw8N910E4WFhQD88Y9/JCEhoeJ1ySWXsHTpUgB+8IMfEB8fz+TJkyvaefTRRyvKRUREmoIme1RkfdATVkXqVnVPBy0rK6Nr164sX76c6OhokpOTWbRoET169Kioc8cdd/Cd73yHO++8k5UrV/L888/z0ksvVWnnH//4BzExMRQWFrJr1y6efPJJ/t//+38MHjyYxYsXU1JSQkZGBq+++mqdz1NERORMdFTkKczsv81sh5nlm9kSM2vX0GMSkeqtX7+emJgYunTpQqtWrRg1ahQ5OVUfirxt2zZuvvlmAAYNGnRaOcDixYv59re/TZs2bWjZsiVHjhyhvLyc0tJSWrRowbRp03j44YfrZU4iIiK15aII3oHlQE93jwf+CvysgccjIjUoKiqic+fOFeno6GiKioqq1OnVqxe/+93vAFiyZAlffPEFn376aZU6WVlZjB49GoDu3bvTsWNHEhMTGT58OLt27aK8vJzExMQ6no2IiEjtqrfg3cxmmdmESunpZnbYzFKD9BIzey64HmdmM4PrpWa2wcy2mllGpfuLzezxIH+FmXWsqW93/4O7nwiS64DoupijiNSPxx57jNWrV9O7d29Wr15NVFQULVq0qCjft28f7777LikpKRV5c+bMYfPmzUyaNImpU6fyyCOPMHPmTNLS0nj22WcbYhoiIiLnrT5X3rOBtErpNGA8MDBIRwEnN7UOBN4Krse5ex8gCZhoZh2C/MuAPHePBVYDPz/HcYwDfl9ToZllmFmemeWVlRw6xyZFpLZERUWxd+/einRhYSFRUVFV6kRGRvK73/2OTZs2MXPmTADatWtXUf7yyy8zYsQIWrZseVr7OTk59OnTh+LiYj744ANefvnlij3wIiIijV29Be/uvgm40swizawXcBBYCQw0sx7ANmC/mXUC+gHvBLdONLMthFbMOwPXBvnlhL4QACwABpxtDGY2BTgB/OYM43zG3ZPcPalFm4jznaaIfEnJycns3LmT3bt3c/z4cbKyskhNTa1S58CBA5SXlwPwi1/8gnHjxlUpX7RoUcWWmcpKS0uZM2cODzzwAEeOHMHMgNCPZI8fP15HMxIREak99b3n/bfASOB7QLa7FwHtgKGEVtrXEFqRL3b3L8zsJuBbQD937wVsAi6poe0zHptjZmOB7wA/cB2xI9JohYeHM3fuXFJSUujevTtpaWnExsYybdo0cnNzAVi1ahXdunWja9eu7N+/nylTplTcX1BQwN69e7nxxhtPa3vevHnceeedtGnThvj4eEpKSoiLi6NPnz5VVu5FREQaq3o9KtLMYoFngSuAG919n5m9ANwcvDoAi4HF7v4TM7sN+KG7Dzez64DNwFB3X2VmDox29ywzewi4yt3/o4Z+hwL/E/T5ybmOV0dFitSt6o6KFBERudicz1GR4XU9mMrcfauZXQ4Uufu+IHsNMMTdd5nZR0D7IA9gGTDezLYD7xPaOnPSYeD6IHD/mNBqfk3mAq2B5cH/Jl/n7uPPNt64qAjyFFyIiIiISCPRZB/SZGbF7t62LvtISkryvLy8uuxCRERERC5yekiTiIiIiEgzVK/bZmpTdavuZjYP6H9K9hPu/nz9jEpEREREpO402eC9Ou4+4ey1RERERESaJm2bERERERFpIhS8i4iIiIg0EQreRaRBLVu2jG7duhETE8OsWbNOK//oo4+45ZZbiI+P56abbqKwsLBK+eeff050dDT33nsvAMeOHWPo0KH07NmTzMzMinoZGRls3LixbicjIiJSx5rVnvfa9m7RIa558PWGHoZIs1DdA5nKysqYMGECy5cvJzo6muTkZFJTU+nRo0dFnfvvv5/09HTuvPNOVq5cyc9+9jNeeumlivKpU6dyww03VKTffPNNBgwYwOTJk+nfvz/33HMPW7ZsoaysjMTExLqdpIiISB27KFbezWy6mRWZ2ebgNayhxyQisH79emJiYujSpQutWrVi1KhR5OTkVKmzbds2br75ZgAGDRpUpXzDhg3s37+fIUOGVOS1bNmSkpISSktLOfkci6lTp/LII4/Uw4xERETq1kURvAced/eE4PVGQw9GRKCoqIjOnTtXpKOjoykqKqpSp1evXvzud78DYMmSJXzxxRd8+umnlJeXM2nSJB577LEq9QcPHkxBQQF9+/Zl4sSJ5ObmkpiYSGRkZN1PSEREpI7VW/BuZrPMbEKl9HQzO2xmqUF6iZk9F1yPM7OZwfVSM9tgZlvNLKPS/cVm9niQv8LMOtbXXESk/jz22GOsXr2a3r17s3r1aqKiomjRogWZmZkMGzaM6OjoKvXDw8NZuHAhmzZt4o477mDOnDlMmjSJ++67j5EjR5Kbm9tAMxEREfny6nPPezYwB5gXpNOA8cBAIBeIAjoFZQOBrOB6nLv/w8wuBf5iZq+4+6fAZUCeu//EzKYBPwfuPUP/95pZOpAHTHL3g9VVCr4gZAC0+Iq+D4jUpaioKPbu3VuRLiwsJCoqqkqdyMjIipX34uJiXnnlFdq1a8fatWtZs2YNmZmZFBcXc/z4cdq2bVvlR6+ZmZmkp6ezbt06IiIiyM7O5uabbyY1NbV+JigiIlLL6m3l3d03AVeaWaSZ9QIOAiuBgWbWA9gG7DezTkA/4J3g1olmtgVYB3QGrg3yywl9IQBYAAw4Q/f/F/gGkADsA2afYZzPuHuSuye1aBNx/hMVkXOWnJzMzp072b17N8ePHycrK+u0wPrAgQOUl5cD8Itf/IJx48YB8Jvf/IY9e/ZQUFDAY489Rnp6epXA/eDBg7z22mukp6dTUlJCWFgYZsaRI0fqb4IiIiK1rL73vP8WGAl8D8h29yKgHTAUeAtYQ2hFvtjdvzCzm4BvAf3cvRewCbikhra9pk7dfb+7l7l7OfAscH2tzEZEvpTw8HDmzp1LSkoK3bt3Jy0tjdjYWKZNm1axvWXVqlV069aNrl27sn//fqZMmXJObc+YMYMpU6YQFhZGSkoKa9asIS4ujjFjxtTllEREROqUnTyNoV46M4slFDxfAdzo7vvM7AXg5uDVAVgMLA62w9wG/NDdh5vZdcBmYKi7rzIzB0a7e5aZPQRc5e7/UUO/ndx9X3D9E+Cb7j7qbONt3ela73TnnC85axGB6o+KFBERETCzDe6edC516/Wcd3ffamaXA0Ung2lCq+1D3H2XmX0EtA/yAJYB481sO/A+oa0zJx0Grg8C948JrebX5P+YWQKh1fkC4N/PZbxxURHkKeAQERERkUaiXlfea5OZFbt727rsIykpyfPy8uqyCxERERG5yJ3PyvvFdM67iIiIiEiTVq/bZmpTdavuZjYP6H9K9hPu/nz9jEpEREREpO402eC9Ou4+4ey1RERERESaJm2bERERERFpIhS8i4iIiIg0EQreRaTeLVu2jG7duhETE1PlqagnffTRR9xyyy3Ex8dz0003UVhYWJGfmJhIQkICsbGxPP300wAcO3aMoUOH0rNnTzIzMyvaycjIYOPGjfUzKRERkXrQZI+KrA96SJPIl1Pdg5nKysro2rUry5cvJzo6muTkZBYtWkSPHj0q6txxxx185zvf4c4772TlypU8//zzvPTSSxw/fhx3p3Xr1hQXF9OzZ0/eeecd8vLyyM/PZ/LkyfTv35+1a9eyZcsWnnzySebPn1+fUxYRETlvOipSRBqt9evXExMTQ5cuXWjVqhWjRo0iJyenSp1t27Zx8803AzBo0KCK8latWtG6dWsgtNpeXl4OQMuWLSkpKaG0tJSTCxJTp07lkUceqa9piYiI1ItmFbyb2X+b2Q4zyzezJWbW7pTyq82s2Mzub6Ahilz0ioqK6Ny5c0U6OjqaoqKiKnV69erF7373OwCWLFnCF198waeffgrA3r17iY+Pp3Pnzvz0pz8lMjKSwYMHU1BQQN++fZk4cSK5ubkkJiYSGRlZfxMTERGpB80qeAeWAz3dPR74K/CzU8r/B/h9vY9KRM7LY489xurVq+nduzerV68mKiqKFi1aANC5c2fy8/PZtWsXv/71r9m/fz/h4eEsXLiQTZs2cccddzBnzhwmTZrEfffdx8iRI8nNzW3gGYmIiNSOWg/ezWyWmU2olJ5uZofNLDVILzGz54LrcWY2M7heamYbzGyrmWVUur/YzB4P8leYWcea+nb3P7j7iSC5Doiu1M53gd3A1rOMP8PM8swsr6zk0HnPX0TOLCoqir1791akCwsLiYqKqlInMjKS3/3ud2zatImZM2cC0K5du9Pq9OzZkzVr1lTJz8zMJD09nXXr1hEREUF2djazZ8+um8mIiIjUs7pYec8G0iql04DxwMAgHQWc/GXaQOCt4Hqcu/cBkoCJZtYhyL8MyHP3WGA18PNzHMc4glV2M2sL/BR4+Gw3ufsz7p7k7kkt2kScY1cicq6Sk5PZuXMnu3fv5vjx42RlZZGamlqlzoEDByr2s//iF79g3LhxQCjQP3LkCAAHDx7k7bffplu3bhX3HTx4kNdee4309HRKSkoICwvDzCruERERaepqPXh3903AlWYWaWa9gIPASmCgmfUAtgH7zawT0A94J7h1opltIbRi3hm4NsgvJ/SFAGABMOBsYzCzKcAJ4DdB1nTgcXcv/pLTE5EvKTw8nLlz55KSkkL37t1JS0sjNjaWadOmVWxvWbVqFd26daNr167s37+fKVOmALB9+3a++c1v0qtXL2688Ubuv/9+4uLiKtqeMWMGU6ZMISwsjJSUFNasWUNcXBxjxoxpkLmKiIjUtjo5KtLMZgAHgK8Bf3f3J81sB/AM8BnQHigFxrh7kpndBDwKDHH3EjNbBUx391VmVga0dvcTZtYFeMXde5+h77HAvwO3uHtJkLeG0BcCgHaEvhBMc/e5Z5qHjooU+XKqOypSREREqjqfoyLD62gM2cCzwBXAjUHeOuDHwM1AB2Bx8AKIAA4Ggft1QN9KbYUBI4Es4PvA2zV1amZDgQeAG08G7gDuPrBSnelA8dkCd4C4qAjyFHyIiIiISCNRJ6fNuPtW4HKgyN33BdlrgHB33wVsJLT6fvKXZsuAcDPbDswiFOifdBi43szeIxT4zzhD13ODfpeb2WYze7q25iQiIiIi0tAa/RNWzazY3ds2RN9JSUmel5fXEF2LiIiIyEVCT1gVEREREWmG6mrPe62pbtXdzOYB/U/JfsLdn6+fUYmIiIiI1L9GH7xXx90nnL2WiIiIiEjzom0zIiIiIiJNhIJ3EREREZEmoklum6kv7xYd4poHX2/oYYg0OjU9fGnZsmX86Ec/oqysjB/+8Ic8+OCDVcr37NnDnXfeyWeffUZZWRmzZs1i2LBhHD9+nH//938nLy+PsLAwnnjiCW666SaOHTvGbbfdRmFhIffccw/33HMPABkZGYwfP57ExMQ6n6uIiEhjopV3EakVZWVlTJgwgd///vds27aNRYsWsW3btip1Hn30UdLS0ti0aRNZWVkVwfizzz4LwLvvvsvy5cuZNGkS5eXlvPnmmwwYMID8/HxeeuklALZs2UJZWZkCdxERuSg1q+DdzO4ws61mVm5mSaeU/czMdpnZ+2aW0lBjFGmu1q9fT0xMDF26dKFVq1aMGjWKnJycKnXMjM8//xyAQ4cOERkZCcC2bdu4+eabAbjyyitp164deXl5tGzZkpKSEkpLSzn5TIqpU6fyyCOP1OPMREREGo9mFbwD7wG3A29VzjSzHsAoIBYYCmSaWYv6H55I81VUVETnzp0r0tHR0RQVFVWpM336dBYsWEB0dDTDhg3jqaeeAqBXr17k5uZy4sQJdu/ezYYNG9i7dy+DBw+moKCAvn37MnHiRHJzc0lMTKwI+kVERC42tR68m9ksM5tQKT3dzA6bWWqQXmJmzwXX48xsZnC91Mw2BCvnGZXuLzazx4P8FWbWsaa+3X27u79fTdFtQJa7H3P33cAu4PrambGInKtFixYxduxYCgsLeeONNxgzZgzl5eWMGzeO6OhokpKS+PGPf8y//uu/0qJFC8LDw1m4cCGbNm3ijjvuYM6cOUyaNIn77ruPkSNHkpub29BTEhERqVd1sfKeDaRVSqcB44GBQToK6BFcD+Sfq+Tj3L0PkARMNLMOQf5lQJ67xwKrgZ9fwJiigL2V0oVB3mnMLMPM8swsr6zk0AV0JXJxioqKYu/ef/41KywsJCqq6l+z+fPnk5YW+uehX79+HD16lAMHDhAeHs7jjz/O5s2bycnJ4bPPPqNr165V7s3MzCQ9PZ1169YRERFBdnY2s2fPrvuJiYiINCK1Hry7+ybgSjOLNLNewEFgJTAw2L6yDdhvZp2AfsA7wa0TzWwLsA7oDFwb5JcT+kIAsAAYUNtjPmX8z7h7krsntWgTUZddiTQrycnJ7Ny5k927d3P8+HGysrJITU2tUufqq69mxYoVAGzfvp2jR4/SsWNHSkpKOHz4MADLly8nPDycHj16VNx38OBBXnvtNdLT0ykpKSEsLAwz48iRI/U3QRERkUagro6K/C0wEvgakO3uRWbWjtB+87eA9oRW5Ivd/Qszuwn4FtDP3UvMbBVwSQ1t+wWMp4jQF4KTooM8Eakl4eHhzJ07l5SUFMrKyhg3bhyxsbFMmzaNpKQkUlNTmT17NnfddRePP/44ZsYLL7yAmfHxxx+TkpJCWFgYUVFRFSfLnDRjxgymTJlCWFgYKSkpzJs3j7i4OMaPH99AsxUREWkYdvIEh1pt1CwWeBa4ArjR3feZ2QvAzcGrA7AYWOzuPzGz24AfuvtwM7sO2AwMdfdVZubAaHfPMrOHgKvc/T/O0v8q4H53z6s0noWE9rlHAiuAa9297EzttO50rXe6c84FvQcizVlN57yLiIjI+TOzDe6edPaadbTy7u5bzexyoMjd9wXZa4Ah7r7LzD4itPq+JihbBow3s+3A+4S2zpx0GLg+CNw/Br5XU79mNgJ4CugIvG5mm909JRjPy4S27JwAJpwtcAeIi4ogT0GKiIiIiDQSdbLyXpvMrNjd2zZE30lJSZ6Xl9cQXYuIiIjIReJ8Vt6b2znvIiIiIiLNVl39YLXWVLfqbmbzgP6nZD/h7s/Xz6hEREREROpfow/eq+PuE85eS0RERESkedG2GRERERGRJkLBu4iIiIhIE6HgXUS+lGXLltGtWzdiYmKYNWvWaeV79uxh0KBB9O7dm/j4eN544w0AfvOb35CQkFDxCgsLY/PmzRw7doyhQ4fSs2dPMjMzK9rJyMhg48aN9TYvERGRxqjRHxXZkPSQJpF/qu7BTGVlZXTt2pXly5cTHR1NcnIyixYtokePHhV1MjIy6N27N3fffTfbtm1j2LBhFBQUVGnn3Xff5bvf/S4ffPABubm55OfnM3nyZPr378/atWvZsmULTz75JPPnz6/raYqIiNS7i/aoSDO7w8y2mlm5mSVVyr/GzI6Y2ebg9XRDjlOkuVi/fj0xMTF06dKFVq1aMWrUKHJycqrUMTM+//xzAA4dOkRkZORp7SxatIhRo0YB0LJlS0pKSigtLeXk4sLUqVN55JFH6ng2IiIijV+zCt6B94DbgbeqKfvA3ROC1/h6HpdIs1RUVETnzp0r0tHR0RQVFVWpM336dBYsWEB0dDTDhg3jqaeeOq2d7OxsRo8eDcDgwYMpKCigb9++TJw4kdzcXBITE6sN+kVERC42tR68m9ksM5tQKT3dzA6bWWqQXmJmzwXX48xsZnC91Mw2BCvnGZXuLzazx4P8FWbWsaa+3X27u79f23MSkQu3aNEixo4dS2FhIW+88QZjxoyhvLy8ovzPf/4zbdq0oWfPngCEh4ezcOFCNm3axB133MGcOXOYNGkS9913HyNHjiQ3N7ehpiIiItLg6mLlPRtIq5ROA8YDA4N0FHByQ+xA/rlKPs7d+wBJwEQz6xDkXwbkuXsssBr4+QWO6+tmtsnMVpvZwJoqmVmGmeWZWV5ZyaEL7Erk4hAVFcXevXsr0oWFhURFRVWpM3/+fNLSQv8k9OvXj6NHj3LgwIGK8qysrIpV91NlZmaSnp7OunXriIiIIDs7m9mzZ9fBTERERJqGWg/e3X0TcKWZRZpZL+AgsBIYaGY9gG3AfjPrBPQD3glunWhmW4B1QGfg2iC/nNAXAoAFwIALGNY+4Gp37w3cByw0s6/UMP5n3D3J3ZNatIm4gK5ELh7Jycns3LmT3bt3c/z4cbKyskhNTa1S5+qrr2bFihUAbN++naNHj9KxY+h/oJWXl/Pyyy9X7Hev7ODBg7z22mukp6dTUlJCWFgYZsaRI0fqfmIiIiKNVF3tef8tMBL4HpDt7kVAO2AooZX2NYRW5Ivd/Qszuwn4FtDP3XsBm4BLamj7vI/Hcfdj7v5pcL0B+ADoer7tiEhV4eHhzJ07l5SUFLp3705aWhqxsbFMmzatYnvL7NmzefbZZ+nVqxejR4/mhRdewMwAeOutt+jcuTNdunQ5re0ZM2YwZcoUwsLCSElJYc2aNcTFxTFmzJh6naOIiEhjUidHRZpZLPAscAVwo7vvM7MXgJuDVwdgMbDY3X9iZrcBP3T34WZ2HbAZGOruq8zMgdHunmVmDwFXuft/nKX/VcD97p4XpDsC/3D3MjPrQujLQ5y7/+NM7eioSJF/qu6oSBEREfnyzueoyPC6GIC7bzWzy4Eid98XZK8Bhrj7LjP7CGgf5AEsA8ab2XbgfUJbZ046DFwfBO4fE1rNr5aZjQCeAjoCr5vZZndPAW4AZphZKaFtOOPPFrgDxEVFkKeARUREREQaiUb/kCYzK3b3tg3Rd1JSkufl5TVE1yIiIiJykbhoH9IkIiIiItKc1cm2mdpU3aq7mc0D+p+S/YS7P18/oxIRERERqX+NPnivjrtPOHstEREREZHmRdtmRERERESaCAXvIiIiIiJNhIJ3ETkvy5Yto1u3bsTExDBr1qzTyvfs2cOgQYPo3bs38fHxvPHGGxVl+fn59OvXj9jYWOLi4jh69CjHjh1j6NCh9OzZk8zMzIq6GRkZbNy4sV7mJCIi0lQ0+qMiG5Ie0iQXs+oeylRWVkbXrl1Zvnw50dHRJCcns2jRInr06FFRJyMjg969e3P33Xezbds2hg0bRkFBASdOnCAxMZGXXnqJXr168emnn9KuXTtef/118vPzmTx5Mv3792ft2rVs2bKFJ598kvnz59fnlEVERBqEjooUkTqxfv16YmJi6NKlC61atWLUqFHk5ORUqWNmfP755wAcOnSIyMhIAP7whz8QHx9Pr169AOjQoQMtWrSgZcuWlJSUUFpaysnFhKlTp/LII4/U48xERESahosmeDez/zCzHWa21cz+T0OPR6QpKioqonPnzhXp6OhoioqKqtSZPn06CxYsIDo6mmHDhvHUU08B8Ne//hUzIyUlhcTERP7P/wn9NRw8eDAFBQX07duXiRMnkpubS2JiYkXQLyIiIv/UJI+KPF9mNgi4Dejl7sfM7MqGHpNIc7Vo0SLGjh3LpEmTWLt2LWPGjOG9997jxIkTvP322/zlL3+hTZs23HLLLfTp04dbbrmFhQsXAlBaWkpKSgo5OTncd9997Nmzh/T0dFJTUxt4ViIiIo1Dva28m9ksM5tQKT3dzA6bWWqQXmJmzwXX48xsZnC91Mw2BCvmGZXuLzazx4P8FWbW8Qzd3w3McvdjAO7+8RnGmWFmeWaWV1Zy6MtNWqSZiYqKYu/evRXpwsJCoqKiqtSZP38+aWlpAPTr14+jR49y4MABoqOjueGGG7jiiito06YNw4YNO+0HqZmZmaSnp7Nu3ToiIiLIzs5m9uzZdT8xERGRJqI+t81kA2mV0mnAeGBgkI4CTv7qbSDwVnA9zt37AEnARDPrEORfBuS5eyywGvj5GfruCgw0sz+b2WozS66pors/4+5J7p7Uok3EeUxPpPlLTk5m586d7N69m+PHj5OVlXXaqvjVV1/NihUrANi+fTtHjx6lY8eOpKSk8O6771JSUsKJEydYvXp1lR+6Hjx4kNdee4309HRKSkoICwvDzDhy5Ei9zlFERKQxq7fg3d03AVeaWaSZ9QIOAisJBdU9gG3AfjPrBPQD3glunWhmW4B1QGfg2iC/nNAXAoAFwIAzdB8OtAf6Av8JvGxmVmuTE7lIhIeHM3fuXFJSUujevTtpaWnExsYybdo0cnNzAZg9ezbPPvssvXr1YvTo0bzwwguYGV/96le57777SE5OJiEhgcTERG699Z8n2syYMYMpU6YQFhZGSkoKa9asIS4ujjFjxjTUdEVERBqdej0q0sxmAAeArwF/d/cnzWwH8AzwGaEAuxQY4+5JZnYT8CgwxN1LzGwVMN3dV5lZGdDa3U+YWRfgFXfvXUO/y4Bfuvsfg/QHQF93/+RM49VRkXIxq+6oSBEREal953NUZH3/YDUbeBa4ArgxyFsH/Bi4GegALA5eABHAwSBwv47QyvlJYcBIIAv4PvD2GfpdCgwC/mhmXYFWhL5EnFFcVAR5CmBEREREpJGo16Mi3X0rcDlQ5O77guw1QLi77wI2Elp9XxOULQPCzWw7MItQoH/SYeB6M3uPUOA/4wxdPwd0CepmAXe6nk4lIiIiIk1Mk33CqpkVu3vbuuwjKSnJ8/Ly6rILEREREbnI6QmrIiIiIiLNUJN9SFN1q+5mNg/of0r2E+7+fP2MSkRERESk7jTZ4L067j7h7LVERERERJombZsREREREWkiFLyLyDlZtmwZ3bp1IyYmhlmzZp1WvmfPHgYNGkTv3r2Jj4/njTfeAKCgoIBLL72UhIQEEhISGD9+PADHjh1j6NCh9OzZk8zMzIp2MjIy2LhxY/1MSkREpIlpVttmatu7RYe45sHXG3oYIvWquoczlZWVMWHCBJYvX050dDTJycmkpqbSo0ePijqPPvooaWlp3H333Wzbto1hw4ZRUFAAwDe+8Q02b95cpc0333yTAQMGMHnyZPr3788999zDli1bKCsrIzExsS6nKCIi0mRp5V1Ezmr9+vXExMTQpUsXWrVqxahRo8jJyalSx8z4/PPPATh06BCRkZFnbLNly5aUlJRQWlrKySNrp06dyiOPPFI3kxAREWkGmlXwbmb/bWY7zCzfzJaYWbsgv6WZ/drM3jWz7Wb2swYeqkiTUlRUROfOnSvS0dHRFBUVVakzffp0FixYQHR0NMOGDeOpp56qKNu9eze9e/fmxhtvZM2a0DPYBg8eTEFBAX379mXixInk5uaSmJh41qBfRETkYtbcts0sB37m7ifM7JfAz4CfAncArd09zszaANvMbJG7FzTgWEWalUWLFjF27FgmTZrE2rVrGTNmDO+99x6dOnViz549dOjQgQ0bNvDd736XrVu38pWvfIWFCxcCUFpaSkpKCjk5Odx3333s2bOH9PR0UlNTG3hWIiIijUutr7yb2Swzm1ApPd3MDptZapBeYmbPBdfjzGxmcL3UzDaY2VYzy6h0f7GZPR7krzCzjjX17e5/cPcTQXIdEH2yCLjMzMKBS4HjwOc1jD/DzPLMLK+s5NAFvw8izUlUVBR79+6tSBcWFhIVFVWlzvz580lLSwOgX79+HD16lAMHDtC6dWs6dOgAQJ8+ffjGN77BX//61yr3ZmZmkp6ezrp164iIiCA7O5vZs2fX8axERESanrrYNpMNpFVKpwHjgYFBOgo4+Su3gcBbwfU4d+8DJAETzaxDkH8ZkOfuscBq4OfnOI5xwO+D68XAYWAfsAd4zN3/Ud1N7v6Muye5e1KLNhHn2JVI85acnMzOnTvZvXs3x48fJysr67RV8auvvpoVK1YAsH37do4ePUrHjh355JNPKCsrA+DDDz9k586ddOnSpeK+gwcP8tprr5Genk5JSQlhYWGYGUeOHKm/CYqIiDQRtR68u/sm4EozizSzXsBBYCUw0Mx6ANuA/WbWCegHvBPcOtHMthBaMe8MXBvklxP6QgCwABhwtjGY2RTgBPCbIOt6oAyIBL4OTDKzLjXcLiKnCA8PZ+7cuaSkpNC9e3fS0tKIjY1l2rRp5ObmAjB79myeffZZevXqxejRo3nhhRcwM9566y3i4+NJSEhg5MiRPP3007Rv376i7RkzZjBlyhTCwsJISUlhzZo1xMXFMWbMmIaaroiISKNlJ095qNVGzWYAB4CvAX939yfNbAfwDPAZ0B4oBca4e5KZ3QQ8Cgxx9xIzWwVMd/dVZlZGaL/6iSDgfsXde5+h77HAvwO3uHtJkDcPWOfuLwXp54Bl7v7ymebRutO13unOORf4Log0TdUdFSkiIiJ1x8w2uHvSudStq9NmsoFRwEjgt0HeOuDHhLbJrAHuD/4EiAAOBoH7dUDfU8Y4Mrj+PvB2TZ2a2VDgASD1ZOAe2APcHNS5LGh/xwXOTURERESkQdTJaTPuvtXMLgeK3H1fkL2G0Mr6LjP7iNDq+8ngfRkw3sy2A+8TCvRPOgxcb2YPAR8D3ztD13OB1sByM4PQavt4YB7wvJltBQx43t3zzzaPuKgI8rQKKSIiIiKNRJ1sm6lNZlbs7m0bou+kpCTPy8triK5FRERE5CLRGLbNiIiIiIhILWv0D2mqbtU9+AFq/1Oyn3D35+tnVCIiIiIi9a/RB+/VcfcJZ68lIiIiItK8aNuMiIiIiEgToeBdRERERKSJUPAuIqdZtmwZ3bp1IyYmhlmzZp1WvmfPHgYNGkTv3r2Jj4/njTfeAGD58uX06dOHuLg4+vTpw8qVKwE4duwYQ4cOpWfPnmRmZla0k5GRwcaNG+tnUiIiIs1Aoz8qsiHpCavS3FX3NNWysjK6du3K8uXLiY6OJjk5mUWLFtGjR4+KOhkZGfTu3Zu7776bbdu2MWzYMAoKCti0aRNXXXUVkZGRvPfee6SkpFBUVERubi75+flMnjyZ/v37s3btWrZs2cKTTz7J/Pnz63PKIiIijc5Fe1Skmd1hZlvNrNzMkirlDzazDWb2bvDnzQ05TpHGbP369cTExNClSxdatWrFqFGjyMnJqVLHzPj8888BOHToEJGRkQD07t274jo2NpYjR45w7NgxWrZsSUlJCaWlpZxcMJg6dSqPPPJIPc5MRESk6WtWwTvwHnA78NYp+QeA4e4eB9wJvFTfAxNpKoqKiujcuXNFOjo6mqKioip1pk+fzoIFC4iOjmbYsGE89dRTp7XzyiuvkJiYSOvWrRk8eDAFBQX07duXiRMnkpubS2JiYkWgLyIiIuem1oN3M5tlZhMqpaeb2WEzSw3SS8zsueB6nJnNDK6XBqviW80so9L9xWb2eJC/wsw61tS3u2939/eryd/k7n8LkluBS82sde3MWOTis2jRIsaOHUthYSFvvPEGY8aMoby8vKJ869at/PSnP+VXv/oVAOHh4SxcuJBNmzZxxx13MGfOHCZNmsR9993HyJEjyc3NbaipiIiINCl1sfKeDaRVSqcB44GBQToKOLl5diD/XCUf5+59gCRgopl1CPIvA/LcPRZYDfz8S47v34CN7n6sukIzyzCzPDPLKys59CW7Eml6oqKi2Lt3b0W6sLCQqKioKnXmz59PWlror3m/fv04evQoBw4cqKg/YsQIXnzxRb7xjW+c1n5mZibp6emsW7eOiIgIsrOzmT17dh3OSEREpPmo9eDd3TcBV5pZpJn1Ag4CK4GBZtYD2AbsN7NOQD/gneDWiWa2BVgHdAauDfLLCX0hAFgADLjQsZlZLPBL4N/PMP5n3D3J3ZNatIm40K5Emqzk5GR27tzJ7t27OX78OFlZWaSmplapc/XVV7NixQoAtm/fztGjR+nYsSOfffYZt956K7NmzaJ//1MfggwHDx7ktddeIz09nZKSEsLCwjAzjhw5Ui9zExERaerqas/7b4GRwPeAbHcvAtoBQwmttK8htCJf7O5fmNlNwLeAfu7eC9gEXFJD2xd0PI6ZRQNLgHR3/+BC2hC5GISHhzN37lxSUlLo3r07aWlpxMbGMm3atIrtLbNnz+bZZ5+lV69ejB49mhdeeAEzY+7cuezatYsZM2aQkJBAQkICH3/8cUXbM2bMYMqUKYSFhZGSksKaNWuIi4tjzJgxDTVdERGRJqVOjooMVrifBa4AbnT3fWb2AnBz8OoALAYWu/tPzOw24IfuPtzMrgM2A0PdfZWZOTDa3bPM7CHgKnf/j7P0vwq4393zgnQ7QltuHnb3353rPHRUpDR31R0VKSIiIvXrfI6KDK+LAbj7VjO7HChy931B9hpgiLvvMrOPgPZBHsAyYLyZbQfeJ7R15qTDwPVB4P4xodX8apnZCOApoCPwupltdvcU4F4gBphmZtOC6kPc/eMamgIgLiqCPAU3IiIiItJINPqHNJlZsbu3bYi+k5KSPC8vryG6FhEREZGLxEX7kCYRERERkeasTrbN1KbqVt3NbB5w6lEWT7j78/UzKhERERGR+tfog/fquPuEs9cSEREREWletG1GRERERKSJUPAuIiIiItJEKHgXkQrLli2jW7duxMTEMGvWrNPK9+zZw6BBg+jduzfx8fG88cYbACxfvpw+ffoQFxdHnz59WLlyJQDHjh1j6NCh9OzZk8zMzIp2MjIy2LhxY/1MSkREpBlp9EdFNiQ9pEmaq+oezlRWVkbXrl1Zvnw50dHRJCcns2jRInr06FFRJyMjg969e3P33Xezbds2hg0bRkFBAZs2beKqq64iMjKS9957j5SUFIqKisjNzSU/P5/JkyfTv39/1q5dy5YtW3jyySeZP39+fU5ZRESk0WrwhzSJSNOzfv16YmJi6NKlCwCjRo0iJyenSvBuZnz++ecAHDp0iMjISAB69+5dUSc2NpYjR45w7NgxWrZsSUlJCaWlpZxcKJg6dSpPP/10fU1LRESkWWlW22bM7A4z22pm5WaWVCn/ejPbHLy2BE9iFZFKioqK6Ny5c0U6OjqaoqKiKnWmT5/OggULiI6OZtiwYTz11FOntfPKK6+QmJhI69atGTx4MAUFBfTt25eJEyeSm5tLYmJiRdAvIiIi56e5rby/B9wO/Kqa/CR3P2FmnYAtZvaqu5+o9xGKNGGLFi1i7NixTJo0ibVr1zJmzBjee+89wsJC6wBbt27lpz/9KX/4wx8ACA8PZ+HChQCUlpaSkpJCTk4O9913H3v27CE9PZ3U1NQGm4+IiEhTU+sr72Y2y8wmVEpPN7PDZpYapJeY2XPB9TgzmxlcLzWzDcHKeUal+4vN7PEgf4WZdaypb3ff7u7vV5NfUilQvwSocaO/mWWYWZ6Z5ZWVHDrf6Ys0WVFRUezdu7ciXVhYSFRUVJU68+fPJy0tDYB+/fpx9OhRDhw4UFF/xIgRvPjii3zjG984rf3MzEzS09NZt24dERERZGdnM3v27DqckYiISPNTF9tmsoG0Suk0YDwwMEhHASc30Q4E3gqux7l7HyAJmGhmHYL8y4A8d48FVgM/v5BBmdk3zWwr8C4wvqZVd3d/xt2T3D2pRZuIC+lKpElKTk5m586d7N69m+PHj5OVlXXaqvjVV1/NihUrANi+fTtHjx6lY8eOfPbZZ9x6663MmjWL/v1PffgxHDx4kNdee4309HRKSkoICwvDzDhy5Ei9zE1ERKS5qPXg3d03AVeaWaSZ9QIOAiuBgWbWA9gG7A+2r/QD3glunWhmW4B1QGfg2iC/nNAXAoAFwIALHNefgy8AycDPzOySC2lHpLkKDw9n7ty5pKSk0L17d9LS0oiNjWXatGnk5uYCMHv2bJ599ll69erF6NGjeeGFFzAz5s6dy65du5gxYwYJCQkkJCTw8ccfV7Q9Y8YMpkyZQlhYGCkpKaxZs4a4uDjGjBnTUNMVERFpkurkqEgzmwEcAL4G/N3dnzSzHcAzwGdAe6AUGOPuSWZ2E/AoMMTdS8xsFTDd3VeZWRnQOtiv3gV4xd17n9Zp1f5XAfe7e14N5SuBB2oqP0lHRUpzVd1RkSIiItIwGsNRkdnAs8AVwI1B3jrgx8DNQAdgcfACiAAOBoH7dUDfSm2FASOBLOD7wNvnOxgz+zqwN/gC8C/AdUDB2e6Li4ogT0GOiIiIiDQSdXJUpLtvBS4Hitx9X5C9Bgh3913ARkKr72uCsmVAuJltB2YRCvRPOgxcb2bvEQr8Z9TUr5mNMLNCQttxXjezN4OiAYROmNkMLAHucfcDX36mIiIiIiL1p9E/YdXMit29bUP0nZSU5Hl5Z9xZIyIiIiLypZzPtplm9ZAmEREREZHmrNE/pKm6VXczmweceh7dE+7+fP2MSkRERESk/jX64L067j7h7LVERERERJoXbZsREREREWkiFLyLXOSWLVtGt27diImJYdasWaeV79mzh0GDBtG7d2/i4+N54403Ksp+8YtfEBMTQ7du3XjzzdDhTp988gkDBgygZ8+eLF26tKLubbfdxt/+9rc6n4+IiEhz1uhPm2lIekiTNCfVPZiprKyMrl27snz5cqKjo0lOTmbRokX06NGjok5GRga9e/fm7rvvZtu2bQwbNoyCggK2bdvG6NGjWb9+PX/729/41re+xV//+lfmzZtH+/btuf322xk2bBirVq3i1VdfZcOGDUyfPr0eZywiItI0NIaHNIlIE7B+/XpiYmLo0qULAKNGjSInJ6dK8G5mfP755wAcOnSIyMhIAHJychg1ahStW7fm61//OjExMaxfv56WLVtSUlLCsWPHaNGiBSdOnGDOnDm8+uqr9T9BERGRZqZZbZsxs/82sx1mlm9mS8ysXZB/jZkdMbPNwevpBh6qSKNQVFRE586dK9LR0dEUFRVVqTN9+nQWLFhAdHQ0w4YN46mnnjrjvd///vfJyclh8ODBTJ48mczMTMaMGUObNm3qZ1IiIiLNWLMK3oHlQE93jwf+CvysUtkH7p4QvMY3zPBEmp5FixYxduxYCgsLeeONNxgzZgzl5eU11o+IiOD1118nLy+PxMREXn31VUaOHMldd93FyJEjWbt2bT2OXkREpHmp9eDdzGaZ2YRK6elmdtjMUoP0EjN7LrgeZ2Yzg+ulZrbBzLaaWUal+4vN7PEgf4WZdaypb3f/g7ufCJLrgOgLGH+GmeWZWV5ZyaHzvV2kSYmKimLv3r0V6cLCQqKioqrUmT9/PmlpaQD069ePo0ePcuDAgXO695FHHmHKlCksWrSIAQMG8Otf/1r73kVERL6Eulh5zwbSKqXTgPHAwCAdBZzcUDsQeCu4HufufYAkYKKZdQjyLwPy3D0WWA38/BzHMQ74faX0181sk5mtNrOBNd3k7s+4e5K7J7VoE3GOXYk0TcnJyezcuZPdu3dz/PhxsrKySE1NrVLn6quvZsWKFQBs376do0eP0rFjR1JTU8nKyuLYsWPs3r2bnTt3cv3111fct3PnTgoLC7npppsoKSkhLCwMM+PIkSP1OkcREZHmpNaDd3ffBFxpZpFm1gs4CKwEBppZD2AbsN/MOgH9gHeCWyea2RZCK+adgWuD/HJCXwgAFgADzjYGM5sCnAB+E2TtA652997AfcBCM/vKl5upSNMXHh7O3LlzSUlJoXv37qSlpREbG8u0adPIzc0FYPbs2Tz77LP06tWL0aNH88ILL2BmxMbGkpaWRo8ePRg6dCjz5s2jRYsWFW1PmTKFmTNnAjB69Gj+7//9vyQnJ/OjH/2oQeYqIiLSHNTJUZFmNgM4AHwN+Lu7P2lmO4BngM+A9kApMMbdk8zsJuBRYIi7l5jZKmC6u68yszKgtbufMLMuwCtBEF5T32OBfwducfeSGuqsAu5397wzzUNHRUpzUt1RkSIiItLwGsNRkdnAs8AVwI1B3jrgx8DNQAdgcfACiAAOBoH7dUDfSm2FASOBLOD7wNs1dWpmQ4EHgBsrB+7BPvl/uHtZ8AXgWuDDs00iLiqCPAU8IiIiItJI1MlpM+6+FbgcKHL3fUH2GiDc3XcBGwmtvq8JypYB4Wa2HZhFKNA/6TBwvZm9Ryjwn3GGrucG/S4/5UjIG4B8M9tM6AvDeHf/x5ecpoiIiIhIvWr0T1g1s2J3b9sQfSclJXle3hl31oiIiIiIfCnns22muZ3zLiIiIiLSbNXVnvdaU92qu5nNA/qfkv2Euz9fP6MSEREREal/jT54r467Tzh7LRERERGR5kXbZkREREREmggF7yIiIiIiTYSCd5GL3LJly+jWrRsxMTHMmjXrtPI9e/YwaNAgevfuTXx8PG+88UZF2S9+8QtiYmLo1q0bb775JgCffPIJAwYMoGfPnixdurSi7m233cbf/va3Op+PiIhIc9boj4psSHrCqjQn1T1htaysjK5du7J8+XKio6NJTk5m0aJF9OjRo6JORkYGvXv35u6772bbtm0MGzaMgoICtm3bxujRo1m/fj1/+9vf+Na3vsVf//pX5s2bR/v27bn99tsZNmwYq1at4tVXX2XDhg1Mnz69HmcsIiLSNFy0R0Wa2X+b2Q4zyzezJWbWLsi/Pnho02Yz22JmIxp4qCKNwvr164mJiaFLly60atWKUaNGkZOTU6WOmfH5558DcOjQISIjIwHIyclh1KhRtG7dmq9//evExMSwfv16WrZsSUlJCceOHaNFixacOHGCOXPm8MADD9T7/ERERJqbZhW8A8uBnu4eD/wV+FmQ/x6Q5O4JwFDgV2bWJE/aEalNRUVFdO7cuSIdHR1NUVFRlTrTp09nwYIFREdHM2zYMJ566qkz3vv973+fnJwcBg8ezOTJk8nMzGTMmDG0adOmfiYlIiLSjNV68G5ms8xsQqX0dDM7bGapQXqJmT0XXI8zs5nB9VIz22BmW80so9L9xWb2eJC/wsw61tS3u//B3U8EyXVAdJBfUin/EkB7hUTO0aJFixg7diyFhYW88cYbjBkzhvLy8hrrR0RE8Prrr5OXl0diYiKvvvoqI0eO5K677mLkyJGsXbu2HkcvIiLSvNTFyns2kFYpnQaMBwYG6Sjg5IbagcBbwfU4d+8DJAETzaxDkH8ZkOfuscBq4OfnOI5xwO9PJszsm2a2FXgXGF8pmK/CzDLMLM/M8spKDp1jVyJNU1RUFHv37q1IFxYWEhUVVaXO/PnzSUsL/ZXu168fR48e5cCBA+d07yOPPMKUKVNYtGgRAwYM4Ne//rX2vYuIiHwJtR68u/sm4EozizSzXsBBYCUw0Mx6ANuA/WbWCegHvBPcOtHMthBaMe8MXBvklxP6QgCwABhwtjGY2RTgBPCbSuP6c/AFIBn4mZldUsP4n3H3JHdPatEm4nymLtLkJCcns3PnTnbv3s3x48fJysoiNTW1Sp2rr76aFStWALB9+3aOHj1Kx44dSU1NJSsri2PHjrF792527tzJ9ddfX3Hfzp07KSws5KabbqKkpISwsDDMjCNHjtTrHEVERJqTutr3/VtgJPA1INvdi4Ifjw4ltNLentCKfLG7f2FmNwHfAvq5e4mZrSK0vaU6Z9zyYmZjge8At3g1R+m4+3YzKwZ6AnnnPTORZiQ8PJy5c+eSkpJCWVkZ48aNIzY2lmnTppGUlERqaiqzZ8/mrrvu4vHHH8fMeOGFFzAzYmNjSUtLo0ePHoSHhzNv3jxatGhR0faUKVOYOXMmAKNHj+a73/0us2bNYsaMGQ01XRERkSavTo6KNLNY4FngCuBGd99nZi8ANwevDsBiYLG7/8TMbgN+6O7Dzew6YDMw1N1XmZkDo909y8weAq5y9/+ood+hwP8EfX5SKf/rwF53P2Fm/wKsBeLd/cCZ5qGjIqU5qe6oSBEREWl453NUZJ2svLv7VjO7HChy931B9hpgiLvvMrOPCK2+rwnKlgHjzWw78D6hrTMnHQauDwL3j4HvnaHruUBrYLmZAaxz9/GEtto8aGalhLbh3HO2wB0gLiqCPAU8IiIiItJINPqHNJlZsbu3bYi+k5KSPC9PO2tEREREpO5ctA9pEhERERFpzhr9g4qqW3U3s3lA/1Oyn3D35+tnVCIiIiIi9a/RB+/VcfcJZ68lIiIiItK8aNuMiIiIiEgToeBdRERERKSJUPAuchFatmwZ3bp1IyYmhlmzZp1W/pOf/ISEhAQSEhLo2rUr7dq1qyj76U9/Ss+ePenZsyfZ2dkV+T/4wQ+Ij49n8uTJFXmPPvooS5curcupiIiIXFSa5J73+vJu0SGuefD1hh6GyAWr7sFMZWVlTJgwgeXLlxMdHU1ycjKpqan06NGjos7jjz9ecf3UU0+xadMmAF5//XU2btzI5s2bOXbsGDfddBPf/va3KSgo4NJLLyU/P5/Bgwdz6NAhSkpK+POf/8xDDz1U9xMVERG5SJx15d3MJprZdjP7TW13bmYJZjasttutpp/2ZrbczHYGf361rvsUaazWr19PTEwMXbp0oVWrVowaNYqcnJwa6y9atIjRo0cDsG3bNm644QbCw8O57LLLiI+PZ9myZbRs2ZIjR45QXl5OaWkpLVq0YNq0aTz88MP1NS0REZGLwrlsm7kHGOzuPziZYWa1tWKfAJxX8G4h57vd50FghbtfC6wI0iIXpaKiIjp37lyRjo6OpqioqNq6H330Ebt37+bmm28GoFevXixbtoySkhIOHDjAH//4R/bu3Uv37t3p2LEjiYmJDB8+nF27dlFeXk5iYmK9zElERORiccYg3MyeBroAvzezq4HcIL3HzCYCTwNXB9V/7O5/MrPLgKeAnkBLYLq7n7asZ2atgBnApWY2APgF0B0odvfHgjrvAd8JbnkT+DPQB7gnGNvbwL8CRcBt7n6khqncBtwUXP8aWAX89ExzFxHIyspi5MiRtGjRAoAhQ4bwl7/8hX/913+lY8eO9OvXr6Jszpw5FfcNHz6cX/3qV8ycOZMtW7YwePBg7rrrroaYgoiISLNyxhVsdx8P/A0YBDwO9AC+5e6jgSeAx909Gfg34P8Ft00BVrr79cF9/x0E9Ke2fRyYBmS7e4K7Z59a5xTXApnuHgt8FKTnBenPgjHU5Cp33xdc/x24qqaKZpZhZnlmlldWcugsQxJpeqKioti7d29FurCwkKioqGrrZmVlVWyZOWnKlCls3ryZ5cuX4+507dq1SnlOTg59+vShuLiYDz74gJdffpnFixdTUlJS+5MRERG5yJzv9pfcSqvb3wJ6mNnJsq+YWVtgCJBqZvcH+ZcQWp3f/iXH+pG7r6uU3u3um4PrDcA159KIu7uZ+RnKnwGeAWjd6doa64k0VcnJyezcuZPdu3cTFRVFVlYWCxcuPK3ejh07OHjwIP369avIKysr47PPPqNDhw7k5+eTn5/PkCFDKspLS0uZM2cOr7/+Ojt37uTkvw9lZWUcP36cNm3a1P0ERUREmrHzDd4PV7oOA/q6+9HKFSz0X+t/c/f3L2A8J6j6fwMuqaFvgGOVrsuAS8/Q7n4z6+Tu+8ysE/DxBYxNpFkIDw9n7ty5pKSkUFZWxrhx44iNjWXatGkkJSWRmpoKhFbdR40aRaUv6JSWljJw4EAAvvKVr7BgwQLCw//5z8i8efO48847adOmDfHx8ZSUlBAXF8ewYcOqHDcpIiIiF8bcz7y4bGYFQBJwL1X3oy8ENrn7fwfpBHffbGb/BXwF+I9glbu3u2+qoe1/A1Ld/c4g/b+A77j7KDNLBP4CfCOo/pq79wzqXXNK+n6grbtPr6Gf/wY+dfdZZvYg0N7dHzjbm9O607Xe6c45Z6sm0mhVd1SkiIiINC5mtsHdk86l7pc5NWYiMM/M8oN23gLGA48Ac4D84FSY3fzzR6en+iPwoJltJvSD1VeAdDPbSujHqX/9EuOrbBbwspn9b0L75dPO5aa4qAjyFPyIiIiISCNx1pX3i1lSUpLn5eU19DBEREREpBk7n5X38z0vXUREREREGkhtPWzpjMwsBfjlKdm73X1ELfczD+h/SvYT7v58bfYjIiIiItIQ6iV4d/c3CT1kqa77mVDXfYiIiIiINBRtmxERERERaSIUvIuIiIiINBEK3kUuAsuWLaNbt27ExMQwa9as08p/8pOfkJCQQEJCAl27dq14oNIf//jHivyEhAQuueQSli5dCsAPfvAD4uPjmTx5ckU7jz76aEW5iIiI1L562fPeVL1bdIhrHny9oYchcs6qeyhTWVkZEyZMYPny5URHR5OcnExqaio9evSoqPP4449XXD/11FNs2hR6rtqgQYPYvHkzAP/4xz+IiYlhyJAh5Ofnc+mll5Kfn8/gwYM5dOgQJSUl/PnPf+ahhx6q20mKiIhcxLTyLtLMrV+/npiYGLp06UKrVq0YNWoUOTk5NdZftGgRo0ePPi1/8eLFfPvb36ZNmza0bNmSI0eOUF5eTmlpKS1atGDatGk8/PDDdTkVERGRi16zCt7N7A4z22pm5WaWVCm/lZk9b2bvmtkWM7up4UYpUr+Kioro3LlzRTo6OpqioqJq63700Ufs3r2bm2+++bSyrKysiqC+e/fudOzYkcTERIYPH86uXbsoLy8nMTGxbiYhIiIiQPPbNvMecDvwq1Py7wJw9zgzuxL4vZklu3t5fQ9QpDHLyspi5MiRtGjRokr+vn37ePfdd0lJSanImzNnTsX18OHD+dWvfsXMmTPZsmULgwcP5q677qqvYYuIiFw0an3l3cxmmdmESunpZnbYzFKD9BIzey64HmdmM4PrpWa2IVg5z6h0f7GZPR7krzCzjjX17e7b3f39aop6ACuDOh8DnwHVPoLWzDLMLM/M8spKDp33/EUam6ioKPbu3VuRLiwsJCoqqtq6lVfXK3v55ZcZMWIELVu2PK0sJyeHPn36UFxczAcffMDLL7/M4sWLKSkpqb1JiIiICFA322aygbRK6TRgPDAwSEcRCqYJ8t4Krse5ex9CQfVEM+sQ5F8G5Ll7LLAa+PkFjGkLkGpm4Wb2daAP0Lm6iu7+jLsnuXtSizYRF9CVSOOSnJzMzp072b17N8ePHycrK4vU1NTT6u3YsYODBw/Sr1+/08pq2gdfWlrKnDlzeOCBBzhy5AhmBoR+JHv8+PHan4yIiMhFrtaDd3ffBFxpZpFm1gs4SGjVe6CZ9QC2AfvNrBPQD3gnuHWimW0B1hEKrK8N8ssJfSEAWAAMuIBhPQcUAnnAnKDPsgtoR6TJCQ8PZ+7cuaSkpNC9e3fS0tKIjY1l2rRp5ObmVtTLyspi1KhRFQH4SQUFBezdu5cbb7zxtLbnzZvHnXfeSZs2bYiPj6ekpIS4uDj69OlTcdykiIiI1B5z99pv1GwGcAD4GvB3d3/SzHYAzxDastIeKAXGuHtS8APSR4Eh7l5iZquA6e6+yszKgNbufsLMugCvuHvvs/S/Crjf3fNqKH8H+KG7bztTO607Xeud7pxzjrMWaXjVHRUpIiIijZuZbXD3ard0n6qufrCaDTwLXAGcXK5bB/wYuBnoACwOXgARwMEgcL8O6FuprTBgJJAFfB94+3wHY2ZtCH1ROWxmg4ETZwvcAeKiIshTMCQiIiIijUSdHBXp7luBy4Eid98XZK8Bwt19F7CR0Or7mqBsGRBuZtuBWYQC/ZMOA9eb2XuEAv8ZNfVrZiPMrJDQdpzXzezNoOhKYGPQ/k+BMbUwTRERERGRelUn22Zqk5kVu3vbhug7KSnJ8/Kq3XkjIiIiIlIrzmfbTLN6SJOIiIiISHPW6B/SVN2qu5nNA/qfkv2Euz9fP6MSEREREal/jT54r467Tzh7LRERERGR5kXbZkREREREmggF7yIiIiIiTYSCd5FmbtmyZXTr1o2YmBhmzZp1WvlPfvITEhISSEhIoGvXrhVPRv3jH/9YkZ+QkMAll1zC0qVLAfjBD35AfHw8kydPrmjn0UcfrSgXERGRutEk97zXl3eLDnHNg6839DBEzkl1T1ctKytjwoQJLF++nOjoaJKTk0lNTaVHjx4VdR5//PGK66eeeopNmzYBMGjQIDZv3gzAP/7xD2JiYhgyZAj5+flceuml5OfnM3jwYA4dOkRJSQl//vOfeeihh+p2kiIiIhe5i2rl3cwmmZmb2RUNPRaR+rB+/XpiYmLo0qULrVq1YtSoUeTk5NRYf9GiRYwePfq0/MWLF/Ptb3+bNm3a0LJlS44cOUJ5eTmlpaW0aNGCadOm8fDDD9flVERERISLKHg3s87AEGBPQ49FpL4UFRXRuXPninR0dDRFRUXV1v3oo4/YvXs3N99882llWVlZFUF99+7d6dixI4mJiQwfPpxdu3ZRXl5OYmJi3UxCREREKtTbthkzmwXsdfd5QXo68J/AaHfPNbMlwEF3H2dm44BvuPsUM1sKdAYuIXSW+zPB/cXAs4QC8r8Do9z9kzMM4XHgAaDmZUeRi1hWVhYjR46kRYsWVfL37dvHu+++S0pKSkXenDlzKq6HDx/Or371K2bOnMmWLVsYPHgwd911V30NW0RE5KJSnyvv2UBapXQaMB4YGKSjgJMbcQcCbwXX49y9D5AETDSzDkH+ZUCeu8cCq4Gf19Sxmd0GFLn7lrMN0swyzCzPzPLKSg6d28xEGqmoqCj27t1bkS4sLCQqKqraupVX1yt7+eWXGTFiBC1btjytLCcnhz59+lBcXMwHH3zAyy+/zOLFiykpKam9SYiIiEiFegve3X0TcKWZRZpZL+AgsBIYaGY9gG3AfjPrBPQD3glunWhmW4B1hFbgrw3yywl9IQBYAAyorl8zawNMBqad4zifcfckd09q0SbifKcp0qgkJyezc+dOdu/ezfHjx8nKyiI1NfW0ejt27ODgwYP069fvtLKa9sGXlpYyZ84cHnjgAY4cOYKZAaEfyR4/frz2JyMiIiL1vuf9t8BI4HtAtrsXAe2AoYRW2tcQWpEvdvcvzOwm4FtAP3fvBWwitH2mOl5D/jeArwNbzKwAiAY2mtnXamE+Io1aeHg4c+fOJSUlhe7du5OWlkZsbCzTpk0jNze3ol5WVhajRo2qCMBPKigoYO/evdx4442ntT1v3jzuvPNO2rRpQ3x8PCUlJcTFxdGnT5+K4yZFRESkdpl7TTFvHXRmFkton/oVwI3uvs/MXgBuDl4dgMXAYnf/SbDd5YfuPtzMrgM2A0PdfZWZOaH98llm9hBwlbv/xzmMoQBIcvcDZ6vbutO13unOORcyVZF6V91RkSIiItL4mdkGd086l7r1es67u281s8sJ7T/fF2SvAYa4+y4z+whoH+QBLAPGm9l24H1CW2dOOgxcHwTuHxNaza9VcVER5CkgEhEREZFGol5X3muTmRW7e9u67CMpKcnz8vLqsgsRERERucidz8r7RXPOu4iIiIhIU1ev22ZqU3Wr7mY2D+h/SvYT7v58/YxKRERERKTuNNltM/XBzL4gtNdemqYrgLP+MFkaLX1+TZ8+w6ZNn1/Tps+vafkXd+94LhWb7Mp7PXn/XPcfSeNjZnn6/JoufX5Nnz7Dpk2fX9Omz6/50p53EREREZEmQsG7iIiIiEgToeD9zJ5p6AHIl6LPr2nT59f06TNs2vT5NW36/Jop/WBVRERERKSJ0Mq7iIiIiEgTcVEE72Y21MzeN7NdZvZgNeWtzSw7KP+zmV1TqexnQf77ZpYS5HU0s7fN7D0z+26lujlmFlkfc7qYnMPnN9bMPjGzzcHrh5XK7jSzncHrziCvtZktCz6/eyrVfcbMEutnVhcPM3vOzD42s/dqKDczezL4fPMrfwb6/BreOXx+N5nZoUp//6ZVKqv2766Z/Sb4rP+rUt5Dlf89ldphZp3N7I9mts3MtprZj6qpo7+DjdQ5fn76O3ixcfdm/QJaAB8AXYBWwBagxyl17gGeDq5HAdnBdY+gfmvg60E7LYCJwP8C2gCrgrrDgekNPd/m9jrHz28sMLeae9sDHwZ/fjW4/iqQCjz0/7d3dyFWVWEYx/+PpaEUJnmRmFGUV0FRgRhBREFFhUMkZUFfVxJGCN1IooViF0FedBWUgUUZYRFOKDIQEUHZx1CZCTHURR9ehJYmhjn2dLHW1DDNmTnYfJwz+/nBhn32WWuz9rzz7r3O3mudQ/nw+lEtexWwbbqPdyYuwA3ANcDXLd6/HdgDCFgO7Ev8OmdpI343Au+Osn3U3AWuBF6qZfqA+cAioHe6j3UmLvVve01dPw/4dpRzaHKwQ5c245ccbNjShDvvy4AB29/Z/hN4A+gZUaYH2F7XdwI3S1Ld/obtk7a/Bwbq/k5ROu7nAKclnQ2sBZ6d7INpoHbi18qtQJ/tI7Z/pZykbuPf+M2mXKwANgMbJrTlAYDtD4AjYxTpAV5x8TFwvqRFJH4doY34tdIqd08BcyXNosTwNLAJeGqCmhzD2D5ku7+u/w4cBBaPKJYc7FBtxq+V5OAM1YTO+2Lgh2Gvf+S///j/lLE9CBwFLhij7uuUBOgDnqHcuX/V9olJaH/TtRM/gLvrI8CdkpaMU7cPuAT4GHhe0gqg3/bPE934aEurOCV+3eM6SV9K2iPpirpt1PjZPgj8AvQDvcDlwKyhDkpMHpUhoVcD+0a8lRzsAmPED5KDjZJfWD0Dto8CdwBIWgCsA+6S9CLlkeJztj+axiY2TS+ww/ZJSaspT1FualW4fkC7H0DSbGAv0CNpK3Ax5Q7UrslvdpyJxK/j9FN+1vu4pNuBd4ClY1WwvXZoXVIvsFrSesrQiz7bL05ec5tJ0rnAW8Ba28f+z76Sg1NvnPglBxumCXfefwKWDHt9Ud02apk6BGY+cLjNuhuALcB9wIfAQ8DTE9P0oI0Y2D5s+2R9+RJwbbt1KU9NXqGM8zwK3As8MSEtj3a1ilPi1wVsH7N9vK7vBmZLWkgb8ZPUA3wOnAtcZvseYKWkeVPS+IaoHey3gNdsvz1KkeRgBxsvfsnB5mlC5/1TYKmkSyXNoUxIHXlHYBel0w2wEnjPtuv2VXVm/aWUT7KfDFWStBS4yPb7lPF/fwEG5k7i8TTNuPGrYzOHrKCMCYRyN+gWSQvqE5Jb6raheguAOykXnsRv+uwCHqzfeLEcOGr7EIlfV5B0YZ0jhKRllOvKYcbJ3dohWUuZKzSXEjsok+zmTNkBzHA1NtuAg7a3tiiWHOxQ7cQvOdg8M37YjO1BSY9RTjhnAS/bPiBpE/BZfbS3DXhV0gBlYtaqWveApDeBb4BBYI3t08N2vwVYX9d3UB5VrQM2EhOizfg9XsdcDlLi93Cte0TSZsoJDGCT7eET7zYCW2z/JWkvsAbYD7wwFcfWFJJ2UL4NYaGkHymTomYD2H4B2E35tosB4ATwSH0v8esAbcRvJfCopEHgD2BVvfkxau4O2/UaYLvtE5K+AuZJ2g/stv3b1BxdI1wPPADsl/RF3fYkZXhLcrDztRO/5GDD5BdWIyIiIiK6RBOGzUREREREzAjpvEdEREREdIl03iMiIiIiukQ67xERERERXSKd94iIiIiILpHOe0REREREl0jnPSIiIiKiS6TzHhERERHRJf4GLPWBEPc4N18AAAAASUVORK5CYII=", 152 | "text/plain": [ 153 | "
" 154 | ] 155 | }, 156 | "metadata": { 157 | "needs_background": "light" 158 | }, 159 | "output_type": "display_data" 160 | } 161 | ], 162 | "source": [ 163 | "# 将特征重要性排序后绘图\n", 164 | "ax = feat_importances.sort_values()[250:].plot(kind='barh', figsize=(12, 6))\n", 165 | "# 设置横坐标格式\n", 166 | "ax.xaxis.set_major_formatter(plt.FuncFormatter(lambda x, pos: \"{:.2%}\".format(x)))\n", 167 | "# 设置标题\n", 168 | "ax.set_title('Importance of Features')\n", 169 | "# 如果不需要显示特征重要性的大小数值，可以使用下面2行代码\n", 170 | "# for container in ax.containers:\n", 171 | "# ax.bar_label(container)\n", 172 | "# 如果需要显示特征重要性的大小数值，可以使用下面的代码\n", 173 | "x_offset = 0\n", 174 | "y_offset = 0\n", 175 | "for p in ax.patches:\n", 176 | " b = p.get_bbox()\n", 177 | " val = \"{:.2%}\".format(b.x1)\n", 178 | " ax.annotate(val, (b.x1 + 0.0002, b.y1 - 0.4))" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 53, 184 | "metadata": {}, 185 | "outputs": [ 186 | { 187 | "data": { 188 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2xElEQVR4nO3deZxN9R/H8ddnxjCDYazZYmSJsURNizU7LagoouwkiUqLpJS00IaQbGlBixBFZAtZx5YtZYsR2cYytjEzn98f9+o3aYw7zJ0zc+/n+XjMwz33fO857zOYz3zP8v2KqmKMMcZ/BTgdwBhjjLOsEBhjjJ+zQmCMMX7OCoExxvg5KwTGGOPnrBAYY4yfs0JgjDF+zgqB8SkiskdEzopIrIgcFJGJIpLzkjbVRWShiJwSkRMiMktEIi5pk0tEhorIXve2drqX819mvyIivURks4icFpFoEflGRCp583iNSQtWCIwvaqqqOYEqQFXgxYsrRKQaMA/4DigClAQ2Ar+IyA3uNlmBBUAFoAmQC6gGHAVuu8w+hwG9gV5AXqAsMAO4J7XhRSRLaj9jzLUQe7LY+BIR2QN0UdX57uUhQAVVvce9vBTYpKo9LvncHOCwqrYTkS7AG0ApVY31YJ9lgN+Aaqq6+jJtFgNfqOo493IHd86a7mUFegJPAVmAH4HTqvpskm18B/ysqu+LSBHgQ6A2EAt8oKrDr/wdMua/rEdgfJaIFAPuAna4l7MD1YFvkmn+NdDQ/boB8KMnRcCtPhB9uSKQCvcBtwMRwBSglYgIgIjkARoBX4pIADALV0+mqHv/T4lI42vcv/FTVgiML5ohIqeAfcAhYID7/by4/s0fSOYzB4CL5//zXabN5aS2/eW8parHVPUssBRQoJZ7XUtghar+BdwKFFDVgaoap6q7gLFA6zTIYPyQFQLji+5T1VCgDlCO//+AjwESgcLJfKYwcMT9+uhl2lxOattfzr6LL9R1zvZL4GH3W22ASe7XJYAiInL84hfQD7guDTIYP2SFwPgsVf0ZmAi8614+DawAHkym+UO4LhADzAcai0gOD3e1ACgmIpEptDkNZE+yXCi5yJcsTwFaikgJXKeMvnW/vw/YraphSb5CVfVuD/Ma8y9WCIyvGwo0FJGb3Mt9gfbuWz1DRSSPiAzCdVfQa+42n+P6YfutiJQTkQARySci/UTkPz9sVfUPYBQwRUTqiEhWEQkWkdYi0tfdbAPwgIhkF5HSQOcrBVfV9bh6KeOAuap63L1qNXBKRF4QkRARCRSRiiJya6q/O8ZghcD4OFU9DHwGvOJeXgY0Bh7AdV7/T1y3mNZ0/0BHVc/jumD8G/ATcBLXD9/8wKrL7KoXMAIYCRwHdgL347qoC/ABEAf8DXzK/0/zXMlkd5bJSY4pAbgX1+2xu/l/scjt4TaN+Re7fdQYY/yc9QiMMcbPWSEwxhg/Z4XAGGP8nBUCY4zxc5lucKv8+fNreHi40zGMMSZTWbt27RFVLZDcukxXCMLDw4mKinI6hjHGZCoi8ufl1tmpIWOM8XNWCIwxxs9ZITDGGD9nhcAYY/ycFQJjjPFzXisEIjJBRA6JyObLrBcRGS4iO0TkVxG52VtZjDHGXJ43ewQTcU38fTl3AWXcX92Aj7yYxRhjMrfEeNBEr2zaa4VAVZcAx1Jo0hz4TF1WAmEikhazPBljjE9ZPesHfn2zDmz5zCvbd/IaQVGSTM0HRLvf+w8R6SYiUSISdfjw4XQJZ4wxTtNzJ3j+wWeo1nw17T+qyIV1H4MXpg7IFBeLVXWMqkaqamSBAsk+IW2MMb7lr5XIF1XhiOsya6M6hUi4fx6IpPmunBxiYj9wfZLlYu73jDHGbx0/dppdsz7g5qOvgibw2iN5af1yY26uV91r+3SyRzATaOe+e+gO4ISqHnAwjzHGOOq7KcuIKP0GzXrHcOJMFrilDyHtf/FqEQAv9ghEZApQB8gvItHAACAIQFVHA7OBu4EdwBmgo7eyGGNMRnbo0Gl6dRzLV7NPANm4o2QMx+t8Te6azdJl/14rBKr68BXWK/CEt/ZvjDEZnaoy6dMoevf6nmOnAsgeFMebjxyg57A3CQwtlG45Mt0w1MYY4ysebzeBj7+IBgJoUHY3Y969iZL3DvLKBeGUZIq7howxxqecOw7ze3Bf8NuEhZxlfMfVzFvyFCWbPp3uRQCsR2CMMenmj9+PsGDKt3S/7lU4fZAmEVnYMxVyN/wGgrI7lssKgTHGeFl8fCLvv/kDA96I4vwFoUrPLNxxRzVo+DG5C1RyOp4VAmOM8aaN66Lp3HYCa39TIIB2t22lTIt+UPsxkIxxdt4KgTHGeMH58/EMemEKb4/YQXxCAMXDjvPx06do0mcY5Ei/O4I8YYXAGGPS2rkYXmz7Jh9MywkE8ETd33hr+KOEVrzb6WTJskJgjDFpRRV++xIWP83zFU+zYm1rhjx3HbW6TYCgEKfTXZYVAmOMSQM/TV/G6MFf89UDI8kSmEihcjVYvqY3UqCi09GuyAqBMcZcg5gjJ3i203AmzEoE8vFJyep0fbEdVOqMZJCLwVdihcAYY67S9HHT6fHsKg6eCCFblngGtD1Oh6FfQVgRp6OlihUCY4xJpYN7onmy3QimLg0BQqhe6hDjP25EufrNnY52VTJHv8UYYzICVdg2me/6PcrUpSHkyBrHh08nsHTLO5m2CID1CIwxxiPnDmwneFlP2DufrlWFXbHhPP5aD8Kr3up0tGtmPQJjjElB4oXzjOjzOiVvHMefG9ZAcB4Cmoxl8HfjfaIIgBUCY4y5rO0//0jtiD48+X4iB0/lZMpf7aDjb1Cpc4YZHiIt2KkhY4y5xIWTh3m395u89kVOzscX4LpcZxk1pCoPPNbK6WheYYXAGGMuUmXzrIm067mW9fsKANDxbuW9iX3JUyDM2WxeZIXAGGMAYg/A/O4kLl3Fpv3dKJH/LGM+akyjlnWdTuZ1VgiMMf5NlS2zJhDx57NI3HEqh+fiu+EFqd2uKzlDszmdLl34ztUOY4xJpVMH9tDznp5UbB7Nt2uKQHgTaL+Zu5/o6TdFAKxHYIzxR6rMHfsx3fr+wd6YgmQJSGBPwW7wQC9H5gx2mhUCY4xfOfbnTp7uMJTPFucHcnHzDacZ/1lbqtS4yelojrFCYIzxD6ps+HYMTTrt5O9T+cmWJZ7XnixAn8H9yRIU6HQ6R1khMMb4vlPRMK8rZf9YQM6sj1M2IpFxn3ei7M0RTifLEKwQGGN8liYmMnnISJoGDCRX4BGyh4ax+OsqFLmzPQGBdq/MRVYIjDE+ac+mLXR7ZBw//RrG49VuZ9TzgdBgNMVyFnY6WoZjhcAY41MS4hMY1X84L35whNNxYeTNfpbqze+F5o/55R1BnrBCYIzxGdvWbKTzI5+x4vdcQFYeqhHDh5/3pmDJkk5Hy9CsEBhjMj9Vds8ZRZVmB4lLyEXhXLGMGlSW+3q+Yr0AD1ghMMZkbif2wLwulNy7gAdveoDgvMV4d2IfwooWdzpZpuHVQiAiTYBhQCAwTlXfvmR9ceBTIMzdpq+qzvZmJmOMbzh75jwDew7n/jxjuK3IDgjOx6efPUhgRCvrBaSS1wqBiAQCI4GGQDSwRkRmqurWJM36A1+r6kciEgHMBsK9lckY4xuWzllJl87T+f1AduYUacC6UVUIaDiSwOwFnY6WKXmzR3AbsENVdwGIyJdAcyBpIVAgl/t1buAvL+YxxmRyJ0+c5cWuwxn1zTkgOxGFjjL6/TsIaN7e6WiZmjcLQVFgX5LlaOD2S9q8CswTkSeBHECD5DYkIt2AbgDFi9t5P2P80eyvl9K9x2z2HQ0mS0AC/R46Rr9R/cmWp5DT0TI9px+texiYqKrFgLuBz0X+OxGoqo5R1UhVjSxQoEC6hzTGOEgTOfHzMNp2cBWByOJ/s3Z6WV6bMsKKQBrxZo9gP3B9kuVi7veS6gw0AVDVFSISDOQHDnkxlzEmE1BV9NgOAuZ3IXf0EobfV5m/g2vy1NBXyZLLrgWkJW8WgjVAGREpiasAtAbaXNJmL1AfmCgi5YFg4LAXMxljMoG/9p+gR9sR1AqbR59aSyB7QR59bQCUecDpaD7Ja4VAVeNFpCcwF9etoRNUdYuIDASiVHUm0AcYKyJP47pw3EFV1VuZjDEZm6oy4cO59HlxGSfOBLEy9DZ6tLuekLuGQUg+p+P5LK8+R+B+JmD2Je+9kuT1VqCGNzMYYzKHXTuO0LXNaBauSQCCuKfiHkaPuouQWg85Hc3n2ZPFxhhHJSQkMvzt73lpYBRn4wLJn+M0w3ucovWr7yLZrReQHqwQGGOck5gAUUOZOv43zsYV4+HIPxg24n4K3N7C6WR+xQqBMSbdxcUlcGrvFvKt7k7ggRWMfzA/f4S0pOkLwyEkr9Px/I4VAmNMulqzah+dH/mEYkE7+KHTCiS0COXuH0O5G+5xOprfskJgjEkXZ85cYMDz3/L+qN9IVOFMvjwcKtqJ6+5/D4LDnI7n16wQGGO8bvHCnXRtP4kd0UqAKM82/JXXhnYie8S9TkczWCEwxniRqtKr22RGjNsBQKXCfzO+r3LrY59AttwOpzMXWSEwxnhHYjyy5l1y7VlIUOAd9L97A33f6U7WG+92Opm5hBUCY0yaOnLkDDvXRnH74T7wdxQv189C24dKEdHuc+sFZFBWCIwxaUJV+WrKrzz5xAyyJMSy9bnN5LnueoIbjSMivJHT8UwKPC4EIpJdVc94M4wxJnOKjj5Jjy5TmDX3IAD1Sh/iTOkO5HlgMGTLdYVPG6ddcT4CEakuIluB39zLN4nIKK8nM8ZkeImJypjRq6lQ7gNmzT1IruBzjH10GfN/eoSiD39kRSCT8KRH8AHQGJgJoKobRaS2V1MZYzKFzm0nMvHLvQA0q/Abo/rnpegD0yBrqMPJTGp4NEOZqu675K0EL2QxxmQWCRdgxUAeCXuTgjlj+bLrYmb80IWirUdaEciEPOkR7BOR6oCKSBDQG9jm3VjGmIxo8+ZDLPjuF3oXfx0Orad+adg1+Qw5Gs+CrDmdjmeukieFoDswDNdk9PuBeUAPb4YyxmQs58/H89YbP/PmW8u4EA+RTxyhxk3h0HgCOYrXdTqeuUaeFIIbVbVt0jdEpAbwi3ciGWMyklWrounc4Ru2/HYSgMerraFSw6Zw19t2GshHeFIIPgRu9uA9Y4wPOX06jpf7L2DosNWoQpn8RxnXfiW1n3wLStR3Op5JQ5ctBCJSDagOFBCRZ5KsyoVrDmJjjA97qc9Uhn38BwGSyHN1lvNqn1KENFpgvQAflFKPICuQ090m6d/8SaClN0MZYxyUGA9R7/FSsbfZVLoZgx/cRGS3wRDe0OlkxksuWwhU9WfgZxGZqKp/pmMmY4wDZs7czugPF/PdI58QdGQVBUJgwciscOdiezDMx3lyjeCMiLwDVACCL76pqvW8lsoYk24OHTpNrydn89XXWwH4NM8FutQvBo3HQXhjh9OZ9OBJIZgEfAXci+tW0vbAYW+GMsZ4n6oyadImevf6gWMxcWQPiuOtuxfQsUMVqP++jRTqRzwpBPlUdbyI9E5yumiNt4MZY7xn794TdH9sFnN+3AlAgzI7GdN+DSXbvA8l73I4nUlvnhSCC+4/D4jIPcBfQF7vRTLGeNu86cuZ8+NOwkLO8n7TuXToUAWpu8LmDvZTnhSCQSKSG+iD6/mBXMBT3gxljEl7p0/HkSMkENYPp7P2Y3+j2+lWP5rCLYfBDfc4Hc846IqFQFW/d788AdSFf54sNsZkAvHxibz//gqGDF7K6pcXcEPCfAQY8FQJqDsdgvM4HdE4LKUHygKBh3CNMfSjqm4WkXuBfkAIUDV9IhpjrtbGjQfp1Ok71q1zTRgzY+Fpnrm7EDQcA6WaOpzOZBQp9QjGA9cDq4HhIvIXEAn0VdUZ6ZDNGHOVzp+PZ9CgJbz99jLi45XiYccZ8+AsGt9XDep+DyF2mc/8X0qFIBKorKqJIhIMHARKqerR9IlmjLka69cfoG3baWzbdgQRpWeN1bzZYhOhTUdA6eZOxzMZUEoT08SpaiKAqp4DdqW2CIhIExHZLiI7RKTvZdo8JCJbRWSLiExOzfaNMf+V7fwBdv5xiBsLHGFJj0/48MU8hHbfYEXAXFZKPYJyIvKr+7UApdzLAqiqVk5pw+5rDCOBhkA0sEZEZqrq1iRtygAvAjVUNUZECl7DsRjjt9atO0DVKgWRjaOJWP0CczoXpHrEOYLvGgll7nc6nsngUioE5a9x27cBO1R1F4CIfAk0B7YmadMVGKmqMQCqeuga92mMX4mJOcuzz85jwoQNTOm1ldbFvwag3r23Q70PIXt+hxOazCClQeeudaC5okDSuY6jgdsvaVMWQER+wTW09auq+uOlGxKRbkA3gOLFi19jLGN8w/Tp2+jRYzYHD8aSLUs8R/fthhsLQIOPoGwLp+OZTMSTB8q8vf8yQB2gGLBERCqp6vGkjVR1DDAGIDIyUtM5ozEZysGDsTz55BymTnV1rmuE72XcQzMpV7Me1P8BshdwOKHJbLxZCPbjuv30omLu95KKBlap6gVgt4j8jqsw2FhGxiRj7dq/aNjwc2JizpEjaxxv3z2fHvV3EdBwFNz4oNPxTCblUSEQkRCguKpuT8W21wBlRKQkrgLQGmhzSZsZwMPAJyKSH9epol2p2IcxfiWi2DkKBMdw2437+bjFLErc3gAazILsdp+FuXpXLAQi0hR4F9eMZSVFpAowUFWbpfQ5VY0XkZ7AXFzn/yeo6hYRGQhEqepM97pGIrIVSACes+cUjPm/xERl3Lh1PPRgBGH7viDk5z4s6ZZIwfzBSIOP4caHQMTpmCaTE9WUT7mLyFqgHrBYVau639ukqpXSId9/REZGalRUlBO7NiZdbd9+hC5dZrFs2V661P+bsXd95FpR+n7XBeEc1zkb0GQqIrJWVSOTW+fRMNSqekL+/VuHXbA1xksuXEjgvfdW8Oqrizl/PoFCuWK5q9hiCM4L9UZAudbWCzBpypNCsEVE2gCB7gfAegHLvRvLGP+0fv0BOneeyfr1rkHiOt66nveaziVPpcbQcCHkKORwQuOLPCkETwIvAeeBybjO6w/yZihj/NHOnce47bZxxMcnEp73BGNafkfDyjFQbxyUa2O9AOM1nhSCcqr6Eq5iYIzxklIFz/Jo7b8JjdvJG3ctJGf5xtDwY8hZ2Oloxsd5UgjeE5FCwFTgK1Xd7OVMxviF2Ng4+vVbwMOtK1At92JY1Jvxdx1HgsNcvYDyj1gvwKQLT2Yoq+suBA8BH4tILlwFwU4PGXOV5s7dQbdu37N37wl+nrmYDT2HIAJS6h7XpDE5izgd0fiRlIah/oeqHlTV4UB3YAPwijdDGeOrjh07S/v2M2jSZBJ7957gluv/5rMWnyLBuaHJRLhvlhUBk+48eaCsPNAKaAEcBb7CNZG9MSYVpk7dyhNPzObQodMEZ03ktYYLeKb2CrKUbgwNx0JoUacjGj/lyTWCCbh++DdW1b+8nMcYn3T8+Dm6dZtFTMw5apfez9gW0yhb9ALUHQcVOti1AOMoT64RVEuPIMb4GlUlMVEJDAwgLOgEozr9RszuzTx2x1oCbmjk6gXkuv7KGzLGyy5bCETka1V9SEQ28e8niT2aocwYf7Znz3G6dZtFvbrh9L1/PyzoSevCR6FEKNQZCxU7WS/AZBgp9Qh6u/+8Nz2CGOMLEhISGTlyDf36LeD06QtsXbedp2QwwUHxUKIhNBoHuWxyJZOxpDRD2QH3yx6q+kLSdSIyGHjhv58yxn9t23aYLl1msXy5a2K+1rf8zrB7vyM4ezDUeQ8qdbVegMmQPLl9tGEy792V1kGMyazi4xN5440lVKnyMcuX76NI3gt813EKUx6eTMEKd0CHzVC5mxUBk2GldI3gcaAHcIOI/JpkVSjwi7eDGZNZBAQI8+btIi4uga41tjCkySzCcmWBOz+Cyo9ZATAZXkrXCCYDc4C3gL5J3j+lqse8msqYDO7s2QucOhVHwYI5CDh3lHFtF7Cv/C/UK7MbiteDRuMhd7jTMY3xSEqFQFV1j4g8cekKEclrxcD4qyVL/qRLl5mEh4cxd0R2ZEEPypw5RJmIHFB7JNzUHcSjh/aNyRCu1CO4F1iL6/bRpP1bBW7wYi5jMpyTJ8/z4ovzGTXKNUNe0PkDHJkyjAI5z8D1daDxBMhd0tmQxlyFlO4autf9p/3LNn5vzpw/eOyx79m37yRZssBLjdbwYu0fyRacDWp/CFV6WC/AZFqejDVUA9igqqdF5BHgZmCoqu71ejpjHKaqdO06i/Hj1wMQWeYcE5pNoFLhQ1CstqsXEFbK4ZTGXBtPfoX5CDgjIjfhGmxuJ/C5V1MZk0GICMWKhhKcTXj3/l9Y0WUIla4/BXWHwUOLrAgYn+DJoHPxqqoi0hwYoarjRaSzt4MZ45S//jrFzp3HqFWrBMQeoF+lD3n06cWUyh8Dxe50PR2cp7TTMY1JM54UglMi8iLwKFBLRAKAIO/GMib9qSoTJqynT595ZM0ayLZvw8i38Tmynj9BqSKhUHs0VO5q1wKMz/GkELQC2gCdVPWgiBQH3vFuLGPS165dMXTtOouFC3cDcO/Nx7gwfyDkioWSd0OD0TZSqPFZngxDfVBEJgG3isi9wGpV/cz70YzxvoSERIYPX0X//os4c+YC+cOE4U1n0rryWiQkH9QbDeXa2NPBxqd5ctfQQ7h6AItxPUvwoYg8p6pTvZzNGK9r124GkydvAqBN9QMMbfS567mAG1tBveGQvaDDCY3xPk9ODb0E3KqqhwBEpAAwH7BCYDK9rp0qs2T+JkY1/Zqm5bdCjsLQYDKUbu50NGPSjSeFIOBiEXA7ioeT3huT0axZs5+FC3fzwgs14e911NnfmR3PbCJblgSo2BnufBeCw5yOaUy68qQQ/Cgic4Ep7uVWwGzvRTIm7Z05c4EBAxbx/vsrSUxUqueYTa24IaAJZMtX0jVtZIn6Tsc0xhGeXCx+TkQeAGq63xqjqtO9G8uYtLN48R66dJnJzp0xBATAs423csup6ZA1EW5+CmoOgqAcTsc0xjEpzUdQBngXKAVsAp5V1f3pFcyYa3XixDmef/4nxoxZB0ClkvGMb/oJtxbfD3nLQ+PxUKSawymNcV5K5/onAN8DLXCNQPphajcuIk1EZLuI7BCRvim0ayEiKiKRqd2HMZfz8suLGDNmHUFBwsBma4l67C1uDf8b7ugPj663ImCMW0qnhkJVdaz79XYRWZeaDYtIIDAS11SX0cAaEZmpqlsvaRcK9AZWpWb7xiRHVRH3Pf+vPFeZ3asW83btT6hQ6DBcd4trwpiCNzmc0piMJaUeQbCIVBWRm0XkZiDkkuUruQ3Yoaq7VDUO+BJI7p6814HBwLlUpzfGTVWZPHkT9ep9RlxcAvz+Lfm/v4VZDw2hQtGTUGswtFlpRcCYZKTUIzgAvJ9k+WCSZQXqXWHbRYF9SZajgduTNnAXlOtV9QcRee5yGxKRbkA3gOLFi19ht8bfREef5PHHf+D7738HYNKz3elYYpxrZdFarkHi8pZ1MKExGVtKE9PU9eaO3YPXvQ90uFJbVR0DjAGIjIxUb+YymUdiojJ27Fqee+4nTp2KI3eo8F7Tn+hQfBkE5YTag23aSGM84MlzBFdrP5B0lK5i7vcuCgUqAovd53QLATNFpJmqRnkxl/EBO3Yco2vXWSxevAeA5rceY1STTyiS+xSEN4GGoyFXCWdDGpNJeLMQrAHKiEhJXAWgNa5RTAFQ1RNA/ovLIrIY1y2qVgTMFS1d+ieLF++hYF5hRLPvaFlhHRKSB+p8ChGP2iBxxqSC1wqBqsaLSE9gLhAITFDVLSIyEIhS1Zne2rfxTcePnyMsLBiADs1COPzwH3QuN418Oc5C2ZZQbwTkuM7hlMZkPqKa8il3cZ23aQvcoKoD3fMRFFLV1ekR8FKRkZEaFWWdBn9y/nw8b765lKFDVxG1uhNljk+AFa9CwnnIUQjqj4QyDzgd05gMTUTWqmqyz2p50iMYBSTiuktoIHAK+Ba4Nc0SGnMZK1dG07nzTLZuPQzA3Dcep8xN01wrK3SEOu9BcB4HExqT+XlSCG5X1ZtFZD2AqsaISFYv5zJ+7vTpOF5+eRFDh65EFcoUU8bf9xm1wne7LgI3HAPhjZyOaYxP8KQQXHA/Jazwz3wEiV5NZfzaqlXRtGkzjV27YggMFJ5ttIUBtacREpQAVZ+Emm9C1pxOxzTGZ3hSCIYD04GCIvIG0BLo79VUxq+FhQWzf/9JbioVz/imE7il2F+Q50bXIHFFazgdzxif48kw1JNEZC1QH9dUlfep6javJzN+ZdmyvdSocT0iwo3BG1j41ExuzbuWoCwCt74I1V6BLMFOxzTGJ3kyZ3Fx4AwwK+l7qrrXm8GMfzh06DS9es3hq6+28Om4RrQrNg42T6B6AaBAFWg8Aa6r6nRMY3yaJ6eGfsB1fUCAYKAksB2o4MVcxsepKpMmbaJ37x85duws2UMCiFv0ElRdBIHZoNoAiHwWAoOcjmqMz/Pk1FClpMvugeJ6eC2R8Xl7956ge/fvmTNnBwANq8Qy5u5xhOc9DkWqu4aKzlfO2ZDG+JFUP1msqutE5PYrtzTmv1atiqZBg8+JjY0jLFcAHzSbS/ubfkGy5oCaw6HqEzZInDHpzJNrBM8kWQwAbgb+8loi49OqVCnE9UWzUy7sICMbT6Bwrlgo0dD1XEDucKfjGeOXPOkRhCZ5HY/rmsG33oljfE18fCIjRqymXbubyJsnG9l+G8cvHQeRJ8sRyBYGdT6BCu1tkDhjHJRiIXA/SBaqqs+mUx7jQzZuPEinTjNZt+4AG1b/zsTmEyD6Z/JkAUrf7xojKGdhp2Ma4/cuWwhEJIt7BFF7gsekyrlz8QwatITBg38hPj6R4oWEh3O9DdHbIHtB9yBxLawXYEwGkVKPYDWu6wEbRGQm8A1w+uJKVZ3m5WwmE1q+fB+dO8/kt9+OIAI9G+7mzTunEBocBxHtoM77EJLP6ZjGmCQ8uUYQDBzFNfroxecJFLBCYP5lx45j1Kr1CYmJyo3FlfHNP6VGiT0Qej00/BhK3uV0RGNMMlIqBAXddwxt5v8F4CKbN9j8R+nSeen2SDHyHpvHyzW+JTgoHm7qAbXfhqyhV96AMcYRKRWCQCAn/y4AF1khMMTEnKVPn3l07FiFWncUgF/6M6rSUEQU8pSBRuOgWG2nYxpjriClQnBAVQemWxKTqUybto0nnpjNwYOxrF35Bxue/gg5uQsJCIDI56DaqxAU4nRMY4wHUioEdkuH+Y+DB2Pp2XM2337rGoC2ZoXzjLvnQ+TkUShQ2TU8RKFkZ8MzxmRQKRWC+umWwmR4qspnn23k6afnEhNzjpw5AhjcdAndb15AQFAQ3PE63Po8BNrkdcZkNpctBKp6LD2DmIzt+PFz9Okzj5iYczS5JZbRjcdSIu8JKHyHa8KYfBFORzTGXKVUDzpn/EdiopKYqGTJEkCesGA+fjmMM+tm8kilX5Cg7FDzA9fUkQGBTkc1xlwDKwQmWb/9doQuXWbSpElp+vcuBfO70yL+e6gMFK/vGiQu7AanYxpj0oAVAvMvFy4k8M47y3nttZ+Ji0tg/+79PBvSjGBOQNZccOd7UKmzDQ9hjA+xQmD+sX79ATp1msmGDQcB6Fx7D+80+JJgzkHp+6DeCAgt6mxIY0yas0JguHAhgQEDFjNkyC8kJCjhheIZ22wyDcruglwloN6HUKqp0zGNMV5ihcCQJUsAq1ZFk5io9K67kUENfiBnSCLc8gJUexmCcjgd0RjjRVYI/NSpU+c5dSqOIkVCkWPbGXf/VxyM2ES18GgoWgsafAT5Kzgd0xiTDqwQ+KG5c3fQrdv33FAyFwtf34VEDaFk4gVKls8PtW3GMGP8jRUCP3L06BmeeWYen322EYACspOjiyaQP8cFqNQFar1tcwUY44e8WghEpAkwDNdIpuNU9e1L1j8DdME1F/JhoJOq/unNTP5IVfn2W9cgcYcOnSY4ayIDG87n6doryXJdBWgwGopWdzqmMcYhXisE7vmORwINgWhgjYjMVNWtSZqtByJV9YyIPA4MAVp5K5M/UlXatp3GlCmbAahdah9jW86gbJFzUH0IVO0FgUEOpzTGOMmbPYLbgB2qugtARL4EmgP/FAJVXZSk/UrgES/m8UsiQkSxc4QGX2DI3XPpdsdaAsreB3WHQa7rnY5njMkAvFkIigL7kixHA7en0L4zMCe5FSLSDegGULx48bTK57N2745h164Y6lcPg2Uv8kL+8XR4NgfFiueF+rPghnucjmiMyUAyxMViEXkEiATuTG69qo4BxgBERkba7GiXkZCQyIgRq+nXbyEhWRPY2nccBbPsJSgoiGKNnoA7+kNQdqdjGmMyGG8Wgv1A0nMPxdzv/YuINABeAu5U1fNezOPTtm49TJcuM1mxIhqAZmU3EXD+CITfCQ1G2TDRxpjL8mYhWAOUEZGSuApAa6BN0gYiUhX4GGiiqoe8mMVnXbiQwODBv/D660uIi0ugSK5TfNTie5pFHoM7P4KIR+2ZAGNMirxWCFQ1XkR6AnNx3T46QVW3iMhAIEpVZwLvADmBb8T1w2qvqjbzViZf1KbNNKZOdV1/73r7Wt65dx65b28PNd+CkLwOpzPGZAZevUagqrOB2Ze890qS1w28uX+fd3IfvW+ZyYbFefi4xffUq54LGiyCItWcTmaMyUQyxMVi47mff97D4oW7GNB0PSwfQM2g02zrF0qW2q+5Zwuzv1JjTOrYT41M4uTJ87zwwk+MHr0WgLqHP6F2qdNQpgVZ6g6F0GLOBjTGZFpWCDKB2bP/4LFuM4neH0tQYAIv1V/CHZWzQKMf4Ia7nY5njMnkrBBkYEeOnOGpp35k0qRNANx2fTTjW8+mYrOOcHs/eybAGJMmrBBkYAP7TWfSpB2EBF1gUJOF9G6TjcCG8yFfeaejGWN8iBWCDEZVkfizsPJ1Xisxgr9vasKb96+n1IMDoPwj9kyAMSbNWSHIIFSVcePWMWHUIhZ1/Yjgs7vIEyx89U4+qLUGgvM4HdEY46OsEGQAO3ceo2vHb1i09CAAXy8Npd1dVaDhaCic0jh9xhhz7awQOCghIZFhHyynf/8FnD0PBXKc5sOWC3moZ3u42Z4JMMakD/tJ45AtWw7R6ZFJrN5wEoC2N//K0GeCyN/sOwgt6nA6Y4w/sULghLNHWf/pIFZvyEfR3Cf5uN0a7nm6H5Rs4nQyY4wfskKQjg4fiqXAoanw87O0LXiU4/dX49HH6pO73lwICnE6njHGT1khSAdnzlzglb4z+WjMRqJ6fUT5644ixevSs+MoyFfO6XjGGD9nhcDLFi3cRdcOU9i5L54AEZbsi6B8h65Qvq09E2CMyRCsEHjJiRPneP7pmYz5ZBsAlQr/zYSnjxHZ4yvIUcjhdMYY839WCLxg2bK9tG75Ofv/jicoMIGXm6zihUEPk/Wm9tYLMMZkOFYI0tqZIxTa0o+jR8O5o8RBxvU+QoUu422YaGNMhmWFIA2oKj/9tIuG4b8iC7pT+swhlj1ViiptniOwSjfrBRhjMjQrBNdo374TPP7YDH6Ys4fxD31Hp9sOwfV1uKXLBMhd0ul4xhhzRVYIrlJiojJ27Fqee3YOp2ITyR18jmxZA6DuMKjaEyTA6YjGGOMRKwRX4Y8/jtK18wx+XhoNwH0VtzGy+98UeXgy5C3rcDpjjEkdKwSptHz5PurXm8i584kUzBnLiBbzaNmjLRLZBwICnY5nTIZ34cIFoqOjOXfunNNRfFJwcDDFihUjKCjI489YIUiNC6eJPDmYMnkTqVrkAO93Pki+BydD/gpOJzMm04iOjiY0NJTw8HDEbqRIU6rK0aNHiY6OpmRJz69RWiG4gvPn43n33eU81jSe/Gu6kfX4Tn55MjuhdfrCbX0h0POqa4yBc+fOWRHwEhEhX758HD58OFWfs0KQgpUro+ncaQZbtx1l27Rf+aLNTihQmdAmn0LBKk7HMybTsiLgPVfzvbVCkIzTp+Po338hw4atQhXKFjjCY9XWwe0vQbVXIDCr0xGNMSbN2D2Ol1iwYBeVKo1i6NBVBEgifestZeOghdR66SuoOciKgDE+IDAwkCpVqlCxYkWaNm3K8ePH/1m3ZcsW6tWrx4033kiZMmV4/fXXUdV/1s+ZM4fIyEgiIiKoWrUqffr0ceAI0pYVgiR+//0oDRt+zu7dJ6hS5ACre43lrZerENwxCgrf5nQ8Y0waCQkJYcOGDWzevJm8efMycuRIAM6ePUuzZs3o27cv27dvZ+PGjSxfvpxRo0YBsHnzZnr27MkXX3zB1q1biYqKonTp0mmaLT4+Pk235wk7NXRRzB+U3d6P3jVPUSDnGZ5rfoCge76GYjWdTmaM73rPS9cK+uiV27hVq1aNX3/9FYDJkydTo0YNGjVqBED27NkZMWIEderU4YknnmDIkCG89NJLlCvnmkckMDCQxx9//D/bjI2N5cknnyQqKgoRYcCAAbRo0YKcOXMSGxsLwNSpU/n++++ZOHEiHTp0IDg4mPXr11OjRg2mTZvGhg0bCAsLA6BMmTIsW7aMgIAAunfvzt69ewEYOnQoNWrUuOpv00V+XQj+/juWXk/MoHu1tdQNGAGJ8XzQIhhufgru6A9BOZyOaIzxooSEBBYsWEDnzp0B12mhW2655V9tSpUqRWxsLCdPnmTz5s0enQp6/fXXyZ07N5s2bQIgJibmip+Jjo5m+fLlBAYGkpCQwPTp0+nYsSOrVq2iRIkSXHfddbRp04ann36amjVrsnfvXho3bsy2bduu4sj/zS8LgaryxcQ1PPXUHI6dhO0rzrH+mUSkYieo/pqNFGpMeknFb+5p6ezZs1SpUoX9+/dTvnx5GjZsmKbbnz9/Pl9++eU/y3ny5LniZx588EECA10PpbZq1YqBAwfSsWNHvvzyS1q1avXPdrdu3frPZ06ePElsbCw5c+a8prxevUYgIk1EZLuI7BCRvsmszyYiX7nXrxKRcG/mAdi75yj31B5Mu06uItCo7A5mvPI30n4jNLbhoo3xBxevEfz555+o6j/XCCIiIli7du2/2u7atYucOXOSK1cuKlSo8J/1qZH01s5Ln6zOkeP/ZyCqVavGjh07OHz4MDNmzOCBBx4AIDExkZUrV7JhwwY2bNjA/v37r7kIgBcLgYgEAiOBu4AI4GERibikWWcgRlVLAx8Ag72VJzE+nlH9P6RCuQ+Ys+w8eULOMrHbBn78qTPhj02D/BW9tWtjTAaVPXt2hg8fznvvvUd8fDxt27Zl2bJlzJ8/H3D1HHr16sXzzz8PwHPPPcebb77J77//Drh+MI8ePfo/223YsOE/xQX+f2rouuuuY9u2bSQmJjJ9+vTL5hIR7r//fp555hnKly9Pvnz5AGjUqBEffvjhP+02bNhwbd8AN2/2CG4DdqjqLlWNA74Eml/Spjnwqfv1VKC+eONJk8R4Tkx+gNeG7SX2fBAtbvmTrfMq0370NKR4nTTfnTEm86hatSqVK1dmypQphISE8N133zFo0CBuvPFGKlWqxK233krPnj0BqFy5MkOHDuXhhx+mfPnyVKxYkV27dv1nm/379ycmJoaKFSty0003sWjRIgDefvtt7r33XqpXr07hwoVTzNWqVSu++OKLf04LAQwfPpyoqCgqV65MREREskXoakjS+2PTkoi0BJqoahf38qPA7araM0mbze420e7lne42Ry7ZVjegG0Dx4sVv+fPPP1MfaOmLzJr0E3HlOtPiyS42NIQxDtm2bRvly5d3OoZPS+57LCJrVTUyufaZ4mKxqo4BxgBERkZeXeWq+QZNq/SE0KJpGc0YYzI9b54a2g9cn2S5mPu9ZNuISBYgN3DUK2kkwIqAMcYkw5uFYA1QRkRKikhWoDUw85I2M4H27tctgYXqrXNVxpgMw/6be8/VfG+9VghUNR7oCcwFtgFfq+oWERkoIs3czcYD+URkB/AM8J9bTI0xviU4OJijR49aMfCCi/MRBAcHp+pzXrtY7C2RkZEaFRXldAxjzFWyGcq863IzlGX6i8XGGN8RFBSUqtmzjPfZ6KPGGOPnrBAYY4yfs0JgjDF+LtNdLBaRw8BVPFoMQH7gyBVb+RY7Zv9gx+wfruWYS6hqgeRWZLpCcC1EJOpyV819lR2zf7Bj9g/eOmY7NWSMMX7OCoExxvg5fysEY5wO4AA7Zv9gx+wfvHLMfnWNwBhjzH/5W4/AGGPMJawQGGOMn/PJQiAiTURku4jsEJH/jGgqItlE5Cv3+lUiEu5AzDTlwTE/IyJbReRXEVkgIiWcyJmWrnTMSdq1EBEVkUx/q6EnxywiD7n/rreIyOT0zpjWPPi3XVxEFonIeve/77udyJlWRGSCiBxyz+CY3HoRkeHu78evInLzNe9UVX3qCwgEdgI3AFmBjUDEJW16AKPdr1sDXzmdOx2OuS6Q3f36cX84Zne7UGAJsBKIdDp3Ovw9lwHWA3ncywWdzp0OxzwGeNz9OgLY43Tuazzm2sDNwObLrL8bmAMIcAew6lr36Ys9gtuAHaq6S1XjgC+B5pe0aQ586n49FagvIpKOGdPaFY9ZVRep6hn34kpcM8ZlZp78PQO8DgwGfGHMY0+OuSswUlVjAFT1UDpnTGueHLMCudyvcwN/pWO+NKeqS4BjKTRpDnymLiuBMBEpfC379MVCUBTYl2Q52v1esm3UNYHOCSBfuqTzDk+OOanOuH6jyMyueMzuLvP1qvpDegbzIk/+nssCZUXkFxFZKSJN0i2dd3hyzK8Cj4hINDAbeDJ9ojkmtf/fr8jmI/AzIvIIEAnc6XQWbxKRAOB9oIPDUdJbFlynh+rg6vUtEZFKqnrcyVBe9jAwUVXfE5FqwOciUlFVE50Olln4Yo9gP3B9kuVi7veSbSMiWXB1J4+mSzrv8OSYEZEGwEtAM1U9n07ZvOVKxxwKVAQWi8geXOdSZ2byC8ae/D1HAzNV9YKq7gZ+x1UYMitPjrkz8DWAqq4AgnENzuarPPr/nhq+WAjWAGVEpKSIZMV1MXjmJW1mAu3dr1sCC9V9FSaTuuIxi0hV4GNcRSCznzeGKxyzqp5Q1fyqGq6q4biuizRT1cw8z6kn/7Zn4OoNICL5cZ0q2pWOGdOaJ8e8F6gPICLlcRWCw+maMn3NBNq57x66AzihqgeuZYM+d2pIVeNFpCcwF9cdBxNUdYuIDASiVHUmMB5X93EHrosyrZ1LfO08POZ3gJzAN+7r4ntVtZljoa+Rh8fsUzw85rlAIxHZCiQAz6lqpu3tenjMfYCxIvI0rgvHHTLzL3YiMgVXMc/vvu4xAAgCUNXRuK6D3A3sAM4AHa95n5n4+2WMMSYN+OKpIWOMMalghcAYY/ycFQJjjPFzVgiMMcbPWSEwxhg/Z4XAZEgikiAiG5J8hafQNjYN9jdRRHa797XO/YRqarcxTkQi3K/7XbJu+bVmdG/n4vdls4jMEpGwK7SvktlH4zTeZ7ePmgxJRGJVNWdat01hGxOB71V1qog0At5V1crXsL1rznSl7YrIp8DvqvpGCu074Bp1tWdaZzG+w3oEJlMQkZzueRTWicgmEfnPSKMiUlhEliT5jbmW+/1GIrLC/dlvRORKP6CXAKXdn33Gva3NIvKU+70cIvKDiGx0v9/K/f5iEYkUkbeBEHeOSe51se4/vxSRe5JknigiLUUkUETeEZE17jHmH/Pg27IC92BjInKb+xjXi8hyEbnR/STuQKCVO0srd/YJIrLa3Ta5EVuNv3F67G37sq/kvnA9FbvB/TUd11Pwudzr8uN6qvJijzbW/Wcf4CX360Bc4w3lx/WDPYf7/ReAV5LZ30Sgpfv1g8Aq4BZgE5AD11PZW4CqQAtgbJLP5nb/uRj3nAcXMyVpczHj/cCn7tdZcY0iGQJ0A/q7388GRAElk8kZm+T4vgGauJdzAVncrxsA37pfdwBGJPn8m8Aj7tdhuMYiyuH037d9Ofvlc0NMGJ9xVlWrXFwQkSDgTRGpDSTi+k34OuBgks+sASa4285Q1Q0icieuyUp+cQ+tkRXXb9LJeUdE+uMap6YzrvFrpqvqaXeGaUAt4EfgPREZjOt00tJUHNccYJiIZAOaAEtU9az7dFRlEWnpbpcb12Bxuy/5fIiIbHAf/zbgpyTtPxWRMriGWQi6zP4bAc1E5Fn3cjBQ3L0t46esEJjMoi1QALhFVS+Ia0TR4KQNVHWJu1DcA0wUkfeBGOAnVX3Yg308p6pTLy6ISP3kGqnq7+Ka6+BuYJCILFDVgZ4chKqeE5HFQGOgFa6JVsA129STqjr3Cps4q6pVRCQ7rvF3ngCG45qAZ5Gq3u++sL74Mp8XoIWqbvckr/EPdo3AZBa5gUPuIlAX+M+cy+Kah/lvVR0LjMM13d9KoIaIXDznn0NEynq4z6XAfSKSXURy4Dqts1REigBnVPULXIP5JTdn7AV3zyQ5X+EaKOxi7wJcP9Qfv/gZESnr3mey1DXbXC+gj/x/KPWLQxF3SNL0FK5TZBfNBZ4Ud/dIXKPSGj9nhcBkFpOASBHZBLQDfkumTR1go4isx/Xb9jBVPYzrB+MUEfkV12mhcp7sUFXX4bp2sBrXNYNxqroeqASsdp+iGQAMSubjY4BfL14svsQ8XBMDzVfX9IvgKlxbgXXimrT8Y67QY3dn+RXXxCxDgLfcx570c4uAiIsXi3H1HILc2ba4l42fs9tHjTHGz1mPwBhj/JwVAmOM8XNWCIwxxs9ZITDGGD9nhcAYY/ycFQJjjPFzVgiMMcbP/Q/zEnodPEAubwAAAABJRU5ErkJggg==", 189 | "text/plain": [ 190 | "
" 191 | ] 192 | }, 193 | "metadata": { 194 | "needs_background": "light" 195 | }, 196 | "output_type": "display_data" 197 | } 198 | ], 199 | "source": [ 200 | "# 绘制ROC曲线\n", 201 | "fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred_proba)\n", 202 | "plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve')\n", 203 | "plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n", 204 | "plt.xlabel('False Positive Rate')\n", 205 | "plt.ylabel('True Positive Rate')\n", 206 | "plt.title('ROC Curve')\n", 207 | "plt.legend(loc=\"lower right\")\n", 208 | "plt.show()" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 63, 214 | "metadata": {}, 215 | "outputs": [], 216 | "source": [ 217 | "# 绘制混淆矩阵\n", 218 | "y_pred = np.where(y_pred > 0.5, 1, 0)\n", 219 | "# 只截取最后943个样本的预测结果\n", 220 | "y_pred = y_pred[-943:]\n", 221 | "y_test = y_test[-943:]\n", 222 | "cm = metrics.confusion_matrix(y_test, y_pred)" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 64, 228 | "metadata": {}, 229 | "outputs": [ 230 | { 231 | "data": { 232 | "text/plain": [ 233 | "array([[655, 181],\n", 234 | " [ 89, 18]])" 235 | ] 236 | }, 237 | "execution_count": 64, 238 | "metadata": {}, 239 | "output_type": "execute_result" 240 | } 241 | ], 242 | "source": [ 243 | "cm" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": 65, 249 | "metadata": {}, 250 | "outputs": [ 251 | { 252 | "data": { 253 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHhCAYAAACSp58BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfxklEQVR4nO3dedRdZX0v8O8viYATo4IIKBYRpXVCRajeOqBWrC3WpRaxNVraXCeqy6o41VkXWufholFQHAqi4gWt1gGxjkwXEBSqRCoSFoOMggiY5Ll/vNvet9xMhJyc933255O1V/Z+9n7Pfg4rWfnx/e3nnGqtBQCgFwumPQEAgI1JcQMAdEVxAwB0RXEDAHRFcQMAdEVxAwB0ZdG0J7A2t3/wi6xThyn45rFvnvYUYJQesfs2tSnvN6l/Z3975gc36fu4JckNANCVOZ3cAAATVH1mHIobABirmmr3aGL6LNkAgNGS3ADAWHXalurzXQEAoyW5AYCx8swNAMDcJ7kBgLHq9JkbxQ0AjJW2FADA3Ce5AYCx6rQt1ee7AgBGS3IDAGPV6TM3ihsAGCttKQCAuU9yAwBj1WlbSnIDAHRFcgMAY9XpMzeKGwAYK20pAIC5T3IDAGPVaVuqz3cFAIyW5AYAxqrT5EZxAwBjtcADxQAAc57kBgDGqtO2VJ/vCgAYLckNAIyVD/EDAJj7JDcAMFadPnOjuAGAsdKWAgCY+xQ3ADBWtWAy27puW7V1VX2+qv6jqs6rqn2ratuq+kZVnT/8vs1wbVXV+6tqWVWdXVV7rev1FTcAwKb2viT/1lq7b5IHJjkvySuTnNha2z3JicNxkuyfZPdhW5Lk8HW9uOIGAMaqajLbWm9ZWyX5kyRHJElr7ebW2jVJDkhy1HDZUUmeMuwfkOSTbcbJSbauqh3Xdg/FDQCM1XTaUvdK8qskH6+qM6vqY1V1xyQ7tNYuGa65NMkOw/5OSS6a9fPLh7E1UtwAABtVVS2pqtNnbUtmnV6UZK8kh7fWHpzkN/l/LagkSWutJWkben9LwQFgrCa0FLy1tjTJ0jWcXp5keWvtlOH485kpbi6rqh1ba5cMbafLh/MXJ9ll1s/vPIytkeQGANhkWmuXJrmoqvYYhvZLcm6SE5IsHsYWJzl+2D8hybOHVVP7JLl2VvtqtSQ3ADBW0/uE4kOSfKaqNktyQZLnZiZwObaqDk5yYZJnDNd+JcmTkixLcsNw7VopbgBgrKb0CcWttbOSPHQ1p/ZbzbUtyQtvzetrSwEAXZHcAMBYdfrFmX2+KwBgtCQ3ADBWkhsAgLlPcgMAYzWl1VKTprgBgLHSlgIAmPskNwAwVp22pSQ3AEBXJDcAMFadPnOjuAGAsdKWAgCY+yQ3ADBSJbkBAJj7JDcAMFK9JjeKGwAYqz5rG20pAKAvkhsAGKle21KSGwCgK5IbABgpyQ0AwDwguQGAkeo1uVHcAMBI9VrcaEsBAF2R3ADAWPUZ3EhuAIC+SG4AYKR6feZGcQMAI9VrcaMtBQB0RXIDACMluQEAmAckNwAwUr0mN4obABirPmsbbSkAoC+SGwAYqV7bUpIbAKArkhsAGCnJDQDAPCC5AYCR6jW5UdwAwFj1WdtoSwEAfZHcAMBI9dqWktwAAF2R3ADASPWa3ChuAGCkei1utKUAgK5IbgBgpCQ3AADzgOQGAMaqz+BGcQMAY6UtBQAwD0huAGCkJDcAAPOA5AYARkpyAwAwD0huAGCs+gxuFDcAMFbaUgAA84DiBgBGqqomsq3HfX9RVedU1VlVdfowtm1VfaOqzh9+32YYr6p6f1Utq6qzq2qvdb2+4gYAmIbHtNYe1Fp76HD8yiQnttZ2T3LicJwk+yfZfdiWJDl8XS/smRs2yFZ3un0Of/1B2XO3HdNa8rw3fiaP2/d++dun/nF+dfX1SZLXf/CEfO175+YeO26bs457bX524eVJklPP+UX+4a3HTHP6MG8d+d635EenfT9bbrVN3vy//iVJ8ssLfpZPfujt+d3NN2fBwoX5m+e/PH+wxx/mkot+kSPf+5Zc+POf5qnPfl6e+NRnTXn2zDVz7JmbA5I8etg/Ksm3kxw6jH+ytdaSnFxVW1fVjq21S9b0QoobNsg7X/G0fP0H5+aglx+R2y1amDtssVket+/98oFPn5T3furE/+/6C5ZfkX0OPGwKM4W+POJxf5b9nvy0fOzdb/qvsc99/IP5i2cenAc89I9z9mk/yOc+/sEcetjhueOdt8xB//OlOePkf5/ijJnLpljctCRfr6qW5COttaVJdphVsFyaZIdhf6ckF8362eXD2BqLG20pbrUt77RFHrnXbvnEF3+YJPndipW59vrfTnlWMA57/NGDc8c7b3mL0cqNN/wmSXLDDddn6+3umiTZcuttc6/77JmFC/1/LJtWVS2pqtNnbUtucckjW2t7Zabl9MKq+pPZJ4eUpm3o/Sf2J76q7puZKGmnYejiJCe01s6b1D3ZNHa9+3a54urrs/SNf53732ennHneRXnZOz6fJHnegX+Sg568d84495d55buPyzXXzRQ9u+60XX549KG57jc35o0f+nK+f+bPp/kWoCvPXPKSvPt1L8lnj/xA2qqWV79z6bSnxHwxoeBmSGLW+AextXbx8PvlVfXFJHsnuez37aaq2jHJ5cPlFyfZZdaP7zyMrdFEkpuqOjTJMZn5z3bqsFWSo6vqlWv7Wea+RYsW5kH33SUf/dx3s+8z354bfntTXva3j89HP/fd7Pnnb8jDDzwsl17x6xz20qcmSS694te5z/6vy77PfHsOfddx+cTbnpM733GLKb8L6MdJXzkuB/7di/OuT5yQA//+xfn4+9467SnBGlXVHavqzr/fT/KEJD9OckKSxcNli5McP+yfkOTZw6qpfZJcu7bnbZLJtaUOTvKw1tphrbVPD9thmanMDl7bD86OslZc8ZMJTY/b4uLLrs7Fl1+T0358YZLki988Kw+67y65/KrrsmpVS2stRx73/Tz0j+6ZJLn5dyty1bUzkfmZ512UC5Zfkd3vuf3U5g+9+cGJX8lD/vgxSZKHPXK//OfPzp3yjJgvprQUfIck36uqH2Um/PjX1tq/JTksyeOr6vwkjxuOk+QrSS5IsizJR5O8YF03mFRbalWSuye58BbjOw7n1mh2lHX7B79og/ttTM5lV16X5Zdend3vuX3Ov/DyPHrvPfIfF1yau91ly1x6xa+TJAc89oE59+czhfVdtrlTrrr2N1m1qmXXnbbLve9x1/zn8ium+RagK1tve5f89Jwzct8HPCTn/ej07HD3Xdb9Q5DpPFDcWrsgyQNXM35lkv1WM96SvPDW3GNSxc1Lkpw4VF+/f8L5HknuneRFE7onm9BL3/65fPxtz8lmixbmFxdfkSWv/3Te9Yqn5wF77JzWWi685Koc8pajkySP3Ove+afn/1l+t2JlVq1qOeStx+TqX98w5XcA89OH3/FP+ek5Z+T6X1+Tf1z85zngWX+fxYe8KkcvfU9WrlyZ2222WRYf8qokybVXX5k3veQ5+e0Nv0ktWJBvHH9M3nL4Mbn9He445XcBk1UzBdEEXrhqQWbaULMfKD6ttbZyfV9DcgPT8c1j3zztKcAoPWL3bTZplHLvl311Iv/OLnvn/lP9AJ2JrZZqra1KcvKkXh8AYHV8+AEAjNQc+4TijcaH+AEAXZHcAMBIdRrcKG4AYKy0pQAA5gHJDQCMVKfBjeQGAOiL5AYARmrBgj6jG8UNAIyUthQAwDwguQGAkbIUHABgHpDcAMBIdRrcKG4AYKy0pQAA5gHJDQCMlOQGAGAekNwAwEh1GtxIbgCAvkhuAGCken3mRnEDACPVaW2jLQUA9EVyAwAj1WtbSnIDAHRFcgMAI9VpcKO4AYCx0pYCAJgHJDcAMFKdBjeSGwCgL5IbABipXp+5UdwAwEh1WttoSwEAfZHcAMBI9dqWktwAAF2R3ADASHUa3EhuAIC+SG4AYKR6feZGcQMAI9VpbaMtBQD0RXIDACPVa1tKcgMAdEVyAwAj1Wtyo7gBgJHqtLbRlgIA+iK5AYCR6rUtJbkBALoiuQGAkeo0uFHcAMBYaUsBAMwDkhsAGKlOgxvJDQDQF8kNAIzUgk6jG8UNAIxUp7WNthQA0BfJDQCMlKXgAADzgOIGAEZqQU1mWx9VtbCqzqyqLw/H96qqU6pqWVV9tqo2G8Y3H46XDed3Xef7ug3/TQAANtSLk5w36/jtSd7TWrt3kquTHDyMH5zk6mH8PcN1a6W4AYCRqqqJbOtx352T/FmSjw3HleSxST4/XHJUkqcM+wcMxxnO71fruIkHigFgpKb4PPF7k7wiyZ2H4+2SXNNaWzEcL0+y07C/U5KLkqS1tqKqrh2uv2JNLy65AQA2qqpaUlWnz9qWzDr35CSXt9b+z6TuL7kBgJGqTCa6aa0tTbJ0DacfkeQvqupJSbZIsmWS9yXZuqoWDenNzkkuHq6/OMkuSZZX1aIkWyW5cm33l9wAAJtMa+1VrbWdW2u7Jjkwybdaa89KclKSpw2XLU5y/LB/wnCc4fy3WmttbfeQ3ADASK3vsu1N5NAkx1TVW5KcmeSIYfyIJJ+qqmVJrspMQbRWihsAGKlpf0Jxa+3bSb497F+QZO/VXHNjkqffmtfVlgIAuiK5AYCR6vSrpSQ3AEBfJDcAMFILOo1uFDcAMFKd1jbaUgBAXyQ3ADBS014KPimSGwCgK5IbABipToMbyQ0A0BfJDQCMlKXgAEBX+ixttKUAgM5IbgBgpCwFBwCYByQ3ADBSC/oMbhQ3ADBW2lIAAPOA5AYARqrT4EZyAwD0RXIDACPV6zM3ihsAGKleV0tpSwEAXZHcAMBI9dqWktwAAF1ZZ3FTVTtU1RFV9dXheM+qOnjyUwMAJqkmtE3b+iQ3n0jytSR3H45/luQlE5oPAMBtsj7FzV1aa8cmWZUkrbUVSVZOdFYAwMQtqJrINm3r80Dxb6pquyQtSapqnyTXTnRWAMDEzYE6ZCLWp7h5aZITkuxWVd9PctckT5vorAAANtA6i5vW2hlV9agke2TmOaGfttZ+N/GZAQAT1etS8HUWN1X17FsM7VVVaa19ckJzAgDYYOvTlnrYrP0tkuyX5IwkihsAmMc6DW7Wqy11yOzjqto6yTGTmhAAsGnMhZVNk7Ahn1D8myT32tgTAQDYGNbnmZsvZVgGnpliaM8kx05yUgDA5HUa3KzXMzfvnLW/IsmFrbXlE5oPAMBtstbipqoWJnlDa+0xm2g+AMAmMsql4K21lVW1qqq2aq1t8k8l/umJ79rUtwSSbL/l5tOeArAJbMiDt/PB+rSlrk9yTlV9IzMPEydJWmv/MLFZAQBsoPUpbo4bttna6i4EAOaPUbalBlu31t43e6CqXjyh+QAA3Cbr025bvJqx52zkeQAAm9iCmsw2bWtMbqrqmUkOSnKvqjph1qk7J7lq0hMDANgQa2tL/SDJJUnukmT2sqXrkpw9yUkBAJM3F1KWSVhjcdNauzDJhUn2XdsLVNUPW2trvQYAmHt6faB4Yyxx32IjvAYAwEaxPqul1sWycACYh3ptS/X64YQAwEits7ipqkOqapu1XbIR5wMAbCJVk9mmbX3aUjskOa2qzkhyZJKvtdZmt6L+ZiIzAwAmasFcqEQmYJ3JTWvttUl2T3JEZj687/yqeltV7Tac//FEZwgAcCus1zM3Q1Jz6bCtSLJNks9X1TsmODcAYIIWTGibtnW2pYbvkXp2kiuSfCzJy1trv6uqBUnOT/KKyU4RAGD9rc8zN9smeerwoX7/pbW2qqqePJlpAQCT1ukjN+sublprr1/LufM27nQAgE1ltA8UAwDMJxvjE4oBgHmo0+BGcgMA9EVyAwAj5bulAABuo6raoqpOraofVdVPquqNw/i9quqUqlpWVZ+tqs2G8c2H42XD+V3XdQ/FDQCM1IKqiWzrcFOSx7bWHpjkQUmeWFX7JHl7kve01u6d5OokBw/XH5zk6mH8PcN1a39fG/afAwCY76bxxZltxvXD4e2GrSV5bJLPD+NHJXnKsH/AcJzh/H5Va7+L4gYA2KiqaklVnT5rW3KL8wur6qwklyf5RpKfJ7mmtbZiuGR5kp2G/Z2SXJQkw/lrk2y3tvt7oBgARmpSDxS31pYmWbqW8yuTPKiqtk7yxST33Zj3l9wAAFPRWrsmyUlJ9k2ydVX9PnTZOcnFw/7FSXZJkuH8VkmuXNvrKm4AYKRqQr/Wes+quw6JTarq9kken+S8zBQ5TxsuW5zk+GH/hOE4w/lvtdba2u6hLQUAIzWlz7nZMclRVbUwMyHLsa21L1fVuUmOqaq3JDkzyRHD9Uck+VRVLUtyVZID13UDxQ0AsMm01s5O8uDVjF+QZO/VjN+Y5Om35h6KGwAYKZ9QDAAwD0huAGCk1vFZePOW4gYARkpbCgBgHpDcAMBIddqVktwAAH2R3ADASC3oNLqR3AAAXZHcAMBI9bpaSnEDACPVaVdKWwoA6IvkBgBGakH6jG4kNwBAVyQ3ADBSvT5zo7gBgJHqdbWUthQA0BXJDQCMlE8oBgCYByQ3ADBSnQY3ihsAGCttKQCAeUByAwAj1WlwI7kBAPoiuQGAkeo14ej1fQEAIyW5AYCRqk4fulHcAMBI9VnaaEsBAJ2R3ADASPkQPwCAeUByAwAj1Wduo7gBgNHqtCulLQUA9EVyAwAj1evn3EhuAICuSG4AYKR6TTgUNwAwUtpSAADzgOQGAEaqz9xGcgMAdEZyAwAj5ZkbAIB5QHIDACPVa8KhuAGAkdKWAgCYByQ3ADBSfeY2khsAoDOSGwAYqU4fuVHcAMBYLei0MaUtBQB0RXIDACPVa1tKcgMAdEVyAwAjVZ0+c6O4AYCR0pYCAJgHJDcAMFKWggMAzAOSGwAYKc/cAABdqZrMtvZ71i5VdVJVnVtVP6mqFw/j21bVN6rq/OH3bYbxqqr3V9Wyqjq7qvZa1/tS3AAAm9KKJP/YWtszyT5JXlhVeyZ5ZZITW2u7JzlxOE6S/ZPsPmxLkhy+rhsobgBgpGpCv9amtXZJa+2MYf+6JOcl2SnJAUmOGi47KslThv0DknyyzTg5ydZVtePa7qG4AQCmoqp2TfLgJKck2aG1dslw6tIkOwz7OyW5aNaPLR/G1sgDxQAwUgsm9EBxVS3JTAvp95a21pbe4po7JflCkpe01n5dsx7Waa21qmoben/FDQCwUQ2FzNI1na+q22WmsPlMa+24YfiyqtqxtXbJ0Ha6fBi/OMkus35852FsjbSlAGCkpvHMTc1ENEckOa+19u5Zp05IsnjYX5zk+Fnjzx5WTe2T5NpZ7avVktwAwEhN6XNuHpHkb5KcU1VnDWOvTnJYkmOr6uAkFyZ5xnDuK0melGRZkhuSPHddN1DcAACbTGvte8ka4539VnN9S/LCW3MPxQ0AjNS6WkjzlWduAICuSG4AYKQmtRR82hQ3ADBS2lIAAPOA5IaN4gtHfypf/dJxqUp23W33vPw1b85PzjkrSz/wrqxY8bvsvsee+cdXvzELF/kjBxvL6177qnzn37+dbbfdLscd/+UkyX+cd17e8qbX5+abbsrCRQvz6te+Ifd/wAOmPFPmqiktBZ84yQ232RWXX5b//bnP5ENHHp2PfuaLWbVyVb719a/kn9/82rzmze/IRz/zxWx/t7vn6185YdpTha4c8JSn5vCPfOy/jb3n3f+c573ghTn2uOPzghe9OO999z9PaXYwPYobNoqVK1fmpptuysoVK3LTjTdmi9vfPotud7vsfI9dkyQP2XuffPfb35zuJKEzD3now7LlVlv9t7FK5frrf5Mkuf6663LXu24/jakxT9SEtmnTI+A2u8v2O+RpBy3Os/7yCdl88y3ykL33zaP2+9N89EPvyU/P+0n2uN8f5jsnfSO/uuzSaU8VuveKV746z19ycN79zrdn1apV+eRnjpn2lJjDFnTal5LccJtd9+tf54ffPSmf+sJXc8yXvpkbb/xtTvzav+Y1b3pHPvy+d+RFf3tQ7nCHO2bBwoXTnip079jPHp2XH/qqfP3Ef8/LD31V3vBPr5n2lGCTm0pxU1Vr/F6IqlpSVadX1en/ctTH1nQZc8gZp52cu+24c7beZtssWnS7PPJR++Xcc87Knvd/YN7z4aPywSP/Jfd/0EOy8y73nPZUoXtfOv6L2e/xT0iSPOFP98+Pzzl7yjNiLuu1LTWt5OaNazrRWlvaWntoa+2hBy3+u005JzbQ9ne7W877ydm58cbfprWWM08/JffY9Q9y9VVXJkluvvnmfPZTR+bJf/n0Kc8U+nfX7bfP6aedmiQ59ZSTc4977jrdCcEUTOyZm6pa0/8uVJIdJnVfNr37/eED8j8e87i8YPFfZeGihdntPvfLkw54Wj7xkQ/k5O9/J62typ//5TPy4Ic+fNpTha4c+rKX5vTTTs0111ydxz/2T/L8Fx6S173hzXnHYW/LyhUrstnmm+d1b3jTtKfJXDYXYpYJqJkv25zAC1ddluRPk1x9y1NJftBau/u6XuOXV900mckBa7X9lptPewowSlss2rTlxsk/v2Yi/87us9vWUy2bJrla6stJ7tRaO+uWJ6rq2xO8LwCwHnr9+oWJFTettYPXcu6gSd0XAFg/na4EtxQcAOiLD/EDgJHqNLiR3AAAfZHcAMBYdRrdKG4AYKR6XS2lLQUAdEVyAwAjZSk4AMA8ILkBgJHqNLhR3ADAaHVa3WhLAQBdkdwAwEhZCg4AMA9IbgBgpCwFBwCYByQ3ADBSnQY3ihsAGK1OqxttKQCgK5IbABgpS8EBAOYByQ0AjFSvS8EVNwAwUp3WNtpSAEBfJDcAMFadRjeSGwCgK5IbABipXpeCK24AYKR6XS2lLQUAdEVyAwAj1WlwI7kBAPoiuQGAseo0upHcAABdkdwAwEhZCg4AdMVScACAeUByAwAj1WlwI7kBAPoiuQGAseo0ulHcAMBI9bpaSlsKAOiK5AYARspScACAeUByAwAj1WlwI7kBgNGqCW3rum3VkVV1eVX9eNbYtlX1jao6f/h9m2G8qur9VbWsqs6uqr3W9fqKGwBgU/tEkifeYuyVSU5sre2e5MThOEn2T7L7sC1Jcvi6XlxxAwAjVRP6tS6tte8kueoWwwckOWrYPyrJU2aNf7LNODnJ1lW149peX3EDAMwFO7TWLhn2L02yw7C/U5KLZl23fBhbI8UNAIxU1aS2WlJVp8/altyaebXWWpK2oe/LaikAYKNqrS1NsvRW/thlVbVja+2Soe10+TB+cZJdZl238zC2RpIbABipKS2WWpMTkiwe9hcnOX7W+LOHVVP7JLl2VvtqtSQ3ADBWU/qgm6o6Osmjk9ylqpYneX2Sw5IcW1UHJ7kwyTOGy7+S5ElJliW5Iclz1/n6M22tuemXV900dycHHdt+y82nPQUYpS0Wbdpy4xdX3jiRf2d33W6LqX4+oOQGAEbKt4IDAMwDkhsAGKlevxVccQMAI9VpbaMtBQD0RXIDACPVa1tKcgMAdEVyAwCj1Wd0o7gBgJHSlgIAmAckNwAwUp0GN5IbAKAvkhsAGCnP3AAAzAOSGwAYqV6/FVxxAwBj1Wdtoy0FAPRFcgMAI9VpcCO5AQD6IrkBgJHqdSm44gYARqrX1VLaUgBAVyQ3ADBWfQY3khsAoC+SGwAYqU6DG8UNAIxVr6ultKUAgK5IbgBgpCwFBwCYByQ3ADBSnrkBAJgHFDcAQFe0pQBgpLSlAADmAckNAIyUpeAAAPOA5AYARqrXZ24UNwAwUp3WNtpSAEBfJDcAMFadRjeSGwCgK5IbABipXpeCK24AYKR6XS2lLQUAdEVyAwAj1WlwI7kBAPoiuQGAseo0ulHcAMBI9bpaSlsKAOiK5AYARspScACAeaBaa9OeAx2qqiWttaXTngeMjb97ILlhcpZMewIwUv7uMXqKGwCgK4obAKArihsmRc8fpsPfPUbPA8UAQFckNwBAVxQ3bHRV9cSq+mlVLauqV057PjAGVXVkVV1eVT+e9lxg2hQ3bFRVtTDJh5Lsn2TPJM+sqj2nOysYhU8keeK0JwFzgeKGjW3vJMtaaxe01m5OckySA6Y8J+hea+07Sa6a9jxgLlDcsLHtlOSiWcfLhzEA2CQUNwBAVxQ3bGwXJ9ll1vHOwxgAbBKKGza205LsXlX3qqrNkhyY5IQpzwmAEVHcsFG11lYkeVGSryU5L8mxrbWfTHdW0L+qOjrJD5PsUVXLq+rgac8JpsUnFAMAXZHcAABdUdwAAF1R3AAAXVHcAABdUdwAAF1R3AAAXVHcABtFVT2nqj447XkAKG6AtaqqhdOeA8CtobiBzlTVm6rqJbOO31pVL17NdY+uqu9U1b9W1U+r6sNVtWA4d31VvauqfpRk36r666o6tarOqqqP/L7gqarnVtXPqurUJI/YRG8RYK0UN9CfI5M8O0mGYuXAJJ9ew7V7JzkkyZ5Jdkvy1GH8jklOaa09MMmVSf4qySNaaw9KsjLJs6pqxyRvzExR88jhNQCmbtG0JwBsXK21X1TVlVX14CQ7JDmztXblGi4/tbV2QfJf3030yCSfz0wB84Xhmv2SPCTJaVWVJLdPcnmShyf5dmvtV8PPfzbJfSbzrgDWn+IG+vSxJM9JcrfMJDlrcssvl/v98Y2ttZXDfiU5qrX2qtkXVtVTbvs0ATY+bSno0xeTPDHJwzLzDe1rsndV3WtoX/1Vku+t5poTkzytqrZPkqratqrumeSUJI+qqu2q6nZJnr5R3wHABpLcQIdaazdX1UlJrpmVwKzOaUk+mOTeSU7KTFF0y9c6t6pem+TrQxH0uyQvbK2dXFVvSPLDJNckOWujvgmADVSt3TKVBua7oQg5I8nTW2vnr+GaRyd5WWvtyZtwagATpy0FnamqPZMsS3LimgobgJ5JbqBzVXX/JJ+6xfBNrbWHT2M+AJOmuAEAuqItBQB0RXEDAHRFcQMAdEVxAwB0RXEDAHTl/wJOY0E9oqE6rgAAAABJRU5ErkJggg==", 254 | "text/plain": [ 255 | "
" 256 | ] 257 | }, 258 | "metadata": { 259 | "needs_background": "light" 260 | }, 261 | "output_type": "display_data" 262 | } 263 | ], 264 | "source": [ 265 | "import seaborn as sn\n", 266 | "plt.figure(figsize = (10,8))\n", 267 | "sn.heatmap(cm, annot=True, cmap='Blues', fmt='d')\n", 268 | "plt.xlabel('y_pred')\n", 269 | "plt.ylabel('y_true')\n", 270 | "plt.show()" 271 | ] 272 | } 273 | ], 274 | "metadata": { 275 | "kernelspec": { 276 | "display_name": "Python 3.8.0 64-bit", 277 | "language": "python", 278 | "name": "python3" 279 | }, 280 | "language_info": { 281 | "codemirror_mode": { 282 | "name": "ipython", 283 | "version": 3 284 | }, 285 | "file_extension": ".py", 286 | "mimetype": "text/x-python", 287 | "name": "python", 288 | "nbconvert_exporter": "python", 289 | "pygments_lexer": "ipython3", 290 | "version": "3.8.0" 291 | }, 292 | "orig_nbformat": 4, 293 | "vscode": { 294 | "interpreter": { 295 | "hash": "767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90" 296 | } 297 | } 298 | }, 299 | "nbformat": 4, 300 | "nbformat_minor": 2 301 | } 302 | --------------------------------------------------------------------------------