├── Models ├── __init__.py ├── utils │ ├── __init__.py │ ├── __pycache__ │ │ ├── layer.cpython-37.pyc │ │ └── __init__.cpython-37.pyc │ └── layer.py ├── Page │ ├── __init__.py │ ├── __pycache__ │ │ ├── PIN.cpython-37.pyc │ │ ├── RACP.cpython-37.pyc │ │ └── __init__.cpython-37.pyc │ └── RACP.py ├── __pycache__ │ ├── engine.cpython-37.pyc │ ├── __init__.cpython-37.pyc │ └── engine.cpython-37.pyc.140248188734896 └── engine.py ├── Runs ├── __init__.py ├── project_path.ini ├── __pycache__ │ ├── queue.cpython-37.pyc │ ├── __init__.cpython-37.pyc │ └── ctr_task.cpython-37.pyc ├── configurations │ └── racp.ini ├── run.py └── ctr_task.py ├── DatasetsLoad ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-37.pyc │ ├── dataset.cpython-37.pyc │ └── sampler.cpython-37.pyc ├── dataset.py └── sampler.py ├── ReadMe.md ├── DataPreprocess ├── first_step.py └── second_step.py └── Data └── sample.csv /Models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Runs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DatasetsLoad/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Models/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Models/Page/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['RACP'] 2 | -------------------------------------------------------------------------------- /Runs/project_path.ini: -------------------------------------------------------------------------------- 1 | [PathSettings] 2 | sys_path=./ -------------------------------------------------------------------------------- /Runs/__pycache__/queue.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Runs/__pycache__/queue.cpython-37.pyc -------------------------------------------------------------------------------- /Models/__pycache__/engine.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Models/__pycache__/engine.cpython-37.pyc -------------------------------------------------------------------------------- /Runs/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Runs/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Runs/__pycache__/ctr_task.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Runs/__pycache__/ctr_task.cpython-37.pyc -------------------------------------------------------------------------------- /Models/Page/__pycache__/PIN.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Models/Page/__pycache__/PIN.cpython-37.pyc -------------------------------------------------------------------------------- /Models/Page/__pycache__/RACP.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Models/Page/__pycache__/RACP.cpython-37.pyc -------------------------------------------------------------------------------- /Models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Models/utils/__pycache__/layer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Models/utils/__pycache__/layer.cpython-37.pyc -------------------------------------------------------------------------------- /DatasetsLoad/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/DatasetsLoad/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /DatasetsLoad/__pycache__/dataset.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/DatasetsLoad/__pycache__/dataset.cpython-37.pyc -------------------------------------------------------------------------------- /DatasetsLoad/__pycache__/sampler.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/DatasetsLoad/__pycache__/sampler.cpython-37.pyc -------------------------------------------------------------------------------- /Models/Page/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Models/Page/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Models/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Models/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Models/__pycache__/engine.cpython-37.pyc.140248188734896: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/racp-submission/racp/HEAD/Models/__pycache__/engine.cpython-37.pyc.140248188734896 -------------------------------------------------------------------------------- /Runs/configurations/racp.ini: -------------------------------------------------------------------------------- 1 | [DataSettings] 2 | data_name=Avito 3 | data_dir=Data/ 4 | dataset_type=Page 5 | 6 | [ModelSettings] 7 | model_name=RACP 8 | engine_type=CTR 9 | 10 | remove_nan=False 11 | embed_dim=10 12 | dnn_dim_list=[200, 80] 13 | mha_head_num=10 14 | page_layer=dynamic_page 15 | alpha_dim_list=[128] 16 | att_func=din_activate 17 | din_att_dim_list=[80, 40] 18 | 19 | 20 | [TrainSettings] 21 | optimizer=adam 22 | ; [rmsprop, adam, adagrad] 23 | criterion=bcelogits 24 | ; [mse, bce, bcelogits] 25 | train_batch_size=2048 26 | ; [128, 1024, 4096] 27 | test_batch_size=4096 28 | 29 | epoch=500 30 | start_epoch=0 31 | pre_train_epoch=0 32 | early_stop_step=5 33 | data_loader_workers=2 34 | 35 | learning_rate=1e-3 36 | weight_decay=1e-5 37 | device=cuda:2 38 | ; device=cpu 39 | 40 | 41 | [ResultSettings] 42 | save_dir=Results/ 43 | 44 | -------------------------------------------------------------------------------- /ReadMe.md: -------------------------------------------------------------------------------- 1 | # Recurrent Attention over Contextualized Page sequence for Click-Through Rate Prediction in Taobao Personalized Search Ranking 2 | ## Introduction 3 | Pipeline: 4 | 1. Prepare Data 5 | 2. Train Model 6 | 7 | ## Running 8 | 9 | We test our code on Python 3.7.5 and PyTorch 1.7.1. 10 | 11 | ### 1. Prepare Data 12 | 13 | ``` 14 | mkdir ./Data/raw/ 15 | mkdir ./Data/new/ 16 | mkdir ./Data/data/ 17 | cd ./Data/raw/ 18 | kaggle competitions download -c avito-context-ad-clicks 19 | py7zr -x *.7z 20 | cd ../../ 21 | python ./DataPreprocess/first_step.py 22 | python ./DataPreprocess/second_step.py 23 | ``` 24 | When you see the files below, you can do the next work. 25 | ``` 26 | ./Data/data/Train_data.csv 27 | ./Data/data/Val_data.csv 28 | ./Data/data/Test_data.csv 29 | ./Data/data/recent_history.pickle 30 | ``` 31 | ### 2. Train Model 32 | Firstly, set the `sys_path` in `./Runs/project_path.ini` as the absoluate path of current project. 33 | 34 | Seconly, run the command `python ./Runs/run.py --model_name racp`. 35 | 36 | -------------------------------------------------------------------------------- /Runs/run.py: -------------------------------------------------------------------------------- 1 | from configparser import ConfigParser 2 | cfg = ConfigParser() 3 | cfg.read('project_path.ini') 4 | sys_path = dict(cfg.items('PathSettings'))['sys_path'] 5 | 6 | import sys, os, argparse 7 | sys.path.append(sys_path) 8 | 9 | from Runs.ctr_task import Run 10 | cfg = ConfigParser() 11 | 12 | 13 | import warnings 14 | warnings.filterwarnings('ignore') 15 | 16 | if __name__ == '__main__': 17 | 18 | parser = argparse.ArgumentParser(description='Welcome to the Experiment Platform Entry') 19 | parser.add_argument('--data_name', nargs='?', help='data name') 20 | parser.add_argument('--model_name', nargs='?', help='model name') 21 | parser.add_argument('--timestamp', default=None, nargs='?', help='timestamp') 22 | parser.add_argument('--mode', default='train', help='train or test') 23 | 24 | args = parser.parse_args() 25 | data = args.data_name 26 | model = args.model_name 27 | timestamp = args.timestamp 28 | mode = args.mode 29 | 30 | 31 | # ======= get the running setting ======== 32 | config_path = sys_path+'Runs/configurations/' 33 | config_file = config_path + model+'.ini' 34 | if os.path.exists(config_file): 35 | cfg.read(config_file) # must set as absolute path 36 | else: 37 | raise FileNotFoundError(config_file +' Not Found !!!') 38 | 39 | print(model) 40 | # ======= run the main file ============ 41 | 42 | Run(DataSettings = dict(cfg.items("DataSettings")), 43 | ModelSettings = dict(cfg.items("ModelSettings")), 44 | TrainSettings = dict(cfg.items("TrainSettings")), 45 | ResultSettings = dict(cfg.items("ResultSettings")), 46 | mode=mode, 47 | timestamp=timestamp 48 | ) 49 | 50 | -------------------------------------------------------------------------------- /DatasetsLoad/dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np, pandas as pd 2 | import pickle 3 | from tqdm import tqdm 4 | import random 5 | import time 6 | 7 | import torch.utils.data 8 | 9 | class PageDataset(torch.utils.data.Dataset): 10 | 11 | def __init__(self, dataset): 12 | print("Init Page features") 13 | dataset = np.array(dataset) 14 | self.labels = dataset[:, 0] 15 | 16 | num_ad_fts = 34 17 | ad_size = 5 18 | page_size = 5 19 | num_user_fts = 5 20 | num_page_fts = num_ad_fts*ad_size+1 21 | index_target_search_id = 5 22 | page_begin = num_user_fts + num_ad_fts - 1 23 | 24 | features = dataset[:, 1:] 25 | del dataset 26 | print(features.shape) 27 | data_len = features.shape[0] 28 | 29 | other_sample_features = features[:, :page_begin] 30 | self.target_page_id = other_sample_features[:, index_target_search_id] 31 | other_sample_features = np.delete(other_sample_features, index_target_search_id, axis=1) # drop target search_id 32 | 33 | pages = features[:, page_begin:-1].reshape(-1, num_page_fts) # only keep page_seq and drop page_num 34 | page_num = features[:, -1] 35 | del features 36 | 37 | # count click ad nums for each sample 38 | click_indexs = [num_page_fts-1-1-i*num_ad_fts for i in range(ad_size)] 39 | page_click_num = sum([pages[:, click_index]==1 for click_index in click_indexs]) 40 | 41 | 42 | ads = pages[:, :-1].reshape(-1, num_ad_fts) # drop ad_num 43 | ad_num = pages[:, -1] 44 | pages = ads[:, 1:].reshape(pages.shape[0], -1) # drop search_id 45 | del ads 46 | features = np.concatenate(( # add ad_num and click num 47 | pages, 48 | ad_num.reshape(-1, 1), 49 | np.array(page_click_num).reshape(-1,1), 50 | ), axis=1).reshape(data_len, -1) # [B, fts_num] 51 | del pages 52 | self.features = np.concatenate(( 53 | other_sample_features, 54 | features, 55 | page_num.reshape(-1, 1) 56 | ), axis=1) 57 | del features 58 | print(self.features.shape) 59 | 60 | 61 | def __getitem__(self, index): 62 | return [self.labels[index], self.features[index], self.target_page_id[index]] 63 | 64 | def __len__(self): 65 | return len(self.labels) 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /DatasetsLoad/sampler.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pandas.io.parsers import ParserBase 3 | import tqdm 4 | import time 5 | import math 6 | import random 7 | import pickle 8 | import numpy as np, pandas as pd 9 | import scipy.sparse as sp 10 | from numpy.random import choice 11 | from copy import deepcopy 12 | from collections import defaultdict 13 | from collections import OrderedDict 14 | 15 | import torch 16 | from torch.utils.data import DataLoader 17 | 18 | from DatasetsLoad.dataset import * 19 | 20 | 21 | class SampleGenerator(object): 22 | 23 | def __init__(self, DataSettings, TrainSettings, mode): 24 | self.mode = mode 25 | # data 26 | self.DataSettings = DataSettings 27 | 28 | # data path 29 | data_name = DataSettings['data_name'] 30 | self.data_dir = DataSettings['data_dir'] 31 | self.dataset_type = DataSettings['dataset_type'] 32 | 33 | 34 | # read data 35 | start_time = time.time() 36 | print("=========== reading ", data_name, " data ===========") 37 | print(self.data_dir) 38 | self._get_main_data(TrainSettings) 39 | self._init_data_statistic() 40 | 41 | print(f'read data file cost {time.time()-start_time} seconds\n') 42 | 43 | 44 | def _get_main_data(self, TrainSettings): 45 | dataset = eval(self.dataset_type+'Dataset') 46 | 47 | self.Train_data, self.Val_data, self.Test_data = None, None, None 48 | if self.dataset_type in ['Page']: 49 | if self.mode == 'train': 50 | self.Train_data = dataset(pd.read_csv(self.data_dir+'Train_data.csv')) 51 | self.Val_data = dataset(pd.read_csv(self.data_dir+'Val_data.csv')) 52 | self.Test_data = dataset(pd.read_csv(self.data_dir+'Test_data.csv')) 53 | else: 54 | raise ValueError('unknow dataset type: ', self.dataset_type) 55 | 56 | if self.mode == 'train': 57 | print('Train size: ', len(self.Train_data)) 58 | print('Val size: ', len(self.Val_data)) 59 | print('Test size: ', len(self.Test_data)) 60 | 61 | # label(1), user(5), target(32), page1(166)*5, page_num(1) 62 | 63 | data_loader_workers = eval(TrainSettings['data_loader_workers']) 64 | batch_size_setting = {'Train':'train', 'Val':'test', 'Test':'test'} 65 | 66 | if self.mode == 'train': 67 | self.Train_data, self.Val_data, self.Test_data = [ 68 | DataLoader( 69 | getattr(self, data_type+'_data'), 70 | batch_size = eval(TrainSettings[batch_size_setting[data_type] + '_batch_size']), 71 | shuffle=True, 72 | num_workers=data_loader_workers 73 | ) 74 | for data_type in ['Train', 'Val', 'Test'] 75 | ] 76 | else: 77 | data_type = 'Test' 78 | self.Test_data = DataLoader( 79 | getattr(self, data_type+'_data'), 80 | batch_size = eval(TrainSettings[batch_size_setting[data_type] + '_batch_size']), 81 | shuffle=True, 82 | num_workers=data_loader_workers 83 | ) 84 | 85 | 86 | def _init_data_statistic(self): 87 | num_features_dict = OrderedDict() 88 | plus_one = lambda o: [ x+1 for x in o ] 89 | 90 | num_features_dict['user'] = plus_one([4339861, 64017, 51, 4213, 89]) 91 | # (user_id, user_agent_id, user_agent_os_id, user_device_id, user_agent_family_id) 92 | 93 | num_features_dict['ad'] = plus_one([1, 2266704, 1, 300] + [36893298, 8, 10, 2]) 94 | # search_id, ip_id, is_user_logged_on, timestamp + ad_id, position, hist_ctr_bin, is_click 95 | 96 | num_features_dict['page_click_num'] = 5 + 1 97 | 98 | num_features_dict['location'] = plus_one([4080, 3, 84, 3723]) 99 | # LocationID, Level, RegionID, CityID 100 | 101 | num_features_dict['category'] = plus_one([68, 3, 12, 57]) 102 | # CategoryID, Level, ParentCategoryID, SubcategoryID 103 | 104 | num_features_dict['ad_title'] = 2943212 + 1 105 | num_features_dict['ad_params'] = 5712 + 1 106 | num_features_dict['search_query'] = 33732 + 1 107 | num_features_dict['search_params'] = 1192 + 1 108 | 109 | num_multi_fts = OrderedDict() 110 | num_multi_fts['search_query'] = 1 111 | num_multi_fts['search_params'] = 3 112 | num_multi_fts['ad_title'] = 5 113 | num_multi_fts['ad_params'] = 5 114 | num_features_dict['multi'] = num_multi_fts 115 | 116 | self.num_features_dict = num_features_dict 117 | 118 | def _sample_negative(self, total_set, pos_set): 119 | j = random.choice(total_set) 120 | while(j in pos_set): 121 | j = random.choice(total_set) 122 | return j -------------------------------------------------------------------------------- /Models/engine.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np, pandas as pd 3 | import math 4 | import random 5 | import time 6 | import tqdm 7 | import pickle 8 | import torch 9 | import torch.nn as nn 10 | from torch.autograd import Variable 11 | import torch.nn.functional as F 12 | 13 | from sklearn import metrics 14 | 15 | 16 | 17 | class Engine(object): 18 | def __init__(self, TrainSettings): 19 | self.TrainSettings = TrainSettings 20 | # self._writer = SummaryWriter(logdir=ResultSettings['save_dir'], comment=ModelSettings['model_name']) 21 | self.optimizer = self._get_optimizer() 22 | self.model.Loss = self._get_criterion() 23 | self.logloss = torch.nn.BCEWithLogitsLoss() 24 | self.device = TrainSettings['device'] 25 | self.clip_grad_norm = False 26 | if 'max_grad_norm' in TrainSettings: 27 | self.clip_grad_norm = True 28 | self.max_grad_norm = eval(TrainSettings['max_grad_norm']) 29 | 30 | def _get_optimizer(self): 31 | optimizer_type = self.TrainSettings['optimizer'] 32 | learning_rate = eval(self.TrainSettings['learning_rate']) 33 | weight_decay = eval(self.TrainSettings['weight_decay']) 34 | 35 | params=self.model.parameters() 36 | if optimizer_type == 'adam': 37 | return torch.optim.Adam(params=params, lr=learning_rate, weight_decay=weight_decay) 38 | elif optimizer_type == 'adagrad': 39 | return torch.optim.Adagrad(params=params, lr=learning_rate, lr_decay=0, weight_decay=weight_decay, initial_accumulator_value=0) 40 | elif optimizer_type == 'rmsprop': 41 | return torch.optim.RMSprop(params=params, lr=learning_rate, alpha=0.9) 42 | else: 43 | raise ValueError('unknow optimizer_type name: ' + optimizer_type) 44 | 45 | def _get_criterion(self): 46 | criterion_type = self.TrainSettings['criterion'] 47 | if criterion_type == 'mse': 48 | return torch.nn.MSELoss() 49 | elif criterion_type == 'bce': 50 | return torch.nn.BCELoss() 51 | elif criterion_type == 'bcelogits': 52 | return torch.nn.BCEWithLogitsLoss() 53 | elif criterion_type == 'ce': 54 | return torch.nn.CrossEntropyLoss() 55 | else: 56 | raise ValueError('unknow criterion_type name: ' + criterion_type) 57 | 58 | 59 | def _get_auc(self, pred_score, label): 60 | return metrics.roc_auc_score(label, pred_score), None, None 61 | 62 | class CTREngine(Engine): 63 | def __init__(self, model, TrainSettings): 64 | self.model = model 65 | # self.model = torch.nn.DataParallel(self.model, device_ids=[0, 1, 2, 7]) 66 | self.model.to(TrainSettings['device']) 67 | super(CTREngine, self).__init__(TrainSettings) 68 | 69 | def train(self, train_loader, epoch_id): 70 | assert hasattr(self, 'model'), 'Please specify the exact model !' 71 | self.model.train() 72 | device = self.device 73 | 74 | loss_sum = 0 75 | loss_array = [] 76 | t0 = time.time() 77 | 78 | for i, input_list in enumerate(tqdm.tqdm(train_loader, desc="train", smoothing=0, mininterval=1.0)): 79 | 80 | # run model 81 | input_list = input_list[:-1] 82 | input_list = [x.long().to(device) for x in input_list] 83 | labels = input_list[0] 84 | self.optimizer.zero_grad() 85 | logits = self.model(input_list[1]) 86 | # logits = self.model(input_list[1],epoch_id) 87 | loss_list = self.model.loss(logits, labels) 88 | 89 | # loss 90 | with torch.autograd.set_detect_anomaly(True): 91 | loss = sum(loss_list) 92 | loss.backward(retain_graph=False) 93 | if self.clip_grad_norm: 94 | torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.max_grad_norm, norm_type=2) 95 | self.optimizer.step() 96 | loss_array.append(loss.item()) 97 | loss_sum += loss.item() 98 | 99 | t1 = time.time() 100 | print("Epoch ", epoch_id, " Train cost:", t1 - t0, " Loss: ", np.mean(loss_array)) 101 | return np.mean(loss_array) 102 | 103 | def evaluate(self, test_loader, epoch_id): 104 | assert hasattr(self, 'model'), 'Please specify the exact model !' 105 | self.model.eval() 106 | device = self.device 107 | 108 | # evaluate source sample 109 | t0 = time.time() 110 | test_logits, test_scores, test_labels, test_page_ids = [], [], [], [] 111 | for i, input_list in enumerate(tqdm.tqdm(test_loader, desc="eval test", smoothing=0, mininterval=1.0)): 112 | batch_label = input_list[0] 113 | batch_page_id = input_list[-1] 114 | batch_input = input_list[1] 115 | test_labels.extend(batch_label) 116 | test_page_ids.extend(batch_page_id) 117 | ## eval 118 | # input_list = [x.long().to(device) for x in input_list[1:]] 119 | batch_input = batch_input.long().to(device) 120 | pred_logits = self.model(batch_input) 121 | test_logits.extend(list(pred_logits.data.cpu().numpy())) 122 | pred_score = F.sigmoid(pred_logits) 123 | test_scores.extend(list(pred_score.data.cpu().numpy())) 124 | test_auc, _, _ = self._get_auc(test_scores, test_labels) 125 | t1 = time.time() 126 | 127 | test_result = '' 128 | test_result += "AUC: " + str(test_auc) + '\n' 129 | print( 130 | "evaluate test cost: ", t1 - t0, 131 | " AUC: " + str(test_auc), 132 | ) 133 | return test_result, test_auc 134 | 135 | 136 | 137 | -------------------------------------------------------------------------------- /Models/utils/layer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class MultiLayerPerceptron(torch.nn.Module): 8 | 9 | def __init__(self, input_dim, mlp_dim_list, dropout, bn=False, activation=None, output_layer=True): 10 | super().__init__() 11 | layers = list() 12 | for mlp_dim in mlp_dim_list: 13 | layers.append(nn.Linear(input_dim, mlp_dim)) 14 | if bn: 15 | layers.append(nn.BatchNorm1d(mlp_dim)) 16 | if activation != None: 17 | layers.append(activation) 18 | layers.append(nn.Dropout(p=dropout)) 19 | input_dim = mlp_dim 20 | if output_layer: 21 | layers.append(nn.Linear(input_dim, 1)) 22 | self.mlp = nn.Sequential(*layers) 23 | 24 | def forward(self, x): 25 | """ 26 | :param x: Float tensor of size ``(batch_size, num_fields, embed_dim)`` 27 | """ 28 | return self.mlp(x) 29 | 30 | 31 | class Attention(torch.nn.Module): 32 | def __init__(self, att_input_dim, ModelSettings): 33 | super(Attention, self).__init__() 34 | hid_dim = att_input_dim 35 | 36 | self.att_func = ModelSettings['att_func'] 37 | if self.att_func == 'dot': 38 | pass 39 | elif self.att_func == 'concat': 40 | self.w = nn.Linear(hid_dim * 2, hid_dim) 41 | self.v = nn.Linear(hid_dim, 1) 42 | elif self.att_func == 'add': 43 | self.w = nn.Linear(hid_dim, hid_dim) 44 | self.u = nn.Linear(hid_dim, hid_dim) 45 | self.v = nn.Linear(hid_dim, 1) 46 | elif self.att_func == 'linear': 47 | self.w = nn.Linear(hid_dim, hid_dim) 48 | elif self.att_func == 'din_activate': 49 | hid_dim_list = eval(ModelSettings['din_att_dim_list']) 50 | # self.mlp = MultiLayerPerceptron(hid_dim*3, hid_dim_list, dropout=0, activation=nn.Sigmoid()) 51 | self.mlp = MultiLayerPerceptron(hid_dim*4, hid_dim_list, dropout=0, activation=nn.Sigmoid()) 52 | self._init_weights() 53 | 54 | def forward(self, query, seq, seq_lens=None, given_mask=None): 55 | if self.att_func == 'dot': 56 | query = query.squeeze().unsqueeze(1) 57 | a = self.mask_softmax(torch.bmm(seq, query.permute(0, 2, 1)), seq_lens, 1) 58 | return a 59 | elif self.att_func == 'concat': 60 | seq_len = len(seq[0]) 61 | batch_size = len(seq) 62 | query = query.squeeze().unsqueeze(1) 63 | a = torch.cat([seq, query.repeat([1, seq_len, 1])], 2).reshape([seq_len * batch_size, -1]) 64 | a = F.relu(self.w(a)) 65 | a = F.relu(self.v(a)) 66 | a = self.mask_softmax(a.reshape([batch_size, seq_len, 1]), seq_lens, 1) 67 | return a 68 | elif self.att_func == 'add': 69 | seq_len = len(seq[0]) 70 | batch_size = len(seq) 71 | seq = self.w(seq.reshape([batch_size * seq_len, -1])) 72 | query = self.u(query).repeat([seq_len, 1]) 73 | a = self.mask_softmax(self.v(F.tanh(seq + query)).reshape([batch_size, seq_len, 1]), seq_lens, 1) 74 | return a 75 | elif self.att_func == 'linear': 76 | seq_len = len(seq[0]) 77 | batch_size = len(seq) 78 | query = query.squeeze() 79 | query = self.w(query).unsqueeze(2) 80 | a = self.mask_softmax(torch.bmm(seq, query), seq_lens, 1) 81 | return a 82 | elif self.att_func == 'din_activate': 83 | seq_len = len(seq[0]) 84 | batch_size = len(seq) 85 | query = query.squeeze().unsqueeze(1) 86 | query = query.repeat([1, seq_len, 1]) 87 | din_activate = torch.cat([query, seq, query - seq, query * seq], dim=-1) 88 | # din_activate = torch.cat([query, seq, query * seq], dim=-1) 89 | din_activate = din_activate.view(batch_size * seq_len, -1) 90 | a = self.mlp(din_activate) 91 | a = a.view(batch_size, seq_len) 92 | # Scale 93 | dim = len(seq[2]) # self.hid_dim 94 | a = a/(dim ** 0.5) 95 | # Mask 96 | # a = self.mask_softmax(a, seq_lens, 1) 97 | if given_mask is None and seq_len is not None: 98 | mask = (torch.arange(seq_len, device=seq_lens.device).repeat( 99 | batch_size, 1) < seq_lens.view(-1, 1)) 100 | elif given_mask is not None: 101 | mask = given_mask 102 | else: 103 | raise ValueError('You should give a seq_len or given_mask in attention layer !!!!') 104 | a[~mask] = -np.inf 105 | # Activation 106 | a = F.softmax(a, dim=1) 107 | return a 108 | 109 | def _init_weights(self): 110 | if self.att_func == 'dot': 111 | pass 112 | elif self.att_func == 'concat': 113 | nn.init.uniform_(self.w.weight, a=-0.1, b=0.1) 114 | nn.init.uniform_(self.v.weight, a=-0.1, b=0.1) 115 | elif self.att_func == 'add': 116 | nn.init.uniform_(self.w.weight, a=-0.1, b=0.1) 117 | nn.init.uniform_(self.u.weight, a=-0.1, b=0.1) 118 | nn.init.uniform_(self.v.weight, a=-0.1, b=0.1) 119 | elif self.att_func == 'linear': 120 | nn.init.uniform_(self.w.weight, a=-0.1, b=0.1) 121 | 122 | def mask_softmax(self, seqs, seq_lens=None, dim=1): 123 | if seq_lens is None: 124 | res = F.softmax(seqs, dim=dim) 125 | else: 126 | max_len = len(seqs[0]) 127 | batch_size = len(seqs) 128 | ones = seq_lens.new_ones(batch_size, max_len, device=seq_lens.device) 129 | range_tensor = ones.cumsum(dim=1) 130 | mask = (seq_lens.unsqueeze(1) >= range_tensor).long() 131 | mask = mask.float() 132 | mask = mask.unsqueeze(2) 133 | # masked_vector = seqs.masked_fill((1 - mask).byte(), -1e32) 134 | masked_vector = seqs.masked_fill((1 - mask).bool(), -1e32) 135 | res = F.softmax(masked_vector, dim=dim) 136 | return res 137 | -------------------------------------------------------------------------------- /Runs/ctr_task.py: -------------------------------------------------------------------------------- 1 | from configparser import ConfigParser 2 | cfg = ConfigParser() 3 | cfg.read('project_path.ini') 4 | sys_path = dict(cfg.items('PathSettings'))['sys_path'] 5 | 6 | 7 | import sys, os 8 | sys.path.append(sys_path) 9 | from shutil import copyfile 10 | 11 | import numpy as np 12 | import random 13 | import time 14 | import torch 15 | import logging 16 | 17 | from DatasetsLoad.sampler import SampleGenerator 18 | from Models.engine import * 19 | from Models.Page import * 20 | 21 | 22 | def get_engine(Sampler, TrainSettings, ModelSettings): 23 | model_name = ModelSettings['model_name'] 24 | engine_type = ModelSettings['engine_type'] 25 | print("=========== model: " + model_name + " engine: " + engine_type + " ===========\n") 26 | module = eval(model_name) 27 | model = getattr(module, model_name)(Sampler, ModelSettings) 28 | engine = eval(engine_type+'Engine')(model, TrainSettings) 29 | return engine 30 | 31 | def save_model_code(source, target, code_name): 32 | try: 33 | copyfile(source, target) 34 | except IOError as e: 35 | print("Unable to copy file. %s" % e) 36 | exit(1) 37 | except: 38 | print("Unexpected error:", sys.exc_info()) 39 | exit(1) 40 | print(code_name+" code copy done!\n") 41 | 42 | def setup_seed(seed): 43 | torch.manual_seed(seed) 44 | torch.cuda.manual_seed_all(seed) 45 | np.random.seed(seed) 46 | random.seed(seed) 47 | torch.backends.cudnn.deterministic = True 48 | 49 | 50 | def Run(DataSettings, ModelSettings, TrainSettings, ResultSettings, 51 | mode='train', 52 | timestamp=None): 53 | 54 | ## =========== setting init =========== 55 | setup_seed(817) # random seed 56 | model_name = ModelSettings['model_name'] 57 | save_dir = sys_path + ResultSettings['save_dir'] + model_name + '/' 58 | 59 | ## =========== data init =========== 60 | Sampler = SampleGenerator(DataSettings, TrainSettings, mode) 61 | 62 | ## =========== model init =========== 63 | Engine = get_engine(Sampler, TrainSettings, ModelSettings) 64 | 65 | 66 | ## ======= train || inference ======= 67 | if timestamp == None: 68 | timestamp = time.time() 69 | localtime = str(time.asctime( time.localtime(int(timestamp)) )) 70 | time_ids = str(localtime) + ' ' + str(int(timestamp)) 71 | model_save_dir = save_dir+'files/'+time_ids+'/' 72 | if not os.path.exists(model_save_dir): 73 | os.makedirs(model_save_dir) 74 | save_model_code(eval(model_name).__file__, model_save_dir+model_name+'.py', code_name='model') 75 | else: 76 | tbias = 8*3600 77 | # localtime = str(time.asctime( time.localtime(int(timestamp)) )) 78 | localtime = str(time.asctime( time.localtime(int(timestamp)+tbias) )) 79 | time_ids = str(localtime) + ' ' + str(int(timestamp)) 80 | print(timestamp) 81 | print(localtime) 82 | model_save_dir = save_dir+'files/'+time_ids+'/' 83 | Engine.model = torch.load(f'{model_save_dir}{model_name}.pt').to(TrainSettings['device']) 84 | 85 | ## =========== log init =========== 86 | log = logging.getLogger(model_name + str(int(timestamp))) 87 | log.setLevel(logging.INFO) 88 | 89 | sh = logging.StreamHandler(sys.stdout) 90 | 91 | ind_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') 92 | ind_fh = logging.FileHandler(save_dir + model_name + 93 | '_%s_%s' % (localtime, str(int(timestamp))) + ".log") 94 | ind_fh.setFormatter(ind_formatter) 95 | 96 | log.addHandler(ind_fh) 97 | log.addHandler(sh) 98 | log.info('========== ' + localtime + " "+ str(int(timestamp)) + ' =========='+'\n') 99 | log.info(str(DataSettings)+'\n'+str(ModelSettings)+'\n'+str(TrainSettings)+'\n') 100 | 101 | 102 | if mode == "train": # train mode 103 | print("=========== Training Start ===========") 104 | 105 | total_epoch = eval(TrainSettings['epoch']) 106 | start_epoch = eval(TrainSettings['start_epoch']) 107 | pre_train_epoch = eval(TrainSettings['pre_train_epoch']) 108 | early_stop_step = eval(TrainSettings['early_stop_step']) 109 | endure_count = 0 110 | 111 | val_auc, best_val_epoch = 0, 0 112 | best_val_result = "" 113 | 114 | # start training 115 | for epoch_i in range(start_epoch, total_epoch): 116 | ### train 117 | Engine.train(Sampler.Train_data, epoch_i) 118 | 119 | ### pre-validate train stage 120 | if epoch_i < pre_train_epoch: 121 | continue 122 | 123 | ### test 124 | result, tmp_auc = Engine.evaluate(Sampler.Val_data, epoch_i) 125 | ### early stop 126 | if tmp_auc > val_auc: 127 | val_auc = tmp_auc 128 | endure_count = 0 129 | best_val_result = result 130 | best_val_epoch = epoch_i 131 | 132 | t_result, t_auc = Engine.evaluate(Sampler.Test_data, epoch_i) 133 | 134 | # save log 135 | log.info(str(int(timestamp))+' epoch: ' + str(epoch_i)) 136 | log.info('better val_result:\n'+best_val_result+'\n') 137 | log.info('current test_result:\n'+t_result+'\n') 138 | # save model file 139 | torch.save(Engine.model, f'{model_save_dir}{model_name}.pt') 140 | else: 141 | endure_count += 1 142 | if endure_count >= early_stop_step: 143 | break 144 | log.info('best val results(epoch: '+str(best_val_epoch)+' timestamp: '+str(int(timestamp))+'):\n'+best_val_result+'\n') 145 | 146 | # start test 147 | print("Inference on the test dataset") 148 | Engine.model = torch.load(f'{model_save_dir}{model_name}.pt').to(TrainSettings['device']) 149 | result, _ = Engine.evaluate(Sampler.Test_data, epoch_id=0) 150 | test_result = result 151 | log.info('test results( timestamp: '+str(int(timestamp))+'):\n'+test_result+'\n') 152 | # mark this timestamp directory to denote a success training 153 | with open(model_save_dir+'Finish', 'w') as f: 154 | f.write('FINISH!!!') 155 | 156 | 157 | elif mode == 'test': # test mode 158 | print("=========== Inference Start ===========") 159 | Engine.model = torch.load(f'{model_save_dir}{model_name}.pt').to(TrainSettings['device']) 160 | result, _ = Engine.evaluate(Sampler.Test_data, epoch_id=0) 161 | test_result = result 162 | log.info('test results( timestamp: '+str(int(timestamp))+'):\n'+test_result+'\n') 163 | 164 | else: 165 | raise NotImplementedError('unkonw mode: ', mode) -------------------------------------------------------------------------------- /DataPreprocess/first_step.py: -------------------------------------------------------------------------------- 1 | import pandas as pd, numpy as np 2 | import pickle, json, ast, re, os, string 3 | import time, datetime 4 | import tqdm 5 | import math 6 | from collections import defaultdict 7 | 8 | 9 | 10 | data_dir = './Data/' 11 | raw_data_dir = data_dir + 'raw/' 12 | new_data_dir = data_dir + 'new/' 13 | 14 | 15 | # ### Load Data 16 | 17 | data_path = raw_data_dir + 'trainSearchStream.tsv' 18 | SearchStream = pd.read_csv(data_path, sep="\t") 19 | 20 | data_path = raw_data_dir + 'SearchInfo.tsv' 21 | SearchInfo = pd.read_csv(data_path, sep="\t") 22 | 23 | ## SearchInfo.shape: (112,159,462, 9) 24 | # SearchID SearchDate IPID UserID IsUserLoggedOn SearchQuery LocationID CategoryID SearchParams 25 | ## SearchInfo['UserID'].nunique(): 4,295,465 26 | ## SearchInfo['SearchID'].nunique(): 112,159,462 27 | 28 | ## SearchStream.shape (392,356,948, 6) 29 | ## SearchStream['SearchID'].nunique(): 107,863,985 30 | ## SearchStream['AdID'].nunique(): 22,848,857 31 | 32 | data_path = raw_data_dir + 'AdsInfo.tsv' 33 | AdsInfo = pd.read_csv(data_path, sep="\t") 34 | 35 | data_path = raw_data_dir + 'UserInfo.tsv' 36 | UserInfo = pd.read_csv(data_path, sep="\t") 37 | data_path = raw_data_dir + 'Category.tsv' 38 | Category = pd.read_csv(data_path, sep="\t") 39 | data_path = raw_data_dir + 'Location.tsv' 40 | Location = pd.read_csv(data_path, sep="\t") 41 | 42 | UserInfo = UserInfo.fillna(0).astype("int") 43 | Category = Category.fillna(0).astype("int") 44 | Location = Location.fillna(0).astype("int") 45 | 46 | 47 | SearchInfo = SearchInfo[['UserID', 'SearchID', 'SearchDate', 'IPID', 'IsUserLoggedOn', 'SearchQuery', 'LocationID', 'CategoryID', 'SearchParams']] 48 | SearchStream = SearchStream[['SearchID', 'AdID', 'Position', 'ObjectType', 'HistCTR', 'IsClick']] 49 | 50 | AdsInfo = AdsInfo[['AdID', 'Title', 'CategoryID', 'Params']] 51 | UserInfo = UserInfo[['UserID', 'UserAgentID', 'UserAgentOSID', 'UserDeviceID', 'UserAgentFamilyID']] 52 | Category = Category[['CategoryID', 'Level', 'ParentCategoryID', 'SubcategoryID']] 53 | Location = Location[['LocationID', 'Level', 'RegionID', 'CityID']] 54 | 55 | """ 56 | features need map: 57 | SearchID, SearchQuery, SearchParams, Titel, Params, CategoryID, LocationID 58 | 59 | DataFrame nedd map: 60 | SearchInfo: SearchID, SearchQuery, SearchParams, CategoryID, LocationID 61 | SearchStream: SearchID, HistCTR 62 | AdsInfo: 'Title', 'CategoryID', 'Params' 63 | Category: CategoryID 64 | Location: LocationID 65 | """ 66 | 67 | # #### 1. implement map function 68 | 69 | zero_list = ['0', 0] 70 | def preprocess_string(input): 71 | input = input.strip() 72 | exclist = string.punctuation + '\t' 73 | table_ = str.maketrans('', '', exclist) 74 | input = input.translate(table_) 75 | return input.strip() 76 | 77 | def param_to_str(param, max_len_per_slot_aux): 78 | param_list = [ preprocess_string(x) for x in param.split(",")][:max_len_per_slot_aux] 79 | return '|'.join(param_list) 80 | 81 | def words_to_str(words, max_len_per_slot_aux): 82 | words = preprocess_string(words) 83 | word_list = words.split(" ")[:max_len_per_slot_aux] 84 | return '|'.join(word_list) 85 | 86 | def word_to_token(words, word_dict, sep, zero_list, max_len_per_slot_aux): 87 | token_list = [] 88 | if words in zero_list: 89 | word_list = [] 90 | else: 91 | word_list = list(filter(None, words.split(sep))) 92 | for i in range(max_len_per_slot_aux): 93 | if i < len(word_list): 94 | word = preprocess_string(word_list[i]) 95 | token = word_dict[word] 96 | token_list.append(token) 97 | else: 98 | token_list.append(0) 99 | return token_list 100 | 101 | 102 | 103 | # #### 2. map Location: LocationID 104 | if not os.path.exists(new_data_dir+'location_ids_dict.pickle'): 105 | location_ids = Location['LocationID'].unique().tolist() 106 | location_ids_dict = dict([(y,x+1) for x,y in enumerate(sorted(location_ids))]) 107 | with open(new_data_dir+'location_ids_dict.pickle', 'wb') as f: 108 | pickle.dump(location_ids_dict, f, protocol=pickle.HIGHEST_PROTOCOL) 109 | else: 110 | f = open(new_data_dir+'location_ids_dict.pickle', 'rb') 111 | location_ids_dict = pickle.load(f) 112 | tqdm.tqdm.pandas() 113 | Location['LocationID'] = Location['LocationID'].progress_apply(lambda x: location_ids_dict[x]) 114 | Location.to_csv(new_data_dir+'Location_new.csv', index=False) 115 | print('Finishing mapping Location') 116 | 117 | 118 | # #### 3. map Category: CategoryID 119 | if not os.path.exists(new_data_dir+'category_ids_dict.pickle'): 120 | category_ids = Category['CategoryID'].unique().tolist() 121 | category_ids_dict = dict([(y,x+1) for x,y in enumerate(sorted(category_ids))]) 122 | with open(new_data_dir+'category_ids_dict.pickle', 'wb') as f: 123 | pickle.dump(category_ids_dict, f, protocol=pickle.HIGHEST_PROTOCOL) 124 | else: 125 | f = open(new_data_dir+'category_ids_dict.pickle', 'rb') 126 | category_ids_dict = pickle.load(f) 127 | tqdm.tqdm.pandas() 128 | Category['CategoryID'] = Category['CategoryID'].progress_apply(lambda x: category_ids_dict[x]) 129 | Category.to_csv(new_data_dir+'Category_new.csv', index=False) 130 | print('Finishing mapping Category') 131 | 132 | 133 | 134 | # #### 4. map AdsInfo: 'Title', 'CategoryID', 'Params' 135 | 136 | # Ads params: convert Multivalent categorical strings to discrete index 137 | ads_params = AdsInfo['Params'][AdsInfo['Params'].notnull()].tolist() 138 | ads_params = [param_to_str(x, 5) for x in ads_params] 139 | AdsInfo['Params'][AdsInfo['Params'].notnull()] = ads_params 140 | if not os.path.exists(new_data_dir+'ads_params_dict.pickle'): 141 | ads_params = "|".join(ads_params) 142 | ads_params = ads_params.split("|") 143 | ads_params_dict = dict([(y,x+1) for x,y in enumerate(sorted(set(ads_params)))]) 144 | with open(new_data_dir+'ads_params_dict.pickle', 'wb') as f: 145 | pickle.dump(ads_params_dict, f, protocol=pickle.HIGHEST_PROTOCOL) 146 | else: 147 | f = open(new_data_dir+'ads_params_dict.pickle', 'rb') 148 | ads_params_dict = pickle.load(f) 149 | 150 | # Ads query: convert Multivalent categorical strings to discrete index 151 | ads_title = AdsInfo['Title'][AdsInfo['Title'].notnull()].tolist() 152 | ads_title = [words_to_str(x, 5) for x in ads_title] 153 | AdsInfo['Title'][AdsInfo['Title'].notnull()] = ads_title 154 | if not os.path.exists(new_data_dir+'ads_title_dict.pickle'): 155 | ads_title = "|".join(ads_title) 156 | ads_title = ads_title.split("|") 157 | ads_title_dict = dict([(y,x+1) for x,y in enumerate(sorted(set(ads_title)))]) 158 | with open(new_data_dir+'ads_title_dict.pickle', 'wb') as f: 159 | pickle.dump(ads_title_dict, f, protocol=pickle.HIGHEST_PROTOCOL) 160 | else: 161 | f = open(new_data_dir+'ads_title_dict.pickle', 'rb') 162 | ads_title_dict = pickle.load(f) 163 | 164 | 165 | tqdm.tqdm.pandas() 166 | AdsInfo = AdsInfo.fillna(0) 167 | AdsInfo['Title'] = AdsInfo['Title'].progress_apply(lambda x: word_to_token(str(x), ads_title_dict, '|', zero_list, 5)) 168 | AdsInfo['Params'] = AdsInfo['Params'].progress_apply(lambda x: word_to_token(str(x), ads_params_dict, '|', zero_list, 5)) 169 | AdsInfo['CategoryID'] = AdsInfo['CategoryID'].progress_apply(lambda x: category_ids_dict[x]) 170 | AdsInfo.to_csv(new_data_dir+'AdsInfo_new.csv', index=False) 171 | print('Finishing mapping AdsInfo') 172 | 173 | 174 | 175 | # #### 5. map SearchStream: SearchID, HistCTR 176 | 177 | num_HistCTR_type = 10 178 | HistCTR_D = SearchStream[(SearchStream['ObjectType']==3)]['HistCTR'].tolist() 179 | HistCTR_D = pd.qcut(HistCTR_D, num_HistCTR_type, labels=False)+1 180 | SearchStream['HistCTR'][(SearchStream['ObjectType']==3)] = HistCTR_D 181 | SearchStream['HistCTR'][(SearchStream['ObjectType']!=3)] = 0 182 | SearchStream['IsClick'][(SearchStream['ObjectType']!=3)] = 2 183 | 184 | 185 | 186 | # #### 6. map SearchInfo: SearchID, SearchQuery, SearchParams, CategoryID, LocationID 187 | 188 | # search params: convert Multivalent categorical strings to discrete index 189 | search_params = SearchInfo['SearchParams'][SearchInfo['SearchParams'].notnull()].tolist() 190 | search_params = [param_to_str(x, 3) for x in search_params] 191 | SearchInfo['SearchParams'][SearchInfo['SearchParams'].notnull()] = search_params 192 | if not os.path.exists(new_data_dir+'search_params_dict.pickle'): 193 | search_params = "|".join(search_params) 194 | search_params = search_params.split("|") 195 | search_params_dict = dict([(y,x+1) for x,y in enumerate(sorted(set(search_params)))]) 196 | with open(new_data_dir+'search_params_dict.pickle', 'wb') as f: 197 | pickle.dump(search_params_dict, f, protocol=pickle.HIGHEST_PROTOCOL) 198 | else: 199 | f = open(new_data_dir+'search_params_dict.pickle', 'rb') 200 | search_params_dict = pickle.load(f) 201 | 202 | 203 | # search query: convert Multivalent categorical strings to discrete index 204 | search_querys = SearchInfo['SearchQuery'][SearchInfo['SearchQuery'].notnull()].tolist() 205 | search_querys = [words_to_str(x, 1) for x in search_querys] 206 | SearchInfo['SearchQuery'][SearchInfo['SearchQuery'].notnull()] = search_querys 207 | if not os.path.exists(new_data_dir+'search_querys_dict.pickle'): 208 | search_querys = "|".join(search_querys) 209 | search_querys = search_querys.split("|") 210 | search_querys_dict = dict([(y,x+1) for x,y in enumerate(sorted(set(search_querys)))]) 211 | with open(new_data_dir+'search_querys_dict.pickle', 'wb') as f: 212 | pickle.dump(search_querys_dict, f, protocol=pickle.HIGHEST_PROTOCOL) 213 | else: 214 | f = open(new_data_dir+'search_querys_dict.pickle', 'rb') 215 | search_querys_dict = pickle.load(f) 216 | 217 | ## search ids 218 | if not os.path.exists(new_data_dir+'search_ids_dict.pickle'): 219 | search_ids = set(SearchInfo['SearchID'].unique().tolist() + SearchStream['SearchID'].unique().tolist()) 220 | search_ids = list(search_ids) 221 | search_ids_dict = dict([(y,x+1) for x,y in enumerate(sorted(search_ids))]) 222 | with open(new_data_dir+'search_ids_dict.pickle', 'wb') as f: 223 | pickle.dump(search_ids_dict, f, protocol=pickle.HIGHEST_PROTOCOL) 224 | else: 225 | f = open(new_data_dir+'search_ids_dict.pickle', 'rb') 226 | search_ids_dict = pickle.load(f) 227 | 228 | 229 | # covert 230 | tqdm.tqdm.pandas() 231 | SearchStream['SearchID'] = SearchStream['SearchID'].progress_apply(lambda x: search_ids_dict[x]) 232 | SearchStream.to_csv(new_data_dir+'SearchStream_new.csv', index=False) 233 | 234 | SearchInfo = SearchInfo.fillna(0) 235 | SearchInfo['SearchID'] = SearchInfo['SearchID'].progress_apply(lambda x: search_ids_dict[x]) 236 | SearchInfo['CategoryID'] = SearchInfo['CategoryID'].progress_apply(lambda x: category_ids_dict[x]) 237 | SearchInfo['LocationID'] = SearchInfo['LocationID'].progress_apply(lambda x: location_ids_dict[x]) 238 | SearchInfo['SearchQuery'] = SearchInfo['SearchQuery'].progress_apply(lambda x: word_to_token(str(x), search_querys_dict, '|', zero_list, 1)) 239 | SearchInfo['SearchParams'] = SearchInfo['SearchParams'].progress_apply(lambda x: word_to_token(str(x), search_params_dict, '|', zero_list, 3)) 240 | SearchInfo['SearchDate'] = pd.to_datetime(SearchInfo['SearchDate']) 241 | SearchInfo.SearchDate = SearchInfo.SearchDate.values.astype(np.int64) // 10 ** 9 242 | num_Date_type = 300 243 | Date_D = SearchInfo['SearchDate'].tolist() 244 | Date_D = pd.qcut(Date_D, num_Date_type, labels=False)+1 245 | SearchInfo['TimeStamp'] = Date_D 246 | print('Finish dealing with SearchInfo') 247 | 248 | 249 | # #### 7. split SearchInfo by datetime 250 | 251 | print('begin split SearchInfo') 252 | train_begin_datetime = "2015-04-28 00:00:00" 253 | val_begin_datetime = "2015-05-19 00:00:00" 254 | test_begin_datetime = "2015-05-20 00:00:00" 255 | train_begin_timestamp = time.mktime(datetime.datetime.strptime(train_begin_datetime, "%Y-%m-%d %H:%M:%S").timetuple()) 256 | val_begin_timestamp = time.mktime(datetime.datetime.strptime(val_begin_datetime, "%Y-%m-%d %H:%M:%S").timetuple()) 257 | test_begin_timestamp = time.mktime(datetime.datetime.strptime(test_begin_datetime, "%Y-%m-%d %H:%M:%S").timetuple()) 258 | val_begin_timestamp = int(val_begin_timestamp) 259 | test_begin_timestamp = int(test_begin_timestamp) 260 | 261 | TrainSearchInfo = SearchInfo[ ~(SearchInfo['SearchDate']0: 353 | print('page_ad_attn:', page_layer['page_ad_attn'][0].data.cpu().numpy()) 354 | print('page_attn:', page_layer['page_attn'][0].data.cpu().numpy()) 355 | 356 | return logits.squeeze() 357 | 358 | 359 | def loss(self, logtis, labels): 360 | loss = self.Loss(logtis.squeeze(), labels.float()) 361 | return [loss] 362 | 363 | 364 | def init_weights(self): 365 | for key_name in ['user', 'ad', 'location', 'category']: 366 | for e in self.embedding_dict[key_name]: 367 | nn.init.xavier_uniform_(e.weight) 368 | for key_name in ['ad_title', 'ad_params', 'search_query', 'search_params']: 369 | nn.init.xavier_uniform_(self.embedding_dict[key_name].weight) 370 | -------------------------------------------------------------------------------- /DataPreprocess/second_step.py: -------------------------------------------------------------------------------- 1 | import pandas as pd, numpy as np 2 | import pickle, json, ast, re, os, string 3 | import time, datetime 4 | import tqdm 5 | import math, random 6 | from collections import defaultdict 7 | from collections import deque 8 | 9 | from tqdm.contrib.concurrent import process_map 10 | 11 | data_dir = '/home/data_ti5_c/fubr/CTRPrediction/CTRDatasets/' 12 | data_dir = './Data/' 13 | raw_data_dir = data_dir + 'raw/' 14 | new_data_dir = data_dir + 'new/' 15 | sample_data_dir = data_dir + 'data/' 16 | random.seed(10) 17 | training_data_type = 'Train' 18 | 19 | flatten = lambda forest: [leaf for tree in forest for leaf in tree] 20 | 21 | # #### 1. Load Data 22 | 23 | TestSearchInfo = pd.read_csv(new_data_dir + 'TestSearchInfo.csv') 24 | ValSearchInfo = pd.read_csv(new_data_dir + 'ValSearchInfo.csv') 25 | TrainSearchInfo = pd.read_csv(new_data_dir + 'TrainSearchInfo.csv') 26 | 27 | TrainSearchInfo = TrainSearchInfo.sort_values(by="SearchDate") 28 | ValSearchInfo = ValSearchInfo.sort_values(by="SearchDate") 29 | TestSearchInfo = TestSearchInfo.sort_values(by="SearchDate") 30 | 31 | AdsInfo = pd.read_csv(new_data_dir + 'AdsInfo_new.csv') 32 | AdsInfo = AdsInfo.set_index('AdID') 33 | SearchStream = pd.read_csv(new_data_dir + 'SearchStream_new.csv') 34 | SearchStream = SearchStream.astype("int") 35 | 36 | data_path = raw_data_dir + 'UserInfo.tsv' 37 | UserInfo = pd.read_csv(data_path, sep="\t") 38 | UserInfo = UserInfo.set_index('UserID') 39 | Category = pd.read_csv(new_data_dir + 'Category_new.csv') 40 | Category = Category.set_index('CategoryID') 41 | Location = pd.read_csv(new_data_dir + 'Location_new.csv') 42 | Location = Location.set_index('LocationID') 43 | UserInfo = UserInfo.fillna(0).astype("int") 44 | Category = Category.fillna(0).astype("int") 45 | Location = Location.fillna(0).astype("int") 46 | 47 | 48 | training_SearchInfo = eval(training_data_type+'SearchInfo') 49 | # #### 2. make search_stream_dict 50 | 51 | SearchStream = SearchStream.sort_values(by='IsClick') 52 | click_search_ids = SearchStream[SearchStream['IsClick']==1]['SearchID'].unique().tolist() 53 | click_dict = dict([(y,True) for x,y in enumerate(sorted(set(click_search_ids)))]) 54 | 55 | search_stream_dict_path = new_data_dir+'search_stream_dict.pickle' 56 | if not os.path.exists(search_stream_dict_path): 57 | tqdm.tqdm.pandas() 58 | search_stream_dict = SearchStream.groupby('SearchID')['AdID'].progress_apply(list).to_dict() 59 | with open(search_stream_dict_path, 'wb') as f: 60 | pickle.dump(search_stream_dict, f, protocol=pickle.HIGHEST_PROTOCOL) 61 | else: 62 | f = open(search_stream_dict_path, 'rb') 63 | search_stream_dict = pickle.load(f) 64 | 65 | 66 | # #### 3. make user seq lists 67 | 68 | num_ado_feature = 18 69 | num_query_feature = 16 70 | 71 | SearchStream = SearchStream.set_index('SearchID').sort_index() 72 | 73 | def deal_with_user_seq(tuples_input): 74 | u_id = tuples_input[0] 75 | search_id_list = tuples_input[1] 76 | drop_search_id = 0 77 | ad_seq_list = [] 78 | user_features = [u_id] 79 | 80 | new_search_id_list = [] 81 | for search_id in search_id_list: 82 | if search_id not in search_stream_dict: 83 | drop_search_id += 1 84 | continue 85 | new_search_id_list.append(search_id) 86 | search_id_list = new_search_id_list 87 | 88 | search_id_list = list(reversed(search_id_list)) 89 | tmp_cnt = 6 90 | for search_id in search_id_list: 91 | if tmp_cnt <= 0: 92 | continue 93 | 94 | if search_id in click_dict: 95 | tmp_cnt = 6 96 | ad_id_list = search_stream_dict[search_id] 97 | query_features = [search_id] 98 | search_ads_list = [query_features, ad_id_list] 99 | ad_seq_list.append(search_ads_list) 100 | tmp_cnt -= 1 101 | 102 | ad_seq_list = list(reversed(ad_seq_list)) 103 | user_seq_list = [user_features, ad_seq_list] 104 | return (user_seq_list, drop_search_id) 105 | 106 | def get_user_seq(data_type): 107 | drop_search_id = 0 108 | tmp_SearchInfo = eval(data_type+'SearchInfo') 109 | data_search_info_dict_path = new_data_dir+data_type+'_search_info_dict.pickle' 110 | if not os.path.exists(data_search_info_dict_path): 111 | print(tmp_SearchInfo.shape) #(95,980,006, 10) (3043334, 10) (1616503, 10) 112 | simple_SearchInfo = tmp_SearchInfo[['UserID', 'SearchID']] 113 | simple_SearchInfo_dict = simple_SearchInfo.groupby('UserID')['SearchID'].apply(list).to_dict() 114 | with open(data_search_info_dict_path, 'wb') as f: 115 | pickle.dump(simple_SearchInfo_dict, f, protocol=pickle.HIGHEST_PROTOCOL) 116 | else: 117 | f = open(data_search_info_dict_path, 'rb') 118 | simple_SearchInfo_dict = pickle.load(f) 119 | 120 | uids = tmp_SearchInfo['UserID'].unique().tolist() 121 | data_user_seq_lists_path = new_data_dir+data_type+'_user_seq_lists.pickle' 122 | if not os.path.exists(data_user_seq_lists_path): 123 | print(data_type, 'produce user seq') 124 | tmp_input = [(uid, simple_SearchInfo_dict[uid]) for uid in uids] 125 | Results = [deal_with_user_seq(x) for x in tmp_input] 126 | # Results = process_map(deal_with_user_seq, tmp_input, max_workers=num_workers, chunksize=10) 127 | user_seq_lists = [x[0] for x in Results] 128 | drop_search_id = sum([x[1] for x in Results]) 129 | with open(data_user_seq_lists_path, 'wb') as f: 130 | pickle.dump(user_seq_lists, f, protocol=pickle.HIGHEST_PROTOCOL) 131 | else: 132 | f = open(data_user_seq_lists_path, 'rb') 133 | user_seq_lists = pickle.load(f) 134 | 135 | print(data_path, " drop search_ids: ", drop_search_id) 136 | # train: 3,438,855 137 | # val: 514,911 138 | # test: 341,711 139 | return user_seq_lists 140 | """ 141 | user_seq_list.pickle: 142 | user_seq = [uid], [ [page1], [page2], ...[pageN] ] 143 | pageN = [sid], [uc_id, uc_id, uc_id, uc_id, c_id] 144 | """ 145 | 146 | # #### 4. produce sample 147 | 148 | def get_query_feature(search_id, search_info): 149 | query_features = [search_id] + search_info[['IPID', 'IsUserLoggedOn', 'TimeStamp']].loc[search_id].astype("int").values.tolist() 150 | # 'LocationID', 'CategoryID' 151 | search_LocationID = search_info['LocationID'].loc[search_id] 152 | search_CategoryID = search_info['CategoryID'].loc[search_id] 153 | search_location_feature = [search_LocationID]+Location[['Level', 'RegionID', 'CityID']].loc[search_LocationID].values.tolist() 154 | search_category_feature = [search_CategoryID]+Category[['Level', 'ParentCategoryID', 'SubcategoryID']].loc[search_CategoryID].values.tolist() 155 | query_words = search_info['SearchQuery'].loc[search_id] 156 | query_words = eval(query_words) 157 | query_params = search_info['SearchParams'].loc[search_id] 158 | query_params = eval(query_params) 159 | query_features = query_features + search_location_feature + search_category_feature + query_words + query_params 160 | # 'SearchID', 'IPID', 'IsUserLoggedOn', 'TimeStamp', 'LocationID', 'CategoryID', 'Query', 'Params' 161 | # 4 + 4 + 4 + 1 + 3 = 16 162 | assert len(query_features)==num_query_feature,'query_features: '+str(len(query_features)) 163 | return query_features 164 | 165 | def get_ad_features(search_stream, ad_id): 166 | ad_CategoryID = AdsInfo['CategoryID'].loc[ad_id] 167 | ad_category_feature = [ad_CategoryID]+Category[['Level', 'ParentCategoryID', 'SubcategoryID']].loc[ad_CategoryID].values.tolist() 168 | ad_title = AdsInfo['Title'].loc[ad_id] 169 | ad_title = eval(ad_title) 170 | ad_params = AdsInfo['Params'].loc[ad_id] 171 | ad_params = eval(ad_params) 172 | 173 | ad_features = [ad_id] + ad_category_feature + ad_title + ad_params 174 | search_stream_features = search_stream[['Position', 'HistCTR', 'IsClick']].loc[ad_id].values.tolist() 175 | ad_features = ad_features + search_stream_features 176 | ## ad_id, position, ad_title_keyword, (ad_cate_id, ad_cate_level, ad_parent_cate_id, ad_sub_cate_id), hist_ctr_bin, ad_params 177 | # 15 + 2 + 1 = 18 178 | assert len(ad_features)==num_ado_feature,'ado_features: '+str(len(ad_features)) 179 | return ad_features 180 | 181 | def get_page_features(page, search_info): 182 | search_id = page[0][0] 183 | ad_id_list = page[1] 184 | 185 | query_features = get_query_feature(search_id, search_info) 186 | 187 | search_stream = SearchStream.loc[[search_id]] 188 | search_stream = search_stream.set_index('AdID').sort_index() 189 | 190 | page_ads_features = [] 191 | cut_len = min(len(ad_id_list), 5) 192 | ad_id_list = ad_id_list[:cut_len] 193 | for ad_id in ad_id_list: 194 | ad_features = get_ad_features(search_stream, ad_id) 195 | current_ad_feature = query_features + ad_features 196 | # 16 + 18 = 34 197 | # assert len(current_ad_feature)==34,'ads_features: '+str(len(current_ad_feature)) 198 | page_ads_features.append(current_ad_feature) 199 | for _ in range(5-len(ad_id_list)): 200 | page_ads_features.append(mask_ad_features) 201 | return flatten(page_ads_features) + [len(ad_id_list)] 202 | 203 | 204 | 205 | sample_page_num = 5 206 | page_ad_num = 5 207 | 208 | num_user_features = 5 209 | num_ad_features = num_query_feature + num_ado_feature 210 | mask_ad_features = [0]*num_ad_features 211 | mask_page_features = mask_ad_features*page_ad_num+[0] 212 | num_sample_features = 1 + num_user_features + (num_ad_features-1) + page_ad_num*len(mask_page_features) + 1 213 | num_drop_user = 0 214 | # training_sample_list = [] 215 | # recent_histroy = defaultdict(list) 216 | 217 | 218 | def deal_with_train_seq_list(user_seq_list): 219 | tmp_samples = [] 220 | tmp_recent = defaultdict(list) 221 | u_id = user_seq_list[0][0] 222 | if u_id in UserInfo.index: 223 | user_features = [u_id]+UserInfo.loc[u_id].values.tolist() 224 | else: 225 | user_features = [u_id]+[0]*(num_user_features-1) 226 | # assert len(user_features)==5, 'user_features: '+str(len(user_features)) 227 | ## UserID UserAgentID UserAgentOSID UserDeviceID UserAgentFamilyID 228 | 229 | page_seq = user_seq_list[1] 230 | 231 | search_info = training_SearchInfo.loc[[u_id]] 232 | search_info = search_info.set_index('SearchID').sort_index() 233 | 234 | begin_index = min(sample_page_num, len(page_seq)-1) 235 | # begin_index = min(5, len(page_seq)) 236 | history_pages_deque = deque([]) 237 | begin_pages = page_seq[:begin_index] 238 | # begin_pages = page_seq[-begin_index:] 239 | # assert len(begin_pages)<=5, "wrong length" 240 | 241 | for history_page in begin_pages: 242 | page_ads_features = get_page_features(history_page, search_info) 243 | history_pages_deque.append(page_ads_features) 244 | res_page_seq = page_seq[begin_index:] 245 | 246 | for i in range(len(res_page_seq)): 247 | page = res_page_seq[i] 248 | search_id = page[0][0] 249 | ad_id_list = page[1] 250 | 251 | search_stream = SearchStream.loc[[search_id]] 252 | search_stream = search_stream.set_index('AdID').sort_index() 253 | current_query_features = get_query_feature(search_id, search_info) 254 | 255 | history_pages_features = list(history_pages_deque) 256 | num_mask_pages = sample_page_num-len(history_pages_features) 257 | for _ in range(num_mask_pages): 258 | history_pages_features.append(mask_page_features) 259 | history_pages_features = flatten(history_pages_features) + [5-num_mask_pages] 260 | 261 | page_ads_features = [] 262 | # cut_len = min(len(ad_id_list), 5) 263 | # ad_id_list = random.shuffle(ad_id_list) 264 | # ad_id_list = ad_id_list[:cut_len] 265 | 266 | is_sample_page = False 267 | if i == len(res_page_seq)-1: 268 | is_sample_page = True 269 | if search_id in click_dict: 270 | is_sample_page = True 271 | 272 | have_neg = False 273 | for ad_id in ad_id_list: 274 | current_ad_features = get_ad_features(search_stream, ad_id) 275 | current_qad_feature = current_query_features + current_ad_features 276 | # assert len(current_qad_feature)==33,'ads_features: '+str(len(current_ad_feature)) 277 | page_ads_features.append(current_qad_feature) 278 | if not is_sample_page: # not click page or not last page 279 | continue 280 | if search_stream['ObjectType'].loc[ad_id] != 3: # remove nan ad 281 | continue 282 | 283 | label = current_ad_features[-1] 284 | if label !=1: 285 | if not have_neg and label == 0: 286 | have_neg = True 287 | else: 288 | continue 289 | 290 | target_ad_features = current_ad_features[:-1] 291 | target_qad_feature = current_query_features + target_ad_features 292 | target_uqad_features = user_features + target_qad_feature 293 | sample = [label] + target_uqad_features + history_pages_features 294 | assert len(sample) == num_sample_features, "sample length wrong!!! with len: "+str(len(sample)) 295 | # training_sample_list.append(sample) 296 | tmp_samples.append(sample) 297 | 298 | for _ in range(page_ad_num-len(ad_id_list)): 299 | page_ads_features.append(mask_ad_features) 300 | page_ads_features = flatten(page_ads_features) + [len(ad_id_list)] 301 | 302 | if len(history_pages_deque) >= sample_page_num: 303 | history_pages_deque.popleft() 304 | history_pages_deque.append(page_ads_features) 305 | 306 | history_pages_features = list(history_pages_deque) 307 | num_mask_pages = sample_page_num-len(history_pages_features) 308 | for _ in range(num_mask_pages): 309 | history_pages_features.append(mask_page_features) 310 | history_features = flatten(history_pages_features) + [sample_page_num-num_mask_pages] 311 | # recent_histroy[u_id] = history_features 312 | tmp_recent[u_id] = history_features 313 | return (tmp_samples, tmp_recent) 314 | 315 | train_sample_data_path = sample_data_dir+training_data_type+'_data.csv' 316 | if not os.path.exists(train_sample_data_path): 317 | print('produce train sample') 318 | training_user_seq_lists = get_user_seq(training_data_type) 319 | training_SearchInfo = training_SearchInfo.set_index('UserID').sort_index() 320 | new_training_user_seq_lists = [] 321 | for user_seq_list in tqdm.tqdm(training_user_seq_lists, desc='filter train seq lists'): 322 | page_seq = user_seq_list[1] 323 | if len(page_seq) < 2: 324 | num_drop_user += 1 325 | continue 326 | new_training_user_seq_lists.append(user_seq_list) 327 | training_user_seq_lists = new_training_user_seq_lists 328 | 329 | # pool = Pool(10) 330 | num_workers = 20 331 | Results = process_map(deal_with_train_seq_list, training_user_seq_lists, max_workers=num_workers, chunksize=1) 332 | training_sample_list = [] 333 | for x in tqdm.tqdm(Results, desc='update training sample list'): 334 | for y in x[0]: 335 | training_sample_list.append(y) 336 | recent_histroy = dict() 337 | for x in tqdm.tqdm(Results, desc='update recent history dict'): 338 | recent_histroy.update(x[1]) 339 | 340 | training_sample_list = np.array(training_sample_list) 341 | training_data = pd.DataFrame(training_sample_list) 342 | print('drop_user: ', num_drop_user) 343 | training_data.to_csv(sample_data_dir+training_data_type+'_data.csv', index=False) 344 | 345 | if not os.path.exists(sample_data_dir+'recent_histroy.pickle'): 346 | with open(sample_data_dir+'recent_history.pickle', 'wb') as f: 347 | pickle.dump(recent_histroy, f) 348 | else: 349 | f = open(sample_data_dir+'recent_history.pickle', 'rb') 350 | recent_histroy = pickle.load(f) 351 | 352 | 353 | def deal_with_validate_seq_list(user_seq_list): 354 | tmp_samples = [] 355 | tmp_recent = defaultdict(list) 356 | u_id = user_seq_list[0][0] 357 | user_features = [u_id]+UserInfo.loc[u_id].values.tolist() 358 | search_info = validate_SearchInfo.loc[[u_id]] 359 | search_info = search_info.set_index('SearchID') 360 | 361 | recent_histroy_features = recent_histroy[u_id] 362 | history_features, history_page_lens = recent_histroy_features[:-1], recent_histroy_features[-1] 363 | history_features = np.array(history_features).reshape(-1, len(mask_page_features)) 364 | history_features = history_features[:history_page_lens] 365 | history_pages_deque = deque(list(history_features)) 366 | 367 | 368 | page_seq = user_seq_list[1] 369 | for i in range(len(page_seq)): 370 | page = page_seq[i] 371 | search_id = page[0][0] 372 | ad_id_list = page[1] 373 | current_query_features = get_query_feature(search_id, search_info) 374 | search_stream = SearchStream.loc[[search_id]] 375 | search_stream = search_stream.set_index('AdID').sort_index() 376 | 377 | history_pages_features = list(history_pages_deque) 378 | num_mask_pages = sample_page_num-len(history_pages_features) 379 | for _ in range(num_mask_pages): 380 | history_pages_features.append(mask_page_features) 381 | history_pages_features = flatten(history_pages_features) + [5-num_mask_pages] 382 | 383 | page_ads_features = [] 384 | # cut_len = min(len(ad_id_list), 5) 385 | # ad_id_list = ad_id_list[:cut_len] 386 | 387 | 388 | is_sample_page = False 389 | if i == len(page_seq)-1: 390 | is_sample_page = True 391 | if search_id in click_dict: 392 | is_sample_page = True 393 | 394 | 395 | have_neg = False 396 | for ad_id in ad_id_list: 397 | current_ad_features = get_ad_features(search_stream, ad_id) 398 | current_qad_feature = current_query_features + current_ad_features 399 | page_ads_features.append(current_qad_feature) 400 | if not is_sample_page: 401 | continue 402 | if search_stream['ObjectType'].loc[ad_id] != 3: 403 | continue 404 | 405 | label = current_ad_features[-1] 406 | if label !=1: 407 | if not have_neg and label == 0: 408 | have_neg = True 409 | else: 410 | continue 411 | 412 | label = current_ad_features[-1] 413 | target_ad_features = current_ad_features[:-1] 414 | target_quad_features = user_features + current_query_features + target_ad_features 415 | sample = [label] + target_quad_features + history_pages_features 416 | assert len(sample) == num_sample_features, "sample length wrong!!! with len: "+str(len(sample)) 417 | tmp_samples.append(sample) 418 | # validate_sample_list.append(sample) 419 | 420 | for _ in range(page_ad_num-len(ad_id_list)): 421 | page_ads_features.append(mask_ad_features) 422 | page_ads_features = flatten(page_ads_features) + [len(ad_id_list)] 423 | 424 | if len(history_pages_deque) >= sample_page_num: 425 | history_pages_deque.popleft() 426 | history_pages_deque.append(page_ads_features) 427 | 428 | history_pages_features = list(history_pages_deque) 429 | num_mask_pages = sample_page_num-len(history_pages_features) 430 | for _ in range(num_mask_pages): 431 | history_pages_features.append(mask_page_features) 432 | history_features = flatten(history_pages_features) + [sample_page_num-num_mask_pages] 433 | # recent_histroy[u_id] = history_features 434 | tmp_recent[u_id] = history_features 435 | return (tmp_samples, tmp_recent) 436 | 437 | print('begin produce val/test sample') 438 | for validate_data_type in ['Val', 'Test']: 439 | validate_SearchInfo = eval(validate_data_type+'SearchInfo') 440 | validate_sample_list = [] 441 | validate_drop_user = 0 442 | 443 | validate_user_seq_lists = get_user_seq(validate_data_type) 444 | validate_SearchInfo = validate_SearchInfo.set_index('UserID').sort_index() 445 | new_validate_user_seq_lists = [] 446 | for user_seq_list in tqdm.tqdm(validate_user_seq_lists, desc='filte val seq lists'): 447 | u_id = user_seq_list[0][0] 448 | if u_id not in recent_histroy.keys(): # remove cold-start user 449 | validate_drop_user += 1 450 | continue 451 | new_validate_user_seq_lists.append(user_seq_list) 452 | validate_user_seq_lists = new_validate_user_seq_lists 453 | 454 | # pool = Pool(10) 455 | num_workers = 20 456 | Results = process_map(deal_with_validate_seq_list, validate_user_seq_lists, max_workers=num_workers, chunksize=1) 457 | validate_sample_list = [] 458 | for x in tqdm.tqdm(Results, desc='update '+validate_data_type+' sample list'): 459 | for y in x[0]: 460 | validate_sample_list.append(y) 461 | recent_histroy = dict() 462 | for x in tqdm.tqdm(Results, desc='update recent history dict'): 463 | recent_histroy.update(x[1]) 464 | 465 | validate_sample_list = np.array(validate_sample_list) 466 | validate_data = pd.DataFrame(validate_sample_list) 467 | print(validate_data_type, ' data:', validate_data.shape) 468 | # current 469 | # Val data: (1,160,184, 895) 470 | # Val drop_user: 327826 471 | # Test data: (173,798, 895) 472 | # Test drop_user: 311017 473 | 474 | # small 475 | # Val data: (308,350, 895) 476 | # Val drop_user: 327,826 477 | # Test data: (43,174, 895) 478 | # Test drop_user: 311,017 479 | 480 | # old 481 | # Val data: (314,663, 895) 482 | # Test data: (43,957, 895) 483 | print(validate_data_type,' drop_user: ', validate_drop_user) 484 | validate_data.to_csv(sample_data_dir+validate_data_type+'_data.csv', index=False) -------------------------------------------------------------------------------- /Data/sample.csv: -------------------------------------------------------------------------------- 1 | 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894 2 | 0,1037836,23170,20,2014,85,275,2031546,1,292,2475,3,26,3603,42,3,12,42,0,198,0,0,11049533,42,3,12,42,1873678,2353686,2030990,2740797,1292370,1445,0,0,0,0,7,2,838781,2031546,1,141,3963,2,26,1043,1,1,10,45,1882,0,0,0,7566569,42,3,12,42,1870329,2821799,1130439,1247498,687063,1442,0,0,0,0,1,10,1,838781,2031546,1,141,3963,2,26,1043,1,1,10,45,1882,0,0,0,7533364,42,3,12,42,2148731,2353686,2822093,2386325,2806723,3833,1446,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,6669666,26,3,12,19,2148656,2394141,2645681,2353686,2504327,1424,0,0,0,0,1,1,0,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,22147078,26,3,12,19,1943361,2779592,2770964,416329,0,1424,0,0,0,0,7,8,1,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,22507108,26,3,12,19,2148837,0,0,0,0,3842,1423,0,0,0,6,0,2,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,11491970,26,3,12,19,1974002,1943361,1113764,769492,979079,1424,0,0,0,0,8,0,2,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,33503899,26,3,12,19,1559991,0,0,0,0,1424,0,0,0,0,2,0,2,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2 3 | 1,1037836,23170,20,2014,85,275,2031546,1,292,2475,3,26,3603,42,3,12,42,0,198,0,0,14215938,42,3,12,42,1378138,341542,263652,579943,0,1445,0,0,0,0,1,7,838781,2031546,1,141,3963,2,26,1043,1,1,10,45,1882,0,0,0,7566569,42,3,12,42,1870329,2821799,1130439,1247498,687063,1442,0,0,0,0,1,10,1,838781,2031546,1,141,3963,2,26,1043,1,1,10,45,1882,0,0,0,7533364,42,3,12,42,2148731,2353686,2822093,2386325,2806723,3833,1446,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,6669666,26,3,12,19,2148656,2394141,2645681,2353686,2504327,1424,0,0,0,0,1,1,0,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,22147078,26,3,12,19,1943361,2779592,2770964,416329,0,1424,0,0,0,0,7,8,1,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,22507108,26,3,12,19,2148837,0,0,0,0,3842,1423,0,0,0,6,0,2,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,11491970,26,3,12,19,1974002,1943361,1113764,769492,979079,1424,0,0,0,0,8,0,2,918681,2031546,1,206,3963,2,26,1043,26,3,12,19,0,0,0,0,33503899,26,3,12,19,1559991,0,0,0,0,1424,0,0,0,0,2,0,2,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2 4 | 0,2905431,16119,20,2014,62,326,1505086,0,294,821,3,63,3383,27,3,12,29,0,173,0,0,24438982,27,3,12,29,290309,0,0,0,0,1321,0,0,0,0,7,5,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,6892762,27,3,12,29,871651,2079542,1113764,880465,0,1321,0,0,0,0,1,9,0,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,24438982,27,3,12,29,290309,0,0,0,0,1321,0,0,0,0,7,5,1,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,18314798,27,3,12,29,1113764,1185262,2662392,2450287,1665898,1321,0,0,0,0,6,0,2,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,24402627,27,3,12,29,1113764,1185211,2662392,2450287,2308303,1321,0,0,0,0,8,0,2,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,29325942,27,3,12,29,2079542,1489353,426570,2775675,2647079,1321,0,0,0,0,2,0,2,5,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,9026365,27,3,12,29,871651,2079542,1172113,1146550,0,1321,0,0,0,0,7,5,0,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,6718520,27,3,12,29,1921894,2282612,2718707,2353686,2689594,1321,0,0,0,0,1,7,1,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,25880508,27,3,12,29,2079542,1113764,0,0,0,1321,0,0,0,0,2,0,2,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,7338633,27,3,12,29,2054357,2685465,2686299,820387,1236823,1321,0,0,0,0,8,0,2,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,11979977,27,3,12,29,303551,0,0,0,0,1321,0,0,0,0,6,0,2,5,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,14727875,39,3,9,44,1571635,2764198,2560844,2509401,0,3784,0,0,0,0,7,7,0,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,9115234,39,3,9,44,1648208,2015964,2058053,0,0,3784,0,0,0,0,1,7,1,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,18480983,39,3,9,44,1991123,2264821,2268398,2235727,2776976,3784,0,0,0,0,6,0,2,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,31687062,39,3,9,44,2060326,2197310,0,0,0,3784,0,0,0,0,8,0,2,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,32128208,39,3,9,44,2110059,2353686,2261124,0,0,3784,0,0,0,0,2,0,2,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3 5 | 1,2905431,16119,20,2014,62,326,1505086,0,294,821,3,63,3383,27,3,12,29,0,173,0,0,6892762,27,3,12,29,871651,2079542,1113764,880465,0,1321,0,0,0,0,1,9,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,6892762,27,3,12,29,871651,2079542,1113764,880465,0,1321,0,0,0,0,1,9,0,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,24438982,27,3,12,29,290309,0,0,0,0,1321,0,0,0,0,7,5,1,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,18314798,27,3,12,29,1113764,1185262,2662392,2450287,1665898,1321,0,0,0,0,6,0,2,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,24402627,27,3,12,29,1113764,1185211,2662392,2450287,2308303,1321,0,0,0,0,8,0,2,684108,1505086,0,152,821,3,63,3383,27,3,12,29,0,173,0,0,29325942,27,3,12,29,2079542,1489353,426570,2775675,2647079,1321,0,0,0,0,2,0,2,5,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,9026365,27,3,12,29,871651,2079542,1172113,1146550,0,1321,0,0,0,0,7,5,0,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,6718520,27,3,12,29,1921894,2282612,2718707,2353686,2689594,1321,0,0,0,0,1,7,1,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,25880508,27,3,12,29,2079542,1113764,0,0,0,1321,0,0,0,0,2,0,2,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,7338633,27,3,12,29,2054357,2685465,2686299,820387,1236823,1321,0,0,0,0,8,0,2,942426,1505086,0,158,821,3,63,3383,27,3,12,29,0,173,0,0,11979977,27,3,12,29,303551,0,0,0,0,1321,0,0,0,0,6,0,2,5,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,14727875,39,3,9,44,1571635,2764198,2560844,2509401,0,3784,0,0,0,0,7,7,0,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,9115234,39,3,9,44,1648208,2015964,2058053,0,0,3784,0,0,0,0,1,7,1,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,18480983,39,3,9,44,1991123,2264821,2268398,2235727,2776976,3784,0,0,0,0,6,0,2,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,31687062,39,3,9,44,2060326,2197310,0,0,0,3784,0,0,0,0,8,0,2,1009293,1505086,0,286,821,3,63,3383,39,3,9,44,0,291,0,0,32128208,39,3,9,44,2110059,2353686,2261124,0,0,3784,0,0,0,0,2,0,2,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3 6 | 0,2105556,41539,9,2014,15,444,1567942,0,298,2646,3,38,2469,28,3,9,41,0,351,367,0,7827939,28,3,9,41,1663875,2646227,853573,790301,398297,3880,3862,0,0,0,1,2,632233,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,33102223,39,3,9,44,1699986,2449656,2851760,71890,1996694,3786,5665,0,0,0,7,3,0,632233,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,7791744,39,3,9,44,1568297,2799561,155604,1678821,0,3786,5665,0,0,0,1,6,1,632233,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,32874123,39,3,9,44,1853041,0,0,0,0,3786,5665,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,28610579,39,3,9,44,1639226,2115953,2348174,0,0,3786,5665,0,0,0,1,6,0,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,22733180,39,3,9,44,1699986,2693252,2200621,9794,0,3786,5665,0,0,0,7,3,1,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,25773732,39,3,9,44,1775531,2474505,2472219,0,0,3786,5665,0,0,0,8,0,2,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,16182786,39,3,9,44,1690854,2472219,1583314,961839,0,3786,5665,0,0,0,6,0,2,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,11590789,39,3,9,44,1852899,1780858,1922218,71943,0,3786,5665,0,0,0,2,0,2,5,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,27275713,39,3,9,44,1807647,2344392,2348174,2368789,2740797,3786,5665,0,0,0,1,6,1,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,20119670,39,3,9,44,1695838,2559702,2348174,1853838,2394141,3786,5665,0,0,0,7,7,1,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,12605455,39,3,9,44,1852899,1775077,2648058,128200,0,3786,5665,0,0,0,8,0,2,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,8987069,39,3,9,44,2126456,2936704,2941900,2202325,0,3786,5665,0,0,0,2,0,2,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,22530114,39,3,9,44,1853023,2394141,2471876,2353686,2321534,3786,5665,0,0,0,6,0,2,5,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,232172,39,3,9,44,2050645,2308685,1699720,2235335,2779592,3786,5665,0,0,0,7,3,0,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,20988845,39,3,9,44,1907607,2348174,2299556,2740797,2865622,3786,5665,0,0,0,1,7,1,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,32727339,39,3,9,44,1695759,2472219,2838793,0,0,3786,5665,0,0,0,2,0,2,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,27103414,39,3,9,44,1899049,2348174,2368786,2353686,2452478,3786,5665,0,0,0,8,0,2,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,8657998,39,3,9,44,1699986,0,0,0,0,3786,5665,0,0,0,6,0,2,5,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,24546183,39,3,9,44,1805564,2560844,2331952,1766401,111804,3786,5665,0,0,0,7,5,0,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,9102893,39,3,9,44,1623595,2348174,2368789,2740797,2865622,3786,5665,0,0,0,1,6,1,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,29517812,39,3,9,44,1852899,2606140,2581329,0,0,3786,5665,0,0,0,6,0,2,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,31738496,39,3,9,44,1852899,2606140,2581329,0,0,3786,5665,0,0,0,8,0,2,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,4683274,39,3,9,44,1695892,2470333,2472219,0,0,3786,5665,0,0,0,2,0,2,5,5 7 | 1,2105556,41539,9,2014,15,444,1567942,0,298,2646,3,38,2469,28,3,9,41,0,351,367,0,36444791,28,3,9,41,1947648,558684,613455,370280,767349,3880,3862,0,0,0,7,4,632233,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,33102223,39,3,9,44,1699986,2449656,2851760,71890,1996694,3786,5665,0,0,0,7,3,0,632233,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,7791744,39,3,9,44,1568297,2799561,155604,1678821,0,3786,5665,0,0,0,1,6,1,632233,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,32874123,39,3,9,44,1853041,0,0,0,0,3786,5665,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,28610579,39,3,9,44,1639226,2115953,2348174,0,0,3786,5665,0,0,0,1,6,0,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,22733180,39,3,9,44,1699986,2693252,2200621,9794,0,3786,5665,0,0,0,7,3,1,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,25773732,39,3,9,44,1775531,2474505,2472219,0,0,3786,5665,0,0,0,8,0,2,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,16182786,39,3,9,44,1690854,2472219,1583314,961839,0,3786,5665,0,0,0,6,0,2,768160,1567942,0,253,2646,3,38,2469,39,3,9,44,0,293,808,0,11590789,39,3,9,44,1852899,1780858,1922218,71943,0,3786,5665,0,0,0,2,0,2,5,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,27275713,39,3,9,44,1807647,2344392,2348174,2368789,2740797,3786,5665,0,0,0,1,6,1,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,20119670,39,3,9,44,1695838,2559702,2348174,1853838,2394141,3786,5665,0,0,0,7,7,1,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,12605455,39,3,9,44,1852899,1775077,2648058,128200,0,3786,5665,0,0,0,8,0,2,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,8987069,39,3,9,44,2126456,2936704,2941900,2202325,0,3786,5665,0,0,0,2,0,2,939606,1567942,0,93,2646,3,38,2469,39,3,9,44,0,293,808,0,22530114,39,3,9,44,1853023,2394141,2471876,2353686,2321534,3786,5665,0,0,0,6,0,2,5,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,232172,39,3,9,44,2050645,2308685,1699720,2235335,2779592,3786,5665,0,0,0,7,3,0,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,20988845,39,3,9,44,1907607,2348174,2299556,2740797,2865622,3786,5665,0,0,0,1,7,1,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,32727339,39,3,9,44,1695759,2472219,2838793,0,0,3786,5665,0,0,0,2,0,2,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,27103414,39,3,9,44,1899049,2348174,2368786,2353686,2452478,3786,5665,0,0,0,8,0,2,1047732,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,8657998,39,3,9,44,1699986,0,0,0,0,3786,5665,0,0,0,6,0,2,5,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,24546183,39,3,9,44,1805564,2560844,2331952,1766401,111804,3786,5665,0,0,0,7,5,0,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,9102893,39,3,9,44,1623595,2348174,2368789,2740797,2865622,3786,5665,0,0,0,1,6,1,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,29517812,39,3,9,44,1852899,2606140,2581329,0,0,3786,5665,0,0,0,6,0,2,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,31738496,39,3,9,44,1852899,2606140,2581329,0,0,3786,5665,0,0,0,8,0,2,1111380,1567942,0,169,2646,3,38,2469,39,3,9,44,0,293,808,0,4683274,39,3,9,44,1695892,2470333,2472219,0,0,3786,5665,0,0,0,2,0,2,5,5 8 | 0,4028512,10275,20,2014,25,635,736839,0,293,1126,2,66,2723,51,3,9,23,0,283,0,0,22968355,51,3,9,23,2088333,2794522,937062,931701,0,3776,0,0,0,0,7,4,249999,736839,0,275,1126,2,66,2723,51,3,9,23,0,283,0,0,10639328,51,3,9,23,1678445,2241868,1587157,1983774,1844891,3776,0,0,0,0,1,10,0,249999,736839,0,275,1126,2,66,2723,51,3,9,23,0,283,0,0,21676486,51,3,9,23,2162451,2561770,360449,2918240,1099526,3776,0,0,0,0,7,8,1,249999,736839,0,275,1126,2,66,2723,51,3,9,23,0,283,0,0,4991003,51,3,9,23,1863799,2795197,2544282,2740797,2384619,3776,0,0,0,0,8,0,2,249999,736839,0,275,1126,2,66,2723,51,3,9,23,0,283,0,0,27461686,51,3,9,23,1695031,2353686,2663399,0,0,3776,0,0,0,0,6,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,23728243,12,3,3,16,2145146,0,0,0,0,11,0,0,0,0,7,6,0,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,13601010,12,3,3,16,2005616,2567543,2898814,2164901,0,11,0,0,0,0,1,8,1,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,25316626,12,3,3,16,822796,85756,1028472,680921,1227353,11,0,0,0,0,6,0,2,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,23369460,12,3,3,16,1118578,431399,1142424,1015728,1227353,11,0,0,0,0,8,0,2,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,12665055,12,3,3,16,1550342,2580712,808682,628208,745698,11,0,0,0,0,2,0,2,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2 9 | 1,4028512,10275,20,2014,25,635,736839,0,293,1126,2,66,2723,51,3,9,23,0,283,0,0,22034073,51,3,9,23,1885476,2879010,2353686,2879077,64276,3776,0,0,0,0,1,10,249999,736839,0,275,1126,2,66,2723,51,3,9,23,0,283,0,0,10639328,51,3,9,23,1678445,2241868,1587157,1983774,1844891,3776,0,0,0,0,1,10,0,249999,736839,0,275,1126,2,66,2723,51,3,9,23,0,283,0,0,21676486,51,3,9,23,2162451,2561770,360449,2918240,1099526,3776,0,0,0,0,7,8,1,249999,736839,0,275,1126,2,66,2723,51,3,9,23,0,283,0,0,4991003,51,3,9,23,1863799,2795197,2544282,2740797,2384619,3776,0,0,0,0,8,0,2,249999,736839,0,275,1126,2,66,2723,51,3,9,23,0,283,0,0,27461686,51,3,9,23,1695031,2353686,2663399,0,0,3776,0,0,0,0,6,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,23728243,12,3,3,16,2145146,0,0,0,0,11,0,0,0,0,7,6,0,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,13601010,12,3,3,16,2005616,2567543,2898814,2164901,0,11,0,0,0,0,1,8,1,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,25316626,12,3,3,16,822796,85756,1028472,680921,1227353,11,0,0,0,0,6,0,2,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,23369460,12,3,3,16,1118578,431399,1142424,1015728,1227353,11,0,0,0,0,8,0,2,962310,736839,0,73,1063,3,66,2170,12,3,3,16,0,10,0,0,12665055,12,3,3,16,1550342,2580712,808682,628208,745698,11,0,0,0,0,2,0,2,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2 10 | 0,1135666,18624,7,2014,64,801,449147,1,298,3427,3,16,3581,61,3,3,43,0,110,16,252,18827475,61,3,3,43,2100756,0,0,0,0,17,1583,0,0,0,1,3,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,22067181,61,3,3,43,2161963,0,0,0,0,13,1583,0,0,0,7,2,0,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,1537147,61,3,3,43,1697851,2853725,0,0,0,13,1583,0,0,0,1,4,1,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,34796409,61,3,3,43,1697851,0,0,0,0,1583,1158,13,0,0,2,0,2,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,17809322,61,3,3,43,2111616,2258335,2560844,2514018,0,1583,1158,13,0,0,6,0,2,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,4511265,61,3,3,43,1697851,2258335,0,0,0,1583,1158,13,0,0,8,0,2,5,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,1301445,48,3,3,28,1854151,1225588,1176779,588027,867770,1245,0,0,0,0,7,4,0,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,1981951,48,3,3,28,1721137,2359272,126677,2343639,2579233,1245,0,0,0,0,1,6,1,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,2016015,48,3,3,28,2158982,2397536,2500914,2464121,2212878,1245,0,0,0,0,6,0,2,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,3713210,48,3,3,28,309819,1870649,2522628,85754,563980,1245,0,0,0,0,8,0,2,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,30090967,48,3,3,28,1678465,950627,2842620,2493674,2227771,1245,0,0,0,0,2,0,2,5,210819,449147,0,283,3427,3,16,3581,61,3,3,43,0,105,17,252,8525410,61,3,3,43,1939823,2740797,2916398,0,0,18,1583,0,0,0,7,1,0,210819,449147,0,283,3427,3,16,3581,61,3,3,43,0,105,17,252,9220615,61,3,3,43,1939823,0,0,0,0,18,1583,0,0,0,1,5,1,210819,449147,0,283,3427,3,16,3581,61,3,3,43,0,105,17,252,33349880,61,3,3,43,1898748,2575225,2475490,2883702,2560844,1195,1583,18,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3 11 | 1,1135666,18624,7,2014,64,801,449147,1,298,3427,3,16,3581,61,3,3,43,0,110,16,252,6915853,61,3,3,43,2021435,1262351,2740797,2381821,2559010,17,1583,0,0,0,7,1,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,22067181,61,3,3,43,2161963,0,0,0,0,13,1583,0,0,0,7,2,0,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,1537147,61,3,3,43,1697851,2853725,0,0,0,13,1583,0,0,0,1,4,1,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,34796409,61,3,3,43,1697851,0,0,0,0,1583,1158,13,0,0,2,0,2,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,17809322,61,3,3,43,2111616,2258335,2560844,2514018,0,1583,1158,13,0,0,6,0,2,612885,449147,0,279,3427,3,16,3581,61,3,3,43,0,76,12,252,4511265,61,3,3,43,1697851,2258335,0,0,0,1583,1158,13,0,0,8,0,2,5,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,1301445,48,3,3,28,1854151,1225588,1176779,588027,867770,1245,0,0,0,0,7,4,0,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,1981951,48,3,3,28,1721137,2359272,126677,2343639,2579233,1245,0,0,0,0,1,6,1,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,2016015,48,3,3,28,2158982,2397536,2500914,2464121,2212878,1245,0,0,0,0,6,0,2,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,3713210,48,3,3,28,309819,1870649,2522628,85754,563980,1245,0,0,0,0,8,0,2,680209,449147,0,216,3427,3,16,3581,48,3,3,28,0,152,0,0,30090967,48,3,3,28,1678465,950627,2842620,2493674,2227771,1245,0,0,0,0,2,0,2,5,210819,449147,0,283,3427,3,16,3581,61,3,3,43,0,105,17,252,8525410,61,3,3,43,1939823,2740797,2916398,0,0,18,1583,0,0,0,7,1,0,210819,449147,0,283,3427,3,16,3581,61,3,3,43,0,105,17,252,9220615,61,3,3,43,1939823,0,0,0,0,18,1583,0,0,0,1,5,1,210819,449147,0,283,3427,3,16,3581,61,3,3,43,0,105,17,252,33349880,61,3,3,43,1898748,2575225,2475490,2883702,2560844,1195,1583,18,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3 12 | 0,3043906,62046,15,2014,64,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,33969662,39,3,9,44,1786268,2058388,2943185,0,0,3793,0,0,0,0,7,3,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,22735388,39,3,9,44,1786268,1879365,2342465,2784239,2740797,3793,0,0,0,0,1,5,0,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,17626124,39,3,9,44,1786268,2058388,2943209,0,0,3793,0,0,0,0,7,4,1,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,22631622,39,3,9,44,1627629,2565381,1899049,2913681,1911035,3793,0,0,0,0,8,0,2,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,32253666,39,3,9,44,2159685,2047053,1791418,2917783,2581329,3793,0,0,0,0,6,0,2,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,36637195,39,3,9,44,2061023,2308685,1698573,0,0,3787,0,0,0,0,2,0,2,5,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,3993204,39,3,9,44,1932940,2238386,2449362,1603053,1587516,3793,0,0,0,0,1,2,0,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,32336711,39,3,9,44,1786268,2003355,1840597,2238386,2740797,3793,0,0,0,0,7,8,1,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,2813274,39,3,9,44,1805564,1850258,2235335,2593098,2779592,3786,5665,0,0,0,6,0,2,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,8096430,39,3,9,44,2159685,0,0,0,0,3793,0,0,0,0,8,0,2,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,12668424,39,3,9,44,1974002,2878866,2525237,1361076,0,3793,0,0,0,0,2,0,2,5,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,30303763,39,3,9,44,1932940,2449362,1583592,443796,1817792,3793,0,0,0,0,7,7,0,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,13369583,39,3,9,44,1786268,1615279,2943188,0,0,3793,0,0,0,0,1,1,1,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,100540,39,3,9,44,2159483,296629,2332187,2353686,2595022,3793,0,0,0,0,6,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,19366131,39,3,9,44,2159483,2481724,1791767,2217769,0,3793,0,0,0,0,2,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,23198596,39,3,9,44,1786217,1126769,1861386,1695759,2525237,3786,5665,0,0,0,8,0,2,5,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,31669306,39,3,9,44,1786268,1840775,1583139,1669549,0,3793,0,0,0,0,7,7,0,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,10578925,39,3,9,44,1786268,2344392,1792876,1675461,0,3793,0,0,0,0,1,1,1,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,26340887,39,3,9,44,2177019,2721091,2525237,0,0,3790,5673,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,5633422,39,3,9,44,1786268,2122787,2016945,2364287,2869049,3793,0,0,0,0,7,7,0,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,3160410,39,3,9,44,1786268,1761539,0,0,0,3793,0,0,0,0,1,1,1,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,9171511,39,3,9,44,2104539,2913681,1805011,2545346,64276,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,5 13 | 1,3043906,62046,15,2014,64,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,22735388,39,3,9,44,1786268,1879365,2342465,2784239,2740797,3793,0,0,0,0,1,4,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,22735388,39,3,9,44,1786268,1879365,2342465,2784239,2740797,3793,0,0,0,0,1,5,0,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,17626124,39,3,9,44,1786268,2058388,2943209,0,0,3793,0,0,0,0,7,4,1,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,22631622,39,3,9,44,1627629,2565381,1899049,2913681,1911035,3793,0,0,0,0,8,0,2,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,32253666,39,3,9,44,2159685,2047053,1791418,2917783,2581329,3793,0,0,0,0,6,0,2,32663,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,36637195,39,3,9,44,2061023,2308685,1698573,0,0,3787,0,0,0,0,2,0,2,5,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,3993204,39,3,9,44,1932940,2238386,2449362,1603053,1587516,3793,0,0,0,0,1,2,0,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,32336711,39,3,9,44,1786268,2003355,1840597,2238386,2740797,3793,0,0,0,0,7,8,1,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,2813274,39,3,9,44,1805564,1850258,2235335,2593098,2779592,3786,5665,0,0,0,6,0,2,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,8096430,39,3,9,44,2159685,0,0,0,0,3793,0,0,0,0,8,0,2,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,12668424,39,3,9,44,1974002,2878866,2525237,1361076,0,3793,0,0,0,0,2,0,2,5,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,30303763,39,3,9,44,1932940,2449362,1583592,443796,1817792,3793,0,0,0,0,7,7,0,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,13369583,39,3,9,44,1786268,1615279,2943188,0,0,3793,0,0,0,0,1,1,1,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,100540,39,3,9,44,2159483,296629,2332187,2353686,2595022,3793,0,0,0,0,6,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,19366131,39,3,9,44,2159483,2481724,1791767,2217769,0,3793,0,0,0,0,2,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,23198596,39,3,9,44,1786217,1126769,1861386,1695759,2525237,3786,5665,0,0,0,8,0,2,5,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,31669306,39,3,9,44,1786268,1840775,1583139,1669549,0,3793,0,0,0,0,7,7,0,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,10578925,39,3,9,44,1786268,2344392,1792876,1675461,0,3793,0,0,0,0,1,1,1,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,26340887,39,3,9,44,2177019,2721091,2525237,0,0,3790,5673,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,5633422,39,3,9,44,1786268,2122787,2016945,2364287,2869049,3793,0,0,0,0,7,7,0,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,3160410,39,3,9,44,1786268,1761539,0,0,0,3793,0,0,0,0,1,1,1,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,9171511,39,3,9,44,2104539,2913681,1805011,2545346,64276,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,5 14 | 0,3043906,62046,15,2014,64,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,27015535,39,3,9,44,1786268,1761528,0,0,0,3793,0,0,0,0,1,3,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,3993204,39,3,9,44,1932940,2238386,2449362,1603053,1587516,3793,0,0,0,0,1,2,0,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,32336711,39,3,9,44,1786268,2003355,1840597,2238386,2740797,3793,0,0,0,0,7,8,1,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,2813274,39,3,9,44,1805564,1850258,2235335,2593098,2779592,3786,5665,0,0,0,6,0,2,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,8096430,39,3,9,44,2159685,0,0,0,0,3793,0,0,0,0,8,0,2,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,12668424,39,3,9,44,1974002,2878866,2525237,1361076,0,3793,0,0,0,0,2,0,2,5,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,30303763,39,3,9,44,1932940,2449362,1583592,443796,1817792,3793,0,0,0,0,7,7,0,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,13369583,39,3,9,44,1786268,1615279,2943188,0,0,3793,0,0,0,0,1,1,1,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,100540,39,3,9,44,2159483,296629,2332187,2353686,2595022,3793,0,0,0,0,6,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,19366131,39,3,9,44,2159483,2481724,1791767,2217769,0,3793,0,0,0,0,2,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,23198596,39,3,9,44,1786217,1126769,1861386,1695759,2525237,3786,5665,0,0,0,8,0,2,5,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,31669306,39,3,9,44,1786268,1840775,1583139,1669549,0,3793,0,0,0,0,7,7,0,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,10578925,39,3,9,44,1786268,2344392,1792876,1675461,0,3793,0,0,0,0,1,1,1,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,26340887,39,3,9,44,2177019,2721091,2525237,0,0,3790,5673,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,5633422,39,3,9,44,1786268,2122787,2016945,2364287,2869049,3793,0,0,0,0,7,7,0,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,3160410,39,3,9,44,1786268,1761539,0,0,0,3793,0,0,0,0,1,1,1,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,9171511,39,3,9,44,2104539,2913681,1805011,2545346,64276,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,33969662,39,3,9,44,1786268,2058388,2943185,0,0,3793,0,0,0,0,7,3,0,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,22735388,39,3,9,44,1786268,1879365,2342465,2784239,2740797,3793,0,0,0,0,1,4,1,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,2548374,39,3,9,44,2159483,243243,2333048,2715065,2490762,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,5 15 | 1,3043906,62046,15,2014,64,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,3021805,39,3,9,44,1786268,1850273,376381,909354,931163,3793,0,0,0,0,7,3,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,3993204,39,3,9,44,1932940,2238386,2449362,1603053,1587516,3793,0,0,0,0,1,2,0,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,32336711,39,3,9,44,1786268,2003355,1840597,2238386,2740797,3793,0,0,0,0,7,8,1,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,2813274,39,3,9,44,1805564,1850258,2235335,2593098,2779592,3786,5665,0,0,0,6,0,2,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,8096430,39,3,9,44,2159685,0,0,0,0,3793,0,0,0,0,8,0,2,81786,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,12668424,39,3,9,44,1974002,2878866,2525237,1361076,0,3793,0,0,0,0,2,0,2,5,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,30303763,39,3,9,44,1932940,2449362,1583592,443796,1817792,3793,0,0,0,0,7,7,0,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,13369583,39,3,9,44,1786268,1615279,2943188,0,0,3793,0,0,0,0,1,1,1,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,100540,39,3,9,44,2159483,296629,2332187,2353686,2595022,3793,0,0,0,0,6,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,19366131,39,3,9,44,2159483,2481724,1791767,2217769,0,3793,0,0,0,0,2,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,23198596,39,3,9,44,1786217,1126769,1861386,1695759,2525237,3786,5665,0,0,0,8,0,2,5,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,31669306,39,3,9,44,1786268,1840775,1583139,1669549,0,3793,0,0,0,0,7,7,0,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,10578925,39,3,9,44,1786268,2344392,1792876,1675461,0,3793,0,0,0,0,1,1,1,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,26340887,39,3,9,44,2177019,2721091,2525237,0,0,3790,5673,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,5633422,39,3,9,44,1786268,2122787,2016945,2364287,2869049,3793,0,0,0,0,7,7,0,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,3160410,39,3,9,44,1786268,1761539,0,0,0,3793,0,0,0,0,1,1,1,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,9171511,39,3,9,44,2104539,2913681,1805011,2545346,64276,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,33969662,39,3,9,44,1786268,2058388,2943185,0,0,3793,0,0,0,0,7,3,0,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,22735388,39,3,9,44,1786268,1879365,2342465,2784239,2740797,3793,0,0,0,0,1,4,1,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,2548374,39,3,9,44,2159483,243243,2333048,2715065,2490762,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,5 16 | 0,3043906,62046,15,2014,64,970579,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,16904117,39,3,9,44,2159483,2644866,1635215,2016167,0,3793,0,0,0,0,1,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,30303763,39,3,9,44,1932940,2449362,1583592,443796,1817792,3793,0,0,0,0,7,7,0,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,13369583,39,3,9,44,1786268,1615279,2943188,0,0,3793,0,0,0,0,1,1,1,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,100540,39,3,9,44,2159483,296629,2332187,2353686,2595022,3793,0,0,0,0,6,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,19366131,39,3,9,44,2159483,2481724,1791767,2217769,0,3793,0,0,0,0,2,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,23198596,39,3,9,44,1786217,1126769,1861386,1695759,2525237,3786,5665,0,0,0,8,0,2,5,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,31669306,39,3,9,44,1786268,1840775,1583139,1669549,0,3793,0,0,0,0,7,7,0,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,10578925,39,3,9,44,1786268,2344392,1792876,1675461,0,3793,0,0,0,0,1,1,1,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,26340887,39,3,9,44,2177019,2721091,2525237,0,0,3790,5673,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,5633422,39,3,9,44,1786268,2122787,2016945,2364287,2869049,3793,0,0,0,0,7,7,0,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,3160410,39,3,9,44,1786268,1761539,0,0,0,3793,0,0,0,0,1,1,1,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,9171511,39,3,9,44,2104539,2913681,1805011,2545346,64276,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,33969662,39,3,9,44,1786268,2058388,2943185,0,0,3793,0,0,0,0,7,3,0,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,22735388,39,3,9,44,1786268,1879365,2342465,2784239,2740797,3793,0,0,0,0,1,4,1,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,2548374,39,3,9,44,2159483,243243,2333048,2715065,2490762,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,27015535,39,3,9,44,1786268,1761528,0,0,0,3793,0,0,0,0,1,3,0,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,3021805,39,3,9,44,1786268,1850273,376381,909354,931163,3793,0,0,0,0,7,3,1,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,818189,39,3,9,44,1897369,1809635,32348,2682572,2490762,3793,0,0,0,0,6,0,2,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,32595174,39,3,9,44,1970943,1577216,0,0,0,3793,0,0,0,0,8,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,5 17 | 1,3043906,62046,15,2014,64,970579,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,17626124,39,3,9,44,1786268,2058388,2943209,0,0,3793,0,0,0,0,7,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,30303763,39,3,9,44,1932940,2449362,1583592,443796,1817792,3793,0,0,0,0,7,7,0,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,13369583,39,3,9,44,1786268,1615279,2943188,0,0,3793,0,0,0,0,1,1,1,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,100540,39,3,9,44,2159483,296629,2332187,2353686,2595022,3793,0,0,0,0,6,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,19366131,39,3,9,44,2159483,2481724,1791767,2217769,0,3793,0,0,0,0,2,0,2,226846,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,23198596,39,3,9,44,1786217,1126769,1861386,1695759,2525237,3786,5665,0,0,0,8,0,2,5,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,31669306,39,3,9,44,1786268,1840775,1583139,1669549,0,3793,0,0,0,0,7,7,0,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,10578925,39,3,9,44,1786268,2344392,1792876,1675461,0,3793,0,0,0,0,1,1,1,581338,1448423,0,198,3447,2,18,623,39,3,9,44,4347,0,0,0,26340887,39,3,9,44,2177019,2721091,2525237,0,0,3790,5673,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,5633422,39,3,9,44,1786268,2122787,2016945,2364287,2869049,3793,0,0,0,0,7,7,0,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,3160410,39,3,9,44,1786268,1761539,0,0,0,3793,0,0,0,0,1,1,1,896447,1448423,0,199,3447,2,18,623,39,3,9,44,4347,0,0,0,9171511,39,3,9,44,2104539,2913681,1805011,2545346,64276,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,33969662,39,3,9,44,1786268,2058388,2943185,0,0,3793,0,0,0,0,7,3,0,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,22735388,39,3,9,44,1786268,1879365,2342465,2784239,2740797,3793,0,0,0,0,1,4,1,854,1448423,0,297,3447,2,18,623,39,3,9,44,0,300,0,0,2548374,39,3,9,44,2159483,243243,2333048,2715065,2490762,3793,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,27015535,39,3,9,44,1786268,1761528,0,0,0,3793,0,0,0,0,1,3,0,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,3021805,39,3,9,44,1786268,1850273,376381,909354,931163,3793,0,0,0,0,7,3,1,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,818189,39,3,9,44,1897369,1809635,32348,2682572,2490762,3793,0,0,0,0,6,0,2,435167,1448423,0,298,3447,2,18,623,39,3,9,44,0,300,0,0,32595174,39,3,9,44,1970943,1577216,0,0,0,3793,0,0,0,0,8,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,5 18 | 0,2375142,55731,20,2014,64,935,491134,0,295,3447,2,18,623,51,3,9,23,0,0,0,0,14396973,51,3,9,23,1767428,1819893,707089,1669342,0,3782,0,0,0,0,7,4,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,7133528,23,3,3,47,2007213,521926,827913,0,0,1545,1580,0,0,0,1,3,0,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,33658801,23,3,3,47,2007213,929386,929134,0,0,1545,1580,0,0,0,7,1,1,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,25432248,23,3,3,47,1879270,2735407,1157023,889445,2237217,1580,1713,1545,0,0,2,0,2,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,26596614,23,3,3,47,1610931,2735407,835071,635140,1747402,1580,1713,1545,0,0,8,0,2,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,32993656,23,3,3,47,2007179,2264821,2434910,0,0,1580,1713,1545,0,0,6,0,2,5,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,337531,23,3,3,47,1642313,2789508,976247,2740797,2360085,1545,1575,0,0,0,1,3,0,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,2066830,23,3,3,47,1814178,2264821,2799386,2536117,0,1545,1575,0,0,0,7,5,1,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,2426822,23,3,3,47,1780089,2482690,636569,1914708,0,5614,1575,1545,0,0,8,0,2,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,13551068,23,3,3,47,1780089,2482690,618129,858933,0,5614,1575,1545,0,0,2,0,2,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,19240435,23,3,3,47,1893079,2568806,2441327,0,0,5614,1575,1545,0,0,6,0,2,5,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,32872848,23,3,3,47,1617146,1262437,1235594,2740797,2360085,1545,1578,0,0,0,1,1,0,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,25982988,23,3,3,47,1617146,770177,0,0,0,1545,1578,0,0,0,7,2,1,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,5260822,23,3,3,47,2053063,2748371,533083,312688,2733506,1578,5635,1545,0,0,8,0,2,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,4262579,23,3,3,47,618541,1288716,1519859,0,0,1578,5635,1545,0,0,2,0,2,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,3700854,23,3,3,47,1625433,976247,927874,0,0,1578,5635,1545,0,0,6,0,2,5,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,8322139,5,3,2,39,1721728,2272746,979374,0,0,1490,0,0,0,0,1,4,0,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,35348941,5,3,2,39,1638295,2473791,742580,2896504,2763012,1489,0,0,0,0,7,3,1,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,20123463,5,3,2,39,1593894,2458148,2740797,2263581,2470037,1492,0,0,0,0,6,0,2,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,17299784,5,3,2,39,1786931,1583517,2016228,0,0,1492,0,0,0,0,8,0,2,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,14616654,5,3,2,39,1695820,2771522,2359557,2273038,0,1490,0,0,0,0,2,0,2,5,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,22711708,23,3,3,47,2100756,780521,0,0,0,1545,1578,0,0,0,1,3,1,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,905707,23,3,3,47,605405,2555441,626280,894147,812374,1578,5635,1545,0,0,8,0,2,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,20100428,23,3,3,47,1871672,0,0,0,0,1578,5635,1545,0,0,6,0,2,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,24516798,23,3,3,47,1958288,2555441,0,0,0,1578,5635,1545,0,0,7,0,2,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,33298384,23,3,3,47,1992905,2608560,533083,1133993,312688,1578,5635,1545,0,0,2,0,2,5,5 19 | 1,2375142,55731,20,2014,64,935,491134,0,295,3447,2,18,623,51,3,9,23,0,0,0,0,15031527,51,3,9,23,1688955,1647911,1602659,1631510,2624887,3775,0,0,0,0,1,7,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,7133528,23,3,3,47,2007213,521926,827913,0,0,1545,1580,0,0,0,1,3,0,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,33658801,23,3,3,47,2007213,929386,929134,0,0,1545,1580,0,0,0,7,1,1,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,25432248,23,3,3,47,1879270,2735407,1157023,889445,2237217,1580,1713,1545,0,0,2,0,2,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,26596614,23,3,3,47,1610931,2735407,835071,635140,1747402,1580,1713,1545,0,0,8,0,2,700455,491134,0,147,3447,2,18,623,23,3,3,47,0,249,242,273,32993656,23,3,3,47,2007179,2264821,2434910,0,0,1580,1713,1545,0,0,6,0,2,5,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,337531,23,3,3,47,1642313,2789508,976247,2740797,2360085,1545,1575,0,0,0,1,3,0,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,2066830,23,3,3,47,1814178,2264821,2799386,2536117,0,1545,1575,0,0,0,7,5,1,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,2426822,23,3,3,47,1780089,2482690,636569,1914708,0,5614,1575,1545,0,0,8,0,2,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,13551068,23,3,3,47,1780089,2482690,618129,858933,0,5614,1575,1545,0,0,2,0,2,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,19240435,23,3,3,47,1893079,2568806,2441327,0,0,5614,1575,1545,0,0,6,0,2,5,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,32872848,23,3,3,47,1617146,1262437,1235594,2740797,2360085,1545,1578,0,0,0,1,1,0,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,25982988,23,3,3,47,1617146,770177,0,0,0,1545,1578,0,0,0,7,2,1,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,5260822,23,3,3,47,2053063,2748371,533083,312688,2733506,1578,5635,1545,0,0,8,0,2,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,4262579,23,3,3,47,618541,1288716,1519859,0,0,1578,5635,1545,0,0,2,0,2,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,3700854,23,3,3,47,1625433,976247,927874,0,0,1578,5635,1545,0,0,6,0,2,5,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,8322139,5,3,2,39,1721728,2272746,979374,0,0,1490,0,0,0,0,1,4,0,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,35348941,5,3,2,39,1638295,2473791,742580,2896504,2763012,1489,0,0,0,0,7,3,1,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,20123463,5,3,2,39,1593894,2458148,2740797,2263581,2470037,1492,0,0,0,0,6,0,2,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,17299784,5,3,2,39,1786931,1583517,2016228,0,0,1492,0,0,0,0,8,0,2,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,14616654,5,3,2,39,1695820,2771522,2359557,2273038,0,1490,0,0,0,0,2,0,2,5,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,22711708,23,3,3,47,2100756,780521,0,0,0,1545,1578,0,0,0,1,3,1,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,905707,23,3,3,47,605405,2555441,626280,894147,812374,1578,5635,1545,0,0,8,0,2,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,20100428,23,3,3,47,1871672,0,0,0,0,1578,5635,1545,0,0,6,0,2,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,24516798,23,3,3,47,1958288,2555441,0,0,0,1578,5635,1545,0,0,7,0,2,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,33298384,23,3,3,47,1992905,2608560,533083,1133993,312688,1578,5635,1545,0,0,2,0,2,5,5 20 | 0,2375142,55731,20,2014,64,1039954,491134,0,294,3447,2,18,623,51,3,9,23,0,289,0,0,13962777,51,3,9,23,2058716,2620914,2626345,2099898,0,3782,0,0,0,0,1,6,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,337531,23,3,3,47,1642313,2789508,976247,2740797,2360085,1545,1575,0,0,0,1,3,0,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,2066830,23,3,3,47,1814178,2264821,2799386,2536117,0,1545,1575,0,0,0,7,5,1,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,2426822,23,3,3,47,1780089,2482690,636569,1914708,0,5614,1575,1545,0,0,8,0,2,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,13551068,23,3,3,47,1780089,2482690,618129,858933,0,5614,1575,1545,0,0,2,0,2,704870,491134,0,51,3447,2,18,623,23,3,3,47,0,244,242,751,19240435,23,3,3,47,1893079,2568806,2441327,0,0,5614,1575,1545,0,0,6,0,2,5,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,32872848,23,3,3,47,1617146,1262437,1235594,2740797,2360085,1545,1578,0,0,0,1,1,0,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,25982988,23,3,3,47,1617146,770177,0,0,0,1545,1578,0,0,0,7,2,1,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,5260822,23,3,3,47,2053063,2748371,533083,312688,2733506,1578,5635,1545,0,0,8,0,2,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,4262579,23,3,3,47,618541,1288716,1519859,0,0,1578,5635,1545,0,0,2,0,2,716853,491134,0,140,3447,2,18,623,23,3,3,47,0,247,242,770,3700854,23,3,3,47,1625433,976247,927874,0,0,1578,5635,1545,0,0,6,0,2,5,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,8322139,5,3,2,39,1721728,2272746,979374,0,0,1490,0,0,0,0,1,4,0,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,35348941,5,3,2,39,1638295,2473791,742580,2896504,2763012,1489,0,0,0,0,7,3,1,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,20123463,5,3,2,39,1593894,2458148,2740797,2263581,2470037,1492,0,0,0,0,6,0,2,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,17299784,5,3,2,39,1786931,1583517,2016228,0,0,1492,0,0,0,0,8,0,2,969111,491134,0,277,3447,2,18,623,5,3,2,39,0,0,0,0,14616654,5,3,2,39,1695820,2771522,2359557,2273038,0,1490,0,0,0,0,2,0,2,5,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,22711708,23,3,3,47,2100756,780521,0,0,0,1545,1578,0,0,0,1,3,1,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,905707,23,3,3,47,605405,2555441,626280,894147,812374,1578,5635,1545,0,0,8,0,2,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,20100428,23,3,3,47,1871672,0,0,0,0,1578,5635,1545,0,0,6,0,2,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,24516798,23,3,3,47,1958288,2555441,0,0,0,1578,5635,1545,0,0,7,0,2,1002195,491134,0,179,3447,2,18,623,23,3,3,47,0,247,242,770,33298384,23,3,3,47,1992905,2608560,533083,1133993,312688,1578,5635,1545,0,0,2,0,2,5,935,491134,0,295,3447,2,18,623,51,3,9,23,0,0,0,0,14396973,51,3,9,23,1767428,1819893,707089,1669342,0,3782,0,0,0,0,7,4,0,935,491134,0,295,3447,2,18,623,51,3,9,23,0,0,0,0,15031527,51,3,9,23,1688955,1647911,1602659,1631510,2624887,3775,0,0,0,0,1,7,1,935,491134,0,295,3447,2,18,623,51,3,9,23,0,0,0,0,30526812,51,3,9,23,888049,1208982,2729798,2764707,433985,3782,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,5 21 | --------------------------------------------------------------------------------