├── .idea ├── .gitignore ├── attenrestler_newchecker.iml ├── inspectionProfiles │ └── profiles_settings.xml ├── modules.xml └── vcs.xml ├── README.md ├── attentionmodel_group ├── attention.py └── generation.py ├── requirements.txt └── restler_bin_atten ├── Compile ├── StdErr.txt ├── StdOut.txt ├── commit_preprocessed.yaml ├── config.json ├── defaultDict.json ├── dependencies.json ├── dependencies_debug.json ├── dict.json ├── engine_settings.json ├── examples.json ├── grammar.json ├── grammar.py ├── restler-20210517-060215.log └── unresolved_dependencies.json ├── __pycache__ └── test2.cpython-38.pyc ├── compiler ├── FSharp.Control.AsyncSeq.dll ├── FSharp.Core.dll ├── Microsoft.ApplicationInsights.dll ├── Microsoft.FSharpLu.Json.dll ├── Microsoft.FSharpLu.dll ├── NJsonSchema.Yaml.dll ├── NJsonSchema.dll ├── NSwag.Core.Yaml.dll ├── NSwag.Core.dll ├── Namotion.Reflection.dll ├── Newtonsoft.Json.Bson.dll ├── Newtonsoft.Json.dll ├── Pluralize.NET.Core.dll ├── Restler.Compiler.dll ├── Restler.Compiler.pdb ├── Restler.CompilerExe ├── Restler.CompilerExe.deps.json ├── Restler.CompilerExe.dll ├── Restler.CompilerExe.pdb ├── Restler.CompilerExe.runtimeconfig.json ├── Sysprep.ps1 ├── System.ComponentModel.Composition.dll ├── System.Configuration.ConfigurationManager.dll ├── System.Net.Http.Formatting.dll ├── System.Security.Cryptography.ProtectedData.dll ├── System.Security.Permissions.dll ├── YamlDotNet.dll ├── cs │ └── FSharp.Core.resources.dll ├── de │ └── FSharp.Core.resources.dll ├── es │ └── FSharp.Core.resources.dll ├── fr │ └── FSharp.Core.resources.dll ├── it │ └── FSharp.Core.resources.dll ├── ja │ └── FSharp.Core.resources.dll ├── ko │ └── FSharp.Core.resources.dll ├── pl │ └── FSharp.Core.resources.dll ├── pt-BR │ └── FSharp.Core.resources.dll ├── ru │ └── FSharp.Core.resources.dll ├── runtimes │ └── win │ │ └── lib │ │ └── netstandard2.0 │ │ └── System.Security.Cryptography.ProtectedData.dll ├── tr │ └── FSharp.Core.resources.dll ├── zh-Hans │ └── FSharp.Core.resources.dll └── zh-Hant │ └── FSharp.Core.resources.dll ├── engine ├── __init__.py ├── checkers │ ├── .checker_base.py.swp │ ├── .data_driven_checker.py.swp │ ├── .invalid_dynamic_object_checker.py.swp │ ├── __init__.py │ ├── body_schema_fuzzer.py │ ├── checker_base.py │ ├── checker_log.py │ ├── data_driven_checker.py │ ├── examples_checker.py │ ├── invalid_dynamic_object_checker.py │ ├── leakage_rule_checker.py │ ├── namespace_rule_checker.py │ ├── payload_body_bucketing.py │ ├── payload_body_checker.py │ ├── resource_hierarchy_checker.py │ ├── response_analyzer.py │ └── use_after_free_checker.py ├── engine │ ├── __init__.py │ ├── bug_bucketing.py │ ├── core │ │ ├── __init__.py │ │ ├── async_request_utilities.py │ │ ├── datacollect.py │ │ ├── driver.py │ │ ├── fuzzer.py │ │ ├── fuzzing_monitor.py │ │ ├── fuzzing_requests.py │ │ ├── postprocessing.py │ │ ├── preprocessing.py │ │ ├── renderings_monitor.py │ │ ├── request_utilities.py │ │ ├── requests.py │ │ ├── sequences.py │ │ └── status_codes_monitor.py │ ├── dependencies.py │ ├── errors.py │ ├── fuzzing_parameters │ │ ├── body_schema.py │ │ ├── fuzzing_config.py │ │ ├── fuzzing_utils.py │ │ ├── request_examples.py │ │ ├── request_params.py │ │ └── request_schema_parser.py │ ├── mime │ │ ├── __init__.py │ │ └── multipart_formdata.py │ ├── primitives.py │ └── transport_layer │ │ ├── __init__.py │ │ ├── messaging.py │ │ └── response.py ├── restler.py ├── restler_settings.py └── utils │ ├── __init__.py │ ├── formatting.py │ ├── hooks.py │ ├── logger.py │ └── saver.py ├── restler ├── Dia2Lib.dll ├── FSharp.Control.AsyncSeq.dll ├── FSharp.Core.dll ├── Microsoft.ApplicationInsights.dll ├── Microsoft.Diagnostics.FastSerialization.dll ├── Microsoft.Diagnostics.Tracing.TraceEvent.dll ├── Microsoft.FSharpLu.Json.dll ├── Microsoft.FSharpLu.Windows.dll ├── Microsoft.FSharpLu.dll ├── Microsoft.Win32.Registry.AccessControl.dll ├── Microsoft.Win32.SystemEvents.dll ├── NJsonSchema.Yaml.dll ├── NJsonSchema.dll ├── NSwag.Core.Yaml.dll ├── NSwag.Core.dll ├── Namotion.Reflection.dll ├── Newtonsoft.Json.Bson.dll ├── Newtonsoft.Json.dll ├── OSExtensions.dll ├── Pluralize.NET.Core.dll ├── Restler ├── Restler.Compiler.dll ├── Restler.Compiler.pdb ├── Restler.deps.json ├── Restler.dll ├── Restler.pdb ├── Restler.runtimeconfig.json ├── Sysprep.ps1 ├── System.CodeDom.dll ├── System.ComponentModel.Composition.dll ├── System.Configuration.ConfigurationManager.dll ├── System.Data.Odbc.dll ├── System.Data.SqlClient.dll ├── System.Diagnostics.EventLog.dll ├── System.Diagnostics.PerformanceCounter.dll ├── System.DirectoryServices.AccountManagement.dll ├── System.DirectoryServices.Protocols.dll ├── System.DirectoryServices.dll ├── System.Drawing.Common.dll ├── System.IO.Packaging.dll ├── System.IO.Ports.dll ├── System.Management.dll ├── System.Net.Http.Formatting.dll ├── System.Runtime.Caching.dll ├── System.Security.Cryptography.Pkcs.dll ├── System.Security.Cryptography.ProtectedData.dll ├── System.Security.Cryptography.Xml.dll ├── System.Security.Permissions.dll ├── System.ServiceModel.Duplex.dll ├── System.ServiceModel.Http.dll ├── System.ServiceModel.NetTcp.dll ├── System.ServiceModel.Primitives.dll ├── System.ServiceModel.Security.dll ├── System.ServiceModel.Syndication.dll ├── System.ServiceModel.dll ├── System.ServiceProcess.ServiceController.dll ├── System.Threading.AccessControl.dll ├── TraceReloggerLib.dll ├── YamlDotNet.dll ├── cs │ └── FSharp.Core.resources.dll ├── de │ └── FSharp.Core.resources.dll ├── es │ └── FSharp.Core.resources.dll ├── fr │ └── FSharp.Core.resources.dll ├── it │ └── FSharp.Core.resources.dll ├── ja │ └── FSharp.Core.resources.dll ├── ko │ └── FSharp.Core.resources.dll ├── pl │ └── FSharp.Core.resources.dll ├── pt-BR │ └── FSharp.Core.resources.dll ├── ru │ └── FSharp.Core.resources.dll ├── runtimes │ ├── linux │ │ └── lib │ │ │ └── netcoreapp2.0 │ │ │ └── System.Data.Odbc.dll │ ├── osx │ │ └── lib │ │ │ └── netcoreapp2.0 │ │ │ └── System.Data.Odbc.dll │ ├── unix │ │ └── lib │ │ │ ├── netcoreapp2.0 │ │ │ ├── System.Drawing.Common.dll │ │ │ └── System.Runtime.Caching.dll │ │ │ ├── netcoreapp2.1 │ │ │ └── System.Data.SqlClient.dll │ │ │ └── netstandard2.0 │ │ │ └── System.Private.ServiceModel.dll │ ├── win-arm64 │ │ └── native │ │ │ └── sni.dll │ ├── win-x64 │ │ └── native │ │ │ └── sni.dll │ ├── win-x86 │ │ └── native │ │ │ └── sni.dll │ └── win │ │ └── lib │ │ ├── netcoreapp2.0 │ │ ├── Microsoft.Win32.SystemEvents.dll │ │ ├── System.Data.Odbc.dll │ │ ├── System.Diagnostics.EventLog.dll │ │ ├── System.Diagnostics.PerformanceCounter.dll │ │ ├── System.DirectoryServices.AccountManagement.dll │ │ ├── System.DirectoryServices.Protocols.dll │ │ ├── System.DirectoryServices.dll │ │ ├── System.Drawing.Common.dll │ │ ├── System.Management.dll │ │ └── System.Runtime.Caching.dll │ │ ├── netcoreapp2.1 │ │ ├── System.Data.SqlClient.dll │ │ └── System.Security.Cryptography.Pkcs.dll │ │ └── netstandard2.0 │ │ ├── Microsoft.Win32.Registry.AccessControl.dll │ │ ├── System.IO.Ports.dll │ │ ├── System.Private.ServiceModel.dll │ │ ├── System.Security.Cryptography.ProtectedData.dll │ │ ├── System.ServiceProcess.ServiceController.dll │ │ └── System.Threading.AccessControl.dll ├── tr │ └── FSharp.Core.resources.dll ├── zh-Hans │ └── FSharp.Core.resources.dll └── zh-Hant │ └── FSharp.Core.resources.dll ├── resultsAnalyzer ├── DiffPlex.dll ├── FSharp.Control.AsyncSeq.dll ├── FSharp.Core.dll ├── Microsoft.ApplicationInsights.dll ├── Microsoft.FSharpLu.Json.dll ├── Microsoft.FSharpLu.dll ├── NJsonSchema.Yaml.dll ├── NJsonSchema.dll ├── NSwag.Core.Yaml.dll ├── NSwag.Core.dll ├── Namotion.Reflection.dll ├── Newtonsoft.Json.Bson.dll ├── Newtonsoft.Json.dll ├── Pluralize.NET.Core.dll ├── Restler.Compiler.dll ├── Restler.Compiler.pdb ├── Restler.ResultsAnalyzer ├── Restler.ResultsAnalyzer.deps.json ├── Restler.ResultsAnalyzer.dll ├── Restler.ResultsAnalyzer.pdb ├── Restler.ResultsAnalyzer.runtimeconfig.json ├── Sysprep.ps1 ├── System.ComponentModel.Composition.dll ├── System.Configuration.ConfigurationManager.dll ├── System.Net.Http.Formatting.dll ├── System.Security.Cryptography.ProtectedData.dll ├── System.Security.Permissions.dll ├── YamlDotNet.dll ├── cs │ └── FSharp.Core.resources.dll ├── de │ └── FSharp.Core.resources.dll ├── es │ └── FSharp.Core.resources.dll ├── fr │ └── FSharp.Core.resources.dll ├── it │ └── FSharp.Core.resources.dll ├── ja │ └── FSharp.Core.resources.dll ├── ko │ └── FSharp.Core.resources.dll ├── pl │ └── FSharp.Core.resources.dll ├── pt-BR │ └── FSharp.Core.resources.dll ├── ru │ └── FSharp.Core.resources.dll ├── runtimes │ └── win │ │ └── lib │ │ └── netstandard2.0 │ │ └── System.Security.Cryptography.ProtectedData.dll ├── tr │ └── FSharp.Core.resources.dll ├── zh-Hans │ └── FSharp.Core.resources.dll └── zh-Hant │ └── FSharp.Core.resources.dll └── ~ └── .config └── microsoft └── restler └── restler.telemetry.uuid /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /.idea/attenrestler_newchecker.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MINER 2 | ### 1. Description 3 | We open source the prototype of MINER. MINER is a REST API fuzzer that utilizes three data-driven designs working together to guide the sequence generation, improve the request generation quality, and capture the unique errors caused by incorrect parameter usage. 4 | More details can be found in the prepublication [PDF](https://arxiv.org/abs/2303.02545) (https://arxiv.org/abs/2303.02545). 5 | 6 | ### 2. Introduction to Usage 7 | First of all, our code must be placed under the '/home/MINER' path, because we set the absolute path in our code. Of cource, you can can modify the code according to your needs. 8 | Since we implement the prototype of MINER based on [RESTler](https://github.com/microsoft/restler-fuzzer), the steps to run MINER are the same as [RESTler](https://github.com/microsoft/restler-fuzzer). For instance, you can follow the guidances of [RESTler](https://github.com/microsoft/restler-fuzzer) to generate the grammar, and run the following cmd to start the test. 9 | 10 | ``` 11 | # /home/MINER/restler_bin_atten/restler/Restler fuzz --grammar_file /path/to/grammar.py --dictionary_file /path/to/dict.json --settings /path/to/engine_settings.json --no_ssl --time_budget 12 12 | ``` 13 | 14 | ### Citation: 15 | ``` 16 | @inproceedings{lyu2023miner, 17 | title={MINER: A Hybrid Data-Driven Approach for REST API Fuzzing}, 18 | author={Lyu, Chenyang and Xu, Jiacheng and Ji, Shouling and Zhang, Xuhong and Wang, Qinying and Zhao, Binbin and Pan, Gaoning and Cao, Wei and Chen, Peng and Beyah, Raheem}, 19 | booktitle = {32th {USENIX} Security Symposium ({USENIX} Security 23)}, 20 | year={2023} 21 | } 22 | ``` 23 | -------------------------------------------------------------------------------- /attentionmodel_group/generation.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import numpy as np 4 | import random 5 | import torch 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | import torch.nn.functional as F 9 | from torch.autograd import Variable 10 | from torch.utils.data import DataLoader 11 | # import spacy 12 | import numpy as np 13 | import random 14 | import math 15 | import time 16 | from sklearn.model_selection import train_test_split 17 | import datetime 18 | import pickle 19 | 20 | max_length = 20 21 | hiddenlyu = 36 22 | 23 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 24 | 25 | 26 | class Attention(nn.Module): 27 | def __init__(self, enc_hid_dim, dec_hid_dim): 28 | super().__init__() 29 | 30 | self.attn = nn.Linear((enc_hid_dim) + dec_hid_dim, dec_hid_dim) 31 | self.v = nn.Linear(dec_hid_dim, 1, bias=False) 32 | 33 | def forward(self, hidden, attention_inputs): 34 | src_len = attention_inputs.shape[0] 35 | if src_len == 1: 36 | hidden = hidden.unsqueeze(1) 37 | attention_inputs = attention_inputs.permute(1, 0, 2) 38 | energy = torch.tanh(self.attn(torch.cat((hidden, attention_inputs), dim=2))) 39 | attention = self.v(energy).squeeze(2) 40 | return F.softmax(attention, dim=1) 41 | else: 42 | hidden = hidden.unsqueeze(1).repeat(1, src_len, 1) 43 | attention_inputs = attention_inputs.permute(1, 0, 2) 44 | 45 | energy = torch.tanh(self.attn(torch.cat((hidden, attention_inputs), dim=2))) 46 | attention = self.v(energy).squeeze(2) 47 | return F.softmax(attention, dim=1) 48 | 49 | 50 | class Decoder(nn.Module): 51 | def __init__(self, output_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout, attention, embeddings): 52 | super().__init__() 53 | 54 | self.output_dim = output_dim 55 | self.attention = attention 56 | self.embedding = embeddings # nn.Embedding(output_dim, emb_dim) 57 | self.rnn = nn.GRU(emb_dim, dec_hid_dim) 58 | self.fc_out = nn.Linear((enc_hid_dim) + dec_hid_dim + emb_dim, output_dim) 59 | self.dropout = nn.Dropout(dropout) 60 | 61 | def forward(self, input, hidden, atten_inputs): 62 | input = input.to(device) 63 | hidden = hidden.to(device) 64 | atten_inputs = atten_inputs.to(device) 65 | input = input.unsqueeze(0) 66 | embedded = self.dropout(self.embedding(input)) 67 | 68 | output, hidden = self.rnn(embedded, hidden.unsqueeze(0)) 69 | a = self.attention(hidden.squeeze(0), atten_inputs) 70 | a = a.unsqueeze(1) 71 | atten_inputs = atten_inputs.permute(1, 0, 2) 72 | weighted = torch.bmm(a, atten_inputs) 73 | weighted = weighted.permute(1, 0, 2) 74 | 75 | embedded = embedded.squeeze(0) 76 | output = output.squeeze(0) 77 | weighted = weighted.squeeze(0) 78 | 79 | prediction = self.fc_out(torch.cat((output, weighted, embedded), dim=1)) 80 | return prediction, hidden.squeeze(0), hidden 81 | 82 | 83 | class attnLSTM(nn.Module): 84 | def __init__(self, decoder, device): 85 | super().__init__() 86 | 87 | self.decoder = decoder 88 | self.device = device 89 | 90 | def forward(self, src, trg, teacher_forcing_ratio=0.5): 91 | batch_size = src.shape[1] 92 | trg_len = trg.shape[0] 93 | 94 | trg_vocab_size = self.decoder.output_dim 95 | 96 | # tensor to store decoder outputs 97 | outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device) 98 | 99 | # first input to the decoder is the tokens 100 | input = src[0, :] 101 | hidden = torch.zeros(batch_size, DEC_HID_DIM) 102 | atten_inputs = torch.zeros(1, batch_size, DEC_HID_DIM) 103 | store_hiddens = atten_inputs 104 | for t in range(1, trg_len): 105 | if t == 1: 106 | output, hidden, decoder_hiddens = self.decoder(input, hidden, atten_inputs) 107 | else: 108 | output, hidden, decoder_hiddens = self.decoder(input, hidden, atten_inputs) 109 | 110 | outputs[t] = output 111 | 112 | # decide if we are going to use teacher forcing or not 113 | teacher_force = random.random() < teacher_forcing_ratio 114 | # get the highest predicted token from our predictions 115 | top1 = output.argmax(1) 116 | # if teacher forcing, use actual next token as next input 117 | # if not, use predicted token 118 | input = src[t] if teacher_force else top1 119 | store_hiddens = torch.cat((store_hiddens, decoder_hiddens), dim=0) 120 | atten_inputs = store_hiddens 121 | 122 | return outputs 123 | 124 | 125 | def generation(service_name, decoder, device): 126 | hidden = ((torch.randn(1, hiddenlyu)) / 25.0).to( 127 | device) # (batch_size, DEC_HID_DIM) #atten_inputs = torch.zeros(1, batch_size, DEC_HID_DIM) 128 | atten_inputs = ((torch.randn(1, 1, hiddenlyu)) / 25.0).to(device) # (1, batch_size, DEC_HID_DIM ) 129 | store_hiddens = atten_inputs 130 | trg_vocab_size = decoder.output_dim 131 | outputs = torch.zeros(max_length, 1, trg_vocab_size).to(device) 132 | top_list = [] 133 | input = torch.tensor([service_name]) 134 | t = 0 135 | while 1: 136 | output, hidden, decoder_hiddens = decoder(input, hidden, atten_inputs) 137 | 138 | outputs[t] = output 139 | t = t + 1 140 | if t >= max_length - 1: 141 | break 142 | tmpoutput = output 143 | top1 = tmpoutput.argmax(1) 144 | for i1 in range(1, 5): 145 | top1 = tmpoutput.argmax(1) 146 | if int(top1) not in top_list: 147 | break 148 | tmpoutput[0][int(top1)] = min(tmpoutput[0]) 149 | top_list.append(int(top1)) 150 | if int(top1) == 0: 151 | break 152 | input = top1 153 | store_hiddens = torch.cat((store_hiddens, decoder_hiddens), dim=0) 154 | atten_inputs = store_hiddens 155 | 156 | return outputs 157 | 158 | 159 | if __name__ == '__main__': 160 | starttime = datetime.datetime.now() 161 | word_to_ix = {} 162 | word_to_ix = np.load('/home/MINER/attentionmodel_group/word_to_ix.npy', allow_pickle=True) 163 | word_to_ix = word_to_ix.item() 164 | ix_to_word = {} 165 | 166 | for key1 in word_to_ix: 167 | ix_to_word[word_to_ix[key1]] = key1 168 | 169 | # embeddings = torch.load('embedding.pt') 170 | if os.path.exists('/home/MINER/attentionmodel_group/apifuzzmodel.pt') == True: 171 | model = torch.load('/home/MINER/attentionmodel_group/apifuzzmodel.pt') 172 | 173 | decoder = model.decoder 174 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 175 | torch.set_grad_enabled(False) 176 | 177 | service_name_list = [] 178 | if os.path.exists("/home/MINER/attentionmodel_group/service_name_list.pkl") == True: 179 | service_name_list = pickle.load(open("/home/MINER/attentionmodel_group/service_name_list.pkl", 'rb')) 180 | else: 181 | trainingset = pickle.load(open("/home/MINER/restler_bin_atten/trainingset.pkl", 'rb')) 182 | for lyui in range(len(trainingset)): 183 | tmpservice_name = str(trainingset[lyui][1]) + trainingset[lyui][4] 184 | if tmpservice_name not in service_name_list: 185 | service_name_list.append(tmpservice_name) 186 | with open("/home/MINER/attentionmodel_group/service_name_list.pkl", 'wb') as f: 187 | pickle.dump(service_name_list, f, pickle.HIGHEST_PROTOCOL) 188 | 189 | listword = np.load('/home/MINER/attentionmodel_group/listword.npy', allow_pickle=True) 190 | listword = listword.tolist() 191 | d = sorted(listword.items(), key=lambda x: x[1], reverse=True) 192 | listwordsort = [key for key, value in d] 193 | 194 | dictoutput = {} 195 | for lyui in range(len(service_name_list)): 196 | service_name = service_name_list[lyui] 197 | if service_name not in dictoutput: 198 | tmplist = [] 199 | dictoutput[service_name] = tmplist 200 | for i in range(200): 201 | outputss = generation(word_to_ix[service_name], decoder, device) 202 | 203 | toplist = [] 204 | toplist2 = [] 205 | totaltop = [] 206 | 207 | for t in range(len(outputss)): 208 | tmpoutput = outputss[t] 209 | 210 | top1 = int(tmpoutput.argmax(1)) 211 | for i2 in range(0, 10): 212 | top1 = int(tmpoutput.argmax(1)) 213 | # add more randomness 214 | if random.random() < 0.03: 215 | tmpoutput[0][int(top1)] = min(tmpoutput[0]) 216 | continue 217 | # add more randomness 218 | if top1 == 0: 219 | break 220 | if top1 not in toplist: 221 | break 222 | tmpoutput[0][int(top1)] = min(tmpoutput[0]) 223 | 224 | toplist.append(top1) 225 | if top1 not in toplist2: 226 | toplist2.append(top1) 227 | totaltop.append(int(outputss[t].argmax(1))) 228 | 229 | mutationlist = [] 230 | for t in range(len(outputss)): 231 | top1 = toplist2[t] 232 | 233 | if int(top1) == 0: 234 | break 235 | mutationlist.append(ix_to_word[int(top1)]) 236 | 237 | mutationlist = [x for _, x in sorted(zip([listwordsort.index(x) for x in mutationlist], mutationlist))] 238 | if mutationlist != [] and mutationlist not in dictoutput[service_name]: 239 | dictoutput[service_name].append(mutationlist) 240 | 241 | # print("\n\nmutationlist:\n\n"+ str(dictoutput)) 242 | 243 | for key1 in dictoutput.keys(): 244 | print("\n " + str(key1) + " " + str(len(dictoutput[key1]))) 245 | 246 | with open("/home/MINER/restler_bin_atten/mutationlist.pkl", "wb") as f: 247 | pickle.dump(dictoutput, f, pickle.HIGHEST_PROTOCOL) 248 | print("generation time: " + str((datetime.datetime.now() - starttime).seconds)) 249 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2020.12.5 2 | charset-normalizer==2.0.6 3 | configparser==5.0.2 4 | goto-statement==1.2 5 | idna==3.2 6 | joblib==1.0.1 7 | lxml==4.6.3 8 | numpy==1.20.2 9 | requests==2.26.0 10 | scikit-learn==0.24.1 11 | scipy==1.6.2 12 | sklearn==0.0 13 | threadpoolctl==2.1.0 14 | torch==1.8.1 15 | typing-extensions==3.7.4.3 16 | urllib3==1.26.7 17 | -------------------------------------------------------------------------------- /restler_bin_atten/Compile/StdErr.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/Compile/StdErr.txt -------------------------------------------------------------------------------- /restler_bin_atten/Compile/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "SwaggerSpecFilePath": [ 3 | "/home/tmprestler/restler_bin_atten_run_group/../yaml/commit.yaml" 4 | ], 5 | "GrammarOutputDirectoryPath": "/home/tmprestler/restler_bin_atten_run_group/Compile", 6 | "CustomDictionaryFilePath": "/home/tmprestler/restler_bin_atten_run_group/Compile/defaultDict.json", 7 | "IncludeOptionalParameters": true, 8 | "UseQueryExamples": true, 9 | "UseBodyExamples": true, 10 | "DiscoverExamples": false, 11 | "ExamplesDirectory": "", 12 | "DataFuzzing": true, 13 | "ReadOnlyFuzz": false, 14 | "ResolveQueryDependencies": true, 15 | "ResolveBodyDependencies": true, 16 | "UseRefreshableToken": true, 17 | "AllowGetProducers": false 18 | } -------------------------------------------------------------------------------- /restler_bin_atten/Compile/defaultDict.json: -------------------------------------------------------------------------------- 1 | { 2 | "restler_fuzzable_string": [ 3 | "fuzzstring" 4 | ], 5 | "restler_fuzzable_string_unquoted": [], 6 | "restler_fuzzable_datetime": [ 7 | "6/25/2019 12:00:00 AM" 8 | ], 9 | "restler_fuzzable_datetime_unquoted": [], 10 | "restler_fuzzable_uuid4": [ 11 | "903bcc44-30cf-4ea7-968a-d9d0da7c072f" 12 | ], 13 | "restler_fuzzable_uuid4_unquoted": [], 14 | "restler_fuzzable_int": [ 15 | "0", 16 | "1" 17 | ], 18 | "restler_fuzzable_number": [ 19 | "0.1", 20 | "1.2" 21 | ], 22 | "restler_fuzzable_bool": [ 23 | "true" 24 | ], 25 | "restler_fuzzable_object": [ 26 | "{}" 27 | ], 28 | "restler_custom_payload": {}, 29 | "restler_custom_payload_unquoted": {}, 30 | "restler_custom_payload_uuid4_suffix": {} 31 | } -------------------------------------------------------------------------------- /restler_bin_atten/Compile/dict.json: -------------------------------------------------------------------------------- 1 | { 2 | "restler_fuzzable_string": [ 3 | "fuzzstring" 4 | ], 5 | "restler_fuzzable_projectname": ["add2", "add3", "add4", "add5", "add6", "add7", "add8", "add9", "add10", "add11", "add12", "add13", "add14", "add15", "add16", "add17", "add18", "add19", "add20", "add21", "add22", "add23", "add24", "add25", "add26", "add27", "add28", "add29", "add30", "add1"], 6 | "restler_fuzzable_useraddprojectname": [ 7 | "useradd1", "useradd2", "useradd3", "useradd4", "useradd5", "useradd6", "useradd7", "useradd8", "useradd9", "useradd10", "useradd11", "useradd12", "useradd13", "useradd14", "useradd15", "useradd16", "useradd17", "useradd18", "useradd19", "useradd20", "useradd21", "useradd22", "useradd23", "useradd24", "useradd25", "useradd26", "useradd27", "useradd28", "useradd29", "useradd30" ], 8 | "restler_fuzzable_starprojectname": ["stitch@zju.edu.cn", "bugzillauser03@gmail.com", "fdahfai@zju.edu", "puppet", "puppet@zju.edu.cn", "stitch"], 9 | "restler_fuzzable_string_unquoted": [], 10 | "restler_fuzzable_datetime": [ 11 | "6/25/2019 12:00:00 AM" 12 | ], 13 | "restler_fuzzable_datetime_unquoted": [], 14 | "restler_fuzzable_uuid4": [ 15 | "903bcc44-30cf-4ea7-968a-d9d0da7c072f" 16 | ], 17 | "restler_fuzzable_uuid4_unquoted": [], 18 | "restler_fuzzable_int": [ 19 | "15", "16", "17", "21", "35", "44", "5634", "-1", "saf", "awd232", "*&(&ssd668&^f" 20 | ], 21 | "restler_fuzzable_number": [ 22 | "0.1", 23 | "1.2" 24 | ], 25 | "restler_fuzzable_bool": [ 26 | "true" 27 | ], 28 | "restler_fuzzable_object": [ 29 | "{}" 30 | ], 31 | "restler_custom_payload": {}, 32 | "restler_custom_payload_unquoted": {}, 33 | "restler_custom_payload_uuid4_suffix": {} 34 | } 35 | -------------------------------------------------------------------------------- /restler_bin_atten/Compile/engine_settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "per_resource_settings": {}, 3 | "max_combinations": 20 4 | } -------------------------------------------------------------------------------- /restler_bin_atten/Compile/examples.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /restler_bin_atten/Compile/restler-20210517-060215.log: -------------------------------------------------------------------------------- 1 | Restler Information: 0 : Starting task Compile... 2 | DateTime=2021-05-17T06:02:15.7004782Z 3 | Restler Information: 0 : Launching 'dotnet "/home/tmprestler/restler_bin_atten_run_group/restler/../compiler/Restler.CompilerExe.dll" "/home/tmprestler/restler_bin_atten_run_group/Compile/config.json"' 4 | DateTime=2021-05-17T06:02:15.9363363Z 5 | Restler Information: 0 : Process execution terminated in 00:00:01.6857677 with exit code 0x0: 'dotnet "/home/tmprestler/restler_bin_atten_run_group/restler/../compiler/Restler.CompilerExe.dll" "/home/tmprestler/restler_bin_atten_run_group/Compile/config.json"' 6 | DateTime=2021-05-17T06:02:17.6285413Z 7 | Restler Information: 0 : (14645) dotnet "/home/tmprestler/restler_bin_atten_run_group/restler/../compiler/Restler.CompilerExe.dll" "/home/tmprestler/restler_bin_atten_run_group/Compile/config.json" exited with code: 0 8 | DateTime=2021-05-17T06:02:17.6351529Z 9 | Restler Information: 0 : Task Compile succeeded. 10 | DateTime=2021-05-17T06:02:17.6457830Z 11 | Restler Information: 0 : Collecting logs... 12 | DateTime=2021-05-17T06:02:17.6509458Z 13 | -------------------------------------------------------------------------------- /restler_bin_atten/__pycache__/test2.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/__pycache__/test2.cpython-38.pyc -------------------------------------------------------------------------------- /restler_bin_atten/compiler/FSharp.Control.AsyncSeq.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/FSharp.Control.AsyncSeq.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/FSharp.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/FSharp.Core.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Microsoft.ApplicationInsights.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Microsoft.ApplicationInsights.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Microsoft.FSharpLu.Json.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Microsoft.FSharpLu.Json.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Microsoft.FSharpLu.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Microsoft.FSharpLu.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/NJsonSchema.Yaml.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/NJsonSchema.Yaml.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/NJsonSchema.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/NJsonSchema.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/NSwag.Core.Yaml.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/NSwag.Core.Yaml.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/NSwag.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/NSwag.Core.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Namotion.Reflection.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Namotion.Reflection.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Newtonsoft.Json.Bson.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Newtonsoft.Json.Bson.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Newtonsoft.Json.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Newtonsoft.Json.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Pluralize.NET.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Pluralize.NET.Core.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Restler.Compiler.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Restler.Compiler.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Restler.Compiler.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Restler.Compiler.pdb -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Restler.CompilerExe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Restler.CompilerExe -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Restler.CompilerExe.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Restler.CompilerExe.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Restler.CompilerExe.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/Restler.CompilerExe.pdb -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Restler.CompilerExe.runtimeconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "runtimeOptions": { 3 | "tfm": "net5.0", 4 | "framework": { 5 | "name": "Microsoft.NETCore.App", 6 | "version": "5.0.0" 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /restler_bin_atten/compiler/Sysprep.ps1: -------------------------------------------------------------------------------- 1 | #.SYNOPSIS 2 | # Sysprep current machine running Windows Server 2008 R2 and Powershell 5.* (providing workarounds introduced by the new Powershell release) 3 | #.DESCRIPTION 4 | # Script Syspreps current machine. It provides needed workarounds to make Sysprep work on Windows 2008 R2 running Powershell 5.*. 5 | # SIL workaround: https://msdn.microsoft.com/en-us/powershell/wmf/5.0/limitation_overview 6 | # RegKey issue workaround: https://windowsserver.uservoice.com/forums/301869-powershell/suggestions/11591262-bug-wmf-5-production-preview-on-windows-server-2 7 | # Script must be run from elevated Powershell session. 8 | #.PARAMETER ShutdownAfterSysprep 9 | # Shutdown the machine after the Sysprep 10 | #.PARAMETER DisableAntivirusBeforeSysprep 11 | # Prevents antivirus from running on VMs based on the sysprepped image, without requiring a reboot. 12 | # This switch should not be used to prepare the VM template image, since antivirus should be 13 | # on by default for these (they are used to create the customer VM). 14 | [CmdletBinding()] 15 | Param( 16 | [switch] $ShutdownAfterSysprep, 17 | [switch] $DisableAntivirusBeforeSysprep 18 | ) 19 | Process { 20 | 21 | $osVersion = [environment]::OSVersion.Version 22 | $windows2008 = ($osVersion.Major -eq 6) -and ($osVersion.Minor -eq 1) 23 | 24 | #### Disable antivirus if required 25 | #### Note: On Win8+, Defender cannot be disabled without a reboot (there does not appear to be 26 | #### a commandlet to cleanly stop defender once it has been disabled). 27 | if (-not $windows2008 -and $DisableAntivirusBeforeSysprep) { 28 | $defenderRegPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows Defender\" 29 | if (-not (Test-Path $defenderRegPath)) { 30 | New-Item $defenderRegPath 31 | } 32 | New-ItemProperty -Path $defenderRegPath -Name DisableAntiSpyware -Value 1 -PropertyType DWord -Force 33 | } 34 | 35 | #### Hack needed to fix PowerShell breaking Sysprep 36 | Write-Output "Apply WMF 5.0 sysprep workaround" 37 | 38 | # The regkey has to be present there on VM boot. Thus need to patch registry before shutdown 39 | New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\StreamProvider" -Name LastFullPayloadTime -Value 0 -PropertyType DWord -Force 40 | # Working around PowerShell 5.0 Bugs: https://msdn.microsoft.com/en-us/powershell/wmf/5.0/limitation_overview 41 | Set-SilLogging -TargetUri https://BlankTarget -CertificateThumbprint 0123456789 42 | Publish-SilData -ErrorAction SilentlyContinue 43 | Remove-Item -Recurse $env:SystemRoot\System32\Logfiles\SIL\ -Force -ErrorAction SilentlyContinue 44 | 45 | #### Run sysprep generalization 46 | $sysprepCommand = "$env:SystemRoot\system32\sysprep\sysprep.exe" 47 | $sysprepParameters = '/generalize /oobe /quiet' 48 | 49 | if ($ShutdownAfterSysprep) { 50 | $sysprepParameters += ' /shutdown' 51 | } else { 52 | $sysprepParameters += ' /quit' 53 | } 54 | 55 | if (-not (Test-Path $sysprepCommand)) { 56 | throw "Failed to locate sysprep command. This is probably due to Wow64 file redirection. Make sure to run VMAgent in the native OS architecture." 57 | } 58 | 59 | # IMPORTANT NOTE: running sysprep.exe directly would fail when running under Remote Powershell (WinRM). 60 | # This is because the sysprep generalization process breaks all running network connections including WinRM itself. 61 | # Subsequently, when the WinRM session is interrupted, all the processes started under that session are automatically terminated. 62 | # This causes sysprep.exe itself to be killed before completion! 63 | # As a workaround, we start sysprep.exe through a Windows scheduled task. The drawback is that it's fire and forget: the remote session 64 | # has no way to detect when sysprep terminates, nor can it retrieve the exit code from sysprep.exe. 65 | Write-Output "Sysprepping machine" 66 | $taskname = "fsharplu-sysprep" 67 | 68 | if ($windows2008) { 69 | Write-Output "Creating scheduled task (Windows 7/Server 2008)" 70 | # On Windows 2008, New-ScheduledTaskAction does not exist. The alternative 71 | # `Register-ScheduledJob` is too limited (works only with PS scripts). 72 | # So we fall back on command `schtasks.exe`. 73 | 74 | # We set the task to run as SYSTEM to avoid having to pass credentials. 75 | # NOTE: In 83 years from now, the following commands will stop working, at which point 76 | # we might consider dropping support for Windows 2008 ;-) 77 | schtasks /CREATE /TN $taskname /RL HIGHEST /F ` 78 | /TR "$sysprepCommand $sysprepParameters" /SC ONCE /ST 00:00 /SD 01/01/2090 ` 79 | /RU SYSTEM 80 | 81 | schtasks /RUN /TN $taskname 82 | 83 | schtasks /DELETE /TN $taskname /F 84 | } else { 85 | Write-Output "Creating scheduled task (Windows >=8)" 86 | $action = New-ScheduledTaskAction -Execute $sysprepCommand -Argument $sysprepParameters 87 | $principal = New-ScheduledTaskPrincipal -RunLevel "Highest" -UserId 'SYSTEM' 88 | New-ScheduledTask -Action $action -Principal $principal ` 89 | | Register-ScheduledTask $taskname -Force ` 90 | | Start-ScheduledTask ` 91 | | Unregister-ScheduledTask -Confirm:$false 92 | } 93 | } -------------------------------------------------------------------------------- /restler_bin_atten/compiler/System.ComponentModel.Composition.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/System.ComponentModel.Composition.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/System.Configuration.ConfigurationManager.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/System.Configuration.ConfigurationManager.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/System.Net.Http.Formatting.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/System.Net.Http.Formatting.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/System.Security.Cryptography.ProtectedData.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/System.Security.Cryptography.ProtectedData.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/System.Security.Permissions.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/System.Security.Permissions.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/YamlDotNet.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/YamlDotNet.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/cs/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/cs/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/de/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/de/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/es/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/es/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/fr/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/fr/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/it/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/it/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/ja/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/ja/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/ko/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/ko/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/pl/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/pl/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/pt-BR/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/pt-BR/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/ru/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/ru/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/runtimes/win/lib/netstandard2.0/System.Security.Cryptography.ProtectedData.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/runtimes/win/lib/netstandard2.0/System.Security.Cryptography.ProtectedData.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/tr/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/tr/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/zh-Hans/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/zh-Hans/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/compiler/zh-Hant/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/compiler/zh-Hant/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/engine/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/engine/__init__.py -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/.checker_base.py.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/engine/checkers/.checker_base.py.swp -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/.data_driven_checker.py.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/engine/checkers/.data_driven_checker.py.swp -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/.invalid_dynamic_object_checker.py.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/engine/checkers/.invalid_dynamic_object_checker.py.swp -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | # The ordering of these checkers is expected to remain consistent. 5 | # If a new checker is added or a new ordering is deemed necessary, 6 | # the unit tests and baseline logs will need to be updated as well. 7 | from checkers.leakage_rule_checker import * 8 | from checkers.resource_hierarchy_checker import * 9 | from checkers.use_after_free_checker import * 10 | from checkers.namespace_rule_checker import * 11 | from checkers.invalid_dynamic_object_checker import * 12 | from checkers.payload_body_checker import * 13 | from checkers.examples_checker import * 14 | from checkers.data_driven_checker import * 15 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/checker_log.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | from __future__ import print_function 5 | 6 | import os 7 | import threading 8 | 9 | import utils.logger as logger 10 | 11 | class CheckerLog(object): 12 | def __init__(self, checker_name): 13 | """ Creates a log file of a specified name 14 | 15 | @param checker_name: The name of the checker that this log is for 16 | @type checker_name: Str 17 | 18 | """ 19 | self._checker_name = checker_name 20 | thread_id = threading.current_thread().ident 21 | self._log_path = os.path.join(logger.LOGS_DIR, f'{self._checker_name}.{thread_id!s}.txt') 22 | if not os.path.exists(self._log_path): 23 | try: 24 | os.makedirs(os.path.dirname(self._log_path)) 25 | except OSError: 26 | return None 27 | 28 | def checker_print(self, msg, print_to_network_log=True): 29 | """ Prints message to the checker log file 30 | 31 | @param msg: The message to print. 32 | @type msg: Str 33 | 34 | """ 35 | msg = logger.remove_tokens_from_logs(msg) 36 | with open(self._log_path, "a+") as log_file: 37 | print(msg, file=log_file) 38 | if print_to_network_log: 39 | logger.raw_network_logging(self._checker_name + ' ' + msg) 40 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/data_driven_checker.py: -------------------------------------------------------------------------------- 1 | 2 | """ Implements logic for data driven checker. """ 3 | from __future__ import print_function 4 | 5 | from checkers.checker_base import * 6 | 7 | import time 8 | import itertools 9 | import copy 10 | 11 | from engine.bug_bucketing import BugBuckets 12 | import engine.dependencies as dependencies 13 | import engine.core.sequences as sequences 14 | import engine.core.datacollect as datacollect 15 | import engine.core.sequences as sequences 16 | 17 | 18 | class DataDrivenChecker(CheckerBase): 19 | def __init__(self, req_collection, fuzzing_requests): 20 | CheckerBase.__init__(self, req_collection, fuzzing_requests, enabled=True) 21 | 22 | 23 | def apply(self, rendered_sequence, lock): 24 | 25 | """ Applies check for data driven generations. 26 | 27 | @param rendered_sequence: Object containing the rendered sequence information 28 | @type rendered_sequence: RenderedSequence 29 | @param lock: Lock object used to sync more than one fuzzing job 30 | @type lock: thread.Lock 31 | 32 | @return: None 33 | @rtype : None 34 | 35 | """ 36 | 37 | if not datacollect.mutationtrainingset: 38 | return 39 | 40 | if len(datacollect.mutationtrainingset) < 1: 41 | return 42 | 43 | if not rendered_sequence.valid: 44 | return 45 | 46 | self._sequence = rendered_sequence.sequence 47 | # render n-1 requests 48 | self._checker_log.checker_print("\nRe-rendering start of original sequence data-driven") 49 | RAW_LOGGING("Re-rendering start of original sequence data-driven") 50 | new_seq = sequences.Sequence([]) 51 | 52 | for request in self._sequence.requests[:-1]: 53 | new_seq = new_seq + sequences.Sequence(request) 54 | response, _ = self._render_and_send_data(new_seq, request, False) 55 | if response: 56 | if response.status_code[0] == '2': 57 | datacollect.data_driven_checker_former_200 += 1 58 | elif response.status_code[0] == '5': 59 | datacollect.data_driven_checker_former_500 += 1 60 | elif response.status_code[0] == '4': 61 | datacollect.data_driven_checker_former_400 += 1 62 | # render n-1 requests finish 63 | 64 | # render last requests start 65 | new_seq = new_seq + sequences.Sequence(self._sequence.requests[-1]) 66 | rendered_data, parser, list_values, list_values_default, whethergen = self._sequence.requests[-1].render_current_lyu3(self._req_collection.candidate_values_pool) 67 | rendered_data = new_seq.resolve_dependencies(rendered_data) 68 | response = self._send_request(parser, rendered_data) 69 | request_utilities.call_response_parser(parser, response) 70 | new_seq.append_data_to_sent_list(rendered_data, parser, response) 71 | if response: 72 | if self._rule_violation(new_seq, response): 73 | sent_request_data_list = new_seq.sent_request_data_list 74 | for req in sent_request_data_list: 75 | data = repr(req.rendered_data).strip("'") 76 | self._print_suspect_sequence(new_seq, response) 77 | BugBuckets.Instance().update_bug_buckets( 78 | new_seq, response.status_code, origin=self.__class__.__name__) #, reproduce=False) 79 | 80 | if response.status_code[0] == '2': 81 | datacollect.data_driven_checker_laster_200 += 1 82 | elif response.status_code[0] == '5': 83 | datacollect.data_driven_checker_laster_500 += 1 84 | elif response.status_code[0] == '4': 85 | datacollect.data_driven_checker_laster_400 += 1 86 | 87 | if '?' in list_values_default: # otherwise there is no node to mutate 88 | if response.status_code[0] == '2': 89 | if whethergen == 1: 90 | datacollect.gen200 += 1 91 | else: 92 | datacollect.nogen200 += 1 93 | datacollect.countid += 1 94 | 95 | if datacollect.seq_which == 1: 96 | datacollect.seq_gen200 += 1 97 | elif datacollect.seq_which == 2: 98 | datacollect.seq_nogen200 += 1 99 | 100 | elif response.status_code[0] == '5': 101 | if whethergen == 1: 102 | datacollect.gen500 += 1 103 | else: 104 | datacollect.nogen500 += 1 105 | 106 | datacollect.countid += 1 107 | 108 | if datacollect.seq_which == 1: 109 | datacollect.seq_gen500 += 1 110 | elif datacollect.seq_which == 2: 111 | datacollect.seq_nogen500 += 1 112 | 113 | elif response.status_code[0] == '4': 114 | if datacollect.seq_which == 1: 115 | datacollect.seq_gen400 += 1 116 | elif datacollect.seq_which == 2: 117 | datacollect.seq_nogen400 += 1 118 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/examples_checker.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import copy 5 | 6 | from checkers.checker_base import * 7 | from engine.fuzzing_parameters.fuzzing_utils import * 8 | from engine.bug_bucketing import BugBuckets 9 | from engine.core.request_utilities import str_to_hex_def 10 | from engine.core.requests import Request 11 | from engine.core.sequences import Sequence 12 | import engine.primitives as primitives 13 | 14 | class ExamplesChecker(CheckerBase): 15 | """ Checker for payload body fuzzing. """ 16 | 17 | def __init__(self, req_collection, fuzzing_requests): 18 | CheckerBase.__init__(self, req_collection, fuzzing_requests) 19 | 20 | # alias the log 21 | self._log = self._checker_log.checker_print 22 | # list of requests that have already been tested 23 | self._tested_requests: set = set() 24 | 25 | def apply(self, rendered_sequence, lock): 26 | """ Applies check for fuzzing request payload body 27 | 28 | @param rendered_sequence: Object containing the rendered sequence information 29 | @type rendered_sequence: RenderedSequence 30 | @param lock: Lock object used to sync more than one fuzzing job 31 | @type lock: thread.Lock 32 | 33 | @return: None 34 | @type : None 35 | 36 | """ 37 | if not rendered_sequence.sequence: 38 | return 39 | 40 | self._sequence = copy.copy(rendered_sequence.sequence) 41 | last_request = self._sequence.last_request 42 | # Remove the last request from the sequence list because it will be replaced 43 | # with new requests. 44 | self._sequence.requests = self._sequence.requests[:-1] 45 | self._sequence._sent_request_data_list = self._sequence._sent_request_data_list[:-1] 46 | 47 | last_request_def = str_to_hex_def(last_request.method) + last_request.request_id 48 | # check to see if we already tested this request 49 | if last_request_def in self._tested_requests: 50 | return 51 | 52 | self._tested_requests.add(last_request_def) 53 | 54 | if last_request.examples: 55 | self._log("Sending examples for request: \n" 56 | f"{last_request.endpoint}\n" 57 | f"Previous attempt was {'valid' if rendered_sequence.valid else 'invalid'}.") 58 | 59 | self._send_each_example(last_request) 60 | 61 | def _send_each_example(self, request): 62 | """ Substitutes each example into the request and sends it 63 | """ 64 | def _send_request(request_to_send): 65 | self._log("Sending example request: \n" 66 | f"{request_to_send.definition}", print_to_network_log=False) 67 | seq = self._sequence + Sequence(request_to_send) 68 | response, _ = self._render_and_send_data(seq, request_to_send) 69 | 70 | code = response.status_code 71 | self._log(f"Status Code: {code}", print_to_network_log=False) 72 | if code not in status_codes: 73 | status_codes[code] = 0 74 | status_codes[code] += 1 75 | 76 | # Check to make sure a bug wasn't uncovered while executing the sequence 77 | if response and response.has_bug_code(): 78 | self._print_suspect_sequence(seq, response) 79 | BugBuckets.Instance().update_bug_buckets(seq, code, origin=self.__class__.__name__, hash_full_request=True) 80 | 81 | status_codes = {} 82 | 83 | # Send new request for each body example 84 | for example in request.examples.body_examples: 85 | blocks = example.get_blocks() 86 | new_request = substitute_body(request, blocks) 87 | if new_request: 88 | _send_request(new_request) 89 | else: 90 | self._log(f"Failed to substitute body for request {request.endpoint}.") 91 | # Send new request for each query example. 92 | # For now don't try to match these up with body examples. 93 | # There will soon be IDs associated with the examples, so they can be matched. 94 | for example in request.examples.query_examples: 95 | q_blocks = [] 96 | for idx, query in enumerate(example.queries): 97 | q_blocks += query.get_blocks() 98 | if idx < len(example) - 1: 99 | # Add the query separator 100 | q_blocks.append(primitives.restler_static_string('&')) 101 | new_request = substitute_query(request, q_blocks) 102 | if new_request: 103 | _send_request(new_request) 104 | else: 105 | self._log('Failed to substitute query') 106 | 107 | self._log("Results:") 108 | for code in status_codes: 109 | self._log(f"{code}: {status_codes[code]}") 110 | self._log("\n", print_to_network_log=False) 111 | 112 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/leakage_rule_checker.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Implements logic for leakage rule checker. """ 5 | from __future__ import print_function 6 | 7 | from checkers.checker_base import * 8 | 9 | import time 10 | import itertools 11 | 12 | from engine.bug_bucketing import BugBuckets 13 | import engine.dependencies as dependencies 14 | import engine.core.sequences as sequences 15 | 16 | class LeakageRuleChecker(CheckerBase): 17 | """ Checker for resource leakage violations. """ 18 | def __init__(self, req_collection, fuzzing_requests): 19 | CheckerBase.__init__(self, req_collection, fuzzing_requests) 20 | 21 | def apply(self, rendered_sequence, lock): 22 | """ Applies check for leakage rule violations. 23 | 24 | @param rendered_sequence: Object containing the rendered sequence information 25 | @type rendered_sequence: RenderedSequence 26 | @param lock: Lock object used to sync more than one fuzzing job 27 | @type lock: thread.Lock 28 | 29 | @return: None 30 | @rtype : None 31 | 32 | """ 33 | # Note that, unlike other checkers, the precondition here is failed renderings. 34 | if rendered_sequence.valid: 35 | return 36 | # Return if the sequence was never fully rendered 37 | if rendered_sequence.sequence is None: 38 | return 39 | self._sequence = rendered_sequence.sequence 40 | 41 | # We skip any sequence that contains DELETE methods so that we 42 | # keep in isolation this checker and the use-after-free checker. 43 | if self._sequence.has_destructor(): 44 | return 45 | 46 | # Type produced by the sequence. Necessary for type-checking 47 | produces = self._sequence.produces 48 | seq_produced_types = set(itertools.chain(*produces)) 49 | 50 | # Target the types produced by the last request of the sequence. 51 | target_types = produces[-1] 52 | 53 | for target_type in target_types: 54 | self._checker_log.checker_print(f"\nTarget type: {target_type}") 55 | # Iterate through each request that has a matching request ID to the the final 56 | # request in the sequence, which is the request that will be checked for leakage. 57 | for req in self._req_collection.request_id_collection[self._sequence.last_request.request_id]: 58 | # Skip requests that are not consumers or don't type-check. 59 | if not req.consumes\ 60 | or req.consumes - seq_produced_types\ 61 | or target_type not in req.consumes: 62 | continue 63 | 64 | self._set_dynamic_variables(self._sequence.sent_request_data_list[-1].rendered_data, req) 65 | self._render_consumer_request(self._sequence + sequences.Sequence(req)) 66 | 67 | if self._mode != 'exhaustive': 68 | break 69 | 70 | def _set_dynamic_variables(self, sent_data, request): 71 | """ Helper that sets dynamic variables to the values that a failed request 72 | was attempting to produce. 73 | 74 | @param sent_data: The data from the sent request 75 | @type sent_data: Str 76 | @param request: A request that consumes the dynamic variable from the sent 77 | request. This is used to find the location of the dynamic 78 | variables within the @param sent_data string 79 | @type request: Request 80 | 81 | @return: None 82 | @rtype : None 83 | 84 | """ 85 | # Get sent data endpoint 86 | sent_data = sent_data.split(" HTTP")[0] 87 | sent_data = sent_data.split("?")[0] 88 | # Split sent data and request endpoint data to try and find dependencies, Ex: 89 | # sent_data = PUT somevar/A-1234/someothervar/B-5678/ 90 | # sent_split = [PUT somevar, A-1234, someothervar, B-5678] 91 | # placeholder_split = [PUT somevar, _READER_DELIM_A_READER_DELIM_, someothervar, _READER_DELIM_A_READER_DELIM_] 92 | sent_split = sent_data.split('/') 93 | placeholder_split = request.endpoint.split('/') 94 | 95 | # Iterate through the request endpoint and set the dynamic variables with the matching 96 | # values that were sent in the request that triggered this checker 97 | for index, val in enumerate(placeholder_split): 98 | if dependencies.RDELIM in val: 99 | dependencies.set_variable_no_gc(val.replace(dependencies.RDELIM, ""), sent_split[index]) 100 | 101 | def _render_consumer_request(self, seq): 102 | """ Render the last request of the sequence and inspect the status 103 | code of the response. If it's not 40x, we have probably hit a bug. 104 | 105 | @param seq: The sequence whose last request we will try to render. 106 | @type seq: Sequence Class object. 107 | 108 | @return: None 109 | @rtype : None 110 | 111 | """ 112 | request = seq.last_request 113 | response, _ = self._render_and_send_data(seq, request, True, 3) 114 | 115 | if response and self._rule_violation(seq, response): 116 | self._print_suspect_sequence(seq, response) 117 | BugBuckets.Instance().update_bug_buckets(seq, response.status_code, origin=self.__class__.__name__) 118 | 119 | if response.status_code[0] == '2': 120 | datacollect.leakage_rule_checker_former_200 += 1 121 | elif response.status_code[0] == '5': 122 | datacollect.leakage_rule_checker_former_500 += 1 123 | elif response.status_code[0] == '4': 124 | datacollect.leakage_rule_checker_former_400 += 1 125 | 126 | 127 | 128 | 129 | def _false_alarm(self, seq, response): 130 | """ Catches leakage rule violation false alarms that occur when 131 | a DELETE request receives a 204 as a response status_code 132 | 133 | @param seq: The sequence that contains the request with the rule violation 134 | @type seq: Sequence 135 | @param response: Body of response. 136 | @type response: Str 137 | 138 | @return: True if false alarm detected 139 | @rtype : Bool 140 | 141 | """ 142 | try: 143 | # If a DELETE request was sent and the status code returned was a 204, 144 | # we can assume that this was not a failure because many services use a 204 145 | # response code when there is nothing to delete 146 | return response.status_code.startswith("204")\ 147 | and seq.last_request.method.startswith('DELETE') 148 | except Exception: 149 | return False 150 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/payload_body_bucketing.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Handles bucketizing payload body checker bugs """ 5 | import os 6 | import utils.logger as logger 7 | import engine.fuzzing_parameters.fuzzing_utils as utils 8 | from engine.transport_layer.messaging import DELIM 9 | 10 | INVALID_JSON_STR = 'InvalidJson' 11 | 12 | class PayloadBodyBuckets(): 13 | def __init__(self): 14 | """ Initializes PayloadBodyBuckets class """ 15 | self._buckets = dict() # {Request, set(error_strs)} 16 | 17 | def add_bug(self, request, new_request_data): 18 | """ Adds a bug to the payload body buckets log if it is unique. 19 | 20 | @param request: The request being fuzzed 21 | @type request: Request 22 | @param new_request_data: The request data of the new request that 23 | includes the fuzzed payload body. 24 | @type new_request_data: Str 25 | 26 | @return: Tuple containing the error string and the response body 27 | @rtype : Tuple(str, str) or None 28 | 29 | """ 30 | # Extract the body from the new request data 31 | new_body = utils.get_response_body(new_request_data) 32 | with open(os.path.join(logger.LOGS_DIR, 'payload_buckets.txt'), 'a') as file: 33 | # Check to see if we have logged any bugs for this request yet 34 | if request.method_endpoint_hex_definition not in self._buckets: 35 | self._buckets[request.method_endpoint_hex_definition] = set() 36 | # Write the header for this particular request to the log 37 | file.write(f'{request.method} {request.endpoint_no_dynamic_objects}\n') 38 | 39 | error_str = self._get_error_str(request, new_body) or 'Other' 40 | if error_str not in self._buckets[request.method_endpoint_hex_definition]: 41 | if error_str == INVALID_JSON_STR: 42 | # body is invalid JSON, so just extract what's at the end of the 43 | # request for logging purposes 44 | new_body = new_request_data.split(DELIM)[-1] 45 | self._buckets[request.method_endpoint_hex_definition].add(error_str) 46 | file.write(f'\t{error_str}\n\t{new_body}\n\n') 47 | return (error_str, new_body) 48 | return None 49 | 50 | def _get_error_str(self, request, new_body): 51 | """ Gets the error string associated with this bug 52 | 53 | @param request: The request being fuzzed 54 | @type request: Request 55 | @param new_body: The fuzzed body of the request to compare 56 | @type new_body: Str 57 | 58 | @return: The error string or None 59 | @rtype : Str or None 60 | 61 | """ 62 | if new_body: 63 | # Compare the new body to the original request for a type mismatch 64 | mismatch = request.body_schema.has_type_mismatch(new_body) 65 | if mismatch: 66 | return f'TypeMismatch_{mismatch}' 67 | # Compare the new body to the original request for a missing struct 68 | missing = request.body_schema.has_struct_missing(new_body) 69 | if missing: 70 | return f'StructMissing_{missing}' 71 | else: 72 | return INVALID_JSON_STR 73 | 74 | return None 75 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/resource_hierarchy_checker.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Implements logic for resource hierarchy rule checker. """ 5 | from __future__ import print_function 6 | 7 | from checkers.checker_base import * 8 | import itertools 9 | 10 | from engine.bug_bucketing import BugBuckets 11 | import engine.dependencies as dependencies 12 | import engine.core.sequences as sequences 13 | 14 | class ResourceHierarchyChecker(CheckerBase): 15 | """ Checker for ResourceHierarchy rule violations. """ 16 | def __init__(self, req_collection, fuzzing_requests): 17 | CheckerBase.__init__(self, req_collection, fuzzing_requests) 18 | 19 | def apply(self, rendered_sequence, lock): 20 | """ Applies check for resource hierarchy rule violations. 21 | 22 | @param rendered_sequence: Object containing the rendered sequence information 23 | @type rendered_sequence: RenderedSequence 24 | @param lock: Lock object used to sync more than one fuzzing job 25 | @type lock: thread.Lock 26 | 27 | @return: None 28 | @rtype : None 29 | 30 | """ 31 | if not rendered_sequence.valid: 32 | return 33 | self._sequence = rendered_sequence.sequence 34 | 35 | # We skip any sequence that contains DELETE methods so that we 36 | # keep in isolation this checker and the use-after-free checker. 37 | if self._sequence.has_destructor(): 38 | return 39 | 40 | consumes = self._sequence.consumes 41 | predecessors_types = consumes[:-1] 42 | # Last request is the victim -- our target! 43 | target_types = consumes[-1] 44 | # In the dictionary of "consumes" constraints, each request of the 45 | # sequence instance has its own dictionary of the dynamic variable 46 | # types produced by each request. We need to flatten this structure. 47 | predecessors_types = set(itertools.chain(*predecessors_types)) 48 | 49 | # Skip sequence if there are no predecessor dependencies or no 50 | # target objects to swap. 51 | if not predecessors_types.intersection(target_types)\ 52 | or not target_types - predecessors_types: 53 | return 54 | 55 | # For the victim types (target dynamic objects), get the lattest 56 | # values which we know will exist due to the previous rendering. 57 | # We will later on use these old values atop a new rendering. 58 | old_values = {} 59 | for target_type in target_types - predecessors_types: 60 | old_values[target_type] = dependencies.get_variable(target_type) 61 | 62 | # Reset tlb of all values and re-render all predecessor up to 63 | # the parent's parent. This will propagate new values for all 64 | # dynamic objects except for those with target type. That's what we 65 | # want and that's why we render up to the parent's parent (i.e., 66 | # up to length(seq) - 2. 67 | dependencies.reset_tlb() 68 | 69 | # Render sequence up to before the first predecessor that produces 70 | # the target type. that is, if any of the types produced by the 71 | # request is in the target types, then do not render this 72 | # predecessor and stop here. 73 | n_predecessors = 0 74 | for req in self._sequence: 75 | if req.produces.intersection(target_types - predecessors_types): 76 | break 77 | n_predecessors += 1 78 | new_seq = self._render_n_predecessor_requests(n_predecessors) 79 | 80 | # log some helpful info 81 | self._checker_log.checker_print("\nTarget types: {}".\ 82 | format(target_types - predecessors_types)) 83 | self._checker_log.checker_print(f"Predecesor types: {predecessors_types}") 84 | self._checker_log.checker_print("Clean tlb: {}".\ 85 | format(dependencies.tlb)) 86 | 87 | # Before rendering the last request, substitute all target types 88 | # (target dynamic object) with a value that does NOT belong to 89 | # the current rendering and should not (?) be accessible through 90 | # the new predecessors' rendering. 91 | for target_type in old_values: 92 | dependencies.set_variable(target_type, old_values[target_type]) 93 | 94 | self._checker_log.checker_print("Poluted tlb: {}".\ 95 | format(dependencies.tlb)) 96 | self._render_last_request(new_seq) 97 | 98 | def _render_n_predecessor_requests(self, n_predecessors): 99 | """ Render up to the parent's parent predecessor request. 100 | 101 | @param n_predecessors: The number of predecessors to render. 102 | @type n_predecessors: Int 103 | 104 | @return: Sequence of n predecessor requests sent to server 105 | @rtype : Sequence 106 | 107 | """ 108 | new_seq = sequences.Sequence([]) 109 | has_500 = False 110 | has_400 = False 111 | has_200 = True 112 | for i in range(n_predecessors): 113 | request = self._sequence.requests[i] 114 | new_seq = new_seq + sequences.Sequence(request) 115 | response, _ = self._render_and_send_data(new_seq, request, False) 116 | 117 | if response.status_code[0] == '2': 118 | datacollect.resource_hierarchy_checker_former_200 += 1 119 | elif response.status_code[0] == '5': 120 | datacollect.resource_hierarchy_checker_former_500 += 1 121 | elif response.status_code[0] == '4': 122 | datacollect.resource_hierarchy_checker_former_400 += 1 123 | 124 | if response: 125 | if response.status_code[0] == '5': 126 | has_500 = True 127 | has_200 = False 128 | elif response.status_code[0] == '4': 129 | has_400 = True 130 | has_200 = False 131 | 132 | if has_500 and not has_400: 133 | datacollect.resource_hierarchy_checker_part_500 += 1 134 | elif has_400 and not has_500: 135 | datacollect.resource_hierarchy_checker_part_400 += 1 136 | elif has_400 and has_500: 137 | datacollect.resource_hierarchy_checker_part_non200 += 1 138 | elif has_200: 139 | datacollect.resource_hierarchy_checker_part_200 += 1 140 | # Check to make sure a bug wasn't uncovered while executing the sequence 141 | if response and response.has_bug_code(): 142 | self._print_suspect_sequence(new_seq, response) 143 | BugBuckets.Instance().update_bug_buckets(new_seq, response.status_code, origin=self.__class__.__name__) 144 | 145 | if has_500 and not has_400: 146 | datacollect.resource_hierarchy_checker_overall_500 += 1 147 | elif has_400 and not has_500: 148 | datacollect.resource_hierarchy_checker_overall_400 += 1 149 | elif has_400 and has_500: 150 | datacollect.resource_hierarchy_checker_overall_non200 += 1 151 | elif has_200: 152 | datacollect.resource_hierarchy_checker_overall_200 += 1 153 | 154 | return new_seq 155 | 156 | def _render_last_request(self, new_seq): 157 | """ Render the last request of the sequence and inspect the status 158 | code of the response. If it's any of 20x, we have probably hit a bug. 159 | 160 | @param new_seq: The new sequence that was rendered with this checker 161 | @type new_seq: Sequence 162 | 163 | @return: None 164 | @rtype : None 165 | 166 | """ 167 | new_seq = new_seq + sequences.Sequence(self._sequence.last_request) 168 | response, _ = self._render_and_send_data(new_seq, self._sequence.last_request, True, 2) 169 | if response and self._rule_violation(new_seq, response): 170 | 171 | if response.status_code[0] == '2': 172 | datacollect.resource_hierarchy_checker_laster_200 += 1 173 | elif response.status_code[0] == '5': 174 | datacollect.resource_hierarchy_checker_laster_500 += 1 175 | elif response.status_code[0] == '4': 176 | datacollect.resource_hierarchy_checker_laster_400 += 1 177 | 178 | 179 | self._print_suspect_sequence(new_seq, response) 180 | BugBuckets.Instance().update_bug_buckets(new_seq, response.status_code, origin=self.__class__.__name__) 181 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/checkers/response_analyzer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | from __future__ import print_function 5 | 6 | import argparse 7 | import json 8 | import re 9 | 10 | from engine.fuzzing_parameters.fuzzing_utils import * 11 | 12 | class ResponseTracker(): 13 | """ Response tracker for error code and error message """ 14 | 15 | def __init__(self, ignore, verbose=False, LOG=print): 16 | """ Initialize the tracker 17 | 18 | @param LOG: Customized log 19 | @type LOG: Function 20 | 21 | @return: None 22 | @rtype: None 23 | 24 | """ 25 | self._log = LOG 26 | self._codes = {} 27 | self._valid_cnt = 0 28 | self._invalid_cnt = 0 29 | self._ignore = ignore 30 | self._verbose = verbose 31 | 32 | @property 33 | def num_requests(self): 34 | """ Return the total number of responses received 35 | 36 | @return: Total number of responses 37 | @rtype: Int 38 | 39 | """ 40 | return self._valid_cnt + self._invalid_cnt 41 | 42 | @property 43 | def num_error_codes(self): 44 | """ Return the number of different error code/message 45 | 46 | @return: Number of different error code/message 47 | @rtype: Int 48 | 49 | """ 50 | total = 0 51 | for c in self._codes: 52 | total += len(self._codes[c]) 53 | return total 54 | 55 | @property 56 | def num_valid(self): 57 | """ Return the number of valid responses 58 | 59 | @return: Number of valid responses 60 | @rtype: Int 61 | 62 | """ 63 | return self._valid_cnt 64 | 65 | def process_response(self, response, test_round=None): 66 | """ Process a response 67 | 68 | @param response: Server's response 69 | @type response: HttpResponse 70 | 71 | @return: None 72 | @rtype: None 73 | 74 | """ 75 | # valid response are not handled for now 76 | if response.has_valid_code(): 77 | self._valid_cnt += 1 78 | return None 79 | 80 | self._invalid_cnt += 1 81 | 82 | # try to get response body (JSON) 83 | body = get_response_body(response.to_str) 84 | 85 | if test_round: 86 | status_code = '{}_{}'.format(test_round, status_code) 87 | else: 88 | status_code = response.status_code 89 | 90 | if body: 91 | self._process_json_error_response(body, response) 92 | else: 93 | self._process_other_error_response(response.to_str, status_code) 94 | 95 | def _process_json_error_response(self, body, response): 96 | """ Process the body of an invalid response 97 | 98 | @param body: Response body 99 | @type body: String 100 | @param response: Server's response 101 | @type response: HttpResponse 102 | 103 | @return: None 104 | @rtype: None 105 | 106 | """ 107 | # get the error code and error message 108 | if 'error' in body: 109 | error = body['error'] 110 | else: 111 | error = body 112 | 113 | if ('code' not in error) or ('message' not in error): 114 | self._update_record('misc', error) 115 | return None 116 | 117 | if response.has_bug_code(): 118 | message = error['message'].split('timestamp')[0] 119 | else: 120 | message = error['message'] 121 | 122 | # update tracker 123 | code = '{0}_{1}'.format(response.status_code, error['code']) 124 | self._update_record(code, message) 125 | 126 | def _process_other_error_response(self, response, status_code): 127 | """ Try to process unexpected response 128 | 129 | @param response: Response 130 | @type response: String 131 | @param status_code: Status code 132 | @type status_code: String 133 | 134 | @return: None 135 | @rtype: None 136 | 137 | """ 138 | # try if is XML 139 | xml_body_start = response.find('') 140 | xml_body_end = response.find('') 141 | if xml_body_start != -1 and xml_body_end != -1: 142 | if xml_body_end > xml_body_start: 143 | xml_body = response[xml_body_start:xml_body_end] 144 | self._update_record('{}_XML'.format(status_code), xml_body) 145 | return 146 | 147 | # don't know 148 | self._update_record('{}_UnknownResponse'.format(status_code), response) 149 | 150 | def _update_record(self, code, msg): 151 | """ Update the tracker record 152 | 153 | @param code: Error code 154 | @type code: String 155 | @param msg: Error message 156 | @type msg: String 157 | 158 | @return: None 159 | @rtype: None 160 | 161 | """ 162 | msg = self._get_message_hash(msg) 163 | 164 | if code in self._codes: 165 | if msg in self._codes[code]: 166 | self._codes[code][msg] += 1 167 | else: 168 | self._codes[code][msg] = 1 169 | else: 170 | self._codes[code] = {msg: 1} 171 | 172 | def _get_message_hash(self, msg): 173 | """ Sanitize the message and return the hash 174 | 175 | @param msg: Error message 176 | @type msg: String 177 | 178 | @return: Hash 179 | @rtype: Int 180 | 181 | """ 182 | # make sure it's string 183 | msg = str(msg) 184 | 185 | # remove ignores 186 | if self._ignore: 187 | for keyword in self._ignore: 188 | msg = msg.replace(keyword, '?') 189 | 190 | # replace guid with '?' 191 | guids = re.findall('[0-9a-f]{10}', msg) 192 | for guid in guids: 193 | msg = msg.replace(guid, '?') 194 | 195 | # replace numbers 196 | msg = replace_number_chunks(msg, '?') 197 | 198 | # hash 199 | if self._verbose: 200 | return msg 201 | else: 202 | return hash(msg) 203 | 204 | def show(self, msg=''): 205 | """ Print out the tracker statistics 206 | 207 | @param msg: Additional message to print along with the statistics 208 | @type msg: String 209 | 210 | @return: None 211 | @rtype: None 212 | 213 | """ 214 | self._log('Tracker begin ({}):'.format(msg)) 215 | self._log(' Valid: {}'.format(self._valid_cnt)) 216 | self._log(' Invalid: {}'.format(self._invalid_cnt)) 217 | for c in self._codes: 218 | detail = '' 219 | total = 0 220 | for msg in self._codes[c]: 221 | cnt = self._codes[c][msg] 222 | detail += ' {}'.format(cnt) 223 | total += cnt 224 | desp = '{}:{}'.format(total, detail) 225 | self._log(' {}: {} ({})'.format(c, len(self._codes[c]), desp)) 226 | self._log('Tracker end') 227 | 228 | def show_detail(self, msg=''): 229 | """ Print out the tracker statistics in detail 230 | 231 | @param msg: Additional message to print along with the statistics 232 | @type msg: String 233 | 234 | @return: None 235 | @rtype: None 236 | 237 | """ 238 | self._log('Tracker begin ({}):'.format(msg)) 239 | self._log(' Valid: {}'.format(self._valid_cnt)) 240 | self._log(' Invalid: {}'.format(self._invalid_cnt)) 241 | for c in self._codes: 242 | total = 0 243 | self._log(' {}: {}'.format(c, len(self._codes[c]))) 244 | for msg in self._codes[c]: 245 | cnt = self._codes[c][msg] 246 | detail = ' ({}) {}'.format(cnt, msg) 247 | self._log(detail) 248 | total += cnt 249 | self._log(' Total: {}'.format(total)) 250 | 251 | self._log('Tracker end') 252 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/engine/engine/__init__.py -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/engine/engine/core/__init__.py -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/core/datacollect.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import random 4 | import datetime 5 | from threading import Thread 6 | import subprocess 7 | import time 8 | 9 | import engine.core.sequences as sequences 10 | seq_collection_origin = [sequences.Sequence()] 11 | 12 | tmptmpcount = 0 13 | countid = 0 14 | countid400 = 0 15 | 16 | requestid = {} 17 | 18 | trainingset = [] 19 | trainingset400 = [] 20 | mutationtrainingset = {} 21 | mutationlist = {} 22 | whether_start_mutation_list = 0 23 | generation_key = 0 24 | seq_gen200 = 0 25 | seq_gen400 = 0 26 | seq_gen500 = 0 27 | seq_nogen200 = 0 28 | seq_nogen400 = 0 29 | seq_nogen500 = 0 30 | seq_if = 0 31 | seq_which = 0 32 | seq_gen_num = 0 33 | seq_nogen_num = 0 34 | 35 | 36 | gen200 = 0 37 | gen400 = 0 38 | gen500 = 0 39 | nogen200 = 0 40 | nogen400 = 0 41 | nogen500 = 0 42 | 43 | 44 | payload_body_checker_former_200 = 0 45 | payload_body_checker_former_500 = 0 46 | payload_body_checker_former_400 = 0 47 | payload_body_checker_laster_200 = 0 48 | payload_body_checker_laster_500 = 0 49 | payload_body_checker_laster_400 = 0 50 | 51 | 52 | invalid_dynamic_object_checker_former_200 = 0 53 | invalid_dynamic_object_checker_former_500 = 0 54 | invalid_dynamic_object_checker_former_400 = 0 55 | 56 | leakage_rule_checker_former_200 = 0 57 | leakage_rule_checker_former_500 = 0 58 | leakage_rule_checker_former_400 = 0 59 | 60 | 61 | 62 | namespace_rule_checker_former_200 = 0 63 | namespace_rule_checker_former_500 = 0 64 | namespace_rule_checker_former_400 = 0 65 | namespace_rule_checker_laster_200 = 0 66 | namespace_rule_checker_laster_500 = 0 67 | namespace_rule_checker_laster_400 = 0 68 | 69 | 70 | 71 | resource_hierarchy_checker_former_200 = 0 72 | resource_hierarchy_checker_former_500 = 0 73 | resource_hierarchy_checker_former_400 = 0 74 | resource_hierarchy_checker_laster_200 = 0 75 | resource_hierarchy_checker_laster_500 = 0 76 | resource_hierarchy_checker_laster_400 = 0 77 | 78 | 79 | 80 | use_after_free_checker_laster_200 = 0 81 | use_after_free_checker_laster_500 = 0 82 | use_after_free_checker_laster_400 = 0 83 | 84 | data_driven_checker_laster_200 = 0 85 | data_driven_checker_laster_500 = 0 86 | data_driven_checker_laster_400 = 0 87 | data_driven_checker_former_200 = 0 88 | data_driven_checker_former_500 = 0 89 | data_driven_checker_former_400 = 0 90 | 91 | starttime = 0 92 | ppid = 0 93 | 94 | seq_overall_200 = 0 95 | seq_overall_400 = 0 96 | seq_overall_500 = 0 97 | seq_overall_non200 = 0 98 | 99 | seq_part_200 = 0 100 | seq_part_400 = 0 101 | seq_part_500 = 0 102 | seq_part_non200 = 0 103 | 104 | invalid_dynamic_object_checker_overall_200 = 0 105 | invalid_dynamic_object_checker_overall_400 = 0 106 | invalid_dynamic_object_checker_overall_500 = 0 107 | invalid_dynamic_object_checker_overall_non200 = 0 108 | invalid_dynamic_object_checker_part_200 = 0 109 | invalid_dynamic_object_checker_part_400 = 0 110 | invalid_dynamic_object_checker_part_500 = 0 111 | invalid_dynamic_object_checker_part_non200 = 0 112 | 113 | namespace_rule_checker_overall_200 = 0 114 | namespace_rule_checker_overall_400 = 0 115 | namespace_rule_checker_overall_500 = 0 116 | namespace_rule_checker_overall_non200 = 0 117 | namespace_rule_checker_part_200 = 0 118 | namespace_rule_checker_part_400 = 0 119 | namespace_rule_checker_part_500 = 0 120 | namespace_rule_checker_part_non200 = 0 121 | 122 | payload_body_checker_overall_200 = 0 123 | payload_body_checker_overall_400 = 0 124 | payload_body_checker_overall_500 = 0 125 | payload_body_checker_overall_non200 = 0 126 | payload_body_checker_part_200 = 0 127 | payload_body_checker_part_400 = 0 128 | payload_body_checker_part_500 = 0 129 | payload_body_checker_part_non200 = 0 130 | 131 | resource_hierarchy_checker_overall_200 = 0 132 | resource_hierarchy_checker_overall_400 = 0 133 | resource_hierarchy_checker_overall_500 = 0 134 | resource_hierarchy_checker_overall_non200 = 0 135 | resource_hierarchy_checker_part_200 = 0 136 | resource_hierarchy_checker_part_400 = 0 137 | resource_hierarchy_checker_part_500 = 0 138 | resource_hierarchy_checker_part_non200 = 0 139 | 140 | 141 | seq_normal_writer = 0 142 | seq_abnormal_writer = 0 143 | seq_abnormal_consumer = 0 144 | 145 | seq_types = [] 146 | valid_seq_types = [] 147 | 148 | writer_stats = {} 149 | 150 | def readdata(): 151 | global mutationlist 152 | global mutationtrainingset 153 | global requestid 154 | tmpmutationlist = {} 155 | if os.path.exists('/home/MINER/restler_bin_atten/mutationlist.pkl'): 156 | tmpmutationlist = pickle.load(open("/home/MINER/restler_bin_atten/mutationlist.pkl", 'rb')) 157 | for key1 in tmpmutationlist.keys(): 158 | if key1 not in mutationlist.keys(): 159 | tmpvalue1 = [] 160 | for value1 in tmpmutationlist[key1]: 161 | tmpvalue1.append(value1) 162 | mutationlist[key1] = tmpvalue1 163 | else: 164 | for value1 in tmpmutationlist[key1]: 165 | mutationlist[key1].append(value1) 166 | 167 | if os.path.exists('/home/MINER/restler_bin_atten/dictrequestid.pkl'): 168 | with open('/home/MINER/restler_bin_atten/dictrequestid.pkl', 'rb') as f: 169 | requestid = pickle.load(f) 170 | 171 | 172 | if os.path.exists('/home/MINER/restler_bin_atten/trainingset.pkl'): 173 | tmptrainingset = pickle.load(open("/home/MINER/restler_bin_atten/trainingset.pkl", 'rb')) 174 | 175 | for case in tmptrainingset: 176 | if case[2] == 'name=': 177 | tmptrainingset.remove(case) 178 | for lyui in range(len(tmptrainingset)): 179 | # buildup dictcount 180 | tmptrainingset[lyui][1] = str(tmptrainingset[lyui][1]) + tmptrainingset[lyui][4] 181 | 182 | tmpword = '' 183 | for lyuk in range(len(tmptrainingset[lyui][3])): 184 | tmpword += tmptrainingset[lyui][3][lyuk] 185 | tup = (tmptrainingset[lyui][2], tmpword) 186 | if tmptrainingset[lyui][1] not in mutationtrainingset: 187 | dict1d = {} 188 | 189 | dict1d[tup] = 1 190 | mutationtrainingset[tmptrainingset[lyui][1]] = dict1d 191 | elif tup not in mutationtrainingset[tmptrainingset[lyui][1]]: 192 | dict1d = mutationtrainingset[tmptrainingset[lyui][1]] 193 | dict1d[tup] = 1 194 | else: 195 | mutationtrainingset[tmptrainingset[lyui][1]][tup] += 1 196 | 197 | def reverseindex(tmplist, tmpword): 198 | start_loc = 0 199 | for lyui in range(0, len(tmplist)): 200 | if tmplist[len(tmplist) - 1 - lyui] == tmpword: 201 | start_loc = len(tmplist) - 1 - lyui 202 | break 203 | return start_loc 204 | 205 | def retrainingmodel(): 206 | t = Thread(target=retrain) 207 | t.start() 208 | 209 | def retrain(): 210 | global mutationlist 211 | global requestid 212 | global trainingset 213 | global whether_start_mutation_list 214 | global seq_gen200 215 | global seq_gen400 216 | global seq_gen500 217 | global seq_nogen200 218 | global seq_nogen400 219 | global seq_nogen500 220 | global seq_gen_num 221 | global seq_nogen_num 222 | whether_start_mutation_list += 1 223 | 224 | score_gen = (seq_gen200*3 + seq_gen500 - seq_gen400*2)*seq_nogen_num 225 | score_nogen = (seq_nogen200*3 + seq_nogen500 - seq_nogen400*2)*seq_gen_num 226 | 227 | print("start retrainingmodel") 228 | retrainingmodel_path = os.path.join(os.getcwd(),'retrainingmodel.txt') 229 | mutationlist_sum = 0 230 | for key in mutationlist: 231 | for l1 in mutationlist[key]: 232 | mutationlist_sum += len(l1) 233 | with open(retrainingmodel_path, 'a') as f: 234 | f.write("start retrainingmodel\n" + str(whether_start_mutation_list) +"\n") 235 | f.write("mutation_list length: "+str(len(mutationlist))+"\nmutationlist_sum: "+str(mutationlist_sum)+"\nnowtime: "+str(datetime.datetime.now())+"\n") 236 | with open('/home/MINER/restler_bin_atten/dictrequestid.pkl', 'wb') as f: 237 | pickle.dump(requestid, f, pickle.HIGHEST_PROTOCOL) 238 | 239 | with open('/home/MINER/restler_bin_atten/trainingset.pkl', 'wb') as f: 240 | pickle.dump(trainingset, f, pickle.HIGHEST_PROTOCOL) 241 | 242 | trainingset = [] 243 | cmd = 'python /home/MINER/attentionmodel_group/attention.py' 244 | os.system(cmd) 245 | cmd = 'python /home/MINER/attentionmodel_group/generation.py' 246 | os.system(cmd) 247 | readdata() 248 | 249 | mutationlist_sum = 0 250 | for key in mutationlist: 251 | for l1 in mutationlist[key]: 252 | mutationlist_sum += len(l1) 253 | with open(retrainingmodel_path, 'a') as f: 254 | if whether_start_mutation_list > 1: 255 | score_gen1 = (seq_gen200*3 + seq_gen500 - seq_gen400*2)*(seq_nogen200+seq_nogen500+seq_nogen400)*1.2 256 | score_nogen1 = (seq_nogen200*3 + seq_nogen500 - seq_nogen400*2)*(seq_gen200+seq_gen500+seq_gen400) 257 | f.write("seq_gen200: " + str(seq_gen200) + " seq_gen500: "+ str(seq_gen500) +" seq_gen400: "+ str(seq_gen400) +" seq_gen_num: "+ str(seq_gen200+seq_gen500+seq_gen400) +"\n") 258 | f.write("seq_nogen200: " + str(seq_nogen200) + " seq_nogen500: "+ str(seq_nogen500) +" seq_nogen400: "+ str(seq_nogen400) +" seq_nogen_num: "+ str(seq_nogen200+seq_nogen500+seq_nogen400) +"\n") 259 | f.write("score_gen: " + str(score_gen1) + " score_nogen: "+ str(score_nogen1) +"\n") 260 | f.write("finish readdata\nmutation_list length: "+str(len(mutationlist))+"\nmutationlist_sum: "+str(mutationlist_sum)+"\nnowtime: "+str(datetime.datetime.now())+"\n\n") 261 | print("finish readdata") 262 | 263 | 264 | def get_writer_stat(req_collection, fuzzing_monitor, global_lock): 265 | from engine.transport_layer.response import VALID_CODES 266 | from engine.transport_layer.response import RESTLER_INVALID_CODE 267 | 268 | num_writer = 0 269 | num_valid_writer = 0 270 | for r in req_collection: 271 | query_result = fuzzing_monitor.query_status_codes_monitor(r, VALID_CODES, [RESTLER_INVALID_CODE], global_lock) 272 | if bool(r.metadata) and 'post_send' in r.metadata\ 273 | and 'parser' in r.metadata['post_send']: 274 | num_writer += 1 275 | if query_result.fully_valid: 276 | num_valid_writer += 1 277 | 278 | return num_valid_writer, num_writer 279 | 280 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/core/fuzzer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import threading 5 | 6 | import engine.core.driver as driver 7 | import utils.logger as logger 8 | 9 | from engine.core.fuzzing_monitor import Monitor 10 | from engine.core.requests import GrammarRequestCollection 11 | from engine.errors import InvalidDictionaryException 12 | 13 | class FuzzingThread(threading.Thread): 14 | """ Fuzzer thread class 15 | """ 16 | def __init__(self, fuzzing_requests, checkers, fuzzing_jobs=1): 17 | """ Constructor for the Fuzzer thread class 18 | 19 | @param fuzzing_requests: The collection of requests to fuzz 20 | @type fuzzing_requests: FuzzingRequestCollection 21 | @param checkers: List of checker objects 22 | @type checkers: List[Checker] 23 | 24 | """ 25 | threading.Thread.__init__(self) 26 | 27 | self._fuzzing_requests = fuzzing_requests 28 | self._checkers = checkers 29 | self._fuzzing_jobs = fuzzing_jobs 30 | self._num_total_sequences = 0 31 | self._exception = None 32 | 33 | @property 34 | def exception(self): 35 | return self._exception 36 | 37 | def run(self): 38 | """ Thread entrance - performs fuzzing 39 | """ 40 | try: 41 | self._num_total_sequences = driver.generate_sequences( 42 | self._fuzzing_requests, self._checkers, self._fuzzing_jobs 43 | ) 44 | 45 | # At the end of everything print out any request that were never 46 | # rendered (because they never had valid constraints). 47 | logger.print_request_rendering_stats_never_rendered_requests( 48 | self._fuzzing_requests, 49 | GrammarRequestCollection().candidate_values_pool, 50 | Monitor() 51 | ) 52 | except InvalidDictionaryException: 53 | pass 54 | except Exception as err: 55 | self._exception = str(err) 56 | 57 | def join(self, *args): 58 | """ Overrides thread join function 59 | 60 | @return: The total number of sequences from the fuzzing run 61 | @rtype : Int 62 | 63 | """ 64 | threading.Thread.join(self, *args) 65 | return self._num_total_sequences 66 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/core/fuzzing_requests.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ A collection of only the requests that fuzzing will be performed on """ 5 | from __future__ import print_function 6 | import time 7 | import engine.core.requests 8 | 9 | import engine.primitives as primitives 10 | 11 | class FuzzingRequestCollection(object): 12 | def __init__(self): 13 | # Reference to Request objects that will be fuzzed 14 | self._requests = [] 15 | # Set of requests that will be processed during pre or post-processing 16 | # These requests will not be included when fuzzing 17 | self._preprocessing_requests = [] 18 | self._postprocessing_requests = [] 19 | 20 | def __iter__(self): 21 | """ Iterate over FuzzingRequests """ 22 | return iter(self._requests) 23 | 24 | def __deepcopy__(self, memo): 25 | """ Don't deepcopy this object, just return its reference""" 26 | return self 27 | 28 | def __contains__(self, request): 29 | """ Returns whether or not a request exists in the request list """ 30 | return request in self._requests 31 | 32 | def add_request(self, request): 33 | """ Adds a new request to the collection of requests to be fuzzed """ 34 | if request not in self._requests: 35 | self._requests.append(request) 36 | 37 | def exclude_preprocessing_request(self, request): 38 | """ Removes a request from the collection of requests to be fuzzed 39 | and adds that request to the preprocessing requests set 40 | 41 | @param request: The preprocessing request 42 | @type request: Request 43 | 44 | @return: None 45 | @rtype : None 46 | 47 | """ 48 | if request in self._requests: 49 | self._requests.remove(request) 50 | if request not in self._preprocessing_requests: 51 | self._preprocessing_requests.append(request) 52 | 53 | def exclude_postprocessing_request(self, request): 54 | """ Removes a request from the collection of requests to be fuzzed 55 | and adds that request to the postprocessing requests set 56 | 57 | @param request: The postprocessing request 58 | @type request: Request 59 | 60 | @return: None 61 | @rtype : None 62 | 63 | """ 64 | if request in self._requests: 65 | self._requests.remove(request) 66 | if request not in self._postprocessing_requests: 67 | self._postprocessing_requests.append(request) 68 | 69 | def set_all_requests(self, requests): 70 | """ Sets all requests in the request list """ 71 | self._requests = list(requests) 72 | 73 | @property 74 | def all_requests(self): 75 | """ Returns the list of all requests that were defined in this 76 | request collection. This included the current request list and 77 | the excluded requests 78 | 79 | @return: The list of requests 80 | @rtype : List[Request] 81 | 82 | """ 83 | return list(self._preprocessing_requests) + list(self._requests) + list(self._postprocessing_requests) 84 | 85 | @property 86 | def preprocessing_requests(self): 87 | """ Returns the list of the requests handled during preprocessing 88 | 89 | @return: The list of preprocessing requests 90 | @rtype : List[Request] 91 | 92 | """ 93 | return list(self._preprocessing_requests) 94 | 95 | @property 96 | def postprocessing_requests(self): 97 | """ Returns the list of the requests handled during postprocessing 98 | 99 | @return: The list of postprocessing requests 100 | @rtype : List[Request] 101 | 102 | """ 103 | return list(self._postprocessing_requests) 104 | 105 | @property 106 | def requests(self): 107 | """ Returns the list of requests being fuzzed 108 | 109 | @return: The list of requests being fuzzed 110 | @rtype : List[Request] 111 | 112 | """ 113 | return list(self._requests) 114 | 115 | @property 116 | def size(self): 117 | """ Returns the number of requests being fuzzed 118 | 119 | @return: The number of request in collection. 120 | @rtype : Int 121 | 122 | """ 123 | return len(self._requests) 124 | 125 | @property 126 | def size_all_requests(self): 127 | """ Returns the number of requests being fuzzed plus the number 128 | of requests used for pre and post-processing stages 129 | 130 | @return: The total number of requests 131 | @rtype : Int 132 | 133 | """ 134 | return self.size + len(self._preprocessing_requests) + len(self._postprocessing_requests) 135 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/core/postprocessing.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import engine.core.requests as requests 5 | import engine.core.sequences as sequences 6 | import time 7 | import engine.core.status_codes_monitor as status_codes_monitor 8 | 9 | from engine.core.requests import GrammarRequestCollection 10 | from restler_settings import Settings 11 | import utils.logger as logger 12 | import utils.formatting as formatting 13 | from engine.errors import TransportLayerException 14 | from engine.transport_layer import messaging as messaging 15 | from engine.core.fuzzing_monitor import Monitor 16 | 17 | def delete_create_once_resources(destructors, fuzzing_requests): 18 | """ Iterates through each destructor request and sends it to the server 19 | 20 | @param destructors: A list of destructor requests to send 21 | @type destructors: list(Request) 22 | @param fuzzing_requests: The global collection of requests to fuzz 23 | @type fuzzing_requests: FuzzingRequestCollection 24 | 25 | @return: None 26 | @rtype : None 27 | 28 | """ 29 | if not destructors: 30 | return 31 | 32 | candidate_values_pool = GrammarRequestCollection().candidate_values_pool 33 | 34 | logger.write_to_main("\nRendering for create-once resource destructors:\n") 35 | 36 | for destructor in destructors: 37 | status_codes = [] 38 | try: 39 | logger.write_to_main(f"{formatting.timestamp()}: Endpoint - {destructor.endpoint_no_dynamic_objects}") 40 | logger.write_to_main(f"{formatting.timestamp()}: Hex Def - {destructor.method_endpoint_hex_definition}") 41 | seq = sequences.Sequence([destructor]) 42 | renderings = seq.render(GrammarRequestCollection().candidate_values_pool, 43 | None, 44 | postprocessing=True) 45 | if not renderings.valid: 46 | logger.write_to_main(f"{formatting.timestamp()}: Rendering INVALID") 47 | else: 48 | logger.write_to_main(f"{formatting.timestamp()}: Rendering VALID") 49 | logger.format_rendering_stats_definition( 50 | destructor, GrammarRequestCollection().candidate_values_pool 51 | ) 52 | if Settings().in_smoke_test_mode(): 53 | destructor.stats.request_order = 'Postprocessing' 54 | destructor.stats.valid = 1 55 | destructor.stats.status_code = renderings.final_request_response.status_code 56 | destructor.stats.status_text = renderings.final_request_response.status_text 57 | except Exception as error: 58 | msg = f"Failed to delete create_once resource: {error!s}" 59 | logger.raw_network_logging(msg) 60 | logger.write_to_main(msg, print_to_console=True) 61 | if Settings().in_smoke_test_mode(): 62 | destructor.stats.request_order = 'Postprocessing' 63 | destructor.stats.valid = 0 64 | if renderings and renderings.final_request_response: 65 | destructor.stats.status_code = renderings.final_request_response.status_code 66 | destructor.stats.status_text = renderings.final_request_response.status_text 67 | destructor.stats.error_msg = renderings.final_request_response.body 68 | pass 69 | 70 | Monitor().current_fuzzing_generation += 1 71 | 72 | logger.print_request_rendering_stats( 73 | candidate_values_pool, 74 | fuzzing_requests, 75 | Monitor(), 76 | fuzzing_requests.size_all_requests, 77 | logger.POSTPROCESSING_GENERATION, 78 | None 79 | ) 80 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/core/renderings_monitor.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Tracks each request rendering with information on whether it is valid or invalid """ 5 | class RenderingsMonitor(object): 6 | def __init__(self): 7 | # Keeps track of the current generation being fuzzed by the driver. 8 | self._current_fuzzing_generation = 0 9 | 10 | # client-side monitor of valid rendering ids for each request 11 | self._rendering_ids = {} 12 | 13 | self._memoize_invalid_past_renderings = False 14 | 15 | 16 | def update(self, request, is_valid): 17 | """ Updates internal request rendering tracker with information on 18 | whether the current rendering is valid or invalid. 19 | 20 | @param request: The request whose current rendering we are registering. 21 | @type request: Request class object. 22 | @param is_valid: Flag indicating whether the current rendering leads to 23 | a valid status code or not. 24 | @type is_valid: Bool 25 | 26 | @return: None 27 | @rtype : None 28 | 29 | Note: The "minus one" rendering because the counter has been increased 30 | before invoking this routine. 31 | """ 32 | if self._current_fuzzing_generation not in self._rendering_ids: 33 | self._rendering_ids[self._current_fuzzing_generation] = {} 34 | 35 | renderings = self._rendering_ids[self._current_fuzzing_generation] 36 | if request.hex_definition not in renderings: 37 | renderings[request.hex_definition] = {'valid': set(), 38 | 'invalid': set()} 39 | 40 | target = 'valid' if is_valid else 'invalid' 41 | renderings[request.hex_definition][target].add( 42 | request._current_combination_id - 1 43 | ) 44 | 45 | def reset(self): 46 | """ Resets the renderings monitor. 47 | This is used specifically to reset the monitor after 48 | proprocessing and prior to starting the fuzzing run. 49 | 50 | @return: None 51 | @rtype : None 52 | 53 | """ 54 | self._rendering_ids.clear() 55 | 56 | def is_invalid_rendering(self, request): 57 | """ Helper to decide whether current request's rendering is registered 58 | in the past with an invalid status code. 59 | 60 | @param request: The request whose current rendering we are registering. 61 | @type request: Request class object. 62 | 63 | @return: True, if rendering is known invalid. 64 | @rtype : Bool 65 | 66 | Note: In order to be sure that we could skip a rendering it must have 67 | completed a whole round of the previous generation having only invalid 68 | renderings and no valid. (Consider the fact that the same rendering of 69 | a request may have both valid and invalid renderings.) 70 | 71 | Note: This function is called with the lock HELD. 72 | 73 | """ 74 | # If option is not set, do not skip anything. 75 | if not self._memoize_invalid_past_renderings: 76 | return False 77 | 78 | # If request has not completed a whole round of the previous generation, 79 | # do not skip 80 | if self._current_fuzzing_generation - 1 not in self._rendering_ids: 81 | return False 82 | 83 | renderings = self._rendering_ids[self._current_fuzzing_generation - 1] 84 | req_hex = request.hex_definition 85 | if req_hex not in renderings: 86 | return False 87 | 88 | # If request has completed previous generation and ONLY has invalid 89 | # renderings, the skip. 90 | invalid = renderings[req_hex]['invalid'] 91 | valid = renderings[req_hex]['valid'] 92 | return request._current_combination_id in invalid - valid 93 | 94 | def is_fully_rendered_request(self, request, lock): 95 | """ Queries internal monitor to decide if the @param request has ever 96 | been rendered by the driver. Note that for a request to have been 97 | rendered it means that in the past it had had its dependencies satisfied 98 | within some sequence. 99 | 100 | @param request: The request in question. 101 | @type request: Request class object. 102 | @param lock: Lock object used for sync of more than one fuzzing jobs. 103 | @type lock: thread.Lock object 104 | 105 | @return: True if the request in question has been rendered in the past. 106 | @rtype : Bool 107 | 108 | """ 109 | if lock is not None: 110 | lock.acquire() 111 | 112 | # If request has not completed a whole round of the previous generation, 113 | # do not skip 114 | if self._current_fuzzing_generation - 1 not in self._rendering_ids: 115 | if lock is not None: 116 | lock.release() 117 | return False 118 | 119 | for generation in range(0, self._current_fuzzing_generation): 120 | renderings = self._rendering_ids[generation] 121 | req_hex = request.hex_definition 122 | result = req_hex in renderings 123 | if result: 124 | break 125 | 126 | if lock is not None: 127 | lock.release() 128 | return result 129 | 130 | def num_fully_rendered_requests(self, request_list, lock): 131 | """ Queries internal monitor and returns the total number of requests 132 | (of request collection) that have been rendered at least once 133 | (regardless of the status code with which the target service responded). 134 | 135 | @param request_list: The complete list of requests to check for full renderings 136 | @type request_list: List[Request] 137 | 138 | @return: The number of requests that have been rendered at least once. 139 | @rtype : Int 140 | 141 | """ 142 | counter = 0 143 | for request in request_list: 144 | if self.is_fully_rendered_request(request, lock): 145 | counter += 1 146 | return counter 147 | 148 | def set_memoize_invalid_past_renderings_on(self): 149 | """ Internal sets feature for skipping known invalid past renderings. 150 | 151 | @return: None 152 | @rype : None 153 | 154 | """ 155 | self._memoize_invalid_past_renderings = True 156 | 157 | @property 158 | def current_fuzzing_generation(self): 159 | return self._current_fuzzing_generation 160 | 161 | @current_fuzzing_generation.setter 162 | def current_fuzzing_generation(self, generation): 163 | self._current_fuzzing_generation = generation 164 | 165 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/core/status_codes_monitor.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Maintains a collection of status codes received by each sequence """ 5 | import time 6 | import collections 7 | 8 | class RequestExecutionStatus(object): 9 | """ RequestExecutionStatus class. """ 10 | def __init__(self, timestamp, request_hex, status_code, is_fully_valid, sequence_failure, num_test_cases=0): 11 | """ Initializes the RequestExecutionStatus object 12 | 13 | @param timestamp: The timestamp of the request 14 | @type timestamp: Int 15 | @param request_hex: The corresponding Request object's hex definition 16 | @type request_hex: Int 17 | @param status_code: The status code returned after sending the request 18 | @type status_code: Str 19 | @param is_fully_valid: If the request received a fully-valid response 20 | @type is_fully_valid: Bool 21 | @param sequence_failure: If the sequence failed before the request was completed 22 | @type sequence_failure: Bool 23 | @param num_test_cases: 24 | @type num_test_cases: Int 25 | 26 | @return: None 27 | @rtype : None 28 | 29 | """ 30 | self.timestamp = timestamp 31 | self.request_hex = request_hex 32 | self.status_code = status_code 33 | self.is_fully_valid = is_fully_valid 34 | self.sequence_failure = sequence_failure 35 | self.num_test_cases = num_test_cases 36 | 37 | class SequenceStatusCodes(object): 38 | """ Collection of status codes for a sequence """ 39 | def __init__(self, length): 40 | self.length = length 41 | # Contains list of RequestExecutionStatus objects with status codes as keys 42 | self.request_statuses = dict() 43 | 44 | class StatusCodesMonitor(object): 45 | def __init__(self, start_time): 46 | # The start time of the fuzzing run 47 | self._start_time = start_time 48 | 49 | # Counter of total requests sent by each type 50 | self._requests_count = {'gc': 0, 'main_driver': 0} 51 | 52 | # Collection of SequenceStatusCodes objects with sequence hex defs as keys 53 | self._sequence_statuses = dict() 54 | 55 | # stuff for experiments with GitLab 56 | self.log_counter = 1 57 | 58 | @property 59 | def sequence_statuses(self): 60 | """ Returns a copy of the sequence_statuses dictionary 61 | 62 | @return A copy of the sequence_statuses dictionary 63 | @rtype Dict(int, SequenceStatusCodes) 64 | 65 | """ 66 | return dict(self._sequence_statuses) 67 | 68 | def increment_requests_count(self, type): 69 | """ Increments the requests count for a specified request type 70 | 71 | @param type: The type of request count to increment (i.e. gc) 72 | @type type: Str 73 | 74 | @return: None 75 | @rtype : None 76 | 77 | """ 78 | if type not in self._requests_count: 79 | self._requests_count[type] = 0 80 | self._requests_count[type] += 1 81 | 82 | def num_requests_sent(self): 83 | """ Returns a copy of the dict containing the request count for 84 | each type. 85 | 86 | @return: The number of requests sent for each type 87 | @rtype : Dict 88 | 89 | """ 90 | return dict(self._requests_count) 91 | 92 | def num_test_cases(self, lock=None): 93 | """ Calculates the total number of test cases executed so far. Locking 94 | may be required in case of more than one fuzzing jobs. 95 | 96 | @param lock: Lock object used for sync of more than one fuzzing jobs. 97 | @type lock: thread.Lock object 98 | 99 | @return: Number of test cases executed so far. 100 | @rtype : Int 101 | 102 | """ 103 | if lock is not None: 104 | lock.acquire() 105 | 106 | total_size = 0 107 | for seq_hash in self._sequence_statuses: 108 | timestamps = [] 109 | for code in self._sequence_statuses[seq_hash].request_statuses: 110 | timestamps.extend(self._sequence_statuses[seq_hash].request_statuses[code]) 111 | total_size += len(timestamps) / self._sequence_statuses[seq_hash].length 112 | 113 | if lock is not None: 114 | lock.release() 115 | 116 | return total_size 117 | 118 | def query_response_codes(self, request, status_codes, fail_codes, lock): 119 | """ Query internal monitor to decide if @param request had received any 120 | of @param status_codes as a service response. 121 | 122 | @param request: The request in question. 123 | @type request: Request class object. 124 | @param status_codes: List of status codes to query for. 125 | @type status_codes: List 126 | @param fail_codes: List of failing status codes to query for. 127 | @type fail_codes: List 128 | @param lock: Lock object used for sync of more than one fuzzing jobs. 129 | @type lock: thread.Lock object 130 | 131 | @return: A namedtuple object, which contains: 132 | whether or not the status code was valid, the request was fully valid, 133 | and if the request failed due to a failed sequence re-render 134 | @rtype : Namedtuple(valid_code, fully_valid, sequence_failure) 135 | 136 | """ 137 | if lock is not None: 138 | lock.acquire() 139 | 140 | QueryResult = collections.namedtuple('QueryResult', ['valid_code', 'fully_valid', 'sequence_failure']) 141 | for seq_hash in self._sequence_statuses: 142 | # iterate over each status code that was detected in this sequence 143 | for code in self._sequence_statuses[seq_hash].request_statuses: 144 | if code in status_codes or code in fail_codes: 145 | for req in self._sequence_statuses[seq_hash].request_statuses[code]: 146 | # Check if the request exists for this status code 147 | if request.hex_definition == req.request_hex: 148 | if lock is not None: 149 | lock.release() 150 | if code in status_codes: 151 | valid_code = True 152 | else: 153 | valid_code = False 154 | return QueryResult(valid_code, req.is_fully_valid, req.sequence_failure) 155 | 156 | if lock is not None: 157 | lock.release() 158 | return QueryResult(valid_code=False, fully_valid=False, sequence_failure=False) 159 | 160 | def update(self, sequence, status_codes, lock): 161 | """ Updates the internal monitor with status codes received. 162 | 163 | @param sequence: The sequence which was just executed and whose status 164 | codes going to be registered in the internal monitor. 165 | @type sequence: Sequence class object. 166 | @param status_codes: List of RequestExecutionStatus objects used when updating 167 | the status codes monitor 168 | @type status_codes: List[RequestExecutionStatus] 169 | @param lock: Lock object used for sync of more than one fuzzing jobs. 170 | @type lock: thread.Lock object 171 | 172 | @return: None 173 | @rtype : None 174 | 175 | Note: Most of this function's book-keeping is for plotting. 176 | 177 | """ 178 | if lock is not None: 179 | lock.acquire() 180 | 181 | seq_length = sequence.length 182 | self._requests_count['main_driver'] += seq_length 183 | seq_definition = sequence.definition 184 | seq_hash = sequence.hex_definition 185 | 186 | if seq_hash not in self._sequence_statuses: 187 | self._sequence_statuses[seq_hash] = SequenceStatusCodes(seq_length) 188 | 189 | # keep counter before looping over a changing dictionary 190 | num_test_cases = self.num_test_cases() + 1 191 | for code in status_codes: 192 | relative_timestamp = code.timestamp - self._start_time 193 | if code.status_code not in self._sequence_statuses[seq_hash].request_statuses: 194 | self._sequence_statuses[seq_hash].request_statuses[code.status_code] = [] 195 | new_req_status = RequestExecutionStatus( 196 | relative_timestamp, code.request_hex, code.status_code, code.is_fully_valid, code.sequence_failure, num_test_cases=num_test_cases) 197 | self._sequence_statuses[seq_hash].request_statuses[code.status_code].append(new_req_status) 198 | 199 | running_time = int((int(time.time()*10**6) - self._start_time)/ 10**6) 200 | INTERVAL = 10 # minutes 201 | if running_time > self.log_counter*INTERVAL*60: 202 | from utils import logger 203 | logger.copy_stats(self.log_counter) 204 | self.log_counter += 1 205 | 206 | if lock is not None: 207 | lock.release() 208 | 209 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Defines exception for customized error handling. """ 5 | 6 | 7 | class TransportLayerException(Exception): 8 | """ Handles transport layer exceptions. """ 9 | def __init__(self, value): 10 | """ Initializes error object for transport layer exception. 11 | 12 | @param value: The error string. 13 | @type value: Str 14 | 15 | @return: None 16 | @rtype : None 17 | 18 | """ 19 | self.parameter = value 20 | 21 | def __str__(self): 22 | """ Return printable object. 23 | 24 | @return: Error message. 25 | @rtype : Str 26 | 27 | """ 28 | return repr(self.parameter) 29 | 30 | 31 | class ResponseParsingException(Exception): 32 | def __init__(self, value): 33 | """ Initializes error object for parsing exception. 34 | 35 | @param value: The error string. 36 | @type value: Str 37 | 38 | @return: None 39 | """ 40 | self.parameter = value 41 | 42 | def __str__(self): 43 | """ Return printable object. 44 | 45 | @return: Error message. 46 | @rtype : Str 47 | 48 | """ 49 | return repr(self.parameter) 50 | 51 | 52 | class TimeOutException(Exception): 53 | def __init__(self, value): 54 | """ Initializes error object for timeout exception. 55 | 56 | @param value: The error string. 57 | @type value: Str 58 | 59 | @return: None 60 | @rtype : None 61 | 62 | """ 63 | self.parameter = value 64 | 65 | def __str__(self): 66 | """ Return printable object. 67 | 68 | @return: Error message. 69 | @rtype : Str 70 | 71 | """ 72 | return repr(self.parameter) 73 | 74 | 75 | class ExhaustSeqCollectionException(Exception): 76 | def __init__(self, value): 77 | """ Initializes error object when running out or renderings. 78 | 79 | @param value: The error string. 80 | @type value: Str 81 | 82 | @return: None 83 | @rtype : None 84 | 85 | """ 86 | self.parameter = value 87 | 88 | def __str__(self): 89 | """ Return printable object. 90 | 91 | @return: Error message. 92 | @rtype : Str 93 | 94 | """ 95 | return repr(self.parameter) 96 | 97 | 98 | class InvalidDictionaryException(Exception): 99 | """ To be raised when an invalid fuzzing dictionary is identified. 100 | """ 101 | pass 102 | 103 | class NoTokenSpecifiedException(Exception): 104 | """ To be raised when a token was expected in a request, 105 | but no token was found when querying for get_token 106 | """ 107 | pass 108 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/fuzzing_parameters/body_schema.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Contains BodySchema class that acts as a wrapper for body parameter nodes """ 5 | import copy 6 | from collections import namedtuple 7 | 8 | from engine.fuzzing_parameters.request_params import * 9 | from engine.fuzzing_parameters.request_schema_parser import * 10 | from engine.fuzzing_parameters.fuzzing_config import * 11 | import utils.logger as logger 12 | import engine.primitives as primitives 13 | 14 | class NoSchemaFound(Exception): 15 | pass 16 | 17 | class BodySchemaVisitor(): 18 | """ BodySchemaVisitor Class. """ 19 | def __init__(self): 20 | # Can be used as an accumulator string while 21 | # traversing the body schema's params. 22 | self.val_str = '' 23 | # Can be used to track the current node depth while traversing 24 | self.depth = 0 25 | 26 | class BodySchema(): 27 | """ Body Schema Class. """ 28 | 29 | def __init__(self, request_schema_json=None, fuzzing_config=None, param=None): 30 | """ Initialize and construct the BodySchema by deserializing the 31 | compiler generated request schema. 32 | 33 | @param request_schema_json: Compiler generated request schema 34 | @type request_schema_json: JSON 35 | @param fuzzing_config: PayloadBodyChecker specific configuration data (can be None) 36 | @type fuzzing_config: Dict 37 | @param param: Body schema as a ParamObject 38 | @type param: ParamObject 39 | 40 | @return: None 41 | @rtype: None 42 | 43 | """ 44 | self._schema = param 45 | # Used by payload body checker 46 | # Config is used for stateful monitoring while traversing the schema 47 | self._config = FuzzingConfig(fuzzing_config) 48 | 49 | if request_schema_json: 50 | try: 51 | self._set_body_schema(request_schema_json['bodyParameters']) 52 | except NoSchemaFound: 53 | raise 54 | except Exception as err: 55 | msg = f'Fail deserializing request schema body parameters: {err!s}' 56 | logger.write_to_main(msg, print_to_console=True) 57 | raise Exception(msg) 58 | 59 | self._node_count = self._schema.count_nodes(self._config) 60 | 61 | def __eq__(self, other): 62 | """ Operator equals 63 | """ 64 | if not isinstance(other, BodySchema): 65 | # don't attempt to compare against unrelated types 66 | return False 67 | 68 | return self._schema == other._schema and\ 69 | self._node_count == other._node_count 70 | 71 | def __hash__(self): 72 | """ Custom hash function """ 73 | return hash(self._schema) + hash(self._node_count) 74 | 75 | @property 76 | def schema(self) -> ParamObject: 77 | """ Returns the body schema 78 | 79 | @return: The body schema 80 | 81 | """ 82 | return self._schema 83 | 84 | @schema.setter 85 | def schema(self, schema: ParamObject): 86 | """ Sets the schema 87 | 88 | @param schema: The body schema to set 89 | @return: None 90 | 91 | """ 92 | self._schema = schema 93 | 94 | @property 95 | def node_count(self) -> int: 96 | """ Gets the number of nodes in the schema (up to config.max_depth) 97 | 98 | @return: Number of nodes in schema 99 | 100 | """ 101 | return self._node_count 102 | 103 | def set_config(self, config): 104 | """ Sets config parameters 105 | 106 | @param config: The config object used to set the new config 107 | @type config: FuzzingConfig 108 | 109 | """ 110 | # call copy constructor of the config object - will reset statefulness 111 | self._config = copy.copy(config) 112 | # Update node count with new schema 113 | self._node_count = self._schema.count_nodes(self._config) 114 | 115 | def get_blocks(self) -> list: 116 | """ Returns the request blocks for this schema 117 | 118 | @return: The request blocks for this schema 119 | @rtype : List[str] 120 | 121 | """ 122 | return self._schema.get_blocks(self._config) 123 | 124 | def get_signature(self) -> str: 125 | """ Returns the signature of this schema 126 | 127 | @return: The signature of this schema 128 | 129 | """ 130 | return self._schema.get_signature(self._config) 131 | 132 | def get_schema_tag_mapping(self) -> dict: 133 | """ Returns the schema tag mapping for this schema. 134 | 135 | @return: The schema tag mapping dict 136 | @rtype : Dictionary format: {"tag": content } 137 | 138 | """ 139 | mapping = dict() 140 | self._schema.get_schema_tag_mapping(mapping, self._config) 141 | return mapping 142 | 143 | def get_fuzzing_pool(self, fuzzer, config) -> list: 144 | """ Returns the fuzzing pool for the schema, created by the fuzzer 145 | 146 | @param fuzzer: The body fuzzer object used for fuzzing and creating the pool 147 | @type fuzzer: BodySchemaStructuralFuzzer 148 | @param config: PayloadBodyChecker specific configuration data (can be None) 149 | @type config: Dict 150 | 151 | @return: The fuzzing pool 152 | @rtype : List[Params] 153 | 154 | """ 155 | # Set the config 156 | self._config = FuzzingConfig(config) 157 | # Get the fuzzing pool of the schema 158 | pool = self._schema.get_fuzzing_pool(fuzzer, self._config) 159 | body_pool = [] 160 | # For each schema in the pool, create a BodySchema object from it 161 | # and copy this config to that object. 162 | for schema in pool: 163 | body = BodySchema(param=schema) 164 | body.set_config(self._config) 165 | body_pool.append(body) 166 | return body_pool 167 | 168 | def fuzz_body_blocks(self, config) -> list: 169 | """ Fuzz the value of interpreted body blocks 170 | 171 | @param config: PayloadBodyChecker specific configuration data (can be None) 172 | @type config: Dict 173 | 174 | @return: The fuzzed request blocks 175 | @rtype : List[str] 176 | 177 | """ 178 | # Set the config 179 | self._config = FuzzingConfig(config) 180 | # Get the fuzzing blocks 181 | blocks = self._schema.get_fuzzing_blocks(self._config) 182 | 183 | if self._config.fuzz_strategy == 'restler': 184 | return [blocks] 185 | 186 | acc = '' 187 | sets = [] 188 | 189 | for block in blocks: 190 | primitive_type = block[0] 191 | value = block[1] 192 | if len(block) > 2: 193 | quoted = block[2] 194 | 195 | # accumulate 196 | if primitive_type == primitives.STATIC_STRING: 197 | if quoted: 198 | value = f'"{value}"' 199 | acc += str(value) 200 | 201 | # fuzzable values 202 | elif primitive_type == primitives.FUZZABLE_GROUP: 203 | choices = [f'{acc}{choice}' for choice in value] 204 | sets.append(choices) 205 | acc = '' 206 | 207 | # not supported yet 208 | else: 209 | logger.raw_network_logging(f'Cannot fuzz type {primitive_type}') 210 | return blocks 211 | 212 | # tailing static string 213 | sets.append([acc]) 214 | 215 | import engine.fuzzing_parameters.fuzzing_utils as fuzzing_utils 216 | # compose 217 | if self._config.fuzz_strategy == 'EX': 218 | pool = fuzzing_utils.get_product_exhaust(sets, self._config.max_combination) 219 | elif self._config.fuzz_strategy == 'D1': 220 | pool = fuzzing_utils.get_product_linear_fair(sets, self._config.max_combination) 221 | else: 222 | pool = fuzzing_utils.get_product_linear_bias(sets, self._config.max_combination) 223 | 224 | strs = [''.join(p) for p in pool] 225 | outs = [[primitives.restler_static_string(string)] for string in strs] 226 | return outs 227 | 228 | def has_type_mismatch(self, new_body): 229 | """ Checks the new_body for a type mismatch in one of the nodes 230 | This is used by the payload body checker for bucketization and logging. 231 | 232 | @param new_body: The body to check for the type mismatch 233 | @type new_body: Str 234 | 235 | @return: The node with the mismatched body, or None 236 | @rtype : Str 237 | 238 | """ 239 | return self._schema.check_type_mismatch(new_body) 240 | 241 | def has_struct_missing(self, new_body): 242 | """ Check the new_body string for a missing struct 243 | This is used by the payload body checker for bucketization and logging. 244 | 245 | @param new_body: The body to check for the missing struct, or None 246 | @type new_body: Str 247 | 248 | @return: A string representing the missing body pieces 249 | @rtype : Str 250 | 251 | """ 252 | visitor = BodySchemaVisitor() 253 | self._schema.check_struct_missing(new_body, visitor) 254 | return visitor.val_str 255 | 256 | def _set_body_schema(self, body_parameters): 257 | """ Deserializes and populates the body schema 258 | 259 | @param body_parameters: Body parameters from request schema 260 | @param body_parameters: JSON 261 | 262 | @return: None 263 | @rtype : None 264 | 265 | """ 266 | for body_parameter in body_parameters: 267 | if body_parameter[0] == 'Schema': 268 | payload = des_body_param(body_parameter[1]) 269 | if payload: 270 | self._schema = des_param_payload(payload) 271 | return 272 | 273 | raise NoSchemaFound 274 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/fuzzing_parameters/fuzzing_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import sys 5 | import engine.primitives as primitives 6 | 7 | class FuzzingConfig(object): 8 | def __init__(self, config_json=None): 9 | """ FuzzingConfig constructor 10 | 11 | @param config_json: PayloadBodyChecker specific configuration data 12 | @type config_json: Dict 13 | 14 | """ 15 | self.use_examples_for_default = False 16 | self.use_response_for_default = False 17 | self.use_embedded = False 18 | self.use_wordbook = True 19 | self.use_examples = False 20 | self.use_response = False 21 | self.get_wordbook_values = None 22 | self.get_examples_values = None 23 | self.get_response_values = None 24 | self.fuzz_strategy = 'restler' 25 | self.max_combination = 30 26 | self.merge_fuzzable_values = False 27 | self.max_depth = sys.maxsize 28 | # Traversal depth state 29 | self.depth = 0 30 | 31 | # config_json used by payload body checker only. 32 | # Some values may be set to payload body checker defaults 33 | # if the config exists. 34 | if config_json: 35 | if 'use_examples_for_default' in config_json: 36 | self.use_examples_for_default = config_json['use_examples_for_default'] 37 | if 'use_response_for_default' in config_json: 38 | self.use_response_for_default = config_json['use_response_for_default'] 39 | 40 | if 'use_embedded_for_fuzzable' in config_json: 41 | self.use_embedded = config_json['use_embedded_for_fuzzable'] 42 | if 'use_wordbook_for_fuzzable' in config_json: 43 | self.use_wordbook = config_json['use_wordbook_for_fuzzable'] 44 | if 'use_examples_for_fuzzable' in config_json: 45 | self.use_examples = config_json['use_examples_for_fuzzable'] 46 | if 'use_response_for_fuzzable' in config_json: 47 | self.use_response = config_json['use_response_for_fuzzable'] 48 | 49 | if 'get_wordbook_values' in config_json: 50 | self.get_wordbook_values = config_json['get_wordbook_values'] 51 | if 'get_examples_values' in config_json: 52 | self.get_examples_values = config_json['get_examples_values'] 53 | if 'get_response_values' in config_json: 54 | self.get_response_values = config_json['get_response_values'] 55 | 56 | if 'fuzz_strategy' in config_json: 57 | self.fuzz_strategy = config_json['fuzz_strategy'] 58 | if 'max_combination' in config_json: 59 | self.max_combination = config_json['max_combination'] 60 | if 'max_depth' in config_json: 61 | self.max_depth = config_json['max_depth'] 62 | else: 63 | self.max_depth = 10 64 | 65 | if self.use_examples or self.use_response or self.use_embedded: 66 | self.merge_fuzzable_values = True 67 | elif self.fuzz_strategy != 'restler': 68 | self.merge_fuzzable_values = True 69 | else: 70 | self.merge_fuzzable_values = False 71 | 72 | def __copy__(self): 73 | """ Copy constructor. Resets stateful variables. """ 74 | new_config = FuzzingConfig() 75 | 76 | new_config.use_examples_for_default = self.use_examples_for_default 77 | new_config.use_response_for_default = self.use_response_for_default 78 | new_config.use_embedded = self.use_embedded 79 | new_config.use_wordbook = self.use_wordbook 80 | new_config.use_examples = self.use_examples 81 | new_config.use_response = self.use_response 82 | new_config.get_wordbook_values = self.get_wordbook_values 83 | new_config.get_examples_values = self.get_examples_values 84 | new_config.get_response_values = self.get_response_values 85 | new_config.fuzz_strategy = self.fuzz_strategy 86 | new_config.max_combination = self.max_combination 87 | new_config.merge_fuzzable_values = self.merge_fuzzable_values 88 | new_config.max_depth = self.max_depth 89 | 90 | return new_config 91 | 92 | def get_default_value(self, tag, primitive_type, hint=None): 93 | """ Return a default value of a parameter by searching from 94 | examples/response 95 | 96 | @param tag: Parameter tag 97 | @type tag: String 98 | @param primitive_type: Primitive type 99 | @type primitive_type: String 100 | 101 | @return: Default value 102 | @rtype: String/Int/Dict 103 | 104 | """ 105 | # initialize 106 | default_value = self.get_default_value_of_type(primitive_type) 107 | 108 | # use example value as default (if exist) 109 | if self.use_examples_for_default and self.get_examples_values: 110 | examples_values = self.get_examples_values(tag) 111 | if examples_values: 112 | default_value = list(examples_values)[0] 113 | 114 | # use response value as default (if exist) 115 | if self.use_response_for_default and self.get_response_values: 116 | response_values = self.get_response_values(tag, hint) 117 | if response_values: 118 | default_value = response_values[0] 119 | 120 | return default_value 121 | 122 | def get_default_value_of_type(self, primitive_type): 123 | """ Return a default value for the primitive type as a json 124 | serialized string 125 | 126 | @param primitive_type: Primitive type 127 | @type primitive_type: String 128 | 129 | @return: Default value 130 | @rtype: String 131 | 132 | """ 133 | if primitive_type == primitives.FUZZABLE_STRING: 134 | return 'fuzzstring' 135 | elif primitive_type == primitives.FUZZABLE_INT: 136 | return '0' 137 | elif primitive_type == primitives.FUZZABLE_BOOL: 138 | return 'false' 139 | elif primitive_type == primitives.FUZZABLE_OBJECT: 140 | return '{ "fuzz" : false }' 141 | else: 142 | logger.raw_network_logging(f'Unknown type {primitive_type} for default') 143 | return 'null' 144 | 145 | def get_fuzzable_values(self, tag, primitive_type): 146 | """ Return a list of fuzzable values of a parameter by searching from 147 | examples/response/wordbook 148 | 149 | @param tag: Parameter tag 150 | @type tag: String 151 | @param primitive_type: Parameter primitive type 152 | @type primitive_type: String 153 | 154 | @return: A list of fuzzable values 155 | @rtype: List 156 | 157 | """ 158 | # initialize 159 | fuzzable_values = [] 160 | 161 | # add examples values 162 | if self.use_examples and self.get_examples_values: 163 | fuzzable_values += self.get_examples_values(tag) 164 | 165 | # add response values 166 | if self.use_response and self.get_response_values: 167 | fuzzable_values += self.get_response_values(tag) 168 | 169 | # add wordbook values 170 | if self.use_wordbook and self.get_wordbook_values: 171 | fuzzable_values += self.get_wordbook_values(primitive_type) 172 | 173 | # add the default value 174 | if self.use_embedded: 175 | fuzzable_values += [ 176 | self.get_default_value_of_type(primitive_type)] 177 | 178 | return fuzzable_values 179 | 180 | def cleanup_fuzzable_group(self, default_value, fuzzable_values): 181 | """ Remove redundant fuzzable values and put default at first place 182 | 183 | @param default_value: Default value 184 | @type default_value: String 185 | @param fuzzable_values: A list of fuzzable values 186 | @type fuzzable_values: List 187 | 188 | @return: Clean fuzzable group 189 | @rtype: List 190 | 191 | """ 192 | # remove overlapping values 193 | x = set(fuzzable_values) 194 | if default_value in x: 195 | x.remove(default_value) 196 | return [default_value] + list(x) 197 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/fuzzing_parameters/request_examples.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | from engine.fuzzing_parameters.request_params import * 5 | from engine.fuzzing_parameters.request_schema_parser import * 6 | from engine.fuzzing_parameters.body_schema import BodySchema 7 | import utils.logger as logger 8 | 9 | class NoExamplesFound(Exception): 10 | pass 11 | 12 | class RequestExamples(): 13 | """ Request Examples Class. """ 14 | 15 | def __init__(self, request_schema_json): 16 | """ Initialize and construct the RequestExamples by deserializing the 17 | compiler generated request schema IL. 18 | 19 | @param request_schema_json: Compiler generated request schema IL 20 | @type request_schema_json: JSON 21 | 22 | @return: None 23 | @rtype: None 24 | 25 | """ 26 | # initialization 27 | self._query_examples: set = set() # {QueryList} 28 | self._body_examples: set = set() # {BodySchema} 29 | 30 | # process the request schema 31 | try: 32 | self._set_query_params(request_schema_json['queryParameters']) 33 | except Exception as err: 34 | msg = f'Fail deserializing request schema query examples: {err!s}' 35 | logger.write_to_main(msg, print_to_console=True) 36 | raise Exception(msg) 37 | 38 | try: 39 | self._set_body_params(request_schema_json['bodyParameters']) 40 | except Exception as err: 41 | msg = f'Fail deserializing request schema body examples: {err!s}' 42 | logger.write_to_main(msg, print_to_console=True) 43 | raise Exception(msg) 44 | 45 | if not self._query_examples and not self._body_examples: 46 | raise NoExamplesFound 47 | 48 | @property 49 | def body_examples(self): 50 | """ Return the body examples 51 | 52 | @return: Body examples 53 | @rtype: Set {BodySchema} 54 | 55 | """ 56 | return self._body_examples 57 | 58 | @property 59 | def query_examples(self): 60 | """ Return the query examples 61 | 62 | @return: Query examples 63 | @rtype: Set {ParamObject} 64 | 65 | """ 66 | return self._query_examples 67 | 68 | def _set_query_params(self, query_parameters): 69 | """ Deserializes and populates the query parameters 70 | 71 | @param query_parameters: Query parameters from request schema 72 | @param query_parameters: JSON 73 | 74 | @return: None 75 | @rtype : None 76 | 77 | """ 78 | # Iterate through each collection of query parameters 79 | for query_parameter in query_parameters: 80 | if query_parameter[0] == 'Examples': 81 | query_list = QueryList() 82 | # Set each query parameter of the query 83 | for query in des_query_param(query_parameter[1]): 84 | query_list.append(query) 85 | self._query_examples.add(query_list) 86 | 87 | def _set_body_params(self, body_parameters): 88 | """ Deserializes and populates the body parameters 89 | 90 | @param body_parameters: Body parameters from request schema 91 | @param body_parameters: JSON 92 | 93 | @return: None 94 | @rtype : None 95 | 96 | """ 97 | # Iterate through each body parameter 98 | for body_parameter in body_parameters: 99 | if body_parameter[0] == 'Examples': 100 | payload = des_body_param(body_parameter[1]) 101 | if payload: 102 | body_example = des_param_payload(payload) 103 | if body_example: 104 | self._body_examples.add(BodySchema(param=body_example)) 105 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/fuzzing_parameters/request_schema_parser.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import utils.logger as logger 5 | import collections 6 | from engine.fuzzing_parameters.request_params import * 7 | 8 | def des_query_param(query_param_payload): 9 | """ Deserialize a query parameter payload 10 | 11 | @param query_param_payload: The query parameter payload to deserialize 12 | @type query_param_payload: JSON 13 | 14 | @return: Yields QueryParam objects that represents the query parameters from json 15 | @rtype : QueryParam(s) 16 | 17 | """ 18 | queries = des_request_param_payload(query_param_payload) 19 | for (key, payload) in queries: 20 | param = des_param_payload(payload, query_param=True) 21 | if param: 22 | yield QueryParam(key, param) 23 | else: 24 | return None 25 | 26 | def des_body_param(request_param_payload_json): 27 | """ Deserializes a body parameter payload 28 | 29 | @param request_param_payload_json: The body parameter payload to deserialize 30 | @type request_param_payload_json: JSON 31 | 32 | @return: The body parameter 33 | @rtype : ParamObject 34 | 35 | """ 36 | des_payload = des_request_param_payload(request_param_payload_json) 37 | if des_payload: 38 | des_payload = des_payload[0] 39 | return des_payload.payload 40 | return None 41 | 42 | def des_request_param_payload(request_param_payload_json): 43 | """ Deserialize RequestParametersPayload type object. 44 | 45 | @param request_param_payload_json: Request parameter from the compiler 46 | @type request_param_payload_json: JSON 47 | 48 | @return: List of tuples containing the keys and payloads 49 | @rtype: List[tuple(str, ParamObject)] 50 | 51 | """ 52 | KeyPayload = collections.namedtuple("KeyPayload", ['key', 'payload']) 53 | payloads = [] 54 | if 'ParameterList' in request_param_payload_json: 55 | param_list_seq = request_param_payload_json['ParameterList'] 56 | for param_payload_pair in param_list_seq: 57 | # pair must have size 2 58 | if len(param_payload_pair) != 2: 59 | logger.write_to_main('string - param payload pair size mismatch') 60 | return [KeyPayload(None, None)] 61 | 62 | key = param_payload_pair[0] 63 | payload = param_payload_pair[1] 64 | payloads.append(KeyPayload(key, payload)) 65 | 66 | return payloads 67 | 68 | return [KeyPayload(None, None)] 69 | 70 | def des_param_payload(param_payload_json, tag='', query_param=False): 71 | """ Deserialize ParameterPayload type object. 72 | 73 | @param param_payload_json: Body parameter from the compiler 74 | @type param_payload_json: JSON 75 | @param tag: Node tag 76 | @type tag: Str 77 | @param query_param: Set to True if this is a query parameter 78 | @type query_param: Bool 79 | 80 | @return: Body parameter schema 81 | @rtype: ParamObject 82 | 83 | """ 84 | param = None 85 | 86 | if 'InternalNode' in param_payload_json: 87 | internal_node = param_payload_json['InternalNode'] 88 | internal_info = internal_node[0] 89 | internal_data = internal_node[1] 90 | 91 | name = internal_info['name'] 92 | property_type = internal_info['propertyType'] 93 | 94 | if tag: 95 | next_tag = tag + '_' + name 96 | else: 97 | next_tag = name 98 | 99 | # Array --> ParamMember { name : ParamArray } 100 | if property_type == 'Array': 101 | values = [] 102 | for data in internal_data: 103 | value = des_param_payload(data, next_tag) 104 | values.append(value) 105 | 106 | array = ParamArray(values) 107 | if query_param or not name: 108 | param = array 109 | else: 110 | param = ParamMember(name, array) 111 | 112 | array.tag = f'{next_tag}_array' 113 | 114 | # Object --> ParamObject { ParamMember, ..., ParamMember } 115 | elif property_type == 'Object': 116 | members = [] 117 | for member_json in internal_data: 118 | member = des_param_payload(member_json, tag) 119 | members.append(member) 120 | 121 | param = ParamObject(members) 122 | 123 | param.tag = f'{next_tag}_object' 124 | 125 | # Property --> ParamMember { name : ParamObject } 126 | elif property_type == 'Property': 127 | if len(internal_data) != 1: 128 | logger.write_to_main(f'Internal Property {name} size != 1') 129 | 130 | value = des_param_payload(internal_data[0], next_tag) 131 | 132 | param = ParamMember(name, value) 133 | 134 | # others 135 | else: 136 | logger.write_to_main(f'Unknown internal type {property_type}') 137 | 138 | elif 'LeafNode' in param_payload_json: 139 | leaf_node = param_payload_json['LeafNode'] 140 | 141 | name = leaf_node['name'] 142 | payload = leaf_node['payload'] 143 | 144 | # payload is a dictionary (or member) with size 1 145 | if len(payload) != 1: 146 | logger.write_to_main(f'Unexpected payload format {payload}') 147 | 148 | content_type = 'Unknown' 149 | content_value = 'Unknown' 150 | custom = False 151 | fuzzable = False 152 | 153 | if 'Fuzzable' in payload: 154 | content_type = payload['Fuzzable'][0] 155 | content_value = payload['Fuzzable'][1] 156 | fuzzable = True 157 | elif 'Constant' in payload: 158 | content_type = payload['Constant'][0] 159 | content_value = payload['Constant'][1] 160 | elif 'DynamicObject' in payload: 161 | content_type = 'Unknown' 162 | content_value = payload['DynamicObject'] 163 | elif 'Custom' in payload: 164 | custom = True 165 | content_type = payload['Custom']['payloadType'] 166 | content_value = payload['Custom']['payloadValue'] 167 | elif 'PayloadParts' in payload: 168 | definition = payload['PayloadParts'][-1] 169 | if 'Custom' in definition: 170 | custom = True 171 | content_type = definition['Custom']['payloadType'] 172 | content_value = definition['Custom']['payloadValue'] 173 | 174 | # create value w.r.t. the type 175 | value = None 176 | if content_type == 'String': 177 | # If query parameter, assign as a value and not a string 178 | # because we don't want to wrap with quotes in the request 179 | if query_param: 180 | value = ParamValue(custom=custom) 181 | else: 182 | value = ParamString(custom) 183 | elif content_type == 'DateTime': 184 | value = ParamString(custom) 185 | elif content_type == 'Int': 186 | value = ParamNumber() 187 | elif content_type == 'Number': 188 | value = ParamNumber() 189 | elif content_type == 'Bool': 190 | value = ParamBoolean() 191 | elif content_type == 'Object': 192 | value = ParamObjectLeaf() 193 | elif content_type == 'UuidSuffix': 194 | value = ParamUuidSuffix() 195 | # Set as unknown for payload body fuzzing purposes. 196 | # This will be fuzzed as a string. 197 | value.set_unknown() 198 | elif 'Enum' in content_type: 199 | # unique case for Enums, as they are defined as 200 | # "fuzzable" types in the schema, but are not fuzzable 201 | # by the same definition as the rest of the fuzzable types. 202 | fuzzable = False 203 | # { 204 | # Enum : [ 205 | # type, 206 | # [ value1, value2, value3 ], 207 | # default_value 208 | # ] 209 | # } 210 | enum_definition = content_type['Enum'] 211 | 212 | if len(enum_definition) == 3: 213 | enum_content_type = enum_definition[0] 214 | contents = enum_definition[1] 215 | value = ParamEnum(contents, enum_content_type) 216 | else: 217 | logger.write_to_main(f'Unexpected enum schema {name}') 218 | else: 219 | value = ParamString(False) 220 | value.set_unknown() 221 | 222 | value.set_fuzzable(fuzzable) 223 | value.content = content_value 224 | 225 | if tag and name: 226 | value.tag = (tag + '_' + name) 227 | elif tag: 228 | value.tag = tag 229 | else: 230 | value.tag = name 231 | 232 | # create the param node 233 | if name: 234 | param = ParamMember(name, value) 235 | else: 236 | # when a LeafNode represent a standard type, e.g., 237 | # string, the name will be empty 238 | param = value 239 | 240 | else: 241 | logger.write_to_main('Neither internal nor leaf property') 242 | 243 | if not param: 244 | logger.write_to_main(f'Fail des param payload {param_payload_json}') 245 | return None 246 | 247 | return param 248 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/mime/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/engine/engine/mime/__init__.py -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/mime/multipart_formdata.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Handling of multipart/form-data MIME type. """ 5 | import time 6 | 7 | 8 | def render(payloads): 9 | """ Render multipart/form-data MIME type. 10 | 11 | @param payloads: The payload to render according to the MIME type. 12 | @type payloads: Str 13 | 14 | @return: The properly rendered payload. 15 | @rtype : String 16 | 17 | Note: To understand why we iterate over multipart/formdata like this one 18 | should first take a look at this example: 19 | https://stackoverflow.com/questions/4526273/what-does-enctype-multipart-form-data-mean 20 | and then read the correspinding RFC: 21 | https://www.ietf.org/rfc/rfc2388.txt 22 | 23 | Overall, there is nothing exotic here but one needs to be careful 24 | positioning the delimiters and the proper structure and headers. 25 | 26 | payloads may contain an arbitrary number of content-disposition and 27 | datasteam path dictionaries, as follows: 28 | 29 | payloads = [ 30 | { 31 | 'content-disposition': 'name="file"; filename="bla1.gz"', 32 | 'datastream': 'bla.gz' 33 | }, 34 | { 35 | 'content-disposition': 'name="file"; filename="bla2.gz"', 36 | 'datastream': 'bla2.gz' 37 | }, 38 | ... 39 | ] 40 | 41 | """ 42 | boundary = "_CUSTOM_BOUNDARY_{}".format(str(int(time.time()))) 43 | 44 | req = "Content-Type: multipart/form-data; boundary={}\r\n\r\n".\ 45 | format(boundary) 46 | req+= '--{}\r\n'.format(boundary) 47 | 48 | for i, payload in enumerate(payloads): 49 | req += 'Content-Disposition: form-data; {}\r\n'.\ 50 | format(payload['content-disposition']) 51 | req += 'Content-Type: {}\r\n\r\n'.\ 52 | format("application/octet-stream") 53 | try: 54 | f = open(payload['datastream'], 'r') 55 | data = f.read() 56 | f.close() 57 | except Exception as error: 58 | print("Unhandled exception reading stream. Error:{}".format(error)) 59 | raise 60 | req += '{}\r\n\r\n'.format(data) 61 | if i == len(payloads) - 1: 62 | req += '--{}--\r\n'.format(boundary) 63 | else: 64 | req += '--{}\r\n\r\n'.format(boundary) 65 | 66 | return req 67 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/transport_layer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/engine/engine/transport_layer/__init__.py -------------------------------------------------------------------------------- /restler_bin_atten/engine/engine/transport_layer/response.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | import string 4 | import re 5 | from restler_settings import Settings 6 | import datetime 7 | 8 | checktime = 0 9 | codes = [] 10 | codes_counter = [0,0,0] 11 | 12 | 13 | DELIM = "\r\n\r\n" 14 | VALID_CODES = {'200', '201', '202', '204', '304'} 15 | TIMEOUT_CODE = '599' 16 | CONNECTION_CLOSED_CODE = '598' 17 | RESTLER_BUG_CODES = [TIMEOUT_CODE, CONNECTION_CLOSED_CODE] 18 | # Code that RESTler may assign to a request that was never sent to the server. 19 | # This is used as a way to identify that a target request was never sent as part 20 | # of a sequence because the sequence failed prior to that request being reached. 21 | RESTLER_INVALID_CODE = '999' 22 | 23 | class HttpResponse(object): 24 | def __init__(self, response_str: str=None): 25 | """ Initializes an HttpResponse object 26 | 27 | @param response_str: The response that was received from the server 28 | 29 | """ 30 | self._str = None 31 | self._status_code = None 32 | 33 | if response_str: 34 | self._str = str(response_str) 35 | 36 | try: 37 | self._status_code = self._str.split(" ")[1] 38 | global checktime, codes, codes_counter 39 | if self._status_code[0] == '2': 40 | codes_counter[0]+=1 41 | elif self._status_code[0] == '4': 42 | codes_counter[1]+=1 43 | elif self._status_code[0] == '5': 44 | codes_counter[2]+=1 45 | #print(f"codes_counter: {codes_counter}") 46 | #print(f"checktime {checktime}") 47 | #print(f"time spand: {(datetime.datetime.now()-checktime).seconds}") 48 | if (datetime.datetime.now()-checktime).seconds > 60*20: 49 | codes.append([codes_counter[0], codes_counter[1],codes_counter[2]]) 50 | checktime = datetime.datetime.now() 51 | except: 52 | pass 53 | 54 | @property 55 | def to_str(self): 56 | """ Stringifies the whole HttpResponse object. 57 | This matches the entire response as it was received from the server. 58 | 59 | @return: The entire response as a string 60 | @rtype: Str 61 | 62 | """ 63 | return self._str 64 | 65 | @property 66 | def status_code(self): 67 | """ The status code of the response 68 | 69 | @return: The status code 70 | @rtype : Str 71 | 72 | """ 73 | return self._status_code 74 | 75 | @property 76 | def body(self): 77 | """ The body of the response 78 | 79 | @return: The body 80 | @rtype : Str 81 | 82 | """ 83 | try: 84 | return self._str.split(DELIM)[1] 85 | except: 86 | return None 87 | 88 | @property 89 | def json_body(self): 90 | """ The json portion of the body if exists. 91 | 92 | @return: The json body 93 | @rtype : Str or None 94 | 95 | """ 96 | def is_invalid(c): 97 | """ Returns True if character is an unexpected value. 98 | This function is called when checking before curly braces in a response. 99 | Hex values and CRLF characters are considered valid, all others are not. 100 | """ 101 | return c not in string.hexdigits and c != '\r' and c != '\n' and c != ' ' 102 | 103 | try: 104 | body = self.body 105 | for idx, c in enumerate(body): 106 | if c == '{': 107 | l_index = idx 108 | r_find = '}' 109 | break 110 | elif c == '[': 111 | l_index = idx 112 | r_find = ']' 113 | break 114 | elif is_invalid(c): 115 | return None 116 | 117 | r_index = body.rindex(r_find) + 1 118 | json_body = body[l_index: r_index] 119 | 120 | left, right = None, None 121 | for idx, c in enumerate(json_body): 122 | if '\r' == c: 123 | if not left: 124 | left = idx 125 | else: 126 | right = idx+2 127 | return json_body[:left]+json_body[right:] 128 | 129 | return json_body 130 | except: 131 | return None 132 | 133 | @property 134 | def status_text(self): 135 | """ Returns the status text of the response 136 | 137 | @return: The status text 138 | @rtype : Str 139 | 140 | """ 141 | try: 142 | # assumed format: HTTP/1.1 STATUS_CODE STATUS TEXT\r\nresponse... 143 | return self._str.split(" ", 2)[2].split('\r\n')[0] 144 | except: 145 | return None 146 | 147 | def has_bug_code(self): 148 | """ Returns True if the status code is considered a bug 149 | 150 | @return: True if the status code is considered a bug 151 | @rtype : Bool 152 | 153 | """ 154 | if self._status_code: 155 | if Settings().custom_non_bug_codes: 156 | # All codes except the ones in the custom_non_bug_codes list should be flagged as bugs. 157 | # Hence, return False only if the status code exists in the list. 158 | for code in Settings().custom_non_bug_codes: 159 | if re.match(code, self._status_code): 160 | return False 161 | else: 162 | return True 163 | if self._status_code.startswith('5'): 164 | return True 165 | for code in Settings().custom_bug_codes: 166 | if re.match(code, self._status_code): 167 | return True 168 | return False 169 | 170 | def has_valid_code(self): 171 | """ Returns True if the status code is a valid status code 172 | 173 | @return: True if the status code is a valid status code 174 | @rtype : Bool 175 | 176 | """ 177 | if self._status_code: 178 | return self._status_code in VALID_CODES 179 | return False 180 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/engine/utils/__init__.py -------------------------------------------------------------------------------- /restler_bin_atten/engine/utils/formatting.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import datetime 5 | import time 6 | 7 | def timestamp(): 8 | epoch = time.time() 9 | ts = datetime.datetime.fromtimestamp(epoch) 10 | # Year-Month-Day Hour:Minute:Second.millisecond 11 | return ts.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] -------------------------------------------------------------------------------- /restler_bin_atten/engine/utils/hooks.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Useful hooks to collect coverage infromation of python services. """ 5 | from __future__ import print_function 6 | import sys 7 | import functools 8 | import linecache 9 | 10 | import pickle 11 | 12 | # Destination of report file 13 | REPORTDIR = os.path.abspath('/tmp/.coverage') 14 | 15 | # Buffer that leeps the line numbers of covered (executed) statements. 16 | DLINES = {} 17 | 18 | def update_report(dlines): 19 | """ Updates file of coverage statistics. 20 | 21 | @param dlines: Dictionary of coverage statistics. 22 | @type frame: Dict 23 | 24 | @return: None 25 | @rtype: None 26 | 27 | """ 28 | with open(REPORTDIR, "wb+") as f: 29 | # print(dlines) 30 | pickle.dump(dlines, f, protocol=pickle.HIGHEST_PROTOCOL) 31 | 32 | 33 | def trace_lines(frame, event, arg): 34 | """ Stack unwinding callback hook to inspects lines within a function call. 35 | 36 | @param frame: The frame object currently inspected (for more 37 | documentation of frame attributes, such as f_code, f_lineno, etc., 38 | see https://docs.python.org/2/library/inspect.html) 39 | @type frame: Python frame object. 40 | @param event: Specifies the intercpeted event, such as "call", "line", 41 | "return", etc. (For more documentation see: 42 | https://docs.python.org/2/library/sys.html#sys.settrace) 43 | @type event: Str 44 | @param arg: settrace expect this argument 45 | @type arg: kwarg 46 | 47 | @return: None 48 | @rtype: None 49 | 50 | """ 51 | if event != 'line': 52 | return 53 | co = frame.f_code 54 | func_name = co.co_name 55 | func_filename = co.co_filename 56 | if func_filename not in DLINES: 57 | DLINES[func_filename] = [] 58 | 59 | line_no = frame.f_lineno 60 | if line_no not in DLINES[func_filename]: 61 | DLINES[func_filename].append(line_no) 62 | update_report(DLINES) 63 | 64 | 65 | def trace_calls(frame, event, arg, to_be_traced=[]): 66 | """ Stack unwinding callback hook to inspects stack frames using settrace. 67 | 68 | Inspect the current frame and unwind function call events. If the function 69 | call originates from a file within a target group trace the funtion lines 70 | executed. 71 | 72 | @param frame: The frame object currently inspected (for more 73 | documentation of frame attributes, such as f_code, f_lineno, etc., 74 | see https://docs.python.org/2/library/inspect.html) 75 | @type frame: Python frame object. 76 | @param event: Specifies the intercpeted event, such as "call", "line", 77 | "return", etc. (For more documentation see: 78 | https://docs.python.org/2/library/sys.html#sys.settrace) 79 | @type event: Str 80 | @param arg: settrace expect this argument 81 | @type arg: kwarg 82 | @param to_be_traced: List of files to be traced 83 | @type to_be_traced: List 84 | 85 | @return: Traced lines. 86 | @rtype: @callback trace_lines 87 | 88 | """ 89 | if event != 'call': 90 | return 91 | co = frame.f_code 92 | func_name = co.co_name 93 | if func_name == 'write': 94 | # Ignore write() calls from printing 95 | return 96 | line_no = frame.f_lineno 97 | filename = co.co_filename 98 | if not filename.endswith(to_be_traced): 99 | return 100 | # print('* Call to {} on line {} of {}'.format( 101 | # func_name, line_no, filename)) 102 | # Trace into this function 103 | return trace_lines 104 | -------------------------------------------------------------------------------- /restler_bin_atten/engine/utils/saver.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | """ Saver module. """ 5 | import os 6 | import pickle 7 | 8 | import utils.logger as logger 9 | import utils.formatting as formatting 10 | 11 | 12 | def save(req_collection, seq_collection, fuzzing_collection, fuzzing_monitor, length): 13 | """ Save in checkpoint fuzzing of latest generation. 14 | 15 | @param req_collection: The request collection. 16 | @type req_collection: RequestCollection class object. 17 | @param seq_collection: List of sequences in sequence collection. 18 | @type seq_collection: List 19 | @param fuzzing_collection: The collection of requests being fuzzed 20 | @type fuzzing_collection: FuzzingRequestCollection 21 | @param fuzzing_monitor: The global monitor for the fuzzing run 22 | @type fuzzing_monitor: FuzzingMonitor 23 | @param length: Length of latest generation. 24 | @type length: Int 25 | 26 | @return: None 27 | @rtype : None 28 | 29 | """ 30 | return 31 | if not os.path.exists(logger.CKPT_DIR): 32 | os.makedirs(logger.CKPT_DIR) 33 | 34 | current_ckpt = os.path.join(logger.CKPT_DIR, "checkpoint-{}".format(length)) 35 | print("{}: Saving checkpoint: {}".format(formatting.timestamp(), current_ckpt)) 36 | 37 | with open(current_ckpt, "wb") as f: 38 | state = { 39 | 'req_collection': req_collection, 40 | 'fuzzing_collection': fuzzing_collection, 41 | 'fuzzing_monitor': fuzzing_monitor, 42 | 'seq_collection': seq_collection, 43 | 'length': length 44 | } 45 | pickle.dump(state, f) 46 | 47 | 48 | def load(req_collection, seq_collection, fuzzing_collection, fuzzing_monitor): 49 | """ Load from checkpoint fuzzing of lattest generation. 50 | 51 | @param req_collection: The target request collection. 52 | @type req_collection: RequestCollection class object. 53 | @param seq_collection: The tareg list of sequences in sequence collection. 54 | @type seq_collection: List 55 | @param length: Length of lattest generation. 56 | @type length: Int 57 | 58 | @return: A tuple of ('length', request collection', 'sequence collection') 59 | @rtype : Tuple 60 | 61 | """ 62 | length = 0 63 | print("No checkpoints used at this phase") 64 | return req_collection, seq_collection, fuzzing_collection, fuzzing_monitor, length 65 | if not os.path.exists(logger.CKPT_DIR): 66 | print("{}: No chekpoint found".format(formatting.timestamp())) 67 | return req_collection, seq_collection, fuzzing_collection, fuzzing_monitor, length 68 | ckpt_files = [os.path.join(logger.CKPT_DIR, f) 69 | for f in os.listdir(logger.CKPT_DIR) 70 | if os.path.isfile(os.path.join(logger.CKPT_DIR, f))] 71 | if not ckpt_files: 72 | print("{}: No chekpoint found".format(formatting.timestamp())) 73 | return req_collection, seq_collection, fuzzing_collection, fuzzing_monitor, length 74 | 75 | lattest_ckpt = sorted(ckpt_files)[-1] 76 | print("{}: Loading state from: {}".format(formatting.timestamp(), 77 | lattest_ckpt)) 78 | with open(lattest_ckpt, "rb") as f: 79 | state = pickle.load(f) 80 | req_collection = state['req_collection'] 81 | seq_collection = state['seq_collection'] 82 | fuzzing_collection = state['fuzzing_collection'] 83 | fuzzing_monitor = state['fuzzing_monitor'] 84 | length = state['length'] 85 | print("{}: Candidate values: {}".\ 86 | format(formatting.timestamp(), 87 | req_collection.candidate_values_pool.candidate_values)) 88 | print("{}: Past test cases: {}".format(formatting.timestamp(), 89 | fuzzing_monitor.num_test_cases())) 90 | fuzzing_monitor.reset_start_time() 91 | return req_collection, seq_collection, fuzzing_collection, fuzzing_monitor, length 92 | -------------------------------------------------------------------------------- /restler_bin_atten/restler/Dia2Lib.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Dia2Lib.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/FSharp.Control.AsyncSeq.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/FSharp.Control.AsyncSeq.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/FSharp.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/FSharp.Core.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Microsoft.ApplicationInsights.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Microsoft.ApplicationInsights.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Microsoft.Diagnostics.FastSerialization.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Microsoft.Diagnostics.FastSerialization.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Microsoft.Diagnostics.Tracing.TraceEvent.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Microsoft.Diagnostics.Tracing.TraceEvent.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Microsoft.FSharpLu.Json.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Microsoft.FSharpLu.Json.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Microsoft.FSharpLu.Windows.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Microsoft.FSharpLu.Windows.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Microsoft.FSharpLu.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Microsoft.FSharpLu.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Microsoft.Win32.Registry.AccessControl.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Microsoft.Win32.Registry.AccessControl.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Microsoft.Win32.SystemEvents.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Microsoft.Win32.SystemEvents.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/NJsonSchema.Yaml.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/NJsonSchema.Yaml.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/NJsonSchema.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/NJsonSchema.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/NSwag.Core.Yaml.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/NSwag.Core.Yaml.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/NSwag.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/NSwag.Core.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Namotion.Reflection.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Namotion.Reflection.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Newtonsoft.Json.Bson.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Newtonsoft.Json.Bson.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Newtonsoft.Json.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Newtonsoft.Json.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/OSExtensions.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/OSExtensions.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Pluralize.NET.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Pluralize.NET.Core.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Restler: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Restler -------------------------------------------------------------------------------- /restler_bin_atten/restler/Restler.Compiler.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Restler.Compiler.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Restler.Compiler.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Restler.Compiler.pdb -------------------------------------------------------------------------------- /restler_bin_atten/restler/Restler.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Restler.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/Restler.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/Restler.pdb -------------------------------------------------------------------------------- /restler_bin_atten/restler/Restler.runtimeconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "runtimeOptions": { 3 | "tfm": "net5.0", 4 | "framework": { 5 | "name": "Microsoft.NETCore.App", 6 | "version": "5.0.0" 7 | }, 8 | "configProperties": { 9 | "restlerLogsUploadDirectory": "" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /restler_bin_atten/restler/Sysprep.ps1: -------------------------------------------------------------------------------- 1 | #.SYNOPSIS 2 | # Sysprep current machine running Windows Server 2008 R2 and Powershell 5.* (providing workarounds introduced by the new Powershell release) 3 | #.DESCRIPTION 4 | # Script Syspreps current machine. It provides needed workarounds to make Sysprep work on Windows 2008 R2 running Powershell 5.*. 5 | # SIL workaround: https://msdn.microsoft.com/en-us/powershell/wmf/5.0/limitation_overview 6 | # RegKey issue workaround: https://windowsserver.uservoice.com/forums/301869-powershell/suggestions/11591262-bug-wmf-5-production-preview-on-windows-server-2 7 | # Script must be run from elevated Powershell session. 8 | #.PARAMETER ShutdownAfterSysprep 9 | # Shutdown the machine after the Sysprep 10 | #.PARAMETER DisableAntivirusBeforeSysprep 11 | # Prevents antivirus from running on VMs based on the sysprepped image, without requiring a reboot. 12 | # This switch should not be used to prepare the VM template image, since antivirus should be 13 | # on by default for these (they are used to create the customer VM). 14 | [CmdletBinding()] 15 | Param( 16 | [switch] $ShutdownAfterSysprep, 17 | [switch] $DisableAntivirusBeforeSysprep 18 | ) 19 | Process { 20 | 21 | $osVersion = [environment]::OSVersion.Version 22 | $windows2008 = ($osVersion.Major -eq 6) -and ($osVersion.Minor -eq 1) 23 | 24 | #### Disable antivirus if required 25 | #### Note: On Win8+, Defender cannot be disabled without a reboot (there does not appear to be 26 | #### a commandlet to cleanly stop defender once it has been disabled). 27 | if (-not $windows2008 -and $DisableAntivirusBeforeSysprep) { 28 | $defenderRegPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows Defender\" 29 | if (-not (Test-Path $defenderRegPath)) { 30 | New-Item $defenderRegPath 31 | } 32 | New-ItemProperty -Path $defenderRegPath -Name DisableAntiSpyware -Value 1 -PropertyType DWord -Force 33 | } 34 | 35 | #### Hack needed to fix PowerShell breaking Sysprep 36 | Write-Output "Apply WMF 5.0 sysprep workaround" 37 | 38 | # The regkey has to be present there on VM boot. Thus need to patch registry before shutdown 39 | New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\StreamProvider" -Name LastFullPayloadTime -Value 0 -PropertyType DWord -Force 40 | # Working around PowerShell 5.0 Bugs: https://msdn.microsoft.com/en-us/powershell/wmf/5.0/limitation_overview 41 | Set-SilLogging -TargetUri https://BlankTarget -CertificateThumbprint 0123456789 42 | Publish-SilData -ErrorAction SilentlyContinue 43 | Remove-Item -Recurse $env:SystemRoot\System32\Logfiles\SIL\ -Force -ErrorAction SilentlyContinue 44 | 45 | #### Run sysprep generalization 46 | $sysprepCommand = "$env:SystemRoot\system32\sysprep\sysprep.exe" 47 | $sysprepParameters = '/generalize /oobe /quiet' 48 | 49 | if ($ShutdownAfterSysprep) { 50 | $sysprepParameters += ' /shutdown' 51 | } else { 52 | $sysprepParameters += ' /quit' 53 | } 54 | 55 | if (-not (Test-Path $sysprepCommand)) { 56 | throw "Failed to locate sysprep command. This is probably due to Wow64 file redirection. Make sure to run VMAgent in the native OS architecture." 57 | } 58 | 59 | # IMPORTANT NOTE: running sysprep.exe directly would fail when running under Remote Powershell (WinRM). 60 | # This is because the sysprep generalization process breaks all running network connections including WinRM itself. 61 | # Subsequently, when the WinRM session is interrupted, all the processes started under that session are automatically terminated. 62 | # This causes sysprep.exe itself to be killed before completion! 63 | # As a workaround, we start sysprep.exe through a Windows scheduled task. The drawback is that it's fire and forget: the remote session 64 | # has no way to detect when sysprep terminates, nor can it retrieve the exit code from sysprep.exe. 65 | Write-Output "Sysprepping machine" 66 | $taskname = "fsharplu-sysprep" 67 | 68 | if ($windows2008) { 69 | Write-Output "Creating scheduled task (Windows 7/Server 2008)" 70 | # On Windows 2008, New-ScheduledTaskAction does not exist. The alternative 71 | # `Register-ScheduledJob` is too limited (works only with PS scripts). 72 | # So we fall back on command `schtasks.exe`. 73 | 74 | # We set the task to run as SYSTEM to avoid having to pass credentials. 75 | # NOTE: In 83 years from now, the following commands will stop working, at which point 76 | # we might consider dropping support for Windows 2008 ;-) 77 | schtasks /CREATE /TN $taskname /RL HIGHEST /F ` 78 | /TR "$sysprepCommand $sysprepParameters" /SC ONCE /ST 00:00 /SD 01/01/2090 ` 79 | /RU SYSTEM 80 | 81 | schtasks /RUN /TN $taskname 82 | 83 | schtasks /DELETE /TN $taskname /F 84 | } else { 85 | Write-Output "Creating scheduled task (Windows >=8)" 86 | $action = New-ScheduledTaskAction -Execute $sysprepCommand -Argument $sysprepParameters 87 | $principal = New-ScheduledTaskPrincipal -RunLevel "Highest" -UserId 'SYSTEM' 88 | New-ScheduledTask -Action $action -Principal $principal ` 89 | | Register-ScheduledTask $taskname -Force ` 90 | | Start-ScheduledTask ` 91 | | Unregister-ScheduledTask -Confirm:$false 92 | } 93 | } -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.CodeDom.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.CodeDom.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.ComponentModel.Composition.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.ComponentModel.Composition.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Configuration.ConfigurationManager.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Configuration.ConfigurationManager.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Data.Odbc.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Data.Odbc.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Data.SqlClient.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Data.SqlClient.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Diagnostics.EventLog.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Diagnostics.EventLog.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Diagnostics.PerformanceCounter.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Diagnostics.PerformanceCounter.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.DirectoryServices.AccountManagement.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.DirectoryServices.AccountManagement.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.DirectoryServices.Protocols.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.DirectoryServices.Protocols.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.DirectoryServices.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.DirectoryServices.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Drawing.Common.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Drawing.Common.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.IO.Packaging.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.IO.Packaging.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.IO.Ports.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.IO.Ports.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Management.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Management.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Net.Http.Formatting.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Net.Http.Formatting.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Runtime.Caching.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Runtime.Caching.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Security.Cryptography.Pkcs.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Security.Cryptography.Pkcs.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Security.Cryptography.ProtectedData.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Security.Cryptography.ProtectedData.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Security.Cryptography.Xml.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Security.Cryptography.Xml.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Security.Permissions.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Security.Permissions.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.ServiceModel.Duplex.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.ServiceModel.Duplex.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.ServiceModel.Http.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.ServiceModel.Http.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.ServiceModel.NetTcp.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.ServiceModel.NetTcp.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.ServiceModel.Primitives.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.ServiceModel.Primitives.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.ServiceModel.Security.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.ServiceModel.Security.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.ServiceModel.Syndication.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.ServiceModel.Syndication.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.ServiceModel.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.ServiceModel.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.ServiceProcess.ServiceController.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.ServiceProcess.ServiceController.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/System.Threading.AccessControl.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/System.Threading.AccessControl.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/TraceReloggerLib.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/TraceReloggerLib.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/YamlDotNet.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/YamlDotNet.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/cs/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/cs/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/de/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/de/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/es/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/es/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/fr/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/fr/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/it/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/it/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/ja/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/ja/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/ko/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/ko/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/pl/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/pl/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/pt-BR/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/pt-BR/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/ru/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/ru/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/linux/lib/netcoreapp2.0/System.Data.Odbc.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/linux/lib/netcoreapp2.0/System.Data.Odbc.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/osx/lib/netcoreapp2.0/System.Data.Odbc.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/osx/lib/netcoreapp2.0/System.Data.Odbc.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/unix/lib/netcoreapp2.0/System.Drawing.Common.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/unix/lib/netcoreapp2.0/System.Drawing.Common.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/unix/lib/netcoreapp2.0/System.Runtime.Caching.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/unix/lib/netcoreapp2.0/System.Runtime.Caching.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/unix/lib/netcoreapp2.1/System.Data.SqlClient.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/unix/lib/netcoreapp2.1/System.Data.SqlClient.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/unix/lib/netstandard2.0/System.Private.ServiceModel.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/unix/lib/netstandard2.0/System.Private.ServiceModel.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win-arm64/native/sni.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win-arm64/native/sni.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win-x64/native/sni.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win-x64/native/sni.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win-x86/native/sni.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win-x86/native/sni.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/Microsoft.Win32.SystemEvents.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/Microsoft.Win32.SystemEvents.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Data.Odbc.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Data.Odbc.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Diagnostics.EventLog.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Diagnostics.EventLog.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Diagnostics.PerformanceCounter.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Diagnostics.PerformanceCounter.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.DirectoryServices.AccountManagement.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.DirectoryServices.AccountManagement.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.DirectoryServices.Protocols.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.DirectoryServices.Protocols.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.DirectoryServices.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.DirectoryServices.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Drawing.Common.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Drawing.Common.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Management.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Management.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Runtime.Caching.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.0/System.Runtime.Caching.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.1/System.Data.SqlClient.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.1/System.Data.SqlClient.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.1/System.Security.Cryptography.Pkcs.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netcoreapp2.1/System.Security.Cryptography.Pkcs.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/Microsoft.Win32.Registry.AccessControl.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/Microsoft.Win32.Registry.AccessControl.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.IO.Ports.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.IO.Ports.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.Private.ServiceModel.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.Private.ServiceModel.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.Security.Cryptography.ProtectedData.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.Security.Cryptography.ProtectedData.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.ServiceProcess.ServiceController.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.ServiceProcess.ServiceController.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.Threading.AccessControl.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/runtimes/win/lib/netstandard2.0/System.Threading.AccessControl.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/tr/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/tr/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/zh-Hans/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/zh-Hans/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/restler/zh-Hant/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/restler/zh-Hant/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/DiffPlex.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/DiffPlex.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/FSharp.Control.AsyncSeq.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/FSharp.Control.AsyncSeq.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/FSharp.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/FSharp.Core.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Microsoft.ApplicationInsights.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Microsoft.ApplicationInsights.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Microsoft.FSharpLu.Json.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Microsoft.FSharpLu.Json.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Microsoft.FSharpLu.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Microsoft.FSharpLu.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/NJsonSchema.Yaml.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/NJsonSchema.Yaml.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/NJsonSchema.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/NJsonSchema.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/NSwag.Core.Yaml.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/NSwag.Core.Yaml.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/NSwag.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/NSwag.Core.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Namotion.Reflection.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Namotion.Reflection.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Newtonsoft.Json.Bson.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Newtonsoft.Json.Bson.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Newtonsoft.Json.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Newtonsoft.Json.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Pluralize.NET.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Pluralize.NET.Core.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Restler.Compiler.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Restler.Compiler.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Restler.Compiler.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Restler.Compiler.pdb -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Restler.ResultsAnalyzer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Restler.ResultsAnalyzer -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Restler.ResultsAnalyzer.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Restler.ResultsAnalyzer.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Restler.ResultsAnalyzer.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/Restler.ResultsAnalyzer.pdb -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Restler.ResultsAnalyzer.runtimeconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "runtimeOptions": { 3 | "tfm": "net5.0", 4 | "framework": { 5 | "name": "Microsoft.NETCore.App", 6 | "version": "5.0.0" 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/Sysprep.ps1: -------------------------------------------------------------------------------- 1 | #.SYNOPSIS 2 | # Sysprep current machine running Windows Server 2008 R2 and Powershell 5.* (providing workarounds introduced by the new Powershell release) 3 | #.DESCRIPTION 4 | # Script Syspreps current machine. It provides needed workarounds to make Sysprep work on Windows 2008 R2 running Powershell 5.*. 5 | # SIL workaround: https://msdn.microsoft.com/en-us/powershell/wmf/5.0/limitation_overview 6 | # RegKey issue workaround: https://windowsserver.uservoice.com/forums/301869-powershell/suggestions/11591262-bug-wmf-5-production-preview-on-windows-server-2 7 | # Script must be run from elevated Powershell session. 8 | #.PARAMETER ShutdownAfterSysprep 9 | # Shutdown the machine after the Sysprep 10 | #.PARAMETER DisableAntivirusBeforeSysprep 11 | # Prevents antivirus from running on VMs based on the sysprepped image, without requiring a reboot. 12 | # This switch should not be used to prepare the VM template image, since antivirus should be 13 | # on by default for these (they are used to create the customer VM). 14 | [CmdletBinding()] 15 | Param( 16 | [switch] $ShutdownAfterSysprep, 17 | [switch] $DisableAntivirusBeforeSysprep 18 | ) 19 | Process { 20 | 21 | $osVersion = [environment]::OSVersion.Version 22 | $windows2008 = ($osVersion.Major -eq 6) -and ($osVersion.Minor -eq 1) 23 | 24 | #### Disable antivirus if required 25 | #### Note: On Win8+, Defender cannot be disabled without a reboot (there does not appear to be 26 | #### a commandlet to cleanly stop defender once it has been disabled). 27 | if (-not $windows2008 -and $DisableAntivirusBeforeSysprep) { 28 | $defenderRegPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows Defender\" 29 | if (-not (Test-Path $defenderRegPath)) { 30 | New-Item $defenderRegPath 31 | } 32 | New-ItemProperty -Path $defenderRegPath -Name DisableAntiSpyware -Value 1 -PropertyType DWord -Force 33 | } 34 | 35 | #### Hack needed to fix PowerShell breaking Sysprep 36 | Write-Output "Apply WMF 5.0 sysprep workaround" 37 | 38 | # The regkey has to be present there on VM boot. Thus need to patch registry before shutdown 39 | New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\StreamProvider" -Name LastFullPayloadTime -Value 0 -PropertyType DWord -Force 40 | # Working around PowerShell 5.0 Bugs: https://msdn.microsoft.com/en-us/powershell/wmf/5.0/limitation_overview 41 | Set-SilLogging -TargetUri https://BlankTarget -CertificateThumbprint 0123456789 42 | Publish-SilData -ErrorAction SilentlyContinue 43 | Remove-Item -Recurse $env:SystemRoot\System32\Logfiles\SIL\ -Force -ErrorAction SilentlyContinue 44 | 45 | #### Run sysprep generalization 46 | $sysprepCommand = "$env:SystemRoot\system32\sysprep\sysprep.exe" 47 | $sysprepParameters = '/generalize /oobe /quiet' 48 | 49 | if ($ShutdownAfterSysprep) { 50 | $sysprepParameters += ' /shutdown' 51 | } else { 52 | $sysprepParameters += ' /quit' 53 | } 54 | 55 | if (-not (Test-Path $sysprepCommand)) { 56 | throw "Failed to locate sysprep command. This is probably due to Wow64 file redirection. Make sure to run VMAgent in the native OS architecture." 57 | } 58 | 59 | # IMPORTANT NOTE: running sysprep.exe directly would fail when running under Remote Powershell (WinRM). 60 | # This is because the sysprep generalization process breaks all running network connections including WinRM itself. 61 | # Subsequently, when the WinRM session is interrupted, all the processes started under that session are automatically terminated. 62 | # This causes sysprep.exe itself to be killed before completion! 63 | # As a workaround, we start sysprep.exe through a Windows scheduled task. The drawback is that it's fire and forget: the remote session 64 | # has no way to detect when sysprep terminates, nor can it retrieve the exit code from sysprep.exe. 65 | Write-Output "Sysprepping machine" 66 | $taskname = "fsharplu-sysprep" 67 | 68 | if ($windows2008) { 69 | Write-Output "Creating scheduled task (Windows 7/Server 2008)" 70 | # On Windows 2008, New-ScheduledTaskAction does not exist. The alternative 71 | # `Register-ScheduledJob` is too limited (works only with PS scripts). 72 | # So we fall back on command `schtasks.exe`. 73 | 74 | # We set the task to run as SYSTEM to avoid having to pass credentials. 75 | # NOTE: In 83 years from now, the following commands will stop working, at which point 76 | # we might consider dropping support for Windows 2008 ;-) 77 | schtasks /CREATE /TN $taskname /RL HIGHEST /F ` 78 | /TR "$sysprepCommand $sysprepParameters" /SC ONCE /ST 00:00 /SD 01/01/2090 ` 79 | /RU SYSTEM 80 | 81 | schtasks /RUN /TN $taskname 82 | 83 | schtasks /DELETE /TN $taskname /F 84 | } else { 85 | Write-Output "Creating scheduled task (Windows >=8)" 86 | $action = New-ScheduledTaskAction -Execute $sysprepCommand -Argument $sysprepParameters 87 | $principal = New-ScheduledTaskPrincipal -RunLevel "Highest" -UserId 'SYSTEM' 88 | New-ScheduledTask -Action $action -Principal $principal ` 89 | | Register-ScheduledTask $taskname -Force ` 90 | | Start-ScheduledTask ` 91 | | Unregister-ScheduledTask -Confirm:$false 92 | } 93 | } -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/System.ComponentModel.Composition.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/System.ComponentModel.Composition.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/System.Configuration.ConfigurationManager.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/System.Configuration.ConfigurationManager.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/System.Net.Http.Formatting.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/System.Net.Http.Formatting.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/System.Security.Cryptography.ProtectedData.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/System.Security.Cryptography.ProtectedData.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/System.Security.Permissions.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/System.Security.Permissions.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/YamlDotNet.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/YamlDotNet.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/cs/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/cs/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/de/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/de/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/es/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/es/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/fr/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/fr/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/it/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/it/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/ja/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/ja/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/ko/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/ko/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/pl/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/pl/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/pt-BR/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/pt-BR/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/ru/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/ru/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/runtimes/win/lib/netstandard2.0/System.Security.Cryptography.ProtectedData.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/runtimes/win/lib/netstandard2.0/System.Security.Cryptography.ProtectedData.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/tr/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/tr/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/zh-Hans/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/zh-Hans/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/resultsAnalyzer/zh-Hant/FSharp.Core.resources.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/puppet-meteor/MINER/e2c68ac7699b475d238d1a425e73d6d72b3ec7cd/restler_bin_atten/resultsAnalyzer/zh-Hant/FSharp.Core.resources.dll -------------------------------------------------------------------------------- /restler_bin_atten/~/.config/microsoft/restler/restler.telemetry.uuid: -------------------------------------------------------------------------------- 1 | 3c8ccae8-a6fb-4eb9-873e-a2d52e297efc --------------------------------------------------------------------------------