├── LICENSE ├── README.md ├── experiments ├── Covariate Shift - Folktables.ipynb ├── Covariate Shift - UCI Airfoil Data.ipynb ├── Exchangeable Data - Multigroup Linear Regression.ipynb ├── Exchangeable Data - Vanilla Linear Regression.ipynb ├── Time Series - Stock Returns.ipynb └── data │ ├── 2018.zip │ ├── AMD.csv │ ├── BB.csv │ ├── FNMA.csv │ ├── NVDA.csv │ └── airfoil_self_noise.dat └── src ├── MultiValidPrediction.py ├── calibrationScorers ├── __init__.py ├── calibrationScorer.py ├── customResidualCalibrationScorer.py ├── predictionSet.py ├── rapsScorer.py ├── residualCalibrationScorer.py └── topSoftMaxScorer.py ├── conformal.py ├── recursiveLeastSquares.py ├── splitConformalPrediction.py └── utils_raps.py /LICENSE: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MVP: Practical Adversarial Multivalid Conformal Prediction 2 | 3 | 4 | This repository contains code associated with the paper [Practical Adversarial Multivalid Conformal Prediction](https://arxiv.org/abs/2206.01067) by O. Bastani, V. Gupta, C. Jung, G. Noarov, R. Ramalingam, and A. Roth. 5 | 6 | We propose MVP (MultiValid Prediction) --- a conformal prediction method for sequential adversarial data that produces prediction sets with valid, stronger-than-marginal empirical coverage that is: 7 | - *Threshold-calibrated:* The coverage is valid conditional on the threshold used to form the prediction set from the conformal score. 8 | - *Group-conditional:* The coverage is valid on each of an arbitrary (e.g. intersecting) user-specified collection of subsets of the feature space. 9 | 10 | ## Contents 11 | 12 | - `src/` Implementation of MVP (class `MultiValidPrediction` contained in `src/MultiValidPrediction.py`), 13 | along with some useful utilities (in particular, a collection of conformal scorers in `src/calibrationScorers/`). 14 | - `experiments/` Jupyter notebooks for the experiments in the corresponding section of the paper. 15 | -------------------------------------------------------------------------------- /experiments/Covariate Shift - Folktables.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "4a314e82", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import warnings\n", 11 | "warnings.filterwarnings('ignore')" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 2, 17 | "id": "09fe74ac", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "import matplotlib.pyplot as plt\n", 22 | "import numpy as np\n", 23 | "import pandas as pd\n", 24 | "import pickle as p\n", 25 | "from sklearn.linear_model import QuantileRegressor\n", 26 | "from sklearn.model_selection import train_test_split\n", 27 | "from folktables import *" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 3, 33 | "id": "9b1e71e7", 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "import sys" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "71747913", 43 | "metadata": {}, 44 | "source": [ 45 | "## Folktables data\n", 46 | "\n", 47 | "NOTE: please unzip file ./data/2018.zip" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "id": "b66fae23", 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "ACSIncomeNew = folktables.BasicProblem(\n", 58 | " features=[\n", 59 | " 'COW',\n", 60 | " 'SCHL',\n", 61 | " 'MAR',\n", 62 | " 'OCCP',\n", 63 | " 'POBP',\n", 64 | " 'RELP',\n", 65 | " 'WKHP',\n", 66 | " 'SEX',\n", 67 | " 'RAC1P',\n", 68 | " ],\n", 69 | " target='PINCP',\n", 70 | " group='SEX',\n", 71 | " preprocess=adult_filter,\n", 72 | " postprocess=lambda x: np.nan_to_num(x, -1),\n", 73 | ")" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "id": "199fbee9", 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "def split_folk_data(state_one, state_two ):\n", 84 | " data_source = ACSDataSource(survey_year='2018', horizon='1-Year', survey='person')\n", 85 | " ca_data = data_source.get_data(states=[state_one], download=True)\n", 86 | " ca_features, ca_label, ca_group = ACSIncomeNew.df_to_numpy(ca_data)\n", 87 | " pa_data = data_source.get_data(states=[state_two], download=True)\n", 88 | " pa_features, pa_label, pa_group = ACSIncomeNew.df_to_numpy(pa_data)\n", 89 | " X_ls, X_calibrate, y_ls, y_calibrate = train_test_split(ca_features, ca_label, test_size=0.6, random_state=0)\n", 90 | " X_eval, X_eval2, y_eval, y_eval2 = train_test_split(pa_features, pa_label, test_size=0.6, random_state=0)\n", 91 | "\n", 92 | " # taking .2 of entire dataset for each fold for time\n", 93 | " X_ls, X_calibrate, y_ls, y_calibrate = train_test_split(X_ls, y_ls, test_size=0.5, random_state=0)\n", 94 | " X_eval, X_eval2, y_eval, y_eval2 = train_test_split(X_eval, y_eval, test_size=0.5, random_state=0)\n", 95 | " \n", 96 | " return X_ls, y_ls, X_calibrate, y_calibrate, X_eval, y_eval" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "id": "7e2a7f30", 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "## folktable data params\n", 107 | "\n", 108 | "dataset_name = 'Folktables'\n", 109 | "\n", 110 | "# residual score normalization \n", 111 | "l = 0\n", 112 | "u = 1217116" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "id": "ee6a8a02", 118 | "metadata": {}, 119 | "source": [ 120 | "## Function definitions" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "id": "cdc6ba3e", 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "def train_qr(X_model, y_model, quant):\n", 131 | " qr = QuantileRegressor(quantile=quant, solver='highs')\n", 132 | " qr.fit(X_model, y_model)\n", 133 | " return qr" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": null, 139 | "id": "0cc2e4a2", 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "def all_points(x):\n", 144 | " return True\n", 145 | "\n", 146 | "def calibrate(x_input, y_input, bucket, r, delta, l, u, T, T_calibrate, myRLS):\n", 147 | " n = bucket\n", 148 | " r = r\n", 149 | " \n", 150 | " # no groups\n", 151 | " groups = [all_points]\n", 152 | "\n", 153 | " eta = 0.5\n", 154 | " delta = delta\n", 155 | "\n", 156 | " myUncertaintyQuantifier = MultiValidPrediction(delta, n, groups, eta, r)\n", 157 | "\n", 158 | " myResidualCalibrationScorer = residualCalibrationScorer.ResidualCalibrationScorer()\n", 159 | "\n", 160 | " myResidualCalibrationScorer.update(myRLS.predict)\n", 161 | "\n", 162 | " y_input = np.asarray(y_input)\n", 163 | "\n", 164 | " covered_arr = []\n", 165 | " width_arr = []\n", 166 | "\n", 167 | " for t in range(T): \n", 168 | "\n", 169 | " x_t = np.matrix(x_input[t])\n", 170 | " y_t = y_input[t]\n", 171 | "\n", 172 | " # calculate the new threshold \n", 173 | " norm_q_t = myUncertaintyQuantifier.predict(x_t)\n", 174 | " \n", 175 | " # rescale threshold\n", 176 | " q_t = norm_q_t * (u - l) + l\n", 177 | "\n", 178 | " # check if the prediction set covers the data\n", 179 | " curr_prediction_set = myResidualCalibrationScorer.get_prediction_set(x_t, q_t)\n", 180 | " covered_t = curr_prediction_set.cover(np.matrix(y_t))\n", 181 | " covered_arr.append(covered_t)\n", 182 | "\n", 183 | " # get prediction interval width\n", 184 | " width_arr.append(curr_prediction_set.interval_width)\n", 185 | " \n", 186 | " if (t > T_calibrate): # evaluate coverage / width after calibration data\n", 187 | " covered_arr.append(covered_t)\n", 188 | " width_arr.append(curr_prediction_set.interval_width)\n", 189 | "\n", 190 | "\n", 191 | " # update the calibrator mutlivalidator \n", 192 | " s_t = myResidualCalibrationScorer.calc_score(x_t, np.matrix(y_t))\n", 193 | "\n", 194 | " # normalize score\n", 195 | " norm_s_t = (s_t - l ) / (u - l)\n", 196 | "\n", 197 | " myUncertaintyQuantifier.update(x_t, norm_q_t, norm_s_t)\n", 198 | "\n", 199 | " # update the calibration scorer\n", 200 | " myResidualCalibrationScorer.update(myRLS.predict)\n", 201 | " \n", 202 | " return covered_arr, width_arr" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": null, 208 | "id": "52ea0f25", 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "def weighted_quantile(values, quantiles, sample_weight=None, \n", 213 | " values_sorted=False, old_style=False):\n", 214 | " \"\"\" Very close to numpy.percentile, but supports weights.\n", 215 | " NOTE: quantiles should be in [0, 1]!\n", 216 | " :param values: numpy.array with data\n", 217 | " :param quantiles: array-like with many quantiles needed\n", 218 | " :param sample_weight: array-like of the same length as `array`\n", 219 | " :param values_sorted: bool, if True, then will avoid sorting of\n", 220 | " initial array\n", 221 | " :param old_style: if True, will correct output to be consistent\n", 222 | " with numpy.percentile.\n", 223 | " :return: numpy.array with computed quantiles.\n", 224 | " \"\"\"\n", 225 | " values = np.array(values)\n", 226 | " quantiles = np.array(quantiles)\n", 227 | " if sample_weight is None:\n", 228 | " sample_weight = np.ones(len(values))\n", 229 | " sample_weight = np.array(sample_weight)\n", 230 | " assert np.all(quantiles >= 0) and np.all(quantiles <= 1), \\\n", 231 | " 'quantiles should be in [0, 1]'\n", 232 | "\n", 233 | " if not values_sorted:\n", 234 | " sorter = np.argsort(values)\n", 235 | " values = values[sorter]\n", 236 | " sample_weight = sample_weight[sorter]\n", 237 | "\n", 238 | " weighted_quantiles = np.cumsum(sample_weight) - 0.5 * sample_weight\n", 239 | " if old_style:\n", 240 | " # To be convenient with numpy.percentile\n", 241 | " weighted_quantiles -= weighted_quantiles[0]\n", 242 | " weighted_quantiles /= weighted_quantiles[-1]\n", 243 | " else:\n", 244 | " weighted_quantiles /= np.sum(sample_weight)\n", 245 | " return np.interp(quantiles, weighted_quantiles, values)\n", 246 | "\n", 247 | "\n", 248 | "def do_weighted_conformal(x, y, weights, alpha, T, T_calibration, model):\n", 249 | "\n", 250 | " # arrays for conformal prediction\n", 251 | " width_conformal = []\n", 252 | " cover_conformal = []\n", 253 | " \n", 254 | " myResidualCalibrationScorer = residualCalibrationScorer.ResidualCalibrationScorer()\n", 255 | "\n", 256 | " y = np.asarray(y)\n", 257 | "\n", 258 | "\n", 259 | " for t in range(T):\n", 260 | " \n", 261 | " x_t = np.matrix(x[t])\n", 262 | " y_t = y[t]\n", 263 | " \n", 264 | " y_pred_conformal_t = model.predict(x_t)\n", 265 | " myResidualCalibrationScorer.update(model.predict)\n", 266 | " new_y = np.reshape(y, -1)\n", 267 | " residuals = myResidualCalibrationScorer.calc_score(x, new_y)\n", 268 | " calibration_size = len(x)\n", 269 | "\n", 270 | " desired_quantile = np.ceil((1-alpha) * (calibration_size + 1)) / calibration_size\n", 271 | " \n", 272 | " w_t_conformal = weighted_quantile(residuals, (1-alpha), weights)\n", 273 | "\n", 274 | " conformal_prediction_set = myResidualCalibrationScorer.get_prediction_set(x_t, w_t_conformal)\n", 275 | "\n", 276 | " covered_conformal_t = conformal_prediction_set.cover(y_t)\n", 277 | "\n", 278 | " if (t > T_calibration): # evaluate coverage / width after calibration data\n", 279 | " width_conformal.append(w_t_conformal)\n", 280 | " cover_conformal.append(covered_conformal_t)\n", 281 | "\n", 282 | " return width_conformal, cover_conformal" 283 | ] 284 | }, 285 | { 286 | "cell_type": "markdown", 287 | "id": "209463b3", 288 | "metadata": { 289 | "heading_collapsed": true 290 | }, 291 | "source": [ 292 | "## Experimental parameters" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": null, 298 | "id": "d57384c8", 299 | "metadata": { 300 | "hidden": true 301 | }, 302 | "outputs": [], 303 | "source": [ 304 | "num_trials = 1\n", 305 | "buckets = [40]\n", 306 | "d = 9\n", 307 | "\n", 308 | "# 1 - coverage\n", 309 | "alpha = .1\n", 310 | "\n", 311 | "model_name = 'Quantile Regression'" 312 | ] 313 | }, 314 | { 315 | "cell_type": "markdown", 316 | "id": "e103aa03", 317 | "metadata": {}, 318 | "source": [ 319 | "## Experiments" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": null, 325 | "id": "5f3e5088", 326 | "metadata": {}, 327 | "outputs": [], 328 | "source": [ 329 | "sys.path.append('../src')\n", 330 | "from MultiValidPrediction import MultiValidPrediction\n", 331 | "from calibrationScorers import residualCalibrationScorer\n", 332 | "import recursiveLeastSquares" 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": null, 338 | "id": "731651cf", 339 | "metadata": {}, 340 | "outputs": [], 341 | "source": [ 342 | "# for each setting of number of buckets, we have num_trials coverage/width values\n", 343 | "folk_coverage = []\n", 344 | "folk_width = []\n", 345 | "\n", 346 | "folk_coverage_conf = []\n", 347 | "folk_width_conf = []" 348 | ] 349 | }, 350 | { 351 | "cell_type": "code", 352 | "execution_count": null, 353 | "id": "2212a9ff", 354 | "metadata": {}, 355 | "outputs": [], 356 | "source": [ 357 | "X_ls, y_ls, X_calibrate, y_calibrate, X_eval, y_eval = split_folk_data('CA', 'PA')" 358 | ] 359 | }, 360 | { 361 | "cell_type": "code", 362 | "execution_count": null, 363 | "id": "c0c1b6cd", 364 | "metadata": {}, 365 | "outputs": [], 366 | "source": [ 367 | "conformal = True\n", 368 | "for num_bucket in buckets:\n", 369 | " \n", 370 | " trial_coverage = []\n", 371 | " trial_width = []\n", 372 | " trial_cov_conf = []\n", 373 | " trial_width_conf = []\n", 374 | " \n", 375 | " for i in range(num_trials):\n", 376 | " # split data\n", 377 | " X_ls, y_ls, X_calibrate, y_calibrate, X_eval, y_eval = split_folk_data('CA', 'PA')\n", 378 | "\n", 379 | " # train model\n", 380 | " reg_model = train_qr(X_ls, y_ls, .5)\n", 381 | "\n", 382 | " X = np.concatenate([X_calibrate, X_eval])\n", 383 | " y = np.concatenate([y_calibrate, y_eval])\n", 384 | " \n", 385 | " # calibrate\n", 386 | " coverage_res, width_res = calibrate(X, y, num_bucket, 80000, alpha, l, u, len(X), len(X_calibrate), reg_model)\n", 387 | "\n", 388 | " # store average coverage and width for this trial\n", 389 | " trial_coverage.append(np.mean(coverage_res))\n", 390 | " trial_width.append(np.mean(width_res))\n", 391 | " \n", 392 | " # conformal\n", 393 | " if conformal: # only do conformal once, not for each different setting of number of buckets\n", 394 | " \n", 395 | " # split conformal on shifted data\n", 396 | " conf_width_res, conf_cov_res = do_weighted_conformal(X, y, (([1] * len(X))), alpha, len(X), len(X_calibrate), reg_model)\n", 397 | "\n", 398 | " trial_cov_conf.append(np.mean(conf_cov_res))\n", 399 | " trial_width_conf.append(np.mean(conf_width_res))\n", 400 | "\n", 401 | "\n", 402 | " if conformal:\n", 403 | " folk_coverage_conf.append(trial_cov_conf)\n", 404 | " folk_width_conf.append(trial_width_conf)\n", 405 | " conformal = False\n", 406 | " \n", 407 | " folk_coverage.append(trial_coverage)\n", 408 | " folk_width.append(trial_width)" 409 | ] 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": null, 414 | "id": "63977c1a", 415 | "metadata": {}, 416 | "outputs": [], 417 | "source": [ 418 | "plt.hist([folk_coverage[0], folk_coverage_conf[0]], label=['MVP', 'split conformal'])\n", 419 | "plt.legend()\n", 420 | "plt.title('Mean Coverage ({0} trials, target coverage .9) \\n {1} \\n {2}'.format(len(folk_coverage[0]), model_name, dataset_name))\n", 421 | "plt.ylabel('Frequency')\n", 422 | "plt.xlabel('Coverage')\n", 423 | "plt.show()\n", 424 | "\n", 425 | "plt.hist([folk_width[0], folk_width_conf[0]], label=['MVP', 'split conformal'])\n", 426 | "plt.legend()\n", 427 | "plt.title('Median Width ({0} trials, target coverage .9) \\n {1} \\n {2}'.format(len(folk_coverage[0]), model_name, dataset_name))\n", 428 | "plt.ylabel('Frequency')\n", 429 | "plt.xlabel('Interval Width (in $)')\n", 430 | "plt.show()" 431 | ] 432 | }, 433 | { 434 | "cell_type": "markdown", 435 | "id": "45fdb795", 436 | "metadata": {}, 437 | "source": [ 438 | "## Plotting" 439 | ] 440 | }, 441 | { 442 | "cell_type": "code", 443 | "execution_count": null, 444 | "id": "6354df43", 445 | "metadata": {}, 446 | "outputs": [], 447 | "source": [ 448 | "fd = open('folk-results/folk-coverage-conformal.pickle', 'rb')\n", 449 | "folk_coverage_conformal = p.load(fd)\n", 450 | "fd.close()\n", 451 | "\n", 452 | "fd = open('folk-results/folk-coverage.pickle', 'rb')\n", 453 | "folk_coverage = p.load(fd)\n", 454 | "fd.close()\n", 455 | "\n", 456 | "fd = open('folk-results/folk-width-conformal.pickle', 'rb')\n", 457 | "folk_width_conformal = p.load(fd)\n", 458 | "fd.close()\n", 459 | "\n", 460 | "fd = open('folk-results/folk-width.pickle', 'rb')\n", 461 | "folk_width = p.load(fd)\n", 462 | "fd.close()" 463 | ] 464 | }, 465 | { 466 | "cell_type": "code", 467 | "execution_count": null, 468 | "id": "e47dfae5", 469 | "metadata": {}, 470 | "outputs": [], 471 | "source": [ 472 | "model_name = 'Quantile Regression'\n", 473 | "dataset_name = 'Folktables Data'" 474 | ] 475 | }, 476 | { 477 | "cell_type": "code", 478 | "execution_count": null, 479 | "id": "91e4b62a", 480 | "metadata": {}, 481 | "outputs": [], 482 | "source": [ 483 | "bins = np.linspace(0.87, 0.95, 60)\n", 484 | "plt.hist([folk_coverage[0]], label=['MVP'], color= 'blue', alpha = 0.4, linewidth=0.5, edgecolor = 'blue', bins=bins)\n", 485 | "plt.hist([folk_coverage_conformal[0]], label=['split conformal'], color= 'orange', alpha = 0.6, linewidth=0.5, edgecolor = 'orange', bins=bins)\n", 486 | "plt.legend()\n", 487 | "plt.axvline(x = .9, color = 'red', linestyle = '--', linewidth = 0.9)\n", 488 | "plt.title('Mean Coverage ({0} trials, target coverage .9) \\n {1} \\n {2}'.format(len(folk_coverage[0]), model_name, dataset_name))\n", 489 | "plt.ylabel('No. of Rounds')\n", 490 | "plt.xlabel('Mean Coverage')\n", 491 | "plt.xlim([.899, .96])\n", 492 | "plt.show()\n", 493 | "\n", 494 | "bins = np.linspace(72700, 73500, 60)\n", 495 | "plt.hist([folk_width[0]], label=['MVP'], color= 'blue', alpha = 0.4, linewidth=0.5, edgecolor = 'blue', bins=bins)\n", 496 | "plt.hist([folk_width_conformal[0]], label=['split conformal'], color= 'orange', alpha = 0.6, linewidth=0.5, edgecolor = 'orange', bins=bins)\n", 497 | "plt.legend()\n", 498 | "plt.legend()\n", 499 | "plt.title('Median Width ({0} trials, target coverage .9) \\n {1} \\n {2}'.format(len(folk_coverage[0]), model_name, dataset_name))\n", 500 | "plt.ylabel('No. Of Rounds')\n", 501 | "plt.xlabel('Median Interval Width (in $)')\n", 502 | "plt.show()" 503 | ] 504 | } 505 | ], 506 | "metadata": { 507 | "kernelspec": { 508 | "display_name": "Python 3 (ipykernel)", 509 | "language": "python", 510 | "name": "python3" 511 | }, 512 | "language_info": { 513 | "codemirror_mode": { 514 | "name": "ipython", 515 | "version": 3 516 | }, 517 | "file_extension": ".py", 518 | "mimetype": "text/x-python", 519 | "name": "python", 520 | "nbconvert_exporter": "python", 521 | "pygments_lexer": "ipython3", 522 | "version": "3.9.12" 523 | } 524 | }, 525 | "nbformat": 4, 526 | "nbformat_minor": 5 527 | } 528 | -------------------------------------------------------------------------------- /experiments/Covariate Shift - UCI Airfoil Data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "66bd5480", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import warnings\n", 11 | "warnings.filterwarnings('ignore', category=DeprecationWarning)" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "id": "62778e9c", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "import matplotlib.pyplot as plt\n", 22 | "import numpy as np\n", 23 | "import pandas as pd\n", 24 | "import sys" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "0acd53bb", 30 | "metadata": { 31 | "tags": [] 32 | }, 33 | "source": [ 34 | "## UCI Airfoil data" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "id": "d6ee1b3d", 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "full_data = pd.read_csv(\"data/airfoil_self_noise.dat\", sep='\\t', header=None)\n", 45 | "\n", 46 | "full_data[0] = np.log(full_data[0])\n", 47 | "full_data[4] = np.log(full_data[4])\n", 48 | "\n", 49 | "dataset_name = 'UCI Airfoil'" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "id": "2ef801b5", 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "# append constant to all data as needed for LS implementation\n", 60 | "full_data.insert(0, \"constant\", [1] * len(full_data))" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "id": "835bc007", 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "def airfoil_split(data):\n", 71 | " \"\"\"\n", 72 | " Split airfoil data into separate folds for least squares data, calibration data, and evaluation data.\n", 73 | "\n", 74 | " Returns:\n", 75 | " ls_x, ls_y: data to train least squares model, 25% of full dataset\n", 76 | " calibration_data: data to feed to calibration algorithm, 25% of full dataset\n", 77 | " evaluation_data: data to feed to calibration algorithm and evaluate performance on, 50% of full dataset\n", 78 | " xs_iid, ys_iid: concatenation of calibration_data and evaluation_data\n", 79 | " \"\"\"\n", 80 | " pre_train = data.sample(frac=.5, axis=0)\n", 81 | "\n", 82 | " evaluation_data = data.drop(index=pre_train.index)\n", 83 | "\n", 84 | " train_model = pre_train.sample(frac=.5, axis=0)\n", 85 | "\n", 86 | " calibration_data = pre_train.drop(index=train_model.index)\n", 87 | "\n", 88 | " ls_x = train_model.iloc[:, :6]\n", 89 | " ls_y = train_model.iloc[:, 6:]\n", 90 | "\n", 91 | " data_no_shift = pd.concat([calibration_data, evaluation_data])\n", 92 | "\n", 93 | " xs_iid = data_no_shift.iloc[:, :6]\n", 94 | " ys_iid = data_no_shift.iloc[:, 6:]\n", 95 | " \n", 96 | " return ls_x, ls_y, calibration_data, evaluation_data, xs_iid, ys_iid\n", 97 | " " 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": null, 103 | "id": "88b86fdd", 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "# residual conformal score normalization \n", 108 | "l = 0\n", 109 | "u = 120" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "id": "c645537f", 115 | "metadata": {}, 116 | "source": [ 117 | "## Function definitions" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "id": "d3b2045d", 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "def train_ls(x_model, y_model):\n", 128 | " myRLS = recursiveLeastSquares.RLS(6, 1.0, 1)\n", 129 | " ls_x = x_model.to_numpy()\n", 130 | " ls_x = [np.matrix(x).T for x in ls_x]\n", 131 | " ls_y = y_model.to_numpy()\n", 132 | " ls_y = [y[0] for y in ls_y]\n", 133 | " myRLS.fit(ls_x, ls_y)\n", 134 | " return myRLS" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "id": "9d1552f2", 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "def all_points(x):\n", 145 | " return True\n", 146 | "\n", 147 | "def calibrate(x_input, y_input, bucket, r, delta, l, u, T, T_calibrate, myRLS):\n", 148 | " n = bucket\n", 149 | " r = r\n", 150 | " \n", 151 | " groups = [all_points]\n", 152 | "\n", 153 | " eta = np.sqrt(np.log(2 * len(groups) * n) / T)\n", 154 | "\n", 155 | " delta = delta\n", 156 | "\n", 157 | " myUncertaintyQuantifier = MultiValidPrediction(delta, n, groups, eta, r)\n", 158 | "\n", 159 | " myResidualCalibrationScorer = residualCalibrationScorer.ResidualCalibrationScorer()\n", 160 | "\n", 161 | " myResidualCalibrationScorer.update(myRLS.predict)\n", 162 | " \n", 163 | " y_input = np.asarray(y_input)\n", 164 | "\n", 165 | " covered_arr = []\n", 166 | " width_arr = []\n", 167 | "\n", 168 | " for t in range(T): \n", 169 | " # data arrival\n", 170 | " x_t = (x_input.iloc[t])\n", 171 | " y_t = y_input[t]\n", 172 | "\n", 173 | " # calculate the new threshold \n", 174 | " norm_q_t = myUncertaintyQuantifier.predict(x_t)\n", 175 | " \n", 176 | " # rescale threshold\n", 177 | " q_t = norm_q_t * (u - l) + l\n", 178 | "\n", 179 | " # check if the prediction set covers the data\n", 180 | " curr_prediction_set = myResidualCalibrationScorer.get_prediction_set(x_t, q_t)\n", 181 | " covered_t = curr_prediction_set.cover(np.matrix(y_t))\n", 182 | " covered_arr.append(covered_t)\n", 183 | "\n", 184 | " # get prediction interval width\n", 185 | " width_arr.append(curr_prediction_set.interval_width)\n", 186 | " \n", 187 | " if (t > T_calibrate): # evaluate coverage / width after calibration data\n", 188 | " covered_arr.append(covered_t)\n", 189 | " width_arr.append(curr_prediction_set.interval_width)\n", 190 | "\n", 191 | "\n", 192 | " # update the calibrator mutlivalidator \n", 193 | " s_t = myResidualCalibrationScorer.calc_score(x_t, np.matrix(y_t))\n", 194 | "\n", 195 | " # normalize score\n", 196 | " norm_s_t = (s_t - l ) / (u - l)\n", 197 | "\n", 198 | " myUncertaintyQuantifier.update(x_t, norm_q_t, norm_s_t)\n", 199 | "\n", 200 | " # update the calibration scorer\n", 201 | " myResidualCalibrationScorer.update(myRLS.predict)\n", 202 | " \n", 203 | " return covered_arr, width_arr\n", 204 | " " 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "id": "2e7a7b53", 210 | "metadata": {}, 211 | "source": [ 212 | "## Experiments" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": null, 218 | "id": "14bfb079", 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "sys.path.append('../src')\n", 223 | "from MultiValidPrediction import MultiValidPrediction\n", 224 | "from calibrationScorers import residualCalibrationScorer\n", 225 | "import recursiveLeastSquares" 226 | ] 227 | }, 228 | { 229 | "cell_type": "markdown", 230 | "id": "8cf64638", 231 | "metadata": {}, 232 | "source": [ 233 | "## Experimental parameters" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": null, 239 | "id": "03f4aa8c", 240 | "metadata": {}, 241 | "outputs": [], 242 | "source": [ 243 | "## airfoil params\n", 244 | "num_trials = 500\n", 245 | "buckets = [40]\n", 246 | "d = 6\n", 247 | "\n", 248 | "# coverage\n", 249 | "alpha = .1\n", 250 | "\n", 251 | "model_name = 'Linear Regression'" 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "id": "722f85e4", 257 | "metadata": {}, 258 | "source": [ 259 | "## Data without covariate shift:" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": null, 265 | "id": "7cadb251", 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [ 269 | "# for each setting of number of buckets, we have num_trials coverage values\n", 270 | "no_shift_coverage = []\n", 271 | "no_shift_width = []" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": null, 277 | "id": "a5cdebcd", 278 | "metadata": {}, 279 | "outputs": [], 280 | "source": [ 281 | "for num_bucket in buckets:\n", 282 | " \n", 283 | " trial_coverage = []\n", 284 | " trial_width = []\n", 285 | " \n", 286 | " for i in range(num_trials):\n", 287 | " \n", 288 | " # split data\n", 289 | " ls_x, ls_y, calibration, evaluation, xs_iid, ys_iid = airfoil_split(full_data) \n", 290 | "\n", 291 | " # retrain LS\n", 292 | " myRLS = train_ls(ls_x, ls_y)\n", 293 | "\n", 294 | " # calibrate\n", 295 | " coverage_res, width_res = calibrate(xs_iid, ys_iid, num_bucket, 1000, alpha, l, u, len(xs_iid), len(calibration), myRLS)\n", 296 | "\n", 297 | " # store average coverage and width for this trial\n", 298 | " trial_coverage.append(np.mean(coverage_res))\n", 299 | " trial_width.append(np.median(width_res))\n", 300 | " \n", 301 | " no_shift_coverage.append(trial_coverage)\n", 302 | " no_shift_width.append(trial_width)" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "id": "b9e8af74", 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [ 312 | "print(\"without shifted data over {0} trials\\n\".format(num_trials))\n", 313 | "for i in range(len(buckets)):\n", 314 | " print(\"coverage with {0} buckets is {1}\".format(buckets[i], np.mean(no_shift_coverage[i])))\n", 315 | " print(\"width with {0} buckets is {1}\\n\".format(buckets[i], np.mean(no_shift_width[i])))" 316 | ] 317 | }, 318 | { 319 | "cell_type": "markdown", 320 | "id": "195265e6", 321 | "metadata": {}, 322 | "source": [ 323 | "## Data with covariate shift: \n", 324 | "### Evaluating calibration on shifted data for us and oracle weights for weighted split conformal\n", 325 | "\n", 326 | "Here our comparison is a warm-start of our uncertainty prediction algorithm,\n", 327 | "by calibrating on data drawn from the shifted distribution we see at evalutation time, \n", 328 | "to simulate a fair comparison to weighted split conformal using likelihood ratios\n", 329 | "of the shifted evaluation data" 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": null, 335 | "id": "d21c27f7", 336 | "metadata": {}, 337 | "outputs": [], 338 | "source": [ 339 | "beta = [0, -1, 0, 0, 0, 1, 0] # weights for exponential tilting shift" 340 | ] 341 | }, 342 | { 343 | "cell_type": "code", 344 | "execution_count": null, 345 | "id": "bed4f031", 346 | "metadata": {}, 347 | "outputs": [], 348 | "source": [ 349 | "# for each setting of number of buckets, we have num_trials coverage values\n", 350 | "shifted_coverage = []\n", 351 | "shifted_width = []" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": null, 357 | "id": "45a0c322", 358 | "metadata": {}, 359 | "outputs": [], 360 | "source": [ 361 | "for num_bucket in buckets:\n", 362 | " \n", 363 | " trial_coverage = []\n", 364 | " trial_width = []\n", 365 | " trial_cov_splitconf = []\n", 366 | " trial_width_splitconf = []\n", 367 | " \n", 368 | " for i in range(num_trials):\n", 369 | " \n", 370 | " # split data\n", 371 | " ls_x, ls_y, calibration, evaluation, xs_iid, ys_iid = airfoil_split(full_data) \n", 372 | " \n", 373 | " # apply shift to evaluation data\n", 374 | " beta = [0, -1, 0, 0, 0, 1, 0] # weights for shift\n", 375 | " shift = evaluation.copy()\n", 376 | " weight = np.exp(np.dot(shift, beta))\n", 377 | " shift = shift.sample(frac=.25, axis=0, weights=weight, replace=True)\n", 378 | " \n", 379 | " weight = np.exp(np.dot(calibration, beta))\n", 380 | " calibration = calibration.sample(frac=1, axis=0, weights=weight, replace=True)\n", 381 | " \n", 382 | " data_with_shift = pd.concat([calibration, shift])\n", 383 | " xs_shift = data_with_shift.iloc[:, :6]\n", 384 | " ys_shift = data_with_shift.iloc[:, 6:]\n", 385 | " \n", 386 | " # retrain LS\n", 387 | " myRLS = train_ls(ls_x, ls_y)\n", 388 | "\n", 389 | " # calibrate\n", 390 | " coverage_res, width_res = calibrate(xs_shift, ys_shift, num_bucket, 1000, alpha, l, u, len(xs_shift), len(calibration), myRLS)\n", 391 | "\n", 392 | " # store average coverage and width for this trial\n", 393 | " trial_coverage.append(np.mean(coverage_res))\n", 394 | " trial_width.append(np.median(width_res))\n", 395 | "\n", 396 | " \n", 397 | " shifted_coverage.append(trial_coverage)\n", 398 | " shifted_width.append(trial_width)" 399 | ] 400 | }, 401 | { 402 | "cell_type": "code", 403 | "execution_count": null, 404 | "id": "2ad5379d", 405 | "metadata": {}, 406 | "outputs": [], 407 | "source": [ 408 | "print(\"with shifted data over {0} trials\\n\".format(num_trials))\n", 409 | "for i in range(len(buckets)):\n", 410 | " print(\"coverage with {0} buckets is {1}\".format(buckets[i], np.mean(shifted_coverage[i])))\n", 411 | " print(\"width with {0} buckets is {1}\\n\".format(buckets[i], np.mean(shifted_width[i])))" 412 | ] 413 | }, 414 | { 415 | "cell_type": "markdown", 416 | "id": "0cf50365", 417 | "metadata": {}, 418 | "source": [ 419 | "## Plotting" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": null, 425 | "id": "f04ca31f", 426 | "metadata": {}, 427 | "outputs": [], 428 | "source": [ 429 | "# read mvp results\n", 430 | "shifted_coverage_res = np.array(pd.read_csv('airfoil-results/coverage-mvp.csv', header=None))\n", 431 | "shifted_width_res = np.array(pd.read_csv('airfoil-results/width-mvp.csv', header=None))" 432 | ] 433 | }, 434 | { 435 | "cell_type": "code", 436 | "execution_count": null, 437 | "id": "f7c02e1e", 438 | "metadata": {}, 439 | "outputs": [], 440 | "source": [ 441 | "# read weighted split conformal results\n", 442 | "shifted_coverage_conf = pd.read_csv('airfoil-results/coverage-tibs.csv')\n", 443 | "shifted_coverage_conf = np.array(shifted_coverage_conf['x'])\n", 444 | "shifted_coverage_conf = np.random.choice(shifted_coverage_conf, 500, replace=False)\n", 445 | "shifted_width_conf = pd.read_csv('airfoil-results/width-tibs.csv')\n", 446 | "shifted_width_conf = np.array(shifted_width_conf['x'])\n", 447 | "shifted_width_conf = np.random.choice(shifted_width_conf, 500, replace=False)" 448 | ] 449 | }, 450 | { 451 | "cell_type": "code", 452 | "execution_count": null, 453 | "id": "63351874", 454 | "metadata": {}, 455 | "outputs": [], 456 | "source": [ 457 | "dataset_name = 'UCI Airfoil'\n", 458 | "model_name = 'Linear Regression'" 459 | ] 460 | }, 461 | { 462 | "cell_type": "code", 463 | "execution_count": null, 464 | "id": "a29436f0", 465 | "metadata": {}, 466 | "outputs": [], 467 | "source": [ 468 | "bins = np.linspace(0.775, 0.975, 40)\n", 469 | "plt.hist(shifted_coverage_res[0], label='MVP', color= 'blue', alpha = 0.4, linewidth=0.5, edgecolor = 'blue', bins=bins)\n", 470 | "plt.hist(shifted_coverage_conf, label='weighted split conformal', color= 'orange', alpha = 0.6, linewidth=0.5, edgecolor = 'orange', bins=bins)\n", 471 | "plt.legend()\n", 472 | "plt.axvline(x = .9, color = 'red', linestyle = '--', linewidth = 0.9)\n", 473 | "plt.title('Mean Coverage ({0} trials, target coverage .9) \\n {1} \\n {2} Data'.format(len(shifted_coverage_res[0]), model_name, dataset_name))\n", 474 | "plt.xlabel('Mean Coverage')\n", 475 | "plt.ylabel('No. of Rounds')\n", 476 | "plt.show()\n", 477 | "\n", 478 | "plt.hist(shifted_width_res[0], label='MVP', color= 'blue', alpha = 0.4, linewidth=0.5, edgecolor = 'blue')\n", 479 | "plt.hist(shifted_width_conf, label='weighted split conformal', color= 'orange', alpha = 0.6, linewidth=0.5, edgecolor = 'orange')\n", 480 | "plt.legend()\n", 481 | "plt.title('Median Width ({0} trials, target coverage .9) \\n {1} \\n {2} Data'.format(len(shifted_coverage_res[0]), model_name, dataset_name))\n", 482 | "plt.xlabel('Median Interval Width (in decibels)')\n", 483 | "plt.ylabel('No. of Rounds')\n", 484 | "plt.show()" 485 | ] 486 | } 487 | ], 488 | "metadata": { 489 | "kernelspec": { 490 | "display_name": "Python 3 (ipykernel)", 491 | "language": "python", 492 | "name": "python3" 493 | }, 494 | "language_info": { 495 | "codemirror_mode": { 496 | "name": "ipython", 497 | "version": 3 498 | }, 499 | "file_extension": ".py", 500 | "mimetype": "text/x-python", 501 | "name": "python", 502 | "nbconvert_exporter": "python", 503 | "pygments_lexer": "ipython3", 504 | "version": "3.9.12" 505 | } 506 | }, 507 | "nbformat": 4, 508 | "nbformat_minor": 5 509 | } 510 | -------------------------------------------------------------------------------- /experiments/Exchangeable Data - Multigroup Linear Regression.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Imports and preliminaries" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": { 14 | "colab": { 15 | "base_uri": "https://localhost:8080/" 16 | }, 17 | "executionInfo": { 18 | "elapsed": 23548, 19 | "status": "ok", 20 | "timestamp": 1648492251609, 21 | "user": { 22 | "displayName": "DaEto Georgy", 23 | "userId": "07578216823519586108" 24 | }, 25 | "user_tz": 240 26 | }, 27 | "id": "zAOPTJsJylLV", 28 | "outputId": "780637ec-b5ad-4bd7-8589-9da90bca9e46" 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import sys, os\n", 33 | "sys.path.append(os.path.join(os.getcwd(), '../src'))\n", 34 | "import numpy as np\n", 35 | "import matplotlib.pyplot as plt\n", 36 | "import pdb\n", 37 | "\n", 38 | "import recursiveLeastSquares\n", 39 | "import splitConformalPrediction\n", 40 | "from MultiValidPrediction import MultiValidPrediction\n", 41 | "from calibrationScorers import residualCalibrationScorer, customResidualCalibrationScorer" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "def produce_group(feat_index, feat_value):\n", 51 | " '''\n", 52 | " Input: \n", 53 | " - feat_index: index of desired input feature\n", 54 | " - feat_value: desired value of that feature\n", 55 | " Output:\n", 56 | " - f - function which defines a group; f(x) returns True\n", 57 | " iff x[feat_index] == feat_value and returns False otherwise.\n", 58 | " '''\n", 59 | " def f(x):\n", 60 | " return True if x[feat_index] == feat_value else False\n", 61 | " \n", 62 | " return f\n", 63 | "\n", 64 | "# Define group that includes all points\n", 65 | "def all_points(x):\n", 66 | " return True\n", 67 | "\n", 68 | "basic_group = [all_points]\n", 69 | "\n", 70 | "# Define 20 overlapping sub-groups - each defined by the value of a single binary feature\n", 71 | "\n", 72 | "twenty_groups = list()\n", 73 | "num_groups = 0\n", 74 | "for i in range(10):\n", 75 | " for j in range(2):\n", 76 | " curr_group = produce_group(i, j)\n", 77 | " twenty_groups.append(curr_group)\n", 78 | "\n", 79 | "num_groups = len(twenty_groups)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "## Synthetic Experiment with groups - Split-Conformal vs. MVP: Single trial" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "##### Set all parameters for a single trial" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": { 100 | "executionInfo": { 101 | "elapsed": 8, 102 | "status": "ok", 103 | "timestamp": 1648492392939, 104 | "user": { 105 | "displayName": "DaEto Georgy", 106 | "userId": "07578216823519586108" 107 | }, 108 | "user_tz": 240 109 | }, 110 | "id": "mNqAcKetylLY" 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "# Parameters for our uncertainty quantifier\n", 115 | "T = 20000\n", 116 | "n = 40\n", 117 | "r = 80000000\n", 118 | "delta = 0.1\n", 119 | "K_e = 2.12\n", 120 | "\n", 121 | "eta = np.sqrt(np.log(num_groups * n) / (2 * K_e * num_groups * n))\n", 122 | "\n", 123 | "# Parameters for data generation\n", 124 | "x_std = 0.1\n", 125 | "y_std = 0.25\n", 126 | "d = 300 # choose d > 10" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": {}, 132 | "source": [ 133 | "##### Generating data according to ordinary least-squares model - with one \"high-noise\" group and one \"low-noise\" group" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": null, 139 | "metadata": { 140 | "executionInfo": { 141 | "elapsed": 11, 142 | "status": "ok", 143 | "timestamp": 1648492395061, 144 | "user": { 145 | "displayName": "DaEto Georgy", 146 | "userId": "07578216823519586108" 147 | }, 148 | "user_tz": 240 149 | }, 150 | "id": "yZXQudeJylLZ" 151 | }, 152 | "outputs": [], 153 | "source": [ 154 | "theta = np.random.normal(loc=np.zeros(d), scale=x_std)\n", 155 | "\n", 156 | "# d-dimension features - first 10 features are binary\n", 157 | "xs_binvars = np.random.randint(low = 0, high = 2, size = (T, 10))\n", 158 | "xs_remvars = np.random.normal(loc=np.zeros(d - 10), scale=x_std, size=(T, d - 10))\n", 159 | "xs = np.concatenate((xs_binvars, xs_remvars), axis = 1)\n", 160 | "std_dev_list = np.array([3.0, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1])\n", 161 | "std_dev = np.dot(xs_binvars, std_dev_list) + y_std\n", 162 | "ys = np.dot(xs, theta) + np.random.normal(loc=0, scale= std_dev, size=T)" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "##### Initializing uncertainty-quantifiers and small initial calibration set for split-conformal prediction" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "metadata": { 176 | "executionInfo": { 177 | "elapsed": 196, 178 | "status": "ok", 179 | "timestamp": 1648492431011, 180 | "user": { 181 | "displayName": "DaEto Georgy", 182 | "userId": "07578216823519586108" 183 | }, 184 | "user_tz": 240 185 | }, 186 | "id": "I0I7N5q7ylLg" 187 | }, 188 | "outputs": [], 189 | "source": [ 190 | "myUncertaintyQuantifier = MultiValidPrediction(delta, n, twenty_groups, eta, r, normalize_by_counts=True)\n", 191 | "myConformalPredictor = splitConformalPrediction.splitConformal(num_groups, twenty_groups, delta)\n", 192 | "myBasicConformalPredictor = splitConformalPrediction.splitConformal(1, basic_group, delta)\n", 193 | "myRLS_conformal = recursiveLeastSquares.RLS(d, 1.0, 1)\n", 194 | "myRLS_ours = recursiveLeastSquares.RLS(d, 1.0, 1)\n", 195 | "\n", 196 | "mult_factor = 8\n", 197 | "myResidualCalibrationScorer = customResidualCalibrationScorer.customResidualCalibrationScorer(mult_factor)\n", 198 | "\n", 199 | "# arrays for MVP\n", 200 | "q_array = []\n", 201 | "y_pred_ours_array = []\n", 202 | "covered_ours_array = []\n", 203 | "\n", 204 | "# arrays (per group) for MVP\n", 205 | "q_array_groups = [list() for i in range(num_groups)] \n", 206 | "y_pred_ours_groups = [list() for i in range(num_groups)] \n", 207 | "covered_ours_groups = [list() for i in range(num_groups)] \n", 208 | "\n", 209 | "# arrays for conformal prediction (with groups)\n", 210 | "y_pred_conformal_array = []\n", 211 | "w_t_conformal_array = []\n", 212 | "covered_conformal_array = []\n", 213 | "\n", 214 | "# arrays (per group) for conformal prediction (with groups)\n", 215 | "y_pred_conformal_groups = [list() for i in range(num_groups)]\n", 216 | "w_t_conformal_groups = [list() for i in range(num_groups)] \n", 217 | "covered_conformal_groups = [list() for i in range(num_groups)]\n", 218 | "\n", 219 | "\n", 220 | "# arrays for basic split-conformal (without groups)\n", 221 | "y_pred_basic_array = []\n", 222 | "w_t_basic_array = []\n", 223 | "covered_basic_array = []\n", 224 | "\n", 225 | "\n", 226 | "# array (per group) for basic split-conformal (without groups)\n", 227 | "y_pred_basic_groups = [list() for i in range(num_groups)]\n", 228 | "w_t_basic_groups = [list() for i in range(num_groups)]\n", 229 | "covered_basic_groups = [list() for i in range(num_groups)]\n", 230 | "\n", 231 | "# keep track of y values per group\n", 232 | "ys_groups = [list() for i in range(num_groups)]\n", 233 | "\n", 234 | "\n", 235 | "# Conformal warm-start calibration set using the same distribution. \n", 236 | "init_size = 10\n", 237 | "conformal_calibration_xs_binvars = np.random.randint(low = 0, high = 2, size = (init_size, 10))\n", 238 | "conformal_calibration_xs_remvars = np.random.normal(loc=np.zeros(d - 10), scale=x_std, size=(init_size, d - 10))\n", 239 | "xs_cc = np.concatenate((conformal_calibration_xs_binvars, conformal_calibration_xs_remvars), axis = 1)\n", 240 | "std_dev = np.dot(conformal_calibration_xs_binvars, std_dev_list) + y_std \n", 241 | "ys_cc = np.dot(xs_cc, theta) + np.random.normal(loc=0, scale= std_dev, size=init_size)\n", 242 | "\n", 243 | "for i, curr_y in enumerate(ys_cc):\n", 244 | " curr_x = xs_cc[i]\n", 245 | " myConformalPredictor.update_calibration_data(curr_x, curr_y)\n", 246 | " myBasicConformalPredictor.update_calibration_data(curr_x, curr_y)" 247 | ] 248 | }, 249 | { 250 | "cell_type": "markdown", 251 | "metadata": {}, 252 | "source": [ 253 | "##### Check if calibration data covers all groups. If it doesn't, run previous cell again (to generate new data) or try increasing init_size (size of warm-start calibration set)" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": null, 259 | "metadata": {}, 260 | "outputs": [], 261 | "source": [ 262 | "print('Does initial calibration data cover all groups: ' + str(myConformalPredictor.all_groups_covered()))" 263 | ] 264 | }, 265 | { 266 | "cell_type": "markdown", 267 | "metadata": {}, 268 | "source": [ 269 | "##### Running MVP and split-conformal prediction across generated data" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": null, 275 | "metadata": { 276 | "executionInfo": { 277 | "elapsed": 4106, 278 | "status": "ok", 279 | "timestamp": 1648492438701, 280 | "user": { 281 | "displayName": "DaEto Georgy", 282 | "userId": "07578216823519586108" 283 | }, 284 | "user_tz": 240 285 | }, 286 | "id": "-zxrB6ijylLh", 287 | "scrolled": true 288 | }, 289 | "outputs": [], 290 | "source": [ 291 | "for t in range(T):\n", 292 | " x_t = xs[t]\n", 293 | " y_t = ys[t]\n", 294 | " \n", 295 | " # 1. CONFORMAL WITH GROUPS\n", 296 | " \n", 297 | " y_pred_conformal_t = myRLS_conformal.predict(x_t)\n", 298 | " myResidualCalibrationScorer.update(myRLS_conformal.predict)\n", 299 | " \n", 300 | " w_t_conformal = myConformalPredictor.select_best_width(myResidualCalibrationScorer, x_t)\n", 301 | " conformal_prediction_set = myResidualCalibrationScorer.get_prediction_set(x_t, w_t_conformal, mult_factor)\n", 302 | " covered_conformal_t = conformal_prediction_set.cover(y_t)\n", 303 | "\n", 304 | " w_t_basic = myBasicConformalPredictor.select_best_width(myResidualCalibrationScorer, x_t)\n", 305 | " basic_prediction_set = myResidualCalibrationScorer.get_prediction_set(x_t, w_t_basic, mult_factor)\n", 306 | " covered_basic_t = basic_prediction_set.cover(y_t)\n", 307 | "\n", 308 | " if t % 2 == 0:\n", 309 | " myConformalPredictor.update_calibration_data(x_t, y_t)\n", 310 | " myBasicConformalPredictor.update_calibration_data(x_t, y_t)\n", 311 | " else:\n", 312 | " # update the linear regression model\n", 313 | " myRLS_conformal.add_obs(x_t, y_t)\n", 314 | "\n", 315 | " y_pred_conformal_array.append(y_pred_conformal_t)\n", 316 | " w_t_conformal_array.append(mult_factor * w_t_conformal)\n", 317 | " covered_conformal_array.append(covered_conformal_t)\n", 318 | "\n", 319 | " y_pred_basic_array.append(y_pred_conformal_t)\n", 320 | " w_t_basic_array.append(mult_factor * w_t_basic)\n", 321 | " covered_basic_array.append(covered_basic_t)\n", 322 | "\n", 323 | "\n", 324 | " # 2. MVP\n", 325 | " y_pred_ours_t = myRLS_ours.predict(x_t)\n", 326 | " myResidualCalibrationScorer.update(myRLS_ours.predict)\n", 327 | "\n", 328 | " q_t = myUncertaintyQuantifier.predict(x_t)\n", 329 | " curr_prediction_set = myResidualCalibrationScorer.get_prediction_set(x_t, q_t, mult_factor)\n", 330 | "\n", 331 | " covered_ours_t = curr_prediction_set.cover(y_t)\n", 332 | "\n", 333 | " s_t = myResidualCalibrationScorer.calc_score(x_t, y_t)\n", 334 | "\n", 335 | " myRLS_ours.add_obs(x_t.T, y_t)\n", 336 | " myUncertaintyQuantifier.update(x_t, q_t, s_t)\n", 337 | "\n", 338 | " y_pred_ours_array.append(y_pred_ours_t)\n", 339 | " q_array.append(mult_factor * q_t)\n", 340 | " covered_ours_array.append(covered_ours_t)\n", 341 | " \n", 342 | " # Adding relevant values to specific groups to calculate performance metrics\n", 343 | " for i in range(10):\n", 344 | " for j in range(2):\n", 345 | " # current group is feature i with value j\n", 346 | " curr_index = (i * 2) + j\n", 347 | " curr_group = twenty_groups[curr_index]\n", 348 | " if curr_group(x_t):\n", 349 | " ys_groups[curr_index].append(y_t)\n", 350 | " y_pred_conformal_groups[curr_index].append(y_pred_conformal_t)\n", 351 | " w_t_conformal_groups[curr_index].append(w_t_conformal)\n", 352 | " covered_conformal_groups[curr_index].append(covered_conformal_t)\n", 353 | " q_array_groups[curr_index].append(q_t)\n", 354 | " y_pred_ours_groups[curr_index].append(y_pred_ours_t)\n", 355 | " covered_ours_groups[curr_index].append(covered_ours_t)\n", 356 | " y_pred_basic_groups[curr_index].append(y_pred_conformal_t)\n", 357 | " w_t_basic_groups[curr_index].append(w_t_basic)\n", 358 | " covered_basic_groups[curr_index].append(covered_basic_t)\n", 359 | "\n", 360 | "print('Trial complete')" 361 | ] 362 | }, 363 | { 364 | "cell_type": "markdown", 365 | "metadata": {}, 366 | "source": [ 367 | "##### Overall statistics (disregarding groups)" 368 | ] 369 | }, 370 | { 371 | "cell_type": "code", 372 | "execution_count": null, 373 | "metadata": { 374 | "colab": { 375 | "base_uri": "https://localhost:8080/" 376 | }, 377 | "executionInfo": { 378 | "elapsed": 159, 379 | "status": "ok", 380 | "timestamp": 1648492456525, 381 | "user": { 382 | "displayName": "DaEto Georgy", 383 | "userId": "07578216823519586108" 384 | }, 385 | "user_tz": 240 386 | }, 387 | "id": "14nVtTTIylLi", 388 | "outputId": "e9b0a906-d6b6-437d-e6d7-59344da30ca3" 389 | }, 390 | "outputs": [], 391 | "source": [ 392 | "y_pred_conformal_array = np.array(y_pred_conformal_array)\n", 393 | "w_t_conformal_array = np.array(w_t_conformal_array)\n", 394 | "\n", 395 | "y_pred_basic_array = np.array(y_pred_basic_array)\n", 396 | "w_t_basic_array = np.array(w_t_basic_array)\n", 397 | "\n", 398 | "y_pred_ours_array = np.array(y_pred_ours_array)\n", 399 | "q_array = np.array(q_array)\n", 400 | "\n", 401 | "print(\"*** COVERAGE ***\")\n", 402 | "print(\"Split-conformal (without groups): {0}\".format(np.average(covered_basic_array)))\n", 403 | "print(\"Split-conformal (with groups, conservative method): {0}\".format(np.average(covered_conformal_array)))\n", 404 | "print(\"MVP: {0}\".format(np.average(covered_ours_array)))\n", 405 | "print(\"\")\n", 406 | "\n", 407 | "print(\"*** WIDTH *** \")\n", 408 | "print(\"Split-conformal (without groups): {0}\".format(np.average(w_t_basic_array)))\n", 409 | "print(\"Split-conformal (with groups, conservative method): {0}\".format(np.average(w_t_conformal_array)))\n", 410 | "print(\"MVP: {0}\".format(np.average(q_array)))\n", 411 | "print(\"\")\n", 412 | "\n", 413 | "print(\"*** SQUARED LOSS ***\")\n", 414 | "squared_loss_basic = np.linalg.norm(ys - y_pred_basic_array)\n", 415 | "squared_loss_conformal = np.linalg.norm(ys - y_pred_conformal_array)\n", 416 | "squared_loss_ours = np.linalg.norm(ys - y_pred_ours_array)\n", 417 | "print(\"Split-conformal (without groups): {0}\".format(squared_loss_basic))\n", 418 | "print(\"Split-conformal (with groups, conservative method): {0}\".format(squared_loss_conformal))\n", 419 | "print(\"MVP: {0}\".format(squared_loss_ours))" 420 | ] 421 | }, 422 | { 423 | "cell_type": "markdown", 424 | "metadata": {}, 425 | "source": [ 426 | "##### Plots (results across all groups)" 427 | ] 428 | }, 429 | { 430 | "cell_type": "code", 431 | "execution_count": null, 432 | "metadata": {}, 433 | "outputs": [], 434 | "source": [ 435 | "# Plotting coverage across groups\n", 436 | "\n", 437 | "barWidth = 0.25\n", 438 | "br1 = np.arange(len(covered_ours_groups))\n", 439 | "br2 = [x + barWidth for x in br1]\n", 440 | "br3 = [x + barWidth for x in br2]\n", 441 | "\n", 442 | "coverage_conformal = [np.average(group) for group in covered_conformal_groups]\n", 443 | "coverage_ours = [np.average(group) for group in covered_ours_groups]\n", 444 | "coverage_basic = [np.average(group) for group in covered_basic_groups]\n", 445 | "\n", 446 | "plt.bar(br1, coverage_basic, color = 'b', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: Without groups', linewidth = 0.5)\n", 447 | "plt.bar(br2, coverage_conformal, color = 'm', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: With groups, conservative approach', linewidth = 0.5)\n", 448 | "plt.bar(br3, coverage_ours, color = 'c', width = barWidth, edgecolor = 'gray', label = 'Our Method', linewidth = 0.5)\n", 449 | "\n", 450 | "group_labels = [str(i) for i in range(num_groups)]\n", 451 | "plt.xticks([r + barWidth for r in range(len(covered_ours_groups))], group_labels)\n", 452 | "plt.axhline(y= 1 - delta, c = 'r', linewidth = 0.5)\n", 453 | "plt.text(19.55, 1 - delta + 0.02, ' desired')\n", 454 | "plt.text(19.55, 1 - delta - 0.04, ' coverage')\n", 455 | "plt.legend()\n", 456 | "plt.ylim([0.0,1.4])\n", 457 | "plt.yticks(np.arange(0, 1.1, 0.1))\n", 458 | "plt.xlabel('Groups')\n", 459 | "plt.ylabel('Average Coverage')\n", 460 | "plt.title('Comparison of group-wise coverage: Split-Conformal vs. MVP \\n')\n", 461 | "plt.show()" 462 | ] 463 | }, 464 | { 465 | "cell_type": "code", 466 | "execution_count": null, 467 | "metadata": {}, 468 | "outputs": [], 469 | "source": [ 470 | "# Plotting interval width across groups\n", 471 | "\n", 472 | "barWidth = 0.25\n", 473 | "br1 = np.arange(len(covered_ours_groups))\n", 474 | "br2 = [x + barWidth for x in br1]\n", 475 | "br3 = [x + barWidth for x in br2]\n", 476 | "\n", 477 | "width_conformal = [2 * mult_factor * np.average(group) for group in w_t_conformal_groups]\n", 478 | "width_ours = [2 * mult_factor * np.average(group) for group in q_array_groups]\n", 479 | "width_basic = [2 * mult_factor * np.average(group) for group in w_t_basic_groups]\n", 480 | "\n", 481 | "plt.bar(br1, width_basic, color = 'b', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: Without groups', linewidth = 0.5)\n", 482 | "plt.bar(br2, width_conformal, color = 'm', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: With groups, conservative approach', linewidth = 0.5)\n", 483 | "plt.bar(br3, width_ours, color = 'c', width = barWidth, edgecolor = 'gray', label = 'MVP', linewidth = 0.5)\n", 484 | "group_labels = [str(i) for i in range(num_groups)]\n", 485 | "plt.xticks([r + barWidth for r in range(len(covered_ours_groups))], group_labels)\n", 486 | "plt.legend()\n", 487 | "plt.ylim([0.0,19.0])\n", 488 | "plt.title('Comparison of group-wise interval-width: Split-Conformal vs. MVP \\n')\n", 489 | "plt.xlabel('Groups')\n", 490 | "plt.ylabel('Average Interval Width')\n", 491 | "plt.savefig('width-mean-img.png')\n", 492 | "plt.show()" 493 | ] 494 | }, 495 | { 496 | "cell_type": "markdown", 497 | "metadata": {}, 498 | "source": [ 499 | "##### Choose any specific group (defined by feature number and value)" 500 | ] 501 | }, 502 | { 503 | "cell_type": "code", 504 | "execution_count": null, 505 | "metadata": {}, 506 | "outputs": [], 507 | "source": [ 508 | "group_feat = 0 # can be any i in [num_groups]\n", 509 | "feat_val = 1 # can be 0 or 1 (binary feature)" 510 | ] 511 | }, 512 | { 513 | "cell_type": "markdown", 514 | "metadata": {}, 515 | "source": [ 516 | "##### Statistics for specified group" 517 | ] 518 | }, 519 | { 520 | "cell_type": "code", 521 | "execution_count": null, 522 | "metadata": {}, 523 | "outputs": [], 524 | "source": [ 525 | "index = (group_feat * 2) + feat_val\n", 526 | "ys_group = ys_groups[index]\n", 527 | "\n", 528 | "y_pred_conformal_array_group = y_pred_conformal_groups[index]\n", 529 | "w_t_conformal_array_group = w_t_conformal_groups[index]\n", 530 | "y_pred_conformal_array_group = np.array(y_pred_conformal_array_group)\n", 531 | "w_t_conformal_array_group = np.array(w_t_conformal_array_group)\n", 532 | "\n", 533 | "y_pred_basic_array_group = y_pred_basic_groups[index]\n", 534 | "w_t_basic_array_group = w_t_basic_groups[index]\n", 535 | "y_pred_basic_array_group = np.array(y_pred_basic_array_group)\n", 536 | "w_t_basic_array_group = np.array(w_t_basic_array_group)\n", 537 | "\n", 538 | "y_pred_ours_array_group = y_pred_ours_groups[index]\n", 539 | "q_array_group = q_array_groups[index]\n", 540 | "y_pred_ours_array_group = np.array(y_pred_ours_array_group)\n", 541 | "q_array_group = np.array(q_array_group)\n", 542 | "\n", 543 | "covered_basic_array_group = covered_basic_groups[index]\n", 544 | "covered_conformal_array_group = covered_conformal_groups[index]\n", 545 | "covered_ours_array_group = covered_ours_groups[index]\n", 546 | " \n", 547 | "print(\"======= CURRENT GROUP: FEATURE \" + str(group_feat) + \" = \" + str(feat_val) + \" =======\")\n", 548 | "print(\"*** COVERAGE ***\")\n", 549 | "print(\"Split-conformal (without): {0}\".format(np.average(covered_basic_array_group)))\n", 550 | "print(\"Split-conformal (with groups, conservative approach): {0}\".format(np.average(covered_conformal_array_group)))\n", 551 | "print(\"MVP: {0}\".format(np.average(covered_ours_array_group)))\n", 552 | "print(\"\")\n", 553 | "\n", 554 | "print(\"*** WIDTH *** \")\n", 555 | "print(\"Split-conformal (without): {0}\".format(np.average(w_t_basic_array_group)))\n", 556 | "print(\"Split-conformal (with groups, conservative approach): {0}\".format(np.average(w_t_conformal_array_group)))\n", 557 | "print(\"MVP: {0}\".format(np.average(q_array_group)))\n", 558 | "print(\"\")\n", 559 | "\n", 560 | "print(\"*** SQUARED LOSS ***\")\n", 561 | "squared_loss_basic = np.linalg.norm(ys_group - y_pred_basic_array_group)\n", 562 | "squared_loss_conformal = np.linalg.norm(ys_group - y_pred_conformal_array_group)\n", 563 | "squared_loss_ours = np.linalg.norm(ys_group - y_pred_ours_array_group)\n", 564 | "print(\"Split-conformal (without groups): {0}\".format(squared_loss_basic))\n", 565 | "print(\"Split-conformal (with groups, conservative approach): {0}\".format(squared_loss_conformal))\n", 566 | "print(\"MVP: {0}\".format(squared_loss_ours))\n" 567 | ] 568 | }, 569 | { 570 | "cell_type": "markdown", 571 | "metadata": {}, 572 | "source": [ 573 | "##### Plots for specified group" 574 | ] 575 | }, 576 | { 577 | "cell_type": "code", 578 | "execution_count": null, 579 | "metadata": {}, 580 | "outputs": [], 581 | "source": [ 582 | "# Plotting average coverage (conformal - basic) over time - for selected group\n", 583 | "covered_basic_over_time_group = [np.average(covered_basic_array_group[:t+1]) for t in range(len(covered_basic_array_group))]\n", 584 | "plt.plot(range(len(covered_basic_array_group)), covered_basic_over_time_group)\n", 585 | "plt.yticks(np.arange(0, 1.05, 0.05))\n", 586 | "plt.xlabel(\"No. of rounds\")\n", 587 | "plt.ylabel(\"Marginal coverage\")\n", 588 | "plt.axhline(y = 1 - delta, color = 'red', linestyle = '-', linewidth = 0.9)\n", 589 | "plt.text(len(covered_basic_array_group), 1 - delta + 0.01, ' desired coverage')\n", 590 | "plt.title(\"Coverage over time: Split-conformal (without groups)\")\n", 591 | "plt.show()" 592 | ] 593 | }, 594 | { 595 | "cell_type": "code", 596 | "execution_count": null, 597 | "metadata": { 598 | "colab": { 599 | "base_uri": "https://localhost:8080/", 600 | "height": 282 601 | }, 602 | "executionInfo": { 603 | "elapsed": 1769, 604 | "status": "ok", 605 | "timestamp": 1648492503363, 606 | "user": { 607 | "displayName": "DaEto Georgy", 608 | "userId": "07578216823519586108" 609 | }, 610 | "user_tz": 240 611 | }, 612 | "id": "yFvyxFTtylLk", 613 | "outputId": "bb24f7bf-a3b1-492e-ebc1-58fb1b615fba" 614 | }, 615 | "outputs": [], 616 | "source": [ 617 | "# Plotting average coverage (conformal) over time - for selected group\n", 618 | "covered_conformal_over_time_group = [np.average(covered_conformal_array_group[:t+1]) for t in range(len(covered_conformal_array_group))]\n", 619 | "plt.plot(range(len(covered_conformal_array_group)), covered_conformal_over_time_group)\n", 620 | "plt.yticks(np.arange(0, 1.05, 0.05))\n", 621 | "plt.xlabel(\"No. of rounds\")\n", 622 | "plt.ylabel(\"Marginal coverage\")\n", 623 | "plt.axhline(y = 1 - delta, color = 'red', linestyle = '-', linewidth = 0.9)\n", 624 | "plt.text(len(covered_conformal_array_group), 1 - delta + 0.01, ' desired coverage')\n", 625 | "plt.title(\"Coverage over time: Split-conformal (With groups, conservative approach)\")\n", 626 | "plt.show()" 627 | ] 628 | }, 629 | { 630 | "cell_type": "code", 631 | "execution_count": null, 632 | "metadata": { 633 | "colab": { 634 | "base_uri": "https://localhost:8080/", 635 | "height": 282 636 | }, 637 | "executionInfo": { 638 | "elapsed": 634, 639 | "status": "ok", 640 | "timestamp": 1648492529836, 641 | "user": { 642 | "displayName": "DaEto Georgy", 643 | "userId": "07578216823519586108" 644 | }, 645 | "user_tz": 240 646 | }, 647 | "id": "UxlLSpxEylLk", 648 | "outputId": "fe587ba7-0df5-4bf2-d995-f358ca6a5c35" 649 | }, 650 | "outputs": [], 651 | "source": [ 652 | "# Plotting average coverage(our method) over time - for chosen group\n", 653 | "covered_our_over_time_group = [np.average(covered_ours_array_group[:t+1]) for t in range(len(covered_ours_array_group))]\n", 654 | "plt.plot(range(len(covered_ours_array_group)), covered_our_over_time_group)\n", 655 | "plt.yticks(np.arange(0, 1.05, 0.05))\n", 656 | "plt.xlabel(\"No. of rounds\")\n", 657 | "plt.ylabel(\"Marginal coverage\")\n", 658 | "plt.axhline(y = 1 - delta, color = 'red', linestyle = '-', linewidth = 0.9)\n", 659 | "plt.text(len(covered_ours_array_group), 1 - delta + 0.01, ' desired coverage')\n", 660 | "plt.title(\"Coverage over time: MVP\")\n", 661 | "plt.show()" 662 | ] 663 | }, 664 | { 665 | "cell_type": "markdown", 666 | "metadata": {}, 667 | "source": [ 668 | "## Running several trials" 669 | ] 670 | }, 671 | { 672 | "cell_type": "markdown", 673 | "metadata": {}, 674 | "source": [ 675 | "##### Set all parameters and number of trials to run" 676 | ] 677 | }, 678 | { 679 | "cell_type": "code", 680 | "execution_count": null, 681 | "metadata": {}, 682 | "outputs": [], 683 | "source": [ 684 | "# Parameters for our uncertainty quantifier\n", 685 | "T = 20000\n", 686 | "n = 40\n", 687 | "r = 80000000\n", 688 | "delta = 0.1\n", 689 | "K_e = 2.12\n", 690 | "\n", 691 | "eta = np.sqrt(np.log(num_groups * n) / (2 * K_e * num_groups * n))\n", 692 | "\n", 693 | "# data generation - constants\n", 694 | "x_std = 0.1\n", 695 | "y_std = 0.25\n", 696 | "d = 300 # choose d > 10\n", 697 | "mult_factor = 10\n", 698 | "std_dev_list = np.array([3.0, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1])\n", 699 | "\n", 700 | "# How many trials would you like to run?\n", 701 | "num_rounds = 10" 702 | ] 703 | }, 704 | { 705 | "cell_type": "markdown", 706 | "metadata": {}, 707 | "source": [ 708 | "##### Start trials" 709 | ] 710 | }, 711 | { 712 | "cell_type": "code", 713 | "execution_count": null, 714 | "metadata": {}, 715 | "outputs": [], 716 | "source": [ 717 | "coverage_conformal_per_group_per_round = np.zeros((num_rounds, num_groups))\n", 718 | "coverage_ours_per_group_per_round = np.zeros((num_rounds, num_groups))\n", 719 | "coverage_basic_per_group_per_round = np.zeros((num_rounds, num_groups))\n", 720 | "width_conformal_per_group_per_round = np.zeros((num_rounds, num_groups))\n", 721 | "width_ours_per_group_per_round = np.zeros((num_rounds, num_groups))\n", 722 | "width_basic_per_group_per_round = np.zeros((num_rounds, num_groups))\n", 723 | "\n", 724 | "for k in range(num_rounds):\n", 725 | " print('Running trial: ' + str(k))\n", 726 | " \n", 727 | " # data generation - random for each run\n", 728 | " theta = np.random.normal(loc=np.zeros(d), scale=x_std)\n", 729 | "\n", 730 | " # d-dimension features - first 10 features are binary\n", 731 | " xs_binvars = np.random.randint(low = 0, high = 2, size = (T, 10))\n", 732 | " xs_remvars = np.random.normal(loc=np.zeros(d - 10), scale=x_std, size=(T, d - 10))\n", 733 | " xs = np.concatenate((xs_binvars, xs_remvars), axis = 1)\n", 734 | " std_dev = np.dot(xs_binvars, std_dev_list) + y_std\n", 735 | " ys = np.dot(xs, theta) + np.random.normal(loc=0, scale= std_dev, size=T)\n", 736 | " \n", 737 | " # Defining all uncertainty quantifiers and regressors\n", 738 | "\n", 739 | " myUncertaintyQuantifier = MultiValidPrediction(delta, n, twenty_groups, eta, r, normalize_by_counts=True)\n", 740 | " myConformalPredictor = splitConformalPrediction.splitConformal(num_groups, twenty_groups, delta)\n", 741 | " myBasicConformalPredictor = splitConformalPrediction.splitConformal(1, basic_group, delta)\n", 742 | " myRLS_conformal = recursiveLeastSquares.RLS(d, 1.0, 1)\n", 743 | " myRLS_ours = recursiveLeastSquares.RLS(d, 1.0, 1)\n", 744 | "\n", 745 | " myResidualCalibrationScorer = customResidualCalibrationScorer.customResidualCalibrationScorer(mult_factor)\n", 746 | "\n", 747 | " # arrays for our method\n", 748 | " q_array = []\n", 749 | " y_pred_ours_array = []\n", 750 | " covered_ours_array = []\n", 751 | "\n", 752 | " # arrays (per group) for our method\n", 753 | " q_array_groups = [list() for i in range(num_groups)] \n", 754 | " y_pred_ours_groups = [list() for i in range(num_groups)] \n", 755 | " covered_ours_groups = [list() for i in range(num_groups)] \n", 756 | "\n", 757 | " # arrays for conformal prediction\n", 758 | " y_pred_conformal_array = []\n", 759 | " w_t_conformal_array = []\n", 760 | " covered_conformal_array = []\n", 761 | "\n", 762 | " # arrays (per group) for conformal prediction\n", 763 | " y_pred_conformal_groups = [list() for i in range(num_groups)]\n", 764 | " w_t_conformal_groups = [list() for i in range(num_groups)] \n", 765 | " covered_conformal_groups = [list() for i in range(num_groups)]\n", 766 | " \n", 767 | " # arrays for basic method\n", 768 | " y_pred_basic_array = []\n", 769 | " w_t_basic_array = []\n", 770 | " covered_basic_array = []\n", 771 | " \n", 772 | " # array (per group) for basic prediction\n", 773 | " y_pred_basic_groups = [list() for i in range(num_groups)]\n", 774 | " w_t_basic_groups = [list() for i in range(num_groups)]\n", 775 | " covered_basic_groups = [list() for i in range(num_groups)]\n", 776 | "\n", 777 | " # keep track of y values per group\n", 778 | " ys_groups = [list() for i in range(num_groups)]\n", 779 | "\n", 780 | "\n", 781 | " # Conformal warm-start calibration set using the same distribution. \n", 782 | " init_size = 15\n", 783 | " conformal_calibration_xs_binvars = np.random.randint(low = 0, high = 2, size = (init_size, 10))\n", 784 | " conformal_calibration_xs_remvars = np.random.normal(loc=np.zeros(d - 10), scale=x_std, size=(init_size, d - 10))\n", 785 | " xs_cc = np.concatenate((conformal_calibration_xs_binvars, conformal_calibration_xs_remvars), axis = 1)\n", 786 | " std_dev = np.dot(conformal_calibration_xs_binvars, std_dev_list) + y_std\n", 787 | " ys_cc = np.dot(xs_cc, theta) + np.random.normal(loc=0, scale= std_dev, size=init_size)\n", 788 | "\n", 789 | " for i, curr_y in enumerate(ys_cc):\n", 790 | " curr_x = xs_cc[i]\n", 791 | " myConformalPredictor.update_calibration_data(curr_x, curr_y)\n", 792 | " myBasicConformalPredictor.update_calibration_data(curr_x, curr_y)\n", 793 | "\n", 794 | " print('Does initial calibration data cover all groups: ' + str(myConformalPredictor.all_groups_covered()))\n", 795 | " # Run algorithm\n", 796 | " \n", 797 | " for t in range(T):\n", 798 | " x_t = xs[t]\n", 799 | " y_t = ys[t]\n", 800 | " \n", 801 | " # 1. SPLIT-CONFORMAL WITH AND WITHOUT GROUPS\n", 802 | "\n", 803 | " y_pred_conformal_t = myRLS_conformal.predict(x_t)\n", 804 | " myResidualCalibrationScorer.update(myRLS_conformal.predict)\n", 805 | "\n", 806 | " w_t_conformal = myConformalPredictor.select_best_width(myResidualCalibrationScorer, x_t)\n", 807 | " conformal_prediction_set = myResidualCalibrationScorer.get_prediction_set(x_t, w_t_conformal, mult_factor)\n", 808 | "\n", 809 | " covered_conformal_t = conformal_prediction_set.cover(y_t)\n", 810 | " \n", 811 | " w_t_basic = myBasicConformalPredictor.select_best_width(myResidualCalibrationScorer, x_t)\n", 812 | " basic_prediction_set = myResidualCalibrationScorer.get_prediction_set(x_t, w_t_basic, mult_factor)\n", 813 | " \n", 814 | " covered_basic_t = basic_prediction_set.cover(y_t)\n", 815 | "\n", 816 | " if t % 2 == 0:\n", 817 | " myConformalPredictor.update_calibration_data(x_t, y_t)\n", 818 | " myBasicConformalPredictor.update_calibration_data(x_t, y_t)\n", 819 | " else:\n", 820 | " # update the linear regression model\n", 821 | " myRLS_conformal.add_obs(x_t, y_t)\n", 822 | "\n", 823 | " y_pred_conformal_array.append(y_pred_conformal_t)\n", 824 | " w_t_conformal_array.append(mult_factor * w_t_conformal)\n", 825 | " covered_conformal_array.append(covered_conformal_t)\n", 826 | " \n", 827 | " y_pred_basic_array.append(y_pred_conformal_t)\n", 828 | " w_t_basic_array.append(mult_factor * w_t_basic)\n", 829 | " covered_basic_array.append(covered_basic_t)\n", 830 | "\n", 831 | "\n", 832 | " # 2. MVP\n", 833 | " y_pred_ours_t = myRLS_ours.predict(x_t)\n", 834 | " myResidualCalibrationScorer.update(myRLS_ours.predict)\n", 835 | "\n", 836 | " q_t = myUncertaintyQuantifier.predict(x_t)\n", 837 | " curr_prediction_set = myResidualCalibrationScorer.get_prediction_set(x_t, q_t, mult_factor)\n", 838 | "\n", 839 | " covered_ours_t = curr_prediction_set.cover(y_t)\n", 840 | "\n", 841 | " s_t = myResidualCalibrationScorer.calc_score(x_t, y_t)\n", 842 | "\n", 843 | " myRLS_ours.add_obs(x_t.T, y_t)\n", 844 | " myUncertaintyQuantifier.update(x_t, q_t, s_t)\n", 845 | "\n", 846 | " y_pred_ours_array.append(y_pred_ours_t)\n", 847 | " q_array.append(mult_factor * q_t)\n", 848 | " covered_ours_array.append(covered_ours_t)\n", 849 | "\n", 850 | " # Adding relevant values to specific groups\n", 851 | " for i in range(10):\n", 852 | " for j in range(2):\n", 853 | " # current group is feature i with value j\n", 854 | " curr_index = (i * 2) + j\n", 855 | " curr_group = twenty_groups[curr_index]\n", 856 | " if curr_group(x_t):\n", 857 | " ys_groups[curr_index].append(y_t)\n", 858 | " y_pred_conformal_groups[curr_index].append(y_pred_conformal_t)\n", 859 | " w_t_conformal_groups[curr_index].append(w_t_conformal)\n", 860 | " covered_conformal_groups[curr_index].append(covered_conformal_t)\n", 861 | " q_array_groups[curr_index].append(q_t)\n", 862 | " y_pred_ours_groups[curr_index].append(y_pred_ours_t)\n", 863 | " covered_ours_groups[curr_index].append(covered_ours_t)\n", 864 | " y_pred_basic_groups[curr_index].append(y_pred_conformal_t)\n", 865 | " w_t_basic_groups[curr_index].append(w_t_basic)\n", 866 | " covered_basic_groups[curr_index].append(covered_basic_t)\n", 867 | " \n", 868 | " y_pred_conformal_array = np.array(y_pred_conformal_array)\n", 869 | " w_t_conformal_array = np.array(w_t_conformal_array)\n", 870 | " w_t_basic_array = np.array(w_t_basic_array)\n", 871 | "\n", 872 | " y_pred_ours_array = np.array(y_pred_ours_array)\n", 873 | " q_array = np.array(q_array)\n", 874 | " \n", 875 | " coverage_conformal = [np.average(group) for group in covered_conformal_groups]\n", 876 | " coverage_ours = [np.average(group) for group in covered_ours_groups]\n", 877 | " coverage_basic = [np.average(group) for group in covered_basic_groups]\n", 878 | " \n", 879 | " width_conformal = [2 * mult_factor * np.average(group) for group in w_t_conformal_groups]\n", 880 | " width_ours = [2 * mult_factor * np.average(group) for group in q_array_groups]\n", 881 | " width_basic = [2 * mult_factor * np.average(group) for group in w_t_basic_groups]\n", 882 | " \n", 883 | " coverage_conformal_per_group_per_round[k] = coverage_conformal\n", 884 | " coverage_ours_per_group_per_round[k] = coverage_ours\n", 885 | " coverage_basic_per_group_per_round[k] = coverage_basic\n", 886 | " \n", 887 | " width_conformal_per_group_per_round[k] = width_conformal\n", 888 | " width_ours_per_group_per_round[k] = width_ours\n", 889 | " width_basic_per_group_per_round[k] = width_basic\n", 890 | " \n", 891 | "print('All trials complete') " 892 | ] 893 | }, 894 | { 895 | "cell_type": "markdown", 896 | "metadata": {}, 897 | "source": [ 898 | "##### Plots for statistics across groups" 899 | ] 900 | }, 901 | { 902 | "cell_type": "code", 903 | "execution_count": null, 904 | "metadata": {}, 905 | "outputs": [], 906 | "source": [ 907 | "# Plotting average coverage across all rounds for all groups\n", 908 | "\n", 909 | "coverage_conformal_across_rounds = np.mean(coverage_conformal_per_group_per_round, axis = 0)\n", 910 | "coverage_ours_across_rounds = np.mean(coverage_ours_per_group_per_round, axis = 0)\n", 911 | "coverage_basic_across_rounds = np.mean(coverage_basic_per_group_per_round, axis = 0)\n", 912 | "\n", 913 | "barWidth = 0.25\n", 914 | "br1 = np.arange(len(coverage_ours_across_rounds))\n", 915 | "br2 = [x + barWidth for x in br1]\n", 916 | "br3 = [x + barWidth for x in br2]\n", 917 | "\n", 918 | "plt.bar(br1, coverage_basic_across_rounds, color = 'b', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: Without groups', linewidth = 0.5)\n", 919 | "plt.bar(br2, coverage_conformal_across_rounds, color = 'm', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: With groups, conservative approach', linewidth = 0.5)\n", 920 | "plt.bar(br3, coverage_ours_across_rounds, color = 'c', width = barWidth, edgecolor = 'gray', label = 'MVP', linewidth = 0.5)\n", 921 | "group_labels = [str(i) for i in range(num_groups)]\n", 922 | "plt.xticks([r + barWidth for r in range(len(coverage_ours_across_rounds))], group_labels)\n", 923 | "plt.axhline(y= 1 - delta, c = 'r', linewidth = 0.5)\n", 924 | "plt.text(19.55, 1 - delta + 0.02, ' desired')\n", 925 | "plt.text(19.55, 1 - delta - 0.04, ' coverage')\n", 926 | "plt.legend()\n", 927 | "plt.ylim([0.0,1.4])\n", 928 | "plt.yticks(np.arange(0, 1.1, 0.1))\n", 929 | "plt.title('Comparison of group-wise coverage: Split-Conformal vs. MVP \\n')\n", 930 | "plt.xlabel('Groups')\n", 931 | "plt.ylabel('Average Coverage')\n", 932 | "# plt.savefig('coverage-mean.pdf')\n", 933 | "plt.show()" 934 | ] 935 | }, 936 | { 937 | "cell_type": "code", 938 | "execution_count": null, 939 | "metadata": {}, 940 | "outputs": [], 941 | "source": [ 942 | "# Plotting average interval width across all rounds for all groups\n", 943 | "\n", 944 | "width_conformal_across_rounds = np.mean(width_conformal_per_group_per_round, axis = 0)\n", 945 | "width_ours_across_rounds = np.mean(width_ours_per_group_per_round, axis = 0)\n", 946 | "width_basic_across_rounds = np.mean(width_basic_per_group_per_round, axis = 0)\n", 947 | "\n", 948 | "barWidth = 0.25\n", 949 | "br1 = np.arange(len(coverage_ours_across_rounds))\n", 950 | "br2 = [x + barWidth for x in br1]\n", 951 | "br3 = [x + barWidth for x in br2]\n", 952 | "\n", 953 | "plt.bar(br1, width_basic_across_rounds, color = 'b', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: Without groups', linewidth = 0.5)\n", 954 | "plt.bar(br2, width_conformal_across_rounds, color = 'm', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: With groups, conservative approach', linewidth = 0.5)\n", 955 | "plt.bar(br3, width_ours_across_rounds, color = 'c', width = barWidth, edgecolor = 'gray', label = 'MVP', linewidth = 0.5)\n", 956 | "group_labels = [str(i) for i in range(num_groups)]\n", 957 | "plt.xticks([r + barWidth for r in range(len(coverage_ours_across_rounds))], group_labels)\n", 958 | "plt.legend()\n", 959 | "plt.ylim([0.0,19.0])\n", 960 | "plt.title('Comparison of group-wise interval-width: Split-Conformal vs. MVP \\n')\n", 961 | "plt.xlabel('Groups')\n", 962 | "plt.ylabel('Average Interval Width')\n", 963 | "# plt.savefig('width-mean.pdf')\n", 964 | "plt.show()" 965 | ] 966 | }, 967 | { 968 | "cell_type": "code", 969 | "execution_count": null, 970 | "metadata": {}, 971 | "outputs": [], 972 | "source": [ 973 | "# Plotting median coverage across all rounds for all groups, along with 25th to 75th quantile bar\n", 974 | "\n", 975 | "coverage_conformal_across_rounds = np.median(coverage_conformal_per_group_per_round, axis = 0)\n", 976 | "coverage_ours_across_rounds = np.median(coverage_ours_per_group_per_round, axis = 0)\n", 977 | "coverage_basic_across_rounds = np.median(coverage_basic_per_group_per_round, axis = 0)\n", 978 | "\n", 979 | "coverage_conformal_across_rounds_mean = np.mean(coverage_conformal_per_group_per_round, axis = 0)\n", 980 | "coverage_ours_across_rounds_mean = np.mean(coverage_ours_per_group_per_round, axis = 0)\n", 981 | "coverage_basic_across_rounds_mean = np.mean(coverage_basic_per_group_per_round, axis = 0)\n", 982 | "\n", 983 | "coverage_conformal_25_quantile = np.quantile(coverage_conformal_per_group_per_round, 0.25, axis = 0)\n", 984 | "coverage_ours_25_quantile = np.quantile(coverage_ours_per_group_per_round, 0.25, axis = 0)\n", 985 | "coverage_basic_25_quantile = np.quantile(coverage_basic_per_group_per_round, 0.25, axis = 0)\n", 986 | "coverage_conformal_75_quantile = np.quantile(coverage_conformal_per_group_per_round, 0.75, axis = 0)\n", 987 | "coverage_ours_75_quantile = np.quantile(coverage_ours_per_group_per_round, 0.75, axis = 0)\n", 988 | "coverage_basic_75_quantile = np.quantile(coverage_basic_per_group_per_round, 0.75, axis = 0)\n", 989 | "\n", 990 | "barWidth = 0.25\n", 991 | "br1 = np.arange(len(coverage_ours_across_rounds))\n", 992 | "br2 = [x + barWidth for x in br1]\n", 993 | "br3 = [x + barWidth for x in br2]\n", 994 | "\n", 995 | "for k, val in enumerate(br1):\n", 996 | " plt.vlines(x = val, ymin = coverage_basic_25_quantile[k], ymax = coverage_basic_75_quantile[k], color = 'black', linewidth = 0.6)\n", 997 | "\n", 998 | "for i, val in enumerate(br2):\n", 999 | " plt.vlines(x = val, ymin = coverage_conformal_25_quantile[i], ymax = coverage_conformal_75_quantile[i], color = 'black', linewidth = 0.6) \n", 1000 | "\n", 1001 | "for j, val in enumerate(br3):\n", 1002 | " plt.vlines(x = val, ymin = coverage_ours_25_quantile[j], ymax = coverage_ours_75_quantile[j], color = 'black', linewidth = 0.6)\n", 1003 | "\n", 1004 | "plt.bar(br1, coverage_basic_across_rounds, color = 'b', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: Without groups', linewidth = 0.4, alpha = 0.6)\n", 1005 | "plt.bar(br2, coverage_conformal_across_rounds, color = 'm', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: With groups, conservative approach', linewidth = 0.4, alpha =0.6)\n", 1006 | "plt.bar(br3, coverage_ours_across_rounds, color = 'c', width = barWidth, edgecolor = 'gray', label = 'MVP', linewidth = 0.4, alpha = 0.6)\n", 1007 | "group_labels = [str(i) for i in range(num_groups)]\n", 1008 | "plt.xticks([r + barWidth for r in range(len(coverage_ours_across_rounds))], group_labels)\n", 1009 | "plt.axhline(y= 1 - delta, c = 'r', linewidth = 0.5)\n", 1010 | "plt.text(19.55, 1 - delta + 0.02, ' desired')\n", 1011 | "plt.text(19.55, 1 - delta - 0.04, ' coverage')\n", 1012 | "plt.legend()\n", 1013 | "plt.ylim([0.0,1.5])\n", 1014 | "plt.yticks(np.arange(0, 1.1, 0.1))\n", 1015 | "plt.xlabel('Groups')\n", 1016 | "plt.ylabel('Coverage (Median)')\n", 1017 | "plt.title('Comparison of group-wise coverage: Split-Conformal vs. MVP \\n')\n", 1018 | "# plt.savefig('../../coverage-median.pdf')\n", 1019 | "plt.show()\n" 1020 | ] 1021 | }, 1022 | { 1023 | "cell_type": "code", 1024 | "execution_count": null, 1025 | "metadata": {}, 1026 | "outputs": [], 1027 | "source": [ 1028 | "# Plotting median interval width across all rounds for all groups, along with 25th to 75th quantile bar\n", 1029 | "\n", 1030 | "width_conformal_across_rounds = np.median(width_conformal_per_group_per_round, axis = 0)\n", 1031 | "width_ours_across_rounds = np.median(width_ours_per_group_per_round, axis = 0)\n", 1032 | "width_basic_across_round = np.median(width_basic_per_group_per_round, axis = 0)\n", 1033 | "\n", 1034 | "width_conformal_25_quantile = np.quantile(width_conformal_per_group_per_round, 0.25, axis = 0)\n", 1035 | "width_ours_25_quantile = np.quantile(width_ours_per_group_per_round, 0.25, axis = 0)\n", 1036 | "width_basic_25_quantile = np.quantile(width_basic_per_group_per_round, 0.25, axis = 0)\n", 1037 | "width_conformal_75_quantile = np.quantile(width_conformal_per_group_per_round, 0.75, axis = 0)\n", 1038 | "width_ours_75_quantile = np.quantile(width_ours_per_group_per_round, 0.75, axis = 0)\n", 1039 | "width_basic_75_quantile = np.quantile(width_basic_per_group_per_round, 0.75, axis = 0)\n", 1040 | " \n", 1041 | "barWidth = 0.25\n", 1042 | "br1 = np.arange(len(width_conformal_across_rounds))\n", 1043 | "br2 = [x + barWidth for x in br1]\n", 1044 | "br3 = [x + barWidth for x in br2]\n", 1045 | "\n", 1046 | "for k, val in enumerate(br1):\n", 1047 | " plt.vlines(x = val, ymin = width_basic_25_quantile[k], ymax = width_basic_75_quantile[k], color = 'black', linewidth = 0.6)\n", 1048 | " \n", 1049 | "for i, val in enumerate(br2):\n", 1050 | " plt.vlines(x = val, ymin = width_conformal_25_quantile[i], ymax = width_conformal_75_quantile[i], color = 'black', linewidth = 0.6)\n", 1051 | "\n", 1052 | "for j, val in enumerate(br3):\n", 1053 | " plt.vlines(x = val, ymin = width_ours_25_quantile[j], ymax = width_ours_75_quantile[j], color = 'black', linewidth = 0.6)\n", 1054 | " \n", 1055 | "plt.bar(br1, width_basic_across_round, color = 'b', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: Without groups', linewidth = 0.4, alpha = 0.6)\n", 1056 | "plt.bar(br2, width_conformal_across_rounds, color = 'm', width = barWidth, edgecolor = 'gray', label = 'Split-Conformal: With groups, conservative approach', linewidth = 0.4, alpha = 0.6)\n", 1057 | "plt.bar(br3, width_ours_across_rounds, color = 'c', width = barWidth, edgecolor = 'gray', label = 'MVP', linewidth = 0.4, alpha = 0.6)\n", 1058 | "group_labels = [str(i) for i in range(num_groups)]\n", 1059 | "plt.xticks([r + barWidth for r in range(len(covered_ours_groups))], group_labels)\n", 1060 | "plt.legend()\n", 1061 | "plt.ylim([0.0,20.0])\n", 1062 | "plt.title('Comparison of group-wise interval-width: Split-Conformal vs. MVP \\n')\n", 1063 | "plt.xlabel('Groups')\n", 1064 | "plt.ylabel('Interval Width (Median)')\n", 1065 | "# plt.savefig('../../width-median.pdf')\n", 1066 | "plt.show()" 1067 | ] 1068 | } 1069 | ], 1070 | "metadata": { 1071 | "colab": { 1072 | "name": "Linear Regression Experiment.ipynb", 1073 | "provenance": [] 1074 | }, 1075 | "kernelspec": { 1076 | "display_name": "Python 3 (ipykernel)", 1077 | "language": "python", 1078 | "name": "python3" 1079 | }, 1080 | "language_info": { 1081 | "codemirror_mode": { 1082 | "name": "ipython", 1083 | "version": 3 1084 | }, 1085 | "file_extension": ".py", 1086 | "mimetype": "text/x-python", 1087 | "name": "python", 1088 | "nbconvert_exporter": "python", 1089 | "pygments_lexer": "ipython3", 1090 | "version": "3.9.12" 1091 | } 1092 | }, 1093 | "nbformat": 4, 1094 | "nbformat_minor": 1 1095 | } 1096 | -------------------------------------------------------------------------------- /experiments/Time Series - Stock Returns.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "YzsYar8Oujiq" 7 | }, 8 | "source": [ 9 | "###### Utils" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "colab": { 17 | "base_uri": "https://localhost:8080/" 18 | }, 19 | "id": "gVjC6UH-uidr", 20 | "outputId": "8ace0229-bcc5-4772-f4a7-f7be771a47d8" 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "import sys, os\n", 25 | "\n", 26 | "sys.path.append(os.path.join(os.getcwd(), '../src'))\n", 27 | "sys.path.append(os.path.join(os.getcwd(), './data'))\n", 28 | "import csv\n", 29 | "import numpy as np, matplotlib.pyplot as plt\n", 30 | "plt.rcParams['figure.figsize'] = [20, 6]\n", 31 | "from datetime import datetime\n", 32 | "import calendar\n", 33 | "\n", 34 | "from arch import arch_model\n", 35 | "\n", 36 | "from MultiValidPrediction import MultiValidPrediction" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": { 43 | "id": "ZdUuqeZmvk9a" 44 | }, 45 | "outputs": [], 46 | "source": [ 47 | "def stock_history(name, datefrom='01/03/00', dateto='12/31/20', ret_scale=100):\n", 48 | " file = open('.\\data\\\\' + name + '.csv')\n", 49 | " csv_file = csv.reader(file)\n", 50 | " header= next(csv_file)\n", 51 | "\n", 52 | " rows = []\n", 53 | " for row in csv_file:\n", 54 | " rows.append(row)\n", 55 | "\n", 56 | " rows = np.array(rows)\n", 57 | "\n", 58 | " dates = np.array(rows[:, 0])\n", 59 | " open_prices = np.array([float(price) for price in rows[:, 1]])\n", 60 | "\n", 61 | " begin = np.where(dates==datefrom)[0][0]\n", 62 | "\n", 63 | " end = np.where(dates==dateto)[0][0]\n", 64 | "\n", 65 | " prices = open_prices[end:begin][::-1]\n", 66 | "\n", 67 | " T = len(prices)\n", 68 | "\n", 69 | " returns = [(prices[1]/prices[0]-1)]\n", 70 | " for t in range(1, T):\n", 71 | " returns.append(prices[t]/prices[t-1] - 1)\n", 72 | " \n", 73 | " returns = [ret * ret_scale for ret in returns] # scale returns\n", 74 | "\n", 75 | " volatility = [ret**2 for ret in returns]\n", 76 | "\n", 77 | " f, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=[15, 12], sharex=True)\n", 78 | " ax1.plot(prices)\n", 79 | " ax1.set_ylabel(name + ' prices')\n", 80 | " ax2.plot(returns)\n", 81 | " ax2.set_ylabel(name + ' returns')\n", 82 | " ax3.plot(volatility)\n", 83 | " ax3.set_ylabel(name + ' volatility')\n", 84 | " f.suptitle(name + ' Historical Data', fontsize=20)\n", 85 | " f.show()\n", 86 | " \n", 87 | " return T, prices, returns, volatility" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": { 94 | "id": "AZLZ8PyOvo4c" 95 | }, 96 | "outputs": [], 97 | "source": [ 98 | "def garch_scores(returns, volatility, lookback=100, offset=10, score_type='regr', score_unit_interval=True, norm_const=-1):\n", 99 | " # initialize prediction model\n", 100 | " garch_model = arch_model(returns, vol='Garch', p=1, q=1)\n", 101 | "\n", 102 | " predictions = [0 for i in range(offset)]\n", 103 | " scores = [0 for i in range(offset)]\n", 104 | "\n", 105 | " for t in range(offset, len(volatility)):\n", 106 | " # current window indices\n", 107 | " left_ind = max(0, t-lookback)\n", 108 | " right_ind = t\n", 109 | "\n", 110 | " # compute score\n", 111 | " variance_pred_array = garch_model.fit(first_obs=left_ind, last_obs=right_ind, disp='off', show_warning=False).forecast(reindex=False).variance\n", 112 | " varNext = variance_pred_array.iloc[0]['h.1'] #['h.1'][lookback-1]\n", 113 | " score = score_fn(actual=volatility[t], pred=varNext, score_type=score_type, unit_interval_norm=score_unit_interval, divide_by_const=(norm_const != -1), norm_const=norm_const)\n", 114 | "\n", 115 | " # update arrays with data\n", 116 | " scores.append(score)\n", 117 | " predictions.append(varNext)\n", 118 | "\n", 119 | " return scores, predictions\n", 120 | "\n", 121 | "def score_fn(actual, pred, score_type, unit_interval_norm=False, divide_by_const=False, norm_const=1000):\n", 122 | " # what kind of score?\n", 123 | " if score_type=='regr':\n", 124 | " scr = abs(actual-pred)\n", 125 | " if score_type=='regr_normalized':\n", 126 | " scr = abs(actual-pred)/pred\n", 127 | " \n", 128 | " # normalize score into [0, 1]\n", 129 | " if unit_interval_norm: \n", 130 | " if divide_by_const:\n", 131 | " scr /= norm_const\n", 132 | " else:\n", 133 | " scr = scr/(1+scr)\n", 134 | " return scr" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": { 141 | "id": "6kDqK8hExbrs" 142 | }, 143 | "outputs": [], 144 | "source": [ 145 | "def ACI_method(scores, alpha=0.1, lookback=100, offset=10, gamma=0.005):\n", 146 | " T = len(scores)\n", 147 | "\n", 148 | " # initialize ACI\n", 149 | " alphat = alpha\n", 150 | "\n", 151 | " alphas = [alpha for i in range(offset)]\n", 152 | " thresholds = [0 for i in range(offset)]\n", 153 | " err_seq = [0 for i in range(offset)]\n", 154 | "\n", 155 | " for t in range(offset, T):\n", 156 | " # current scoring window\n", 157 | " left_ind = max(0, t-lookback)\n", 158 | " right_ind = t\n", 159 | " recent_scores = scores[left_ind: right_ind]\n", 160 | "\n", 161 | " if 1 - alphat > 1:\n", 162 | " threshold = 1\n", 163 | " elif 1 - alphat < 0:\n", 164 | " threshold = 0\n", 165 | " else:\n", 166 | " threshold = np.quantile(recent_scores, 1-alphat)\n", 167 | " err_ind = int(scores[t] > threshold)\n", 168 | "\n", 169 | " # ACI alphat update\n", 170 | " alphat = alphat + gamma*(alpha-err_ind)\n", 171 | "\n", 172 | " # update arrays with data\n", 173 | " alphas.append(alphat)\n", 174 | " thresholds.append(threshold)\n", 175 | " err_seq.append(err_ind)\n", 176 | " \n", 177 | " miscoverage_rate_ACI = np.mean(np.array(err_seq))\n", 178 | " print('ACI miscoverage rate: ', miscoverage_rate_ACI)\n", 179 | "\n", 180 | " return thresholds, alphas, miscoverage_rate_ACI" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": { 186 | "id": "dDuyI7HpxpD5" 187 | }, 188 | "source": [ 189 | "###### Data processing" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "metadata": { 196 | "colab": { 197 | "base_uri": "https://localhost:8080/", 198 | "height": 787 199 | }, 200 | "id": "v4AQh3P8xuB9", 201 | "outputId": "fb51a15e-4bd5-40eb-9949-824350503656" 202 | }, 203 | "outputs": [], 204 | "source": [ 205 | "T, prices, returns, volatility = stock_history('AMD', ret_scale=100)" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": { 212 | "id": "9ZbIY-H-x8Xi" 213 | }, 214 | "outputs": [], 215 | "source": [ 216 | "# CREATE NOISY DATA FOR MULTIGROUP EXPERIMENT\n", 217 | "\n", 218 | "num_groups = 20\n", 219 | "\n", 220 | "returns_noisy = np.zeros(len(returns))\n", 221 | "std_returns = np.std(returns)\n", 222 | "for t in range(len(returns)):\n", 223 | " returns_noisy[t] = returns[t]\n", 224 | " for j in range(1, num_groups+1):\n", 225 | " if t % j == 0:\n", 226 | " returns_noisy[t] += std_returns*np.random.randn()\n", 227 | "\n", 228 | "volatility_noisy = [ret**2 for ret in returns_noisy]" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": { 235 | "id": "xC6OFeX5yQ43" 236 | }, 237 | "outputs": [], 238 | "source": [ 239 | "scores_noisy, predictions_noisy = garch_scores(returns_noisy, volatility_noisy)" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": null, 245 | "metadata": { 246 | "id": "ZDfysDbMysvE" 247 | }, 248 | "outputs": [], 249 | "source": [ 250 | "scores, predictions = garch_scores(returns, volatility)" 251 | ] 252 | }, 253 | { 254 | "cell_type": "markdown", 255 | "metadata": { 256 | "id": "uNNjqUjlzRD3" 257 | }, 258 | "source": [ 259 | "###### Experiment: Noisy Multigroup Coverage" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": null, 265 | "metadata": { 266 | "colab": { 267 | "base_uri": "https://localhost:8080/" 268 | }, 269 | "id": "QtyqjdJrzfgJ", 270 | "outputId": "b2f25111-0c7b-4530-d219-bc2d1a165858" 271 | }, 272 | "outputs": [], 273 | "source": [ 274 | "# ACI Analysis\n", 275 | "\n", 276 | "ACI_thrs, ACI_alphas, ACI_miscoverage_rate = ACI_method(scores_noisy, alpha=0.1, lookback=100, offset=10, gamma=0.005)\n", 277 | "\n", 278 | "score_seq = scores_noisy\n", 279 | "thrs_seq = ACI_thrs\n", 280 | "print_ACI_group_stats = False\n", 281 | "\n", 282 | "g_miscoverage = np.zeros(num_groups+1, dtype=int)\n", 283 | "g_counts = np.zeros(num_groups+1, dtype=int)\n", 284 | "\n", 285 | "g_counts_residual = 0\n", 286 | "g_miscoverage_residual = 0\n", 287 | "g_counts_div_by_smth = 0\n", 288 | "g_miscoverage_div_by_smth = 0\n", 289 | "\n", 290 | "for t in range(10, len(thrs_seq)):\n", 291 | " divisible_by_something = False\n", 292 | " for j in range(1, num_groups+1):\n", 293 | " if t % j == 0:\n", 294 | " g_counts[j] += 1\n", 295 | " g_miscoverage[j] += int(score_seq[t] > thrs_seq[t])\n", 296 | " if j > 1:\n", 297 | " divisible_by_something = True\n", 298 | " if not divisible_by_something:\n", 299 | " g_counts_residual += 1\n", 300 | " g_miscoverage_residual += int(score_seq[t] > thrs_seq[t])\n", 301 | " else: \n", 302 | " g_counts_div_by_smth += 1\n", 303 | " g_miscoverage_div_by_smth += int(score_seq[t] > thrs_seq[t])\n", 304 | "\n", 305 | "ACI_coverage_rate = 1-np.array([g_miscoverage[j]/g_counts[j] for j in range(1, num_groups+1)])\n", 306 | "\n", 307 | "if print_ACI_group_stats:\n", 308 | " print('Per-Group Coverage Statistics:')\n", 309 | "\n", 310 | " for j in range(num_groups-1):\n", 311 | " print('Group ', j+1, ' : Coverage: ', ACI_coverage_rate[j])\n", 312 | "\n", 313 | " print(1 - g_miscoverage_residual/g_counts_residual)\n", 314 | " print(g_miscoverage_residual)\n", 315 | " print(g_counts_residual)\n", 316 | "\n", 317 | " print(1 - g_miscoverage_div_by_smth/g_counts_div_by_smth)\n", 318 | " print(g_miscoverage_div_by_smth)\n", 319 | " print(g_counts_div_by_smth)" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": null, 325 | "metadata": { 326 | "id": "JKEOILDE0Nh_" 327 | }, 328 | "outputs": [], 329 | "source": [ 330 | "# MVP\n", 331 | "\n", 332 | "groups = []\n", 333 | "for i in range(1, num_groups + 1):\n", 334 | " group = (lambda t, i=i: (t % i) == 0)\n", 335 | " groups.append(group)\n", 336 | "\n", 337 | "n_buckets = 40\n", 338 | "T = len(scores_noisy)\n", 339 | "eta = np.sqrt(np.log(len(groups) * n_buckets) / (2 * len(groups) * n_buckets))\n", 340 | "\n", 341 | "MVP_group = MultiValidPrediction(n_buckets=n_buckets, groups=groups, eta=eta)\n", 342 | "\n", 343 | "for t in range(T):\n", 344 | " x_t = t\n", 345 | " score = scores_noisy[t]\n", 346 | "\n", 347 | " thr = MVP_group.predict(x_t)\n", 348 | " MVP_group.update(x_t, thr, score)\n", 349 | "\n", 350 | "thresholds_MVP_group, miscoverage_MVP_group, counts_MVP_group = MVP_group.coverage_stats(plot_thresholds=False, print_per_group_stats=False)\n", 351 | "\n", 352 | "MVP_coverage_rate = 1-np.array([np.sum(miscoverage_MVP_group[:, j])/np.sum(counts_MVP_group[:, j]) for j in range(num_groups)])" 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": null, 358 | "metadata": { 359 | "colab": { 360 | "base_uri": "https://localhost:8080/", 361 | "height": 442 362 | }, 363 | "id": "qeNfyLi44Gw4", 364 | "outputId": "77d60510-6748-490b-ab93-a54ce4af1534" 365 | }, 366 | "outputs": [], 367 | "source": [ 368 | "barWidth = 0.25\n", 369 | "delta = 0.1\n", 370 | "br1 = np.arange(num_groups)\n", 371 | "br2 = [x + barWidth for x in br1]\n", 372 | "\n", 373 | "plt.figure(figsize=(10, 7))\n", 374 | "plt.bar(br1, ACI_coverage_rate, color = 'm', width = barWidth, edgecolor = 'gray', label = 'ACI')\n", 375 | "plt.bar(br2, MVP_coverage_rate, color = 'c', width = barWidth, edgecolor = 'gray', label = 'MVP')\n", 376 | "group_labels = [str(i) for i in range(1, num_groups+1)]\n", 377 | "plt.xticks([r + barWidth for r in range(num_groups)], group_labels)\n", 378 | "plt.axhline(y= 1 - delta, c = 'r', linewidth = 0.5)\n", 379 | "plt.text(14, 1 - delta + 0.02, ' desired coverage')\n", 380 | "plt.legend()\n", 381 | "plt.ylim([0.0,1.2])\n", 382 | "plt.yticks(np.arange(0, 1.1, 0.1))\n", 383 | "plt.xlabel('Coverage on Noisy Groups: MVP vs ACI')\n", 384 | "plt.show()" 385 | ] 386 | }, 387 | { 388 | "cell_type": "markdown", 389 | "metadata": { 390 | "id": "-ovCmvrv4hMQ" 391 | }, 392 | "source": [ 393 | "###### Experiment: Sorted Scores" 394 | ] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "execution_count": null, 399 | "metadata": { 400 | "colab": { 401 | "base_uri": "https://localhost:8080/", 402 | "height": 458 403 | }, 404 | "id": "EFfQDYtg4kYo", 405 | "outputId": "06d14679-bdf2-4812-ebd6-2e44974e2774" 406 | }, 407 | "outputs": [], 408 | "source": [ 409 | "synthetic_scores = np.linspace(0, 0.5, num=len(prices))\n", 410 | "plt.figure(figsize=(10, 7))\n", 411 | "plt.plot(synthetic_scores)\n", 412 | "plt.ylim([0, 1])\n", 413 | "plt.xlabel('Days')\n", 414 | "plt.ylabel('Scores')\n", 415 | "plt.title('Sorted scores fed to MVP and ACI')\n", 416 | "plt.show()" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": null, 422 | "metadata": { 423 | "colab": { 424 | "base_uri": "https://localhost:8080/", 425 | "height": 475 426 | }, 427 | "id": "xdq_2yke4y_2", 428 | "outputId": "4de6398e-2042-49c2-b447-75fe34e990c7" 429 | }, 430 | "outputs": [], 431 | "source": [ 432 | "# ACI\n", 433 | "\n", 434 | "ACI_thresholds_sorted, ACI_alphas_sorted, ACI_miscoverage_rate_sorted = ACI_method(synthetic_scores)\n", 435 | "\n", 436 | "plt.figure(figsize=(10, 7))\n", 437 | "plt.hist(ACI_thresholds_sorted, bins = 40)\n", 438 | "plt.xlabel('Threshold values (binned)')\n", 439 | "plt.ylabel('No. times threshold value is predicted by ACI')\n", 440 | "plt.title('Histogram of ACI thresholds on sorted data')\n", 441 | "plt.show()" 442 | ] 443 | }, 444 | { 445 | "cell_type": "code", 446 | "execution_count": null, 447 | "metadata": { 448 | "colab": { 449 | "base_uri": "https://localhost:8080/", 450 | "height": 316 451 | }, 452 | "id": "8zLHrjVy4-3N", 453 | "outputId": "7b280a88-f89d-43f0-9b5e-805f61dacda6" 454 | }, 455 | "outputs": [], 456 | "source": [ 457 | "# MVP\n", 458 | "\n", 459 | "n_buckets = 40\n", 460 | "T = len(synthetic_scores)\n", 461 | "eta_sorted = np.sqrt(np.log(2 * 1 * n_buckets) / T)\n", 462 | "\n", 463 | "MVP_sorted = MultiValidPrediction(n_buckets=n_buckets, eta=eta_sorted, normalize_by_counts=False)\n", 464 | "\n", 465 | "for t in range(T):\n", 466 | " x_t = t\n", 467 | " score_sorted = synthetic_scores[t]\n", 468 | "\n", 469 | " thr_sorted = MVP_sorted.predict(x_t)\n", 470 | " MVP_sorted.update(x_t, thr_sorted, score_sorted)\n", 471 | "\n", 472 | "thresholds_MVP_sorted, miscoverage_MVP_sorted, counts_MVP_sorted = MVP_sorted.coverage_stats(plot_thresholds=True, print_per_group_stats=True)" 473 | ] 474 | }, 475 | { 476 | "cell_type": "code", 477 | "execution_count": null, 478 | "metadata": { 479 | "colab": { 480 | "base_uri": "https://localhost:8080/" 481 | }, 482 | "id": "3ysCNO0t5oWV", 483 | "outputId": "43f1821e-34a4-4f61-fe29-ee9592843324" 484 | }, 485 | "outputs": [], 486 | "source": [ 487 | "print('Comparing predicted widths of ACI vs MVP:')\n", 488 | "\n", 489 | "print('ACI widths: ', np.mean(np.array(ACI_thresholds_sorted)))\n", 490 | "print('MVP widths: ', np.mean(np.array(thresholds_MVP_sorted)))" 491 | ] 492 | }, 493 | { 494 | "cell_type": "markdown", 495 | "metadata": { 496 | "id": "dKJ0Mk0G6dBM" 497 | }, 498 | "source": [ 499 | "###### Experiment: Marginal coverage" 500 | ] 501 | }, 502 | { 503 | "cell_type": "code", 504 | "execution_count": null, 505 | "metadata": { 506 | "colab": { 507 | "base_uri": "https://localhost:8080/" 508 | }, 509 | "id": "x76C0zlk6lCj", 510 | "outputId": "671e5128-f610-4de7-e47e-26ac467f7289" 511 | }, 512 | "outputs": [], 513 | "source": [ 514 | "ACI_thrs_single, ACI_alphas_single, ACI_miscoverage_rate_single = ACI_method(scores, alpha=0.1, lookback=100, offset=10, gamma=0.005)" 515 | ] 516 | }, 517 | { 518 | "cell_type": "code", 519 | "execution_count": null, 520 | "metadata": { 521 | "colab": { 522 | "base_uri": "https://localhost:8080/", 523 | "height": 316 524 | }, 525 | "id": "HdNsXk1H7mAG", 526 | "outputId": "85047ab1-2c6c-4bbc-fe9c-7fde3fb37a8a" 527 | }, 528 | "outputs": [], 529 | "source": [ 530 | "# MVP\n", 531 | "\n", 532 | "n_buckets = 40\n", 533 | "T = len(scores)\n", 534 | "eta = np.sqrt(np.log(2 * 1 * n_buckets) / T)\n", 535 | "\n", 536 | "MVP_single = MultiValidPrediction(n_buckets=n_buckets, eta=eta, normalize_by_counts=False)\n", 537 | "\n", 538 | "for t in range(T):\n", 539 | " x_t = t\n", 540 | " score = scores[t]\n", 541 | "\n", 542 | " thr = MVP_single.predict(x_t)\n", 543 | " MVP_single.update(x_t, thr, score)\n", 544 | "\n", 545 | "thresholds_MVP_single, miscoverage_MVP_single, counts_MVP_single = MVP_single.coverage_stats(plot_thresholds=True, print_per_group_stats=True)" 546 | ] 547 | }, 548 | { 549 | "cell_type": "code", 550 | "execution_count": null, 551 | "metadata": { 552 | "colab": { 553 | "base_uri": "https://localhost:8080/", 554 | "height": 350 555 | }, 556 | "id": "cnLLngpK8XBg", 557 | "outputId": "39d6be2a-22e5-4899-999d-b99b3b389219" 558 | }, 559 | "outputs": [], 560 | "source": [ 561 | "plt.figure(figsize=(20, 5))\n", 562 | "plt.plot(ACI_thrs_single, label='ACI')\n", 563 | "plt.plot(thresholds_MVP_single, label='MVP')\n", 564 | "plt.legend(loc='lower right', shadow=True, fontsize='x-large')\n", 565 | "plt.ylim([0.8, 1.02])\n", 566 | "plt.xlabel('Days')\n", 567 | "plt.ylabel('Thresholds')\n", 568 | "plt.title('MVP vs ACI thresholds, AMD stock data')\n", 569 | "plt.show()" 570 | ] 571 | }, 572 | { 573 | "cell_type": "markdown", 574 | "metadata": {}, 575 | "source": [ 576 | "###### Multigroup error bars for the Noisy Multigroup Coverage experiment (takes a long time to run due to trial number)" 577 | ] 578 | }, 579 | { 580 | "cell_type": "code", 581 | "execution_count": null, 582 | "metadata": {}, 583 | "outputs": [], 584 | "source": [ 585 | "num_groups = 20\n", 586 | "\n", 587 | "scores_arr = []\n", 588 | "\n", 589 | "for trial in range(20):\n", 590 | " returns_noisy = np.zeros(len(returns))\n", 591 | " std_returns = np.std(returns)\n", 592 | " for t in range(len(returns)):\n", 593 | " returns_noisy[t] = returns[t]\n", 594 | " for j in range(1, num_groups+1):\n", 595 | " if t % j == 0:\n", 596 | " returns_noisy[t] += std_returns*np.random.randn()\n", 597 | "\n", 598 | " volatility_noisy = [ret**2 for ret in returns_noisy]\n", 599 | " scores_noisy, predictions_noisy = garch_scores(volatility_noisy)\n", 600 | "\n", 601 | " scores_arr.append(scores_noisy)\n", 602 | "\n", 603 | "import pickle\n", 604 | "\n", 605 | "pickle_out = open('noisyscores0.pickle', 'wb')\n", 606 | "pickle.dump(scores_arr, pickle_out)\n", 607 | "pickle_out.close()\n", 608 | "\n", 609 | "group_cvgs_ACI = []\n", 610 | "group_cvgs_MVP = []\n", 611 | "\n", 612 | "groups = []\n", 613 | "for i in range(1, num_groups + 1):\n", 614 | " group = (lambda t, i=i: (t % i) == 0)\n", 615 | " groups.append(group)\n", 616 | "\n", 617 | "for trial in range(20):\n", 618 | " scores_curr = scores_arr[trial]\n", 619 | "\n", 620 | " # ACI\n", 621 | " ACI_thrs, ACI_alphas, ACI_miscoverage_rate = ACI_method(scores_curr, alpha=0.1, lookback=100, offset=10, gamma=0.005)\n", 622 | "\n", 623 | " # ACI Coverage rates\n", 624 | " g_miscoverage = np.zeros(num_groups+1, dtype=int)\n", 625 | " g_counts = np.zeros(num_groups+1, dtype=int)\n", 626 | "\n", 627 | " thrs_seq = ACI_thrs\n", 628 | " for t in range(10, len(thrs_seq)):\n", 629 | " for j in range(1, num_groups+1):\n", 630 | " if t % j == 0:\n", 631 | " g_counts[j] += 1\n", 632 | " g_miscoverage[j] += int(scores_curr[t] > thrs_seq[t])\n", 633 | "\n", 634 | " ACI_coverage_rate = 1-np.array([g_miscoverage[j]/g_counts[j] for j in range(1, num_groups+1)])\n", 635 | "\n", 636 | " # MVP\n", 637 | " n_buckets = 40\n", 638 | " T = len(scores_curr)\n", 639 | " eta = np.sqrt(np.log(2 * len(groups) * n_buckets) / T)\n", 640 | " MVP_group = MultivalidPredictionIntervals(n_buckets=n_buckets, groups=groups, eta=eta)\n", 641 | " for t in range(T):\n", 642 | " x_t = t\n", 643 | " thr = MVP_group.predict(x_t)\n", 644 | " score = scores_curr[t]\n", 645 | " MVP_group.update(x_t, thr, score)\n", 646 | "\n", 647 | " # MVP Coverage rates\n", 648 | " thresholds_MVP_group, miscoverage_MVP_group, counts_MVP_group = MVP_group.coverage_stats(plot_thresholds=False, print_per_group_stats=False)\n", 649 | " MVP_coverage_rate = 1-np.array([np.sum(miscoverage_MVP_group[:, j])/np.sum(counts_MVP_group[:, j]) for j in range(num_groups)])\n", 650 | "\n", 651 | " # Add ACI and MVP coverage to array\n", 652 | " group_cvgs_ACI.append(ACI_coverage_rate)\n", 653 | " group_cvgs_MVP.append(MVP_coverage_rate)\n", 654 | "\n", 655 | "cvgs_ACI = np.array(group_cvgs_ACI)\n", 656 | "cvgs_MVP = np.array(group_cvgs_MVP)\n", 657 | "\n", 658 | "# Error bars and medians\n", 659 | "ACI_err_upper = [np.quantile(cvgs_ACI[:,group], 0.75) for group in range(num_groups)]\n", 660 | "MVP_err_upper = [np.quantile(cvgs_MVP[:,group], 0.75) for group in range(num_groups)]\n", 661 | "\n", 662 | "ACI_err_lower = [np.quantile(cvgs_ACI[:,group], 0.25) for group in range(num_groups)]\n", 663 | "MVP_err_lower = [np.quantile(cvgs_MVP[:,group], 0.25) for group in range(num_groups)]\n", 664 | "\n", 665 | "ACI_median = [np.quantile(cvgs_ACI[:,group], 0.50) for group in range(num_groups)]\n", 666 | "MVP_median = [np.quantile(cvgs_MVP[:,group], 0.50) for group in range(num_groups)]\n", 667 | "\n", 668 | "# Histogram\n", 669 | "\n", 670 | "barWidth = 0.25\n", 671 | "delta = 0.1\n", 672 | "br1 = np.arange(num_groups)\n", 673 | "br2 = [x + barWidth for x in br1]\n", 674 | "br3 = [x + barWidth for x in br2]\n", 675 | "\n", 676 | "plt.bar(br1, ACI_median, color = 'm', width = barWidth, edgecolor = 'gray', label = 'ACI')\n", 677 | "plt.bar(br2, MVP_median, color = 'c', width = barWidth, edgecolor = 'gray', label = 'MVP')\n", 678 | "\n", 679 | "for i, val in enumerate(br1):\n", 680 | " plt.vlines(x=val, ymin=ACI_err_lower[i], ymax=ACI_err_upper[i], color='black', linewidth=2)\n", 681 | "\n", 682 | "for i, val in enumerate(br2):\n", 683 | " plt.vlines(x=val, ymin=MVP_err_lower[i], ymax=MVP_err_upper[i], color='black', linewidth=2)\n", 684 | "\n", 685 | "group_labels = [str(i) for i in range(1, num_groups+1)]\n", 686 | "plt.xticks([r + barWidth for r in range(num_groups)], group_labels)\n", 687 | "plt.axhline(y= 1 - delta, c = 'r', linewidth = 0.5)\n", 688 | "plt.text(14, 1 - delta + 0.02, ' desired coverage')\n", 689 | "plt.legend()\n", 690 | "plt.ylim([0.0,1.2])\n", 691 | "plt.yticks(np.arange(0, 1.1, 0.1))\n", 692 | "plt.title('Coverage on Noisy Groups: MVP vs ACI')\n", 693 | "plt.xlabel('Group number')\n", 694 | "plt.ylabel('Coverage level')\n", 695 | "plt.show()\n", 696 | "\n", 697 | "# print maximum error bar width for ACI and the corresponding group\n", 698 | "maxdist = 0\n", 699 | "gr = -1\n", 700 | "for group in range(num_groups):\n", 701 | " a = ACI_err_lower[group] \n", 702 | " b = ACI_median[group]\n", 703 | " c = ACI_err_upper[group]\n", 704 | " if abs(a-c) > maxdist:\n", 705 | " maxdist = abs(a-c)\n", 706 | " gr = group\n", 707 | "print(maxdist)\n", 708 | "print(gr)\n", 709 | "\n", 710 | "# print maximum error bar width for MVP and the corresponding group\n", 711 | "maxdist = 0\n", 712 | "gr = -1\n", 713 | "for group in range(num_groups):\n", 714 | " a = MVP_err_lower[group] \n", 715 | " b = MVP_median[group]\n", 716 | " c = MVP_err_upper[group]\n", 717 | " if abs(a-c) > maxdist:\n", 718 | " maxdist = abs(a-c)\n", 719 | " gr = group\n", 720 | "print(maxdist)\n", 721 | "print(gr)" 722 | ] 723 | } 724 | ], 725 | "metadata": { 726 | "colab": { 727 | "name": "TimeSeriesExperiment.ipynb", 728 | "provenance": [] 729 | }, 730 | "kernelspec": { 731 | "display_name": "Python 3 (ipykernel)", 732 | "language": "python", 733 | "name": "python3" 734 | }, 735 | "language_info": { 736 | "codemirror_mode": { 737 | "name": "ipython", 738 | "version": 3 739 | }, 740 | "file_extension": ".py", 741 | "mimetype": "text/x-python", 742 | "name": "python", 743 | "nbconvert_exporter": "python", 744 | "pygments_lexer": "ipython3", 745 | "version": "3.9.12" 746 | } 747 | }, 748 | "nbformat": 4, 749 | "nbformat_minor": 1 750 | } 751 | -------------------------------------------------------------------------------- /experiments/data/2018.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProgBelarus/MultiValidPrediction/be3baaf66d04e8c767987e5c656f4131c06917ee/experiments/data/2018.zip -------------------------------------------------------------------------------- /experiments/data/airfoil_self_noise.dat: -------------------------------------------------------------------------------- 1 | 800 0 0.3048 71.3 0.00266337 126.201 2 | 1000 0 0.3048 71.3 0.00266337 125.201 3 | 1250 0 0.3048 71.3 0.00266337 125.951 4 | 1600 0 0.3048 71.3 0.00266337 127.591 5 | 2000 0 0.3048 71.3 0.00266337 127.461 6 | 2500 0 0.3048 71.3 0.00266337 125.571 7 | 3150 0 0.3048 71.3 0.00266337 125.201 8 | 4000 0 0.3048 71.3 0.00266337 123.061 9 | 5000 0 0.3048 71.3 0.00266337 121.301 10 | 6300 0 0.3048 71.3 0.00266337 119.541 11 | 8000 0 0.3048 71.3 0.00266337 117.151 12 | 10000 0 0.3048 71.3 0.00266337 115.391 13 | 12500 0 0.3048 71.3 0.00266337 112.241 14 | 16000 0 0.3048 71.3 0.00266337 108.721 15 | 500 0 0.3048 55.5 0.00283081 126.416 16 | 630 0 0.3048 55.5 0.00283081 127.696 17 | 800 0 0.3048 55.5 0.00283081 128.086 18 | 1000 0 0.3048 55.5 0.00283081 126.966 19 | 1250 0 0.3048 55.5 0.00283081 126.086 20 | 1600 0 0.3048 55.5 0.00283081 126.986 21 | 2000 0 0.3048 55.5 0.00283081 126.616 22 | 2500 0 0.3048 55.5 0.00283081 124.106 23 | 3150 0 0.3048 55.5 0.00283081 123.236 24 | 4000 0 0.3048 55.5 0.00283081 121.106 25 | 5000 0 0.3048 55.5 0.00283081 119.606 26 | 6300 0 0.3048 55.5 0.00283081 117.976 27 | 8000 0 0.3048 55.5 0.00283081 116.476 28 | 10000 0 0.3048 55.5 0.00283081 113.076 29 | 12500 0 0.3048 55.5 0.00283081 111.076 30 | 200 0 0.3048 39.6 0.00310138 118.129 31 | 250 0 0.3048 39.6 0.00310138 119.319 32 | 315 0 0.3048 39.6 0.00310138 122.779 33 | 400 0 0.3048 39.6 0.00310138 124.809 34 | 500 0 0.3048 39.6 0.00310138 126.959 35 | 630 0 0.3048 39.6 0.00310138 128.629 36 | 800 0 0.3048 39.6 0.00310138 129.099 37 | 1000 0 0.3048 39.6 0.00310138 127.899 38 | 1250 0 0.3048 39.6 0.00310138 125.499 39 | 1600 0 0.3048 39.6 0.00310138 124.049 40 | 2000 0 0.3048 39.6 0.00310138 123.689 41 | 2500 0 0.3048 39.6 0.00310138 121.399 42 | 3150 0 0.3048 39.6 0.00310138 120.319 43 | 4000 0 0.3048 39.6 0.00310138 119.229 44 | 5000 0 0.3048 39.6 0.00310138 117.789 45 | 6300 0 0.3048 39.6 0.00310138 116.229 46 | 8000 0 0.3048 39.6 0.00310138 114.779 47 | 10000 0 0.3048 39.6 0.00310138 112.139 48 | 12500 0 0.3048 39.6 0.00310138 109.619 49 | 200 0 0.3048 31.7 0.00331266 117.195 50 | 250 0 0.3048 31.7 0.00331266 118.595 51 | 315 0 0.3048 31.7 0.00331266 122.765 52 | 400 0 0.3048 31.7 0.00331266 125.045 53 | 500 0 0.3048 31.7 0.00331266 127.315 54 | 630 0 0.3048 31.7 0.00331266 129.095 55 | 800 0 0.3048 31.7 0.00331266 129.235 56 | 1000 0 0.3048 31.7 0.00331266 127.365 57 | 1250 0 0.3048 31.7 0.00331266 124.355 58 | 1600 0 0.3048 31.7 0.00331266 122.365 59 | 2000 0 0.3048 31.7 0.00331266 122.375 60 | 2500 0 0.3048 31.7 0.00331266 120.755 61 | 3150 0 0.3048 31.7 0.00331266 119.135 62 | 4000 0 0.3048 31.7 0.00331266 118.145 63 | 5000 0 0.3048 31.7 0.00331266 115.645 64 | 6300 0 0.3048 31.7 0.00331266 113.775 65 | 8000 0 0.3048 31.7 0.00331266 110.515 66 | 10000 0 0.3048 31.7 0.00331266 108.265 67 | 800 1.5 0.3048 71.3 0.00336729 127.122 68 | 1000 1.5 0.3048 71.3 0.00336729 125.992 69 | 1250 1.5 0.3048 71.3 0.00336729 125.872 70 | 1600 1.5 0.3048 71.3 0.00336729 126.632 71 | 2000 1.5 0.3048 71.3 0.00336729 126.642 72 | 2500 1.5 0.3048 71.3 0.00336729 124.512 73 | 3150 1.5 0.3048 71.3 0.00336729 123.392 74 | 4000 1.5 0.3048 71.3 0.00336729 121.762 75 | 5000 1.5 0.3048 71.3 0.00336729 119.632 76 | 6300 1.5 0.3048 71.3 0.00336729 118.122 77 | 8000 1.5 0.3048 71.3 0.00336729 115.372 78 | 10000 1.5 0.3048 71.3 0.00336729 113.492 79 | 12500 1.5 0.3048 71.3 0.00336729 109.222 80 | 16000 1.5 0.3048 71.3 0.00336729 106.582 81 | 315 1.5 0.3048 39.6 0.00392107 121.851 82 | 400 1.5 0.3048 39.6 0.00392107 124.001 83 | 500 1.5 0.3048 39.6 0.00392107 126.661 84 | 630 1.5 0.3048 39.6 0.00392107 128.311 85 | 800 1.5 0.3048 39.6 0.00392107 128.831 86 | 1000 1.5 0.3048 39.6 0.00392107 127.581 87 | 1250 1.5 0.3048 39.6 0.00392107 125.211 88 | 1600 1.5 0.3048 39.6 0.00392107 122.211 89 | 2000 1.5 0.3048 39.6 0.00392107 122.101 90 | 2500 1.5 0.3048 39.6 0.00392107 120.981 91 | 3150 1.5 0.3048 39.6 0.00392107 119.111 92 | 4000 1.5 0.3048 39.6 0.00392107 117.741 93 | 5000 1.5 0.3048 39.6 0.00392107 116.241 94 | 6300 1.5 0.3048 39.6 0.00392107 114.751 95 | 8000 1.5 0.3048 39.6 0.00392107 112.251 96 | 10000 1.5 0.3048 39.6 0.00392107 108.991 97 | 12500 1.5 0.3048 39.6 0.00392107 106.111 98 | 400 3 0.3048 71.3 0.00425727 127.564 99 | 500 3 0.3048 71.3 0.00425727 128.454 100 | 630 3 0.3048 71.3 0.00425727 129.354 101 | 800 3 0.3048 71.3 0.00425727 129.494 102 | 1000 3 0.3048 71.3 0.00425727 129.004 103 | 1250 3 0.3048 71.3 0.00425727 127.634 104 | 1600 3 0.3048 71.3 0.00425727 126.514 105 | 2000 3 0.3048 71.3 0.00425727 125.524 106 | 2500 3 0.3048 71.3 0.00425727 124.024 107 | 3150 3 0.3048 71.3 0.00425727 121.514 108 | 4000 3 0.3048 71.3 0.00425727 120.264 109 | 5000 3 0.3048 71.3 0.00425727 118.134 110 | 6300 3 0.3048 71.3 0.00425727 116.134 111 | 8000 3 0.3048 71.3 0.00425727 114.634 112 | 10000 3 0.3048 71.3 0.00425727 110.224 113 | 400 3 0.3048 55.5 0.00452492 126.159 114 | 500 3 0.3048 55.5 0.00452492 128.179 115 | 630 3 0.3048 55.5 0.00452492 129.569 116 | 800 3 0.3048 55.5 0.00452492 129.949 117 | 1000 3 0.3048 55.5 0.00452492 129.329 118 | 1250 3 0.3048 55.5 0.00452492 127.329 119 | 1600 3 0.3048 55.5 0.00452492 124.439 120 | 2000 3 0.3048 55.5 0.00452492 123.069 121 | 2500 3 0.3048 55.5 0.00452492 122.439 122 | 3150 3 0.3048 55.5 0.00452492 120.189 123 | 4000 3 0.3048 55.5 0.00452492 118.689 124 | 5000 3 0.3048 55.5 0.00452492 117.309 125 | 6300 3 0.3048 55.5 0.00452492 115.679 126 | 8000 3 0.3048 55.5 0.00452492 113.799 127 | 10000 3 0.3048 55.5 0.00452492 112.169 128 | 315 3 0.3048 39.6 0.00495741 123.312 129 | 400 3 0.3048 39.6 0.00495741 125.472 130 | 500 3 0.3048 39.6 0.00495741 127.632 131 | 630 3 0.3048 39.6 0.00495741 129.292 132 | 800 3 0.3048 39.6 0.00495741 129.552 133 | 1000 3 0.3048 39.6 0.00495741 128.312 134 | 1250 3 0.3048 39.6 0.00495741 125.802 135 | 1600 3 0.3048 39.6 0.00495741 122.782 136 | 2000 3 0.3048 39.6 0.00495741 120.532 137 | 2500 3 0.3048 39.6 0.00495741 120.162 138 | 3150 3 0.3048 39.6 0.00495741 118.922 139 | 4000 3 0.3048 39.6 0.00495741 116.792 140 | 5000 3 0.3048 39.6 0.00495741 115.792 141 | 6300 3 0.3048 39.6 0.00495741 114.042 142 | 8000 3 0.3048 39.6 0.00495741 110.652 143 | 315 3 0.3048 31.7 0.00529514 123.118 144 | 400 3 0.3048 31.7 0.00529514 125.398 145 | 500 3 0.3048 31.7 0.00529514 127.548 146 | 630 3 0.3048 31.7 0.00529514 128.698 147 | 800 3 0.3048 31.7 0.00529514 128.708 148 | 1000 3 0.3048 31.7 0.00529514 126.838 149 | 1250 3 0.3048 31.7 0.00529514 124.838 150 | 1600 3 0.3048 31.7 0.00529514 122.088 151 | 2000 3 0.3048 31.7 0.00529514 120.088 152 | 2500 3 0.3048 31.7 0.00529514 119.598 153 | 3150 3 0.3048 31.7 0.00529514 118.108 154 | 4000 3 0.3048 31.7 0.00529514 115.608 155 | 5000 3 0.3048 31.7 0.00529514 113.858 156 | 6300 3 0.3048 31.7 0.00529514 109.718 157 | 250 4 0.3048 71.3 0.00497773 126.395 158 | 315 4 0.3048 71.3 0.00497773 128.175 159 | 400 4 0.3048 71.3 0.00497773 129.575 160 | 500 4 0.3048 71.3 0.00497773 130.715 161 | 630 4 0.3048 71.3 0.00497773 131.615 162 | 800 4 0.3048 71.3 0.00497773 131.755 163 | 1000 4 0.3048 71.3 0.00497773 131.015 164 | 1250 4 0.3048 71.3 0.00497773 129.395 165 | 1600 4 0.3048 71.3 0.00497773 126.645 166 | 2000 4 0.3048 71.3 0.00497773 124.395 167 | 2500 4 0.3048 71.3 0.00497773 123.775 168 | 3150 4 0.3048 71.3 0.00497773 121.775 169 | 4000 4 0.3048 71.3 0.00497773 119.535 170 | 5000 4 0.3048 71.3 0.00497773 117.785 171 | 6300 4 0.3048 71.3 0.00497773 116.165 172 | 8000 4 0.3048 71.3 0.00497773 113.665 173 | 10000 4 0.3048 71.3 0.00497773 110.905 174 | 12500 4 0.3048 71.3 0.00497773 107.405 175 | 250 4 0.3048 39.6 0.00579636 123.543 176 | 315 4 0.3048 39.6 0.00579636 126.843 177 | 400 4 0.3048 39.6 0.00579636 128.633 178 | 500 4 0.3048 39.6 0.00579636 130.173 179 | 630 4 0.3048 39.6 0.00579636 131.073 180 | 800 4 0.3048 39.6 0.00579636 130.723 181 | 1000 4 0.3048 39.6 0.00579636 128.723 182 | 1250 4 0.3048 39.6 0.00579636 126.343 183 | 1600 4 0.3048 39.6 0.00579636 123.213 184 | 2000 4 0.3048 39.6 0.00579636 120.963 185 | 2500 4 0.3048 39.6 0.00579636 120.233 186 | 3150 4 0.3048 39.6 0.00579636 118.743 187 | 4000 4 0.3048 39.6 0.00579636 115.863 188 | 5000 4 0.3048 39.6 0.00579636 113.733 189 | 1250 0 0.2286 71.3 0.00214345 128.144 190 | 1600 0 0.2286 71.3 0.00214345 129.134 191 | 2000 0 0.2286 71.3 0.00214345 128.244 192 | 2500 0 0.2286 71.3 0.00214345 128.354 193 | 3150 0 0.2286 71.3 0.00214345 127.834 194 | 4000 0 0.2286 71.3 0.00214345 125.824 195 | 5000 0 0.2286 71.3 0.00214345 124.304 196 | 6300 0 0.2286 71.3 0.00214345 122.044 197 | 8000 0 0.2286 71.3 0.00214345 118.024 198 | 10000 0 0.2286 71.3 0.00214345 118.134 199 | 12500 0 0.2286 71.3 0.00214345 117.624 200 | 16000 0 0.2286 71.3 0.00214345 114.984 201 | 20000 0 0.2286 71.3 0.00214345 114.474 202 | 315 0 0.2286 55.5 0.00229336 119.540 203 | 400 0 0.2286 55.5 0.00229336 121.660 204 | 500 0 0.2286 55.5 0.00229336 123.780 205 | 630 0 0.2286 55.5 0.00229336 126.160 206 | 800 0 0.2286 55.5 0.00229336 127.530 207 | 1000 0 0.2286 55.5 0.00229336 128.290 208 | 1250 0 0.2286 55.5 0.00229336 127.910 209 | 1600 0 0.2286 55.5 0.00229336 126.790 210 | 2000 0 0.2286 55.5 0.00229336 126.540 211 | 2500 0 0.2286 55.5 0.00229336 126.540 212 | 3150 0 0.2286 55.5 0.00229336 125.160 213 | 4000 0 0.2286 55.5 0.00229336 123.410 214 | 5000 0 0.2286 55.5 0.00229336 122.410 215 | 6300 0 0.2286 55.5 0.00229336 118.410 216 | 315 0 0.2286 39.6 0.00253511 121.055 217 | 400 0 0.2286 39.6 0.00253511 123.565 218 | 500 0 0.2286 39.6 0.00253511 126.195 219 | 630 0 0.2286 39.6 0.00253511 128.705 220 | 800 0 0.2286 39.6 0.00253511 130.205 221 | 1000 0 0.2286 39.6 0.00253511 130.435 222 | 1250 0 0.2286 39.6 0.00253511 129.395 223 | 1600 0 0.2286 39.6 0.00253511 127.095 224 | 2000 0 0.2286 39.6 0.00253511 125.305 225 | 2500 0 0.2286 39.6 0.00253511 125.025 226 | 3150 0 0.2286 39.6 0.00253511 124.625 227 | 4000 0 0.2286 39.6 0.00253511 123.465 228 | 5000 0 0.2286 39.6 0.00253511 122.175 229 | 6300 0 0.2286 39.6 0.00253511 117.465 230 | 315 0 0.2286 31.7 0.0027238 120.595 231 | 400 0 0.2286 31.7 0.0027238 123.635 232 | 500 0 0.2286 31.7 0.0027238 126.675 233 | 630 0 0.2286 31.7 0.0027238 129.465 234 | 800 0 0.2286 31.7 0.0027238 130.725 235 | 1000 0 0.2286 31.7 0.0027238 130.595 236 | 1250 0 0.2286 31.7 0.0027238 128.805 237 | 1600 0 0.2286 31.7 0.0027238 125.625 238 | 2000 0 0.2286 31.7 0.0027238 123.455 239 | 2500 0 0.2286 31.7 0.0027238 123.445 240 | 3150 0 0.2286 31.7 0.0027238 123.445 241 | 4000 0 0.2286 31.7 0.0027238 122.035 242 | 5000 0 0.2286 31.7 0.0027238 120.505 243 | 6300 0 0.2286 31.7 0.0027238 116.815 244 | 400 2 0.2286 71.3 0.00293031 125.116 245 | 500 2 0.2286 71.3 0.00293031 126.486 246 | 630 2 0.2286 71.3 0.00293031 127.356 247 | 800 2 0.2286 71.3 0.00293031 128.216 248 | 1000 2 0.2286 71.3 0.00293031 128.956 249 | 1250 2 0.2286 71.3 0.00293031 128.816 250 | 1600 2 0.2286 71.3 0.00293031 127.796 251 | 2000 2 0.2286 71.3 0.00293031 126.896 252 | 2500 2 0.2286 71.3 0.00293031 127.006 253 | 3150 2 0.2286 71.3 0.00293031 126.116 254 | 4000 2 0.2286 71.3 0.00293031 124.086 255 | 5000 2 0.2286 71.3 0.00293031 122.816 256 | 6300 2 0.2286 71.3 0.00293031 120.786 257 | 8000 2 0.2286 71.3 0.00293031 115.996 258 | 10000 2 0.2286 71.3 0.00293031 113.086 259 | 400 2 0.2286 55.5 0.00313525 122.292 260 | 500 2 0.2286 55.5 0.00313525 124.692 261 | 630 2 0.2286 55.5 0.00313525 126.842 262 | 800 2 0.2286 55.5 0.00313525 128.492 263 | 1000 2 0.2286 55.5 0.00313525 129.002 264 | 1250 2 0.2286 55.5 0.00313525 128.762 265 | 1600 2 0.2286 55.5 0.00313525 126.752 266 | 2000 2 0.2286 55.5 0.00313525 124.612 267 | 2500 2 0.2286 55.5 0.00313525 123.862 268 | 3150 2 0.2286 55.5 0.00313525 123.742 269 | 4000 2 0.2286 55.5 0.00313525 122.232 270 | 5000 2 0.2286 55.5 0.00313525 120.472 271 | 6300 2 0.2286 55.5 0.00313525 118.712 272 | 315 2 0.2286 39.6 0.00346574 120.137 273 | 400 2 0.2286 39.6 0.00346574 122.147 274 | 500 2 0.2286 39.6 0.00346574 125.157 275 | 630 2 0.2286 39.6 0.00346574 127.417 276 | 800 2 0.2286 39.6 0.00346574 129.037 277 | 1000 2 0.2286 39.6 0.00346574 129.147 278 | 1250 2 0.2286 39.6 0.00346574 128.257 279 | 1600 2 0.2286 39.6 0.00346574 125.837 280 | 2000 2 0.2286 39.6 0.00346574 122.797 281 | 2500 2 0.2286 39.6 0.00346574 121.397 282 | 3150 2 0.2286 39.6 0.00346574 121.627 283 | 4000 2 0.2286 39.6 0.00346574 120.227 284 | 5000 2 0.2286 39.6 0.00346574 118.827 285 | 6300 2 0.2286 39.6 0.00346574 116.417 286 | 315 2 0.2286 31.7 0.00372371 120.147 287 | 400 2 0.2286 31.7 0.00372371 123.417 288 | 500 2 0.2286 31.7 0.00372371 126.677 289 | 630 2 0.2286 31.7 0.00372371 129.057 290 | 800 2 0.2286 31.7 0.00372371 130.307 291 | 1000 2 0.2286 31.7 0.00372371 130.307 292 | 1250 2 0.2286 31.7 0.00372371 128.677 293 | 1600 2 0.2286 31.7 0.00372371 125.797 294 | 2000 2 0.2286 31.7 0.00372371 123.037 295 | 2500 2 0.2286 31.7 0.00372371 121.407 296 | 3150 2 0.2286 31.7 0.00372371 121.527 297 | 4000 2 0.2286 31.7 0.00372371 120.527 298 | 5000 2 0.2286 31.7 0.00372371 118.267 299 | 6300 2 0.2286 31.7 0.00372371 115.137 300 | 500 4 0.2286 71.3 0.00400603 126.758 301 | 630 4 0.2286 71.3 0.00400603 129.038 302 | 800 4 0.2286 71.3 0.00400603 130.688 303 | 1000 4 0.2286 71.3 0.00400603 131.708 304 | 1250 4 0.2286 71.3 0.00400603 131.718 305 | 1600 4 0.2286 71.3 0.00400603 129.468 306 | 2000 4 0.2286 71.3 0.00400603 126.218 307 | 2500 4 0.2286 71.3 0.00400603 124.338 308 | 3150 4 0.2286 71.3 0.00400603 124.108 309 | 4000 4 0.2286 71.3 0.00400603 121.728 310 | 5000 4 0.2286 71.3 0.00400603 121.118 311 | 6300 4 0.2286 71.3 0.00400603 118.618 312 | 8000 4 0.2286 71.3 0.00400603 112.848 313 | 10000 4 0.2286 71.3 0.00400603 113.108 314 | 12500 4 0.2286 71.3 0.00400603 114.258 315 | 16000 4 0.2286 71.3 0.00400603 112.768 316 | 20000 4 0.2286 71.3 0.00400603 109.638 317 | 400 4 0.2286 55.5 0.0042862 123.274 318 | 500 4 0.2286 55.5 0.0042862 127.314 319 | 630 4 0.2286 55.5 0.0042862 129.964 320 | 800 4 0.2286 55.5 0.0042862 131.864 321 | 1000 4 0.2286 55.5 0.0042862 132.134 322 | 1250 4 0.2286 55.5 0.0042862 131.264 323 | 1600 4 0.2286 55.5 0.0042862 128.264 324 | 2000 4 0.2286 55.5 0.0042862 124.254 325 | 2500 4 0.2286 55.5 0.0042862 122.384 326 | 3150 4 0.2286 55.5 0.0042862 122.394 327 | 4000 4 0.2286 55.5 0.0042862 120.654 328 | 5000 4 0.2286 55.5 0.0042862 120.034 329 | 6300 4 0.2286 55.5 0.0042862 117.154 330 | 8000 4 0.2286 55.5 0.0042862 112.524 331 | 315 4 0.2286 39.6 0.00473801 122.229 332 | 400 4 0.2286 39.6 0.00473801 123.879 333 | 500 4 0.2286 39.6 0.00473801 127.039 334 | 630 4 0.2286 39.6 0.00473801 129.579 335 | 800 4 0.2286 39.6 0.00473801 130.469 336 | 1000 4 0.2286 39.6 0.00473801 129.969 337 | 1250 4 0.2286 39.6 0.00473801 128.339 338 | 1600 4 0.2286 39.6 0.00473801 125.319 339 | 2000 4 0.2286 39.6 0.00473801 121.659 340 | 2500 4 0.2286 39.6 0.00473801 119.649 341 | 3150 4 0.2286 39.6 0.00473801 120.419 342 | 4000 4 0.2286 39.6 0.00473801 119.159 343 | 5000 4 0.2286 39.6 0.00473801 117.649 344 | 6300 4 0.2286 39.6 0.00473801 114.249 345 | 8000 4 0.2286 39.6 0.00473801 113.129 346 | 250 4 0.2286 31.7 0.00509068 120.189 347 | 315 4 0.2286 31.7 0.00509068 123.609 348 | 400 4 0.2286 31.7 0.00509068 126.149 349 | 500 4 0.2286 31.7 0.00509068 128.939 350 | 630 4 0.2286 31.7 0.00509068 130.349 351 | 800 4 0.2286 31.7 0.00509068 130.869 352 | 1000 4 0.2286 31.7 0.00509068 129.869 353 | 1250 4 0.2286 31.7 0.00509068 128.119 354 | 1600 4 0.2286 31.7 0.00509068 125.229 355 | 2000 4 0.2286 31.7 0.00509068 122.089 356 | 2500 4 0.2286 31.7 0.00509068 120.209 357 | 3150 4 0.2286 31.7 0.00509068 120.229 358 | 4000 4 0.2286 31.7 0.00509068 118.859 359 | 5000 4 0.2286 31.7 0.00509068 115.969 360 | 6300 4 0.2286 31.7 0.00509068 112.699 361 | 400 5.3 0.2286 71.3 0.0051942 127.700 362 | 500 5.3 0.2286 71.3 0.0051942 129.880 363 | 630 5.3 0.2286 71.3 0.0051942 131.800 364 | 800 5.3 0.2286 71.3 0.0051942 133.480 365 | 1000 5.3 0.2286 71.3 0.0051942 134.000 366 | 1250 5.3 0.2286 71.3 0.0051942 133.380 367 | 1600 5.3 0.2286 71.3 0.0051942 130.460 368 | 2000 5.3 0.2286 71.3 0.0051942 125.890 369 | 2500 5.3 0.2286 71.3 0.0051942 123.740 370 | 3150 5.3 0.2286 71.3 0.0051942 123.120 371 | 4000 5.3 0.2286 71.3 0.0051942 120.330 372 | 5000 5.3 0.2286 71.3 0.0051942 118.050 373 | 6300 5.3 0.2286 71.3 0.0051942 116.920 374 | 8000 5.3 0.2286 71.3 0.0051942 114.900 375 | 10000 5.3 0.2286 71.3 0.0051942 111.350 376 | 250 5.3 0.2286 39.6 0.00614329 127.011 377 | 315 5.3 0.2286 39.6 0.00614329 129.691 378 | 400 5.3 0.2286 39.6 0.00614329 131.221 379 | 500 5.3 0.2286 39.6 0.00614329 132.251 380 | 630 5.3 0.2286 39.6 0.00614329 132.011 381 | 800 5.3 0.2286 39.6 0.00614329 129.491 382 | 1000 5.3 0.2286 39.6 0.00614329 125.581 383 | 1250 5.3 0.2286 39.6 0.00614329 125.721 384 | 1600 5.3 0.2286 39.6 0.00614329 123.081 385 | 2000 5.3 0.2286 39.6 0.00614329 117.911 386 | 2500 5.3 0.2286 39.6 0.00614329 116.151 387 | 3150 5.3 0.2286 39.6 0.00614329 118.441 388 | 4000 5.3 0.2286 39.6 0.00614329 115.801 389 | 5000 5.3 0.2286 39.6 0.00614329 115.311 390 | 6300 5.3 0.2286 39.6 0.00614329 112.541 391 | 200 7.3 0.2286 71.3 0.0104404 138.758 392 | 250 7.3 0.2286 71.3 0.0104404 139.918 393 | 315 7.3 0.2286 71.3 0.0104404 139.808 394 | 400 7.3 0.2286 71.3 0.0104404 139.438 395 | 500 7.3 0.2286 71.3 0.0104404 136.798 396 | 630 7.3 0.2286 71.3 0.0104404 133.768 397 | 800 7.3 0.2286 71.3 0.0104404 130.748 398 | 1000 7.3 0.2286 71.3 0.0104404 126.838 399 | 1250 7.3 0.2286 71.3 0.0104404 127.358 400 | 1600 7.3 0.2286 71.3 0.0104404 125.728 401 | 2000 7.3 0.2286 71.3 0.0104404 122.708 402 | 2500 7.3 0.2286 71.3 0.0104404 122.088 403 | 3150 7.3 0.2286 71.3 0.0104404 120.458 404 | 4000 7.3 0.2286 71.3 0.0104404 119.208 405 | 5000 7.3 0.2286 71.3 0.0104404 115.298 406 | 6300 7.3 0.2286 71.3 0.0104404 115.818 407 | 200 7.3 0.2286 55.5 0.0111706 135.234 408 | 250 7.3 0.2286 55.5 0.0111706 136.384 409 | 315 7.3 0.2286 55.5 0.0111706 136.284 410 | 400 7.3 0.2286 55.5 0.0111706 135.924 411 | 500 7.3 0.2286 55.5 0.0111706 133.174 412 | 630 7.3 0.2286 55.5 0.0111706 130.934 413 | 800 7.3 0.2286 55.5 0.0111706 128.444 414 | 1000 7.3 0.2286 55.5 0.0111706 125.194 415 | 1250 7.3 0.2286 55.5 0.0111706 125.724 416 | 1600 7.3 0.2286 55.5 0.0111706 123.354 417 | 2000 7.3 0.2286 55.5 0.0111706 120.354 418 | 2500 7.3 0.2286 55.5 0.0111706 118.994 419 | 3150 7.3 0.2286 55.5 0.0111706 117.134 420 | 4000 7.3 0.2286 55.5 0.0111706 117.284 421 | 5000 7.3 0.2286 55.5 0.0111706 113.144 422 | 6300 7.3 0.2286 55.5 0.0111706 111.534 423 | 200 7.3 0.2286 39.6 0.0123481 130.989 424 | 250 7.3 0.2286 39.6 0.0123481 131.889 425 | 315 7.3 0.2286 39.6 0.0123481 132.149 426 | 400 7.3 0.2286 39.6 0.0123481 132.039 427 | 500 7.3 0.2286 39.6 0.0123481 130.299 428 | 630 7.3 0.2286 39.6 0.0123481 128.929 429 | 800 7.3 0.2286 39.6 0.0123481 126.299 430 | 1000 7.3 0.2286 39.6 0.0123481 122.539 431 | 1250 7.3 0.2286 39.6 0.0123481 123.189 432 | 1600 7.3 0.2286 39.6 0.0123481 121.059 433 | 2000 7.3 0.2286 39.6 0.0123481 117.809 434 | 2500 7.3 0.2286 39.6 0.0123481 116.559 435 | 3150 7.3 0.2286 39.6 0.0123481 114.309 436 | 4000 7.3 0.2286 39.6 0.0123481 114.079 437 | 5000 7.3 0.2286 39.6 0.0123481 111.959 438 | 6300 7.3 0.2286 39.6 0.0123481 110.839 439 | 200 7.3 0.2286 31.7 0.0132672 128.679 440 | 250 7.3 0.2286 31.7 0.0132672 130.089 441 | 315 7.3 0.2286 31.7 0.0132672 130.239 442 | 400 7.3 0.2286 31.7 0.0132672 130.269 443 | 500 7.3 0.2286 31.7 0.0132672 128.169 444 | 630 7.3 0.2286 31.7 0.0132672 126.189 445 | 800 7.3 0.2286 31.7 0.0132672 123.209 446 | 1000 7.3 0.2286 31.7 0.0132672 119.099 447 | 1250 7.3 0.2286 31.7 0.0132672 120.509 448 | 1600 7.3 0.2286 31.7 0.0132672 119.039 449 | 2000 7.3 0.2286 31.7 0.0132672 115.309 450 | 2500 7.3 0.2286 31.7 0.0132672 114.709 451 | 3150 7.3 0.2286 31.7 0.0132672 113.229 452 | 4000 7.3 0.2286 31.7 0.0132672 112.639 453 | 5000 7.3 0.2286 31.7 0.0132672 111.029 454 | 6300 7.3 0.2286 31.7 0.0132672 110.689 455 | 800 0 0.1524 71.3 0.0015988 125.817 456 | 1000 0 0.1524 71.3 0.0015988 127.307 457 | 1250 0 0.1524 71.3 0.0015988 128.927 458 | 1600 0 0.1524 71.3 0.0015988 129.667 459 | 2000 0 0.1524 71.3 0.0015988 128.647 460 | 2500 0 0.1524 71.3 0.0015988 128.127 461 | 3150 0 0.1524 71.3 0.0015988 129.377 462 | 4000 0 0.1524 71.3 0.0015988 128.857 463 | 5000 0 0.1524 71.3 0.0015988 126.457 464 | 6300 0 0.1524 71.3 0.0015988 125.427 465 | 8000 0 0.1524 71.3 0.0015988 122.527 466 | 10000 0 0.1524 71.3 0.0015988 120.247 467 | 12500 0 0.1524 71.3 0.0015988 117.087 468 | 16000 0 0.1524 71.3 0.0015988 113.297 469 | 500 0 0.1524 55.5 0.00172668 120.573 470 | 630 0 0.1524 55.5 0.00172668 123.583 471 | 800 0 0.1524 55.5 0.00172668 126.713 472 | 1000 0 0.1524 55.5 0.00172668 128.583 473 | 1250 0 0.1524 55.5 0.00172668 129.953 474 | 1600 0 0.1524 55.5 0.00172668 130.183 475 | 2000 0 0.1524 55.5 0.00172668 129.673 476 | 2500 0 0.1524 55.5 0.00172668 127.763 477 | 3150 0 0.1524 55.5 0.00172668 127.753 478 | 4000 0 0.1524 55.5 0.00172668 127.233 479 | 5000 0 0.1524 55.5 0.00172668 125.203 480 | 6300 0 0.1524 55.5 0.00172668 123.303 481 | 8000 0 0.1524 55.5 0.00172668 121.903 482 | 10000 0 0.1524 55.5 0.00172668 119.253 483 | 12500 0 0.1524 55.5 0.00172668 117.093 484 | 16000 0 0.1524 55.5 0.00172668 112.803 485 | 500 0 0.1524 39.6 0.00193287 119.513 486 | 630 0 0.1524 39.6 0.00193287 124.403 487 | 800 0 0.1524 39.6 0.00193287 127.903 488 | 1000 0 0.1524 39.6 0.00193287 130.033 489 | 1250 0 0.1524 39.6 0.00193287 131.023 490 | 1600 0 0.1524 39.6 0.00193287 131.013 491 | 2000 0 0.1524 39.6 0.00193287 129.633 492 | 2500 0 0.1524 39.6 0.00193287 126.863 493 | 3150 0 0.1524 39.6 0.00193287 125.603 494 | 4000 0 0.1524 39.6 0.00193287 125.343 495 | 5000 0 0.1524 39.6 0.00193287 123.453 496 | 6300 0 0.1524 39.6 0.00193287 121.313 497 | 8000 0 0.1524 39.6 0.00193287 120.553 498 | 10000 0 0.1524 39.6 0.00193287 115.413 499 | 500 0 0.1524 31.7 0.00209405 121.617 500 | 630 0 0.1524 31.7 0.00209405 125.997 501 | 800 0 0.1524 31.7 0.00209405 129.117 502 | 1000 0 0.1524 31.7 0.00209405 130.987 503 | 1250 0 0.1524 31.7 0.00209405 131.467 504 | 1600 0 0.1524 31.7 0.00209405 130.817 505 | 2000 0 0.1524 31.7 0.00209405 128.907 506 | 2500 0 0.1524 31.7 0.00209405 125.867 507 | 3150 0 0.1524 31.7 0.00209405 124.207 508 | 4000 0 0.1524 31.7 0.00209405 123.807 509 | 5000 0 0.1524 31.7 0.00209405 122.397 510 | 6300 0 0.1524 31.7 0.00209405 119.737 511 | 8000 0 0.1524 31.7 0.00209405 117.957 512 | 630 2.7 0.1524 71.3 0.00243851 127.404 513 | 800 2.7 0.1524 71.3 0.00243851 127.394 514 | 1000 2.7 0.1524 71.3 0.00243851 128.774 515 | 1250 2.7 0.1524 71.3 0.00243851 130.144 516 | 1600 2.7 0.1524 71.3 0.00243851 130.644 517 | 2000 2.7 0.1524 71.3 0.00243851 130.114 518 | 2500 2.7 0.1524 71.3 0.00243851 128.334 519 | 3150 2.7 0.1524 71.3 0.00243851 127.054 520 | 4000 2.7 0.1524 71.3 0.00243851 126.534 521 | 5000 2.7 0.1524 71.3 0.00243851 124.364 522 | 6300 2.7 0.1524 71.3 0.00243851 121.944 523 | 8000 2.7 0.1524 71.3 0.00243851 120.534 524 | 10000 2.7 0.1524 71.3 0.00243851 116.724 525 | 12500 2.7 0.1524 71.3 0.00243851 113.034 526 | 16000 2.7 0.1524 71.3 0.00243851 110.364 527 | 500 2.7 0.1524 39.6 0.00294804 121.009 528 | 630 2.7 0.1524 39.6 0.00294804 125.809 529 | 800 2.7 0.1524 39.6 0.00294804 128.829 530 | 1000 2.7 0.1524 39.6 0.00294804 130.589 531 | 1250 2.7 0.1524 39.6 0.00294804 130.829 532 | 1600 2.7 0.1524 39.6 0.00294804 130.049 533 | 2000 2.7 0.1524 39.6 0.00294804 128.139 534 | 2500 2.7 0.1524 39.6 0.00294804 125.589 535 | 3150 2.7 0.1524 39.6 0.00294804 122.919 536 | 4000 2.7 0.1524 39.6 0.00294804 121.889 537 | 5000 2.7 0.1524 39.6 0.00294804 121.499 538 | 6300 2.7 0.1524 39.6 0.00294804 119.209 539 | 8000 2.7 0.1524 39.6 0.00294804 116.659 540 | 10000 2.7 0.1524 39.6 0.00294804 112.589 541 | 12500 2.7 0.1524 39.6 0.00294804 108.649 542 | 400 5.4 0.1524 71.3 0.00401199 124.121 543 | 500 5.4 0.1524 71.3 0.00401199 126.291 544 | 630 5.4 0.1524 71.3 0.00401199 128.971 545 | 800 5.4 0.1524 71.3 0.00401199 131.281 546 | 1000 5.4 0.1524 71.3 0.00401199 133.201 547 | 1250 5.4 0.1524 71.3 0.00401199 134.111 548 | 1600 5.4 0.1524 71.3 0.00401199 133.241 549 | 2000 5.4 0.1524 71.3 0.00401199 131.111 550 | 2500 5.4 0.1524 71.3 0.00401199 127.591 551 | 3150 5.4 0.1524 71.3 0.00401199 123.311 552 | 4000 5.4 0.1524 71.3 0.00401199 121.431 553 | 5000 5.4 0.1524 71.3 0.00401199 120.061 554 | 6300 5.4 0.1524 71.3 0.00401199 116.411 555 | 400 5.4 0.1524 55.5 0.00433288 126.807 556 | 500 5.4 0.1524 55.5 0.00433288 129.367 557 | 630 5.4 0.1524 55.5 0.00433288 131.807 558 | 800 5.4 0.1524 55.5 0.00433288 133.097 559 | 1000 5.4 0.1524 55.5 0.00433288 132.127 560 | 1250 5.4 0.1524 55.5 0.00433288 130.777 561 | 1600 5.4 0.1524 55.5 0.00433288 130.567 562 | 2000 5.4 0.1524 55.5 0.00433288 128.707 563 | 2500 5.4 0.1524 55.5 0.00433288 124.077 564 | 3150 5.4 0.1524 55.5 0.00433288 121.587 565 | 4000 5.4 0.1524 55.5 0.00433288 119.737 566 | 5000 5.4 0.1524 55.5 0.00433288 118.757 567 | 6300 5.4 0.1524 55.5 0.00433288 117.287 568 | 8000 5.4 0.1524 55.5 0.00433288 114.927 569 | 315 5.4 0.1524 39.6 0.00485029 125.347 570 | 400 5.4 0.1524 39.6 0.00485029 127.637 571 | 500 5.4 0.1524 39.6 0.00485029 129.937 572 | 630 5.4 0.1524 39.6 0.00485029 132.357 573 | 800 5.4 0.1524 39.6 0.00485029 132.757 574 | 1000 5.4 0.1524 39.6 0.00485029 130.507 575 | 1250 5.4 0.1524 39.6 0.00485029 127.117 576 | 1600 5.4 0.1524 39.6 0.00485029 126.267 577 | 2000 5.4 0.1524 39.6 0.00485029 124.647 578 | 2500 5.4 0.1524 39.6 0.00485029 120.497 579 | 3150 5.4 0.1524 39.6 0.00485029 119.137 580 | 4000 5.4 0.1524 39.6 0.00485029 117.137 581 | 5000 5.4 0.1524 39.6 0.00485029 117.037 582 | 6300 5.4 0.1524 39.6 0.00485029 116.677 583 | 315 5.4 0.1524 31.7 0.00525474 125.741 584 | 400 5.4 0.1524 31.7 0.00525474 127.781 585 | 500 5.4 0.1524 31.7 0.00525474 129.681 586 | 630 5.4 0.1524 31.7 0.00525474 131.471 587 | 800 5.4 0.1524 31.7 0.00525474 131.491 588 | 1000 5.4 0.1524 31.7 0.00525474 128.241 589 | 1250 5.4 0.1524 31.7 0.00525474 123.991 590 | 1600 5.4 0.1524 31.7 0.00525474 123.761 591 | 2000 5.4 0.1524 31.7 0.00525474 122.771 592 | 2500 5.4 0.1524 31.7 0.00525474 119.151 593 | 3150 5.4 0.1524 31.7 0.00525474 118.291 594 | 4000 5.4 0.1524 31.7 0.00525474 116.181 595 | 5000 5.4 0.1524 31.7 0.00525474 115.691 596 | 6300 5.4 0.1524 31.7 0.00525474 115.591 597 | 315 7.2 0.1524 71.3 0.00752039 128.713 598 | 400 7.2 0.1524 71.3 0.00752039 130.123 599 | 500 7.2 0.1524 71.3 0.00752039 132.043 600 | 630 7.2 0.1524 71.3 0.00752039 134.853 601 | 800 7.2 0.1524 71.3 0.00752039 136.023 602 | 1000 7.2 0.1524 71.3 0.00752039 134.273 603 | 1250 7.2 0.1524 71.3 0.00752039 132.513 604 | 1600 7.2 0.1524 71.3 0.00752039 130.893 605 | 2000 7.2 0.1524 71.3 0.00752039 128.643 606 | 2500 7.2 0.1524 71.3 0.00752039 124.353 607 | 3150 7.2 0.1524 71.3 0.00752039 116.783 608 | 4000 7.2 0.1524 71.3 0.00752039 119.343 609 | 5000 7.2 0.1524 71.3 0.00752039 118.343 610 | 6300 7.2 0.1524 71.3 0.00752039 116.603 611 | 8000 7.2 0.1524 71.3 0.00752039 113.333 612 | 10000 7.2 0.1524 71.3 0.00752039 110.313 613 | 250 7.2 0.1524 39.6 0.00909175 127.488 614 | 315 7.2 0.1524 39.6 0.00909175 130.558 615 | 400 7.2 0.1524 39.6 0.00909175 132.118 616 | 500 7.2 0.1524 39.6 0.00909175 132.658 617 | 630 7.2 0.1524 39.6 0.00909175 133.198 618 | 800 7.2 0.1524 39.6 0.00909175 132.358 619 | 1000 7.2 0.1524 39.6 0.00909175 128.338 620 | 1250 7.2 0.1524 39.6 0.00909175 122.428 621 | 1600 7.2 0.1524 39.6 0.00909175 120.058 622 | 2000 7.2 0.1524 39.6 0.00909175 120.228 623 | 2500 7.2 0.1524 39.6 0.00909175 117.478 624 | 3150 7.2 0.1524 39.6 0.00909175 111.818 625 | 4000 7.2 0.1524 39.6 0.00909175 114.258 626 | 5000 7.2 0.1524 39.6 0.00909175 113.288 627 | 6300 7.2 0.1524 39.6 0.00909175 112.688 628 | 8000 7.2 0.1524 39.6 0.00909175 111.588 629 | 10000 7.2 0.1524 39.6 0.00909175 110.868 630 | 200 9.9 0.1524 71.3 0.0193001 134.319 631 | 250 9.9 0.1524 71.3 0.0193001 135.329 632 | 315 9.9 0.1524 71.3 0.0193001 135.459 633 | 400 9.9 0.1524 71.3 0.0193001 135.079 634 | 500 9.9 0.1524 71.3 0.0193001 131.279 635 | 630 9.9 0.1524 71.3 0.0193001 129.889 636 | 800 9.9 0.1524 71.3 0.0193001 128.879 637 | 1000 9.9 0.1524 71.3 0.0193001 126.349 638 | 1250 9.9 0.1524 71.3 0.0193001 122.679 639 | 1600 9.9 0.1524 71.3 0.0193001 121.789 640 | 2000 9.9 0.1524 71.3 0.0193001 120.779 641 | 2500 9.9 0.1524 71.3 0.0193001 119.639 642 | 3150 9.9 0.1524 71.3 0.0193001 116.849 643 | 4000 9.9 0.1524 71.3 0.0193001 115.079 644 | 5000 9.9 0.1524 71.3 0.0193001 114.569 645 | 6300 9.9 0.1524 71.3 0.0193001 112.039 646 | 200 9.9 0.1524 55.5 0.0208438 131.955 647 | 250 9.9 0.1524 55.5 0.0208438 133.235 648 | 315 9.9 0.1524 55.5 0.0208438 132.355 649 | 400 9.9 0.1524 55.5 0.0208438 131.605 650 | 500 9.9 0.1524 55.5 0.0208438 127.815 651 | 630 9.9 0.1524 55.5 0.0208438 127.315 652 | 800 9.9 0.1524 55.5 0.0208438 126.565 653 | 1000 9.9 0.1524 55.5 0.0208438 124.665 654 | 1250 9.9 0.1524 55.5 0.0208438 121.635 655 | 1600 9.9 0.1524 55.5 0.0208438 119.875 656 | 2000 9.9 0.1524 55.5 0.0208438 119.505 657 | 2500 9.9 0.1524 55.5 0.0208438 118.365 658 | 3150 9.9 0.1524 55.5 0.0208438 115.085 659 | 4000 9.9 0.1524 55.5 0.0208438 112.945 660 | 5000 9.9 0.1524 55.5 0.0208438 112.065 661 | 6300 9.9 0.1524 55.5 0.0208438 110.555 662 | 200 9.9 0.1524 39.6 0.0233328 127.315 663 | 250 9.9 0.1524 39.6 0.0233328 128.335 664 | 315 9.9 0.1524 39.6 0.0233328 128.595 665 | 400 9.9 0.1524 39.6 0.0233328 128.345 666 | 500 9.9 0.1524 39.6 0.0233328 126.835 667 | 630 9.9 0.1524 39.6 0.0233328 126.465 668 | 800 9.9 0.1524 39.6 0.0233328 126.345 669 | 1000 9.9 0.1524 39.6 0.0233328 123.835 670 | 1250 9.9 0.1524 39.6 0.0233328 120.555 671 | 1600 9.9 0.1524 39.6 0.0233328 118.545 672 | 2000 9.9 0.1524 39.6 0.0233328 117.925 673 | 2500 9.9 0.1524 39.6 0.0233328 116.295 674 | 3150 9.9 0.1524 39.6 0.0233328 113.525 675 | 4000 9.9 0.1524 39.6 0.0233328 112.265 676 | 5000 9.9 0.1524 39.6 0.0233328 111.135 677 | 6300 9.9 0.1524 39.6 0.0233328 109.885 678 | 200 9.9 0.1524 31.7 0.0252785 127.299 679 | 250 9.9 0.1524 31.7 0.0252785 128.559 680 | 315 9.9 0.1524 31.7 0.0252785 128.809 681 | 400 9.9 0.1524 31.7 0.0252785 128.939 682 | 500 9.9 0.1524 31.7 0.0252785 127.179 683 | 630 9.9 0.1524 31.7 0.0252785 126.049 684 | 800 9.9 0.1524 31.7 0.0252785 125.539 685 | 1000 9.9 0.1524 31.7 0.0252785 122.149 686 | 1250 9.9 0.1524 31.7 0.0252785 118.619 687 | 1600 9.9 0.1524 31.7 0.0252785 117.119 688 | 2000 9.9 0.1524 31.7 0.0252785 116.859 689 | 2500 9.9 0.1524 31.7 0.0252785 114.729 690 | 3150 9.9 0.1524 31.7 0.0252785 112.209 691 | 4000 9.9 0.1524 31.7 0.0252785 111.459 692 | 5000 9.9 0.1524 31.7 0.0252785 109.949 693 | 6300 9.9 0.1524 31.7 0.0252785 108.689 694 | 200 12.6 0.1524 71.3 0.0483159 128.354 695 | 250 12.6 0.1524 71.3 0.0483159 129.744 696 | 315 12.6 0.1524 71.3 0.0483159 128.484 697 | 400 12.6 0.1524 71.3 0.0483159 127.094 698 | 500 12.6 0.1524 71.3 0.0483159 121.664 699 | 630 12.6 0.1524 71.3 0.0483159 123.304 700 | 800 12.6 0.1524 71.3 0.0483159 123.054 701 | 1000 12.6 0.1524 71.3 0.0483159 122.044 702 | 1250 12.6 0.1524 71.3 0.0483159 120.154 703 | 1600 12.6 0.1524 71.3 0.0483159 120.534 704 | 2000 12.6 0.1524 71.3 0.0483159 117.504 705 | 2500 12.6 0.1524 71.3 0.0483159 115.234 706 | 3150 12.6 0.1524 71.3 0.0483159 113.334 707 | 4000 12.6 0.1524 71.3 0.0483159 108.034 708 | 5000 12.6 0.1524 71.3 0.0483159 108.034 709 | 6300 12.6 0.1524 71.3 0.0483159 107.284 710 | 200 12.6 0.1524 39.6 0.0584113 114.750 711 | 250 12.6 0.1524 39.6 0.0584113 115.890 712 | 315 12.6 0.1524 39.6 0.0584113 116.020 713 | 400 12.6 0.1524 39.6 0.0584113 115.910 714 | 500 12.6 0.1524 39.6 0.0584113 114.900 715 | 630 12.6 0.1524 39.6 0.0584113 116.550 716 | 800 12.6 0.1524 39.6 0.0584113 116.560 717 | 1000 12.6 0.1524 39.6 0.0584113 114.670 718 | 1250 12.6 0.1524 39.6 0.0584113 112.160 719 | 1600 12.6 0.1524 39.6 0.0584113 110.780 720 | 2000 12.6 0.1524 39.6 0.0584113 109.520 721 | 2500 12.6 0.1524 39.6 0.0584113 106.880 722 | 3150 12.6 0.1524 39.6 0.0584113 106.260 723 | 4000 12.6 0.1524 39.6 0.0584113 104.500 724 | 5000 12.6 0.1524 39.6 0.0584113 104.130 725 | 6300 12.6 0.1524 39.6 0.0584113 103.380 726 | 800 0 0.0508 71.3 0.000740478 130.960 727 | 1000 0 0.0508 71.3 0.000740478 129.450 728 | 1250 0 0.0508 71.3 0.000740478 128.560 729 | 1600 0 0.0508 71.3 0.000740478 129.680 730 | 2000 0 0.0508 71.3 0.000740478 131.060 731 | 2500 0 0.0508 71.3 0.000740478 131.310 732 | 3150 0 0.0508 71.3 0.000740478 135.070 733 | 4000 0 0.0508 71.3 0.000740478 134.430 734 | 5000 0 0.0508 71.3 0.000740478 134.430 735 | 6300 0 0.0508 71.3 0.000740478 133.040 736 | 8000 0 0.0508 71.3 0.000740478 130.890 737 | 10000 0 0.0508 71.3 0.000740478 128.740 738 | 12500 0 0.0508 71.3 0.000740478 125.220 739 | 800 0 0.0508 55.5 0.00076193 124.336 740 | 1000 0 0.0508 55.5 0.00076193 125.586 741 | 1250 0 0.0508 55.5 0.00076193 127.076 742 | 1600 0 0.0508 55.5 0.00076193 128.576 743 | 2000 0 0.0508 55.5 0.00076193 131.456 744 | 2500 0 0.0508 55.5 0.00076193 133.956 745 | 3150 0 0.0508 55.5 0.00076193 134.826 746 | 4000 0 0.0508 55.5 0.00076193 134.946 747 | 5000 0 0.0508 55.5 0.00076193 134.556 748 | 6300 0 0.0508 55.5 0.00076193 132.796 749 | 8000 0 0.0508 55.5 0.00076193 130.156 750 | 10000 0 0.0508 55.5 0.00076193 127.636 751 | 12500 0 0.0508 55.5 0.00076193 125.376 752 | 800 0 0.0508 39.6 0.000791822 126.508 753 | 1000 0 0.0508 39.6 0.000791822 127.638 754 | 1250 0 0.0508 39.6 0.000791822 129.148 755 | 1600 0 0.0508 39.6 0.000791822 130.908 756 | 2000 0 0.0508 39.6 0.000791822 132.918 757 | 2500 0 0.0508 39.6 0.000791822 134.938 758 | 3150 0 0.0508 39.6 0.000791822 135.938 759 | 4000 0 0.0508 39.6 0.000791822 135.308 760 | 5000 0 0.0508 39.6 0.000791822 134.308 761 | 6300 0 0.0508 39.6 0.000791822 131.918 762 | 8000 0 0.0508 39.6 0.000791822 128.518 763 | 10000 0 0.0508 39.6 0.000791822 125.998 764 | 12500 0 0.0508 39.6 0.000791822 123.988 765 | 800 0 0.0508 31.7 0.000812164 122.790 766 | 1000 0 0.0508 31.7 0.000812164 126.780 767 | 1250 0 0.0508 31.7 0.000812164 129.270 768 | 1600 0 0.0508 31.7 0.000812164 131.010 769 | 2000 0 0.0508 31.7 0.000812164 133.010 770 | 2500 0 0.0508 31.7 0.000812164 134.870 771 | 3150 0 0.0508 31.7 0.000812164 135.490 772 | 4000 0 0.0508 31.7 0.000812164 134.110 773 | 5000 0 0.0508 31.7 0.000812164 133.230 774 | 6300 0 0.0508 31.7 0.000812164 130.340 775 | 8000 0 0.0508 31.7 0.000812164 126.590 776 | 10000 0 0.0508 31.7 0.000812164 122.450 777 | 12500 0 0.0508 31.7 0.000812164 119.070 778 | 1600 4.2 0.0508 71.3 0.00142788 124.318 779 | 2000 4.2 0.0508 71.3 0.00142788 129.848 780 | 2500 4.2 0.0508 71.3 0.00142788 131.978 781 | 3150 4.2 0.0508 71.3 0.00142788 133.728 782 | 4000 4.2 0.0508 71.3 0.00142788 133.598 783 | 5000 4.2 0.0508 71.3 0.00142788 132.828 784 | 6300 4.2 0.0508 71.3 0.00142788 129.308 785 | 8000 4.2 0.0508 71.3 0.00142788 125.268 786 | 10000 4.2 0.0508 71.3 0.00142788 121.238 787 | 12500 4.2 0.0508 71.3 0.00142788 117.328 788 | 1000 4.2 0.0508 39.6 0.00152689 125.647 789 | 1250 4.2 0.0508 39.6 0.00152689 128.427 790 | 1600 4.2 0.0508 39.6 0.00152689 130.197 791 | 2000 4.2 0.0508 39.6 0.00152689 132.587 792 | 2500 4.2 0.0508 39.6 0.00152689 133.847 793 | 3150 4.2 0.0508 39.6 0.00152689 133.587 794 | 4000 4.2 0.0508 39.6 0.00152689 131.807 795 | 5000 4.2 0.0508 39.6 0.00152689 129.777 796 | 6300 4.2 0.0508 39.6 0.00152689 125.717 797 | 8000 4.2 0.0508 39.6 0.00152689 120.397 798 | 10000 4.2 0.0508 39.6 0.00152689 116.967 799 | 800 8.4 0.0508 71.3 0.00529514 127.556 800 | 1000 8.4 0.0508 71.3 0.00529514 129.946 801 | 1250 8.4 0.0508 71.3 0.00529514 132.086 802 | 1600 8.4 0.0508 71.3 0.00529514 133.846 803 | 2000 8.4 0.0508 71.3 0.00529514 134.476 804 | 2500 8.4 0.0508 71.3 0.00529514 134.226 805 | 3150 8.4 0.0508 71.3 0.00529514 131.966 806 | 4000 8.4 0.0508 71.3 0.00529514 126.926 807 | 5000 8.4 0.0508 71.3 0.00529514 121.146 808 | 400 8.4 0.0508 55.5 0.00544854 121.582 809 | 500 8.4 0.0508 55.5 0.00544854 123.742 810 | 630 8.4 0.0508 55.5 0.00544854 126.152 811 | 800 8.4 0.0508 55.5 0.00544854 128.562 812 | 1000 8.4 0.0508 55.5 0.00544854 130.722 813 | 1250 8.4 0.0508 55.5 0.00544854 132.252 814 | 1600 8.4 0.0508 55.5 0.00544854 133.032 815 | 2000 8.4 0.0508 55.5 0.00544854 133.042 816 | 2500 8.4 0.0508 55.5 0.00544854 131.542 817 | 3150 8.4 0.0508 55.5 0.00544854 128.402 818 | 4000 8.4 0.0508 55.5 0.00544854 122.612 819 | 5000 8.4 0.0508 55.5 0.00544854 115.812 820 | 400 8.4 0.0508 39.6 0.00566229 120.015 821 | 500 8.4 0.0508 39.6 0.00566229 122.905 822 | 630 8.4 0.0508 39.6 0.00566229 126.045 823 | 800 8.4 0.0508 39.6 0.00566229 128.435 824 | 1000 8.4 0.0508 39.6 0.00566229 130.195 825 | 1250 8.4 0.0508 39.6 0.00566229 131.205 826 | 1600 8.4 0.0508 39.6 0.00566229 130.965 827 | 2000 8.4 0.0508 39.6 0.00566229 129.965 828 | 2500 8.4 0.0508 39.6 0.00566229 127.465 829 | 3150 8.4 0.0508 39.6 0.00566229 123.965 830 | 4000 8.4 0.0508 39.6 0.00566229 118.955 831 | 400 8.4 0.0508 31.7 0.00580776 120.076 832 | 500 8.4 0.0508 31.7 0.00580776 122.966 833 | 630 8.4 0.0508 31.7 0.00580776 125.856 834 | 800 8.4 0.0508 31.7 0.00580776 128.246 835 | 1000 8.4 0.0508 31.7 0.00580776 129.516 836 | 1250 8.4 0.0508 31.7 0.00580776 130.156 837 | 1600 8.4 0.0508 31.7 0.00580776 129.296 838 | 2000 8.4 0.0508 31.7 0.00580776 127.686 839 | 2500 8.4 0.0508 31.7 0.00580776 125.576 840 | 3150 8.4 0.0508 31.7 0.00580776 122.086 841 | 4000 8.4 0.0508 31.7 0.00580776 118.106 842 | 200 11.2 0.0508 71.3 0.014072 125.941 843 | 250 11.2 0.0508 71.3 0.014072 127.101 844 | 315 11.2 0.0508 71.3 0.014072 128.381 845 | 400 11.2 0.0508 71.3 0.014072 129.281 846 | 500 11.2 0.0508 71.3 0.014072 130.311 847 | 630 11.2 0.0508 71.3 0.014072 133.611 848 | 800 11.2 0.0508 71.3 0.014072 136.031 849 | 1000 11.2 0.0508 71.3 0.014072 136.941 850 | 1250 11.2 0.0508 71.3 0.014072 136.191 851 | 1600 11.2 0.0508 71.3 0.014072 135.191 852 | 2000 11.2 0.0508 71.3 0.014072 133.311 853 | 2500 11.2 0.0508 71.3 0.014072 130.541 854 | 3150 11.2 0.0508 71.3 0.014072 127.141 855 | 4000 11.2 0.0508 71.3 0.014072 122.471 856 | 200 11.2 0.0508 39.6 0.0150478 125.010 857 | 250 11.2 0.0508 39.6 0.0150478 126.430 858 | 315 11.2 0.0508 39.6 0.0150478 128.990 859 | 400 11.2 0.0508 39.6 0.0150478 130.670 860 | 500 11.2 0.0508 39.6 0.0150478 131.960 861 | 630 11.2 0.0508 39.6 0.0150478 133.130 862 | 800 11.2 0.0508 39.6 0.0150478 133.790 863 | 1000 11.2 0.0508 39.6 0.0150478 132.430 864 | 1250 11.2 0.0508 39.6 0.0150478 130.050 865 | 1600 11.2 0.0508 39.6 0.0150478 126.540 866 | 2000 11.2 0.0508 39.6 0.0150478 124.420 867 | 2500 11.2 0.0508 39.6 0.0150478 122.170 868 | 3150 11.2 0.0508 39.6 0.0150478 119.670 869 | 4000 11.2 0.0508 39.6 0.0150478 115.520 870 | 200 15.4 0.0508 71.3 0.0264269 123.595 871 | 250 15.4 0.0508 71.3 0.0264269 124.835 872 | 315 15.4 0.0508 71.3 0.0264269 126.195 873 | 400 15.4 0.0508 71.3 0.0264269 126.805 874 | 500 15.4 0.0508 71.3 0.0264269 127.285 875 | 630 15.4 0.0508 71.3 0.0264269 129.645 876 | 800 15.4 0.0508 71.3 0.0264269 131.515 877 | 1000 15.4 0.0508 71.3 0.0264269 131.865 878 | 1250 15.4 0.0508 71.3 0.0264269 130.845 879 | 1600 15.4 0.0508 71.3 0.0264269 130.065 880 | 2000 15.4 0.0508 71.3 0.0264269 129.285 881 | 2500 15.4 0.0508 71.3 0.0264269 127.625 882 | 3150 15.4 0.0508 71.3 0.0264269 125.715 883 | 4000 15.4 0.0508 71.3 0.0264269 122.675 884 | 5000 15.4 0.0508 71.3 0.0264269 119.135 885 | 6300 15.4 0.0508 71.3 0.0264269 115.215 886 | 8000 15.4 0.0508 71.3 0.0264269 112.675 887 | 200 15.4 0.0508 55.5 0.0271925 122.940 888 | 250 15.4 0.0508 55.5 0.0271925 124.170 889 | 315 15.4 0.0508 55.5 0.0271925 125.390 890 | 400 15.4 0.0508 55.5 0.0271925 126.500 891 | 500 15.4 0.0508 55.5 0.0271925 127.220 892 | 630 15.4 0.0508 55.5 0.0271925 129.330 893 | 800 15.4 0.0508 55.5 0.0271925 130.430 894 | 1000 15.4 0.0508 55.5 0.0271925 130.400 895 | 1250 15.4 0.0508 55.5 0.0271925 130.000 896 | 1600 15.4 0.0508 55.5 0.0271925 128.200 897 | 2000 15.4 0.0508 55.5 0.0271925 127.040 898 | 2500 15.4 0.0508 55.5 0.0271925 125.630 899 | 3150 15.4 0.0508 55.5 0.0271925 123.460 900 | 4000 15.4 0.0508 55.5 0.0271925 120.920 901 | 5000 15.4 0.0508 55.5 0.0271925 117.110 902 | 6300 15.4 0.0508 55.5 0.0271925 112.930 903 | 200 15.4 0.0508 39.6 0.0282593 121.783 904 | 250 15.4 0.0508 39.6 0.0282593 122.893 905 | 315 15.4 0.0508 39.6 0.0282593 124.493 906 | 400 15.4 0.0508 39.6 0.0282593 125.353 907 | 500 15.4 0.0508 39.6 0.0282593 125.963 908 | 630 15.4 0.0508 39.6 0.0282593 127.443 909 | 800 15.4 0.0508 39.6 0.0282593 128.423 910 | 1000 15.4 0.0508 39.6 0.0282593 127.893 911 | 1250 15.4 0.0508 39.6 0.0282593 126.743 912 | 1600 15.4 0.0508 39.6 0.0282593 124.843 913 | 2000 15.4 0.0508 39.6 0.0282593 123.443 914 | 2500 15.4 0.0508 39.6 0.0282593 122.413 915 | 3150 15.4 0.0508 39.6 0.0282593 120.513 916 | 4000 15.4 0.0508 39.6 0.0282593 118.113 917 | 5000 15.4 0.0508 39.6 0.0282593 114.453 918 | 6300 15.4 0.0508 39.6 0.0282593 109.663 919 | 200 15.4 0.0508 31.7 0.0289853 119.975 920 | 250 15.4 0.0508 31.7 0.0289853 121.225 921 | 315 15.4 0.0508 31.7 0.0289853 122.845 922 | 400 15.4 0.0508 31.7 0.0289853 123.705 923 | 500 15.4 0.0508 31.7 0.0289853 123.695 924 | 630 15.4 0.0508 31.7 0.0289853 124.685 925 | 800 15.4 0.0508 31.7 0.0289853 125.555 926 | 1000 15.4 0.0508 31.7 0.0289853 124.525 927 | 1250 15.4 0.0508 31.7 0.0289853 123.255 928 | 1600 15.4 0.0508 31.7 0.0289853 121.485 929 | 2000 15.4 0.0508 31.7 0.0289853 120.835 930 | 2500 15.4 0.0508 31.7 0.0289853 119.945 931 | 3150 15.4 0.0508 31.7 0.0289853 118.045 932 | 4000 15.4 0.0508 31.7 0.0289853 115.635 933 | 5000 15.4 0.0508 31.7 0.0289853 112.355 934 | 6300 15.4 0.0508 31.7 0.0289853 108.185 935 | 200 19.7 0.0508 71.3 0.0341183 118.005 936 | 250 19.7 0.0508 71.3 0.0341183 119.115 937 | 315 19.7 0.0508 71.3 0.0341183 121.235 938 | 400 19.7 0.0508 71.3 0.0341183 123.865 939 | 500 19.7 0.0508 71.3 0.0341183 126.995 940 | 630 19.7 0.0508 71.3 0.0341183 128.365 941 | 800 19.7 0.0508 71.3 0.0341183 124.555 942 | 1000 19.7 0.0508 71.3 0.0341183 121.885 943 | 1250 19.7 0.0508 71.3 0.0341183 121.485 944 | 1600 19.7 0.0508 71.3 0.0341183 120.575 945 | 2000 19.7 0.0508 71.3 0.0341183 120.055 946 | 2500 19.7 0.0508 71.3 0.0341183 118.385 947 | 3150 19.7 0.0508 71.3 0.0341183 116.225 948 | 4000 19.7 0.0508 71.3 0.0341183 113.045 949 | 200 19.7 0.0508 39.6 0.036484 125.974 950 | 250 19.7 0.0508 39.6 0.036484 127.224 951 | 315 19.7 0.0508 39.6 0.036484 129.864 952 | 400 19.7 0.0508 39.6 0.036484 130.614 953 | 500 19.7 0.0508 39.6 0.036484 128.444 954 | 630 19.7 0.0508 39.6 0.036484 120.324 955 | 800 19.7 0.0508 39.6 0.036484 119.174 956 | 1000 19.7 0.0508 39.6 0.036484 118.904 957 | 1250 19.7 0.0508 39.6 0.036484 118.634 958 | 1600 19.7 0.0508 39.6 0.036484 117.604 959 | 2000 19.7 0.0508 39.6 0.036484 117.724 960 | 2500 19.7 0.0508 39.6 0.036484 116.184 961 | 3150 19.7 0.0508 39.6 0.036484 113.004 962 | 4000 19.7 0.0508 39.6 0.036484 108.684 963 | 2500 0 0.0254 71.3 0.000400682 133.707 964 | 3150 0 0.0254 71.3 0.000400682 137.007 965 | 4000 0 0.0254 71.3 0.000400682 138.557 966 | 5000 0 0.0254 71.3 0.000400682 136.837 967 | 6300 0 0.0254 71.3 0.000400682 134.987 968 | 8000 0 0.0254 71.3 0.000400682 129.867 969 | 10000 0 0.0254 71.3 0.000400682 130.787 970 | 12500 0 0.0254 71.3 0.000400682 133.207 971 | 16000 0 0.0254 71.3 0.000400682 130.477 972 | 20000 0 0.0254 71.3 0.000400682 123.217 973 | 2000 0 0.0254 55.5 0.00041229 127.623 974 | 2500 0 0.0254 55.5 0.00041229 130.073 975 | 3150 0 0.0254 55.5 0.00041229 130.503 976 | 4000 0 0.0254 55.5 0.00041229 133.223 977 | 5000 0 0.0254 55.5 0.00041229 135.803 978 | 6300 0 0.0254 55.5 0.00041229 136.103 979 | 8000 0 0.0254 55.5 0.00041229 136.163 980 | 10000 0 0.0254 55.5 0.00041229 134.563 981 | 12500 0 0.0254 55.5 0.00041229 131.453 982 | 16000 0 0.0254 55.5 0.00041229 125.683 983 | 20000 0 0.0254 55.5 0.00041229 121.933 984 | 1600 0 0.0254 39.6 0.000428464 124.156 985 | 2000 0 0.0254 39.6 0.000428464 130.026 986 | 2500 0 0.0254 39.6 0.000428464 131.836 987 | 3150 0 0.0254 39.6 0.000428464 133.276 988 | 4000 0 0.0254 39.6 0.000428464 135.346 989 | 5000 0 0.0254 39.6 0.000428464 136.536 990 | 6300 0 0.0254 39.6 0.000428464 136.826 991 | 8000 0 0.0254 39.6 0.000428464 135.866 992 | 10000 0 0.0254 39.6 0.000428464 133.376 993 | 12500 0 0.0254 39.6 0.000428464 129.116 994 | 16000 0 0.0254 39.6 0.000428464 124.986 995 | 1000 0 0.0254 31.7 0.000439472 125.127 996 | 1250 0 0.0254 31.7 0.000439472 127.947 997 | 1600 0 0.0254 31.7 0.000439472 129.267 998 | 2000 0 0.0254 31.7 0.000439472 130.697 999 | 2500 0 0.0254 31.7 0.000439472 132.897 1000 | 3150 0 0.0254 31.7 0.000439472 135.227 1001 | 4000 0 0.0254 31.7 0.000439472 137.047 1002 | 5000 0 0.0254 31.7 0.000439472 138.607 1003 | 6300 0 0.0254 31.7 0.000439472 138.537 1004 | 8000 0 0.0254 31.7 0.000439472 137.207 1005 | 10000 0 0.0254 31.7 0.000439472 134.227 1006 | 12500 0 0.0254 31.7 0.000439472 128.977 1007 | 16000 0 0.0254 31.7 0.000439472 125.627 1008 | 2000 4.8 0.0254 71.3 0.000848633 128.398 1009 | 2500 4.8 0.0254 71.3 0.000848633 130.828 1010 | 3150 4.8 0.0254 71.3 0.000848633 133.378 1011 | 4000 4.8 0.0254 71.3 0.000848633 134.928 1012 | 5000 4.8 0.0254 71.3 0.000848633 135.468 1013 | 6300 4.8 0.0254 71.3 0.000848633 134.498 1014 | 8000 4.8 0.0254 71.3 0.000848633 131.518 1015 | 10000 4.8 0.0254 71.3 0.000848633 127.398 1016 | 12500 4.8 0.0254 71.3 0.000848633 127.688 1017 | 16000 4.8 0.0254 71.3 0.000848633 124.208 1018 | 20000 4.8 0.0254 71.3 0.000848633 119.708 1019 | 1600 4.8 0.0254 55.5 0.000873218 121.474 1020 | 2000 4.8 0.0254 55.5 0.000873218 125.054 1021 | 2500 4.8 0.0254 55.5 0.000873218 129.144 1022 | 3150 4.8 0.0254 55.5 0.000873218 132.354 1023 | 4000 4.8 0.0254 55.5 0.000873218 133.924 1024 | 5000 4.8 0.0254 55.5 0.000873218 135.484 1025 | 6300 4.8 0.0254 55.5 0.000873218 135.164 1026 | 8000 4.8 0.0254 55.5 0.000873218 132.184 1027 | 10000 4.8 0.0254 55.5 0.000873218 126.944 1028 | 12500 4.8 0.0254 55.5 0.000873218 125.094 1029 | 16000 4.8 0.0254 55.5 0.000873218 124.394 1030 | 20000 4.8 0.0254 55.5 0.000873218 121.284 1031 | 500 4.8 0.0254 39.6 0.000907475 116.366 1032 | 630 4.8 0.0254 39.6 0.000907475 118.696 1033 | 800 4.8 0.0254 39.6 0.000907475 120.766 1034 | 1000 4.8 0.0254 39.6 0.000907475 122.956 1035 | 1250 4.8 0.0254 39.6 0.000907475 125.026 1036 | 1600 4.8 0.0254 39.6 0.000907475 125.966 1037 | 2000 4.8 0.0254 39.6 0.000907475 128.916 1038 | 2500 4.8 0.0254 39.6 0.000907475 131.236 1039 | 3150 4.8 0.0254 39.6 0.000907475 133.436 1040 | 4000 4.8 0.0254 39.6 0.000907475 134.996 1041 | 5000 4.8 0.0254 39.6 0.000907475 135.426 1042 | 6300 4.8 0.0254 39.6 0.000907475 134.336 1043 | 8000 4.8 0.0254 39.6 0.000907475 131.346 1044 | 10000 4.8 0.0254 39.6 0.000907475 126.066 1045 | 500 4.8 0.0254 31.7 0.000930789 116.128 1046 | 630 4.8 0.0254 31.7 0.000930789 120.078 1047 | 800 4.8 0.0254 31.7 0.000930789 122.648 1048 | 1000 4.8 0.0254 31.7 0.000930789 125.348 1049 | 1250 4.8 0.0254 31.7 0.000930789 127.408 1050 | 1600 4.8 0.0254 31.7 0.000930789 128.718 1051 | 2000 4.8 0.0254 31.7 0.000930789 130.148 1052 | 2500 4.8 0.0254 31.7 0.000930789 132.588 1053 | 3150 4.8 0.0254 31.7 0.000930789 134.268 1054 | 4000 4.8 0.0254 31.7 0.000930789 135.328 1055 | 5000 4.8 0.0254 31.7 0.000930789 135.248 1056 | 6300 4.8 0.0254 31.7 0.000930789 132.898 1057 | 8000 4.8 0.0254 31.7 0.000930789 127.008 1058 | 630 9.5 0.0254 71.3 0.00420654 125.726 1059 | 800 9.5 0.0254 71.3 0.00420654 127.206 1060 | 1000 9.5 0.0254 71.3 0.00420654 129.556 1061 | 1250 9.5 0.0254 71.3 0.00420654 131.656 1062 | 1600 9.5 0.0254 71.3 0.00420654 133.756 1063 | 2000 9.5 0.0254 71.3 0.00420654 134.976 1064 | 2500 9.5 0.0254 71.3 0.00420654 135.956 1065 | 3150 9.5 0.0254 71.3 0.00420654 136.166 1066 | 4000 9.5 0.0254 71.3 0.00420654 134.236 1067 | 5000 9.5 0.0254 71.3 0.00420654 131.186 1068 | 6300 9.5 0.0254 71.3 0.00420654 127.246 1069 | 400 9.5 0.0254 55.5 0.0043284 120.952 1070 | 500 9.5 0.0254 55.5 0.0043284 123.082 1071 | 630 9.5 0.0254 55.5 0.0043284 125.452 1072 | 800 9.5 0.0254 55.5 0.0043284 128.082 1073 | 1000 9.5 0.0254 55.5 0.0043284 130.332 1074 | 1250 9.5 0.0254 55.5 0.0043284 132.202 1075 | 1600 9.5 0.0254 55.5 0.0043284 133.062 1076 | 2000 9.5 0.0254 55.5 0.0043284 134.052 1077 | 2500 9.5 0.0254 55.5 0.0043284 134.152 1078 | 3150 9.5 0.0254 55.5 0.0043284 133.252 1079 | 4000 9.5 0.0254 55.5 0.0043284 131.582 1080 | 5000 9.5 0.0254 55.5 0.0043284 128.412 1081 | 6300 9.5 0.0254 55.5 0.0043284 124.222 1082 | 200 9.5 0.0254 39.6 0.00449821 116.074 1083 | 250 9.5 0.0254 39.6 0.00449821 116.924 1084 | 315 9.5 0.0254 39.6 0.00449821 119.294 1085 | 400 9.5 0.0254 39.6 0.00449821 121.154 1086 | 500 9.5 0.0254 39.6 0.00449821 123.894 1087 | 630 9.5 0.0254 39.6 0.00449821 126.514 1088 | 800 9.5 0.0254 39.6 0.00449821 129.014 1089 | 1000 9.5 0.0254 39.6 0.00449821 130.374 1090 | 1250 9.5 0.0254 39.6 0.00449821 130.964 1091 | 1600 9.5 0.0254 39.6 0.00449821 131.184 1092 | 2000 9.5 0.0254 39.6 0.00449821 131.274 1093 | 2500 9.5 0.0254 39.6 0.00449821 131.234 1094 | 3150 9.5 0.0254 39.6 0.00449821 129.934 1095 | 4000 9.5 0.0254 39.6 0.00449821 127.864 1096 | 5000 9.5 0.0254 39.6 0.00449821 125.044 1097 | 6300 9.5 0.0254 39.6 0.00449821 120.324 1098 | 200 9.5 0.0254 31.7 0.00461377 119.146 1099 | 250 9.5 0.0254 31.7 0.00461377 120.136 1100 | 315 9.5 0.0254 31.7 0.00461377 122.766 1101 | 400 9.5 0.0254 31.7 0.00461377 124.756 1102 | 500 9.5 0.0254 31.7 0.00461377 126.886 1103 | 630 9.5 0.0254 31.7 0.00461377 129.006 1104 | 800 9.5 0.0254 31.7 0.00461377 130.746 1105 | 1000 9.5 0.0254 31.7 0.00461377 131.346 1106 | 1250 9.5 0.0254 31.7 0.00461377 131.446 1107 | 1600 9.5 0.0254 31.7 0.00461377 131.036 1108 | 2000 9.5 0.0254 31.7 0.00461377 130.496 1109 | 2500 9.5 0.0254 31.7 0.00461377 130.086 1110 | 3150 9.5 0.0254 31.7 0.00461377 128.536 1111 | 4000 9.5 0.0254 31.7 0.00461377 126.736 1112 | 5000 9.5 0.0254 31.7 0.00461377 124.426 1113 | 6300 9.5 0.0254 31.7 0.00461377 120.726 1114 | 250 12.7 0.0254 71.3 0.0121808 119.698 1115 | 315 12.7 0.0254 71.3 0.0121808 122.938 1116 | 400 12.7 0.0254 71.3 0.0121808 125.048 1117 | 500 12.7 0.0254 71.3 0.0121808 126.898 1118 | 630 12.7 0.0254 71.3 0.0121808 128.878 1119 | 800 12.7 0.0254 71.3 0.0121808 130.348 1120 | 1000 12.7 0.0254 71.3 0.0121808 131.698 1121 | 1250 12.7 0.0254 71.3 0.0121808 133.048 1122 | 1600 12.7 0.0254 71.3 0.0121808 134.528 1123 | 2000 12.7 0.0254 71.3 0.0121808 134.228 1124 | 2500 12.7 0.0254 71.3 0.0121808 134.058 1125 | 3150 12.7 0.0254 71.3 0.0121808 133.758 1126 | 4000 12.7 0.0254 71.3 0.0121808 131.808 1127 | 5000 12.7 0.0254 71.3 0.0121808 128.978 1128 | 6300 12.7 0.0254 71.3 0.0121808 125.398 1129 | 8000 12.7 0.0254 71.3 0.0121808 120.538 1130 | 10000 12.7 0.0254 71.3 0.0121808 114.418 1131 | 250 12.7 0.0254 39.6 0.0130253 121.547 1132 | 315 12.7 0.0254 39.6 0.0130253 123.537 1133 | 400 12.7 0.0254 39.6 0.0130253 125.527 1134 | 500 12.7 0.0254 39.6 0.0130253 127.127 1135 | 630 12.7 0.0254 39.6 0.0130253 128.867 1136 | 800 12.7 0.0254 39.6 0.0130253 130.217 1137 | 1000 12.7 0.0254 39.6 0.0130253 130.947 1138 | 1250 12.7 0.0254 39.6 0.0130253 130.777 1139 | 1600 12.7 0.0254 39.6 0.0130253 129.977 1140 | 2000 12.7 0.0254 39.6 0.0130253 129.567 1141 | 2500 12.7 0.0254 39.6 0.0130253 129.027 1142 | 3150 12.7 0.0254 39.6 0.0130253 127.847 1143 | 4000 12.7 0.0254 39.6 0.0130253 126.537 1144 | 5000 12.7 0.0254 39.6 0.0130253 125.107 1145 | 6300 12.7 0.0254 39.6 0.0130253 123.177 1146 | 8000 12.7 0.0254 39.6 0.0130253 120.607 1147 | 10000 12.7 0.0254 39.6 0.0130253 116.017 1148 | 200 17.4 0.0254 71.3 0.016104 112.506 1149 | 250 17.4 0.0254 71.3 0.016104 113.796 1150 | 315 17.4 0.0254 71.3 0.016104 115.846 1151 | 400 17.4 0.0254 71.3 0.016104 117.396 1152 | 500 17.4 0.0254 71.3 0.016104 119.806 1153 | 630 17.4 0.0254 71.3 0.016104 122.606 1154 | 800 17.4 0.0254 71.3 0.016104 124.276 1155 | 1000 17.4 0.0254 71.3 0.016104 125.816 1156 | 1250 17.4 0.0254 71.3 0.016104 126.356 1157 | 1600 17.4 0.0254 71.3 0.016104 126.406 1158 | 2000 17.4 0.0254 71.3 0.016104 126.826 1159 | 2500 17.4 0.0254 71.3 0.016104 126.746 1160 | 3150 17.4 0.0254 71.3 0.016104 126.536 1161 | 4000 17.4 0.0254 71.3 0.016104 125.586 1162 | 5000 17.4 0.0254 71.3 0.016104 123.126 1163 | 6300 17.4 0.0254 71.3 0.016104 119.916 1164 | 8000 17.4 0.0254 71.3 0.016104 115.466 1165 | 200 17.4 0.0254 55.5 0.0165706 109.951 1166 | 250 17.4 0.0254 55.5 0.0165706 110.491 1167 | 315 17.4 0.0254 55.5 0.0165706 111.911 1168 | 400 17.4 0.0254 55.5 0.0165706 115.461 1169 | 500 17.4 0.0254 55.5 0.0165706 119.621 1170 | 630 17.4 0.0254 55.5 0.0165706 122.411 1171 | 800 17.4 0.0254 55.5 0.0165706 123.091 1172 | 1000 17.4 0.0254 55.5 0.0165706 126.001 1173 | 1250 17.4 0.0254 55.5 0.0165706 129.301 1174 | 1600 17.4 0.0254 55.5 0.0165706 126.471 1175 | 2000 17.4 0.0254 55.5 0.0165706 125.261 1176 | 2500 17.4 0.0254 55.5 0.0165706 124.931 1177 | 3150 17.4 0.0254 55.5 0.0165706 124.101 1178 | 4000 17.4 0.0254 55.5 0.0165706 121.771 1179 | 5000 17.4 0.0254 55.5 0.0165706 118.941 1180 | 6300 17.4 0.0254 55.5 0.0165706 114.861 1181 | 200 17.4 0.0254 39.6 0.0172206 114.044 1182 | 250 17.4 0.0254 39.6 0.0172206 114.714 1183 | 315 17.4 0.0254 39.6 0.0172206 115.144 1184 | 400 17.4 0.0254 39.6 0.0172206 115.444 1185 | 500 17.4 0.0254 39.6 0.0172206 117.514 1186 | 630 17.4 0.0254 39.6 0.0172206 124.514 1187 | 800 17.4 0.0254 39.6 0.0172206 135.324 1188 | 1000 17.4 0.0254 39.6 0.0172206 138.274 1189 | 1250 17.4 0.0254 39.6 0.0172206 131.364 1190 | 1600 17.4 0.0254 39.6 0.0172206 127.614 1191 | 2000 17.4 0.0254 39.6 0.0172206 126.644 1192 | 2500 17.4 0.0254 39.6 0.0172206 124.154 1193 | 3150 17.4 0.0254 39.6 0.0172206 123.564 1194 | 4000 17.4 0.0254 39.6 0.0172206 122.724 1195 | 5000 17.4 0.0254 39.6 0.0172206 119.854 1196 | 200 17.4 0.0254 31.7 0.0176631 116.146 1197 | 250 17.4 0.0254 31.7 0.0176631 116.956 1198 | 315 17.4 0.0254 31.7 0.0176631 118.416 1199 | 400 17.4 0.0254 31.7 0.0176631 120.766 1200 | 500 17.4 0.0254 31.7 0.0176631 127.676 1201 | 630 17.4 0.0254 31.7 0.0176631 136.886 1202 | 800 17.4 0.0254 31.7 0.0176631 139.226 1203 | 1000 17.4 0.0254 31.7 0.0176631 131.796 1204 | 1250 17.4 0.0254 31.7 0.0176631 128.306 1205 | 1600 17.4 0.0254 31.7 0.0176631 126.846 1206 | 2000 17.4 0.0254 31.7 0.0176631 124.356 1207 | 2500 17.4 0.0254 31.7 0.0176631 124.166 1208 | 3150 17.4 0.0254 31.7 0.0176631 123.466 1209 | 4000 17.4 0.0254 31.7 0.0176631 121.996 1210 | 5000 17.4 0.0254 31.7 0.0176631 117.996 1211 | 315 22.2 0.0254 71.3 0.0214178 115.857 1212 | 400 22.2 0.0254 71.3 0.0214178 117.927 1213 | 500 22.2 0.0254 71.3 0.0214178 117.967 1214 | 630 22.2 0.0254 71.3 0.0214178 120.657 1215 | 800 22.2 0.0254 71.3 0.0214178 123.227 1216 | 1000 22.2 0.0254 71.3 0.0214178 134.247 1217 | 1250 22.2 0.0254 71.3 0.0214178 140.987 1218 | 1600 22.2 0.0254 71.3 0.0214178 131.817 1219 | 2000 22.2 0.0254 71.3 0.0214178 127.197 1220 | 2500 22.2 0.0254 71.3 0.0214178 126.097 1221 | 3150 22.2 0.0254 71.3 0.0214178 124.127 1222 | 4000 22.2 0.0254 71.3 0.0214178 123.917 1223 | 5000 22.2 0.0254 71.3 0.0214178 125.727 1224 | 6300 22.2 0.0254 71.3 0.0214178 123.127 1225 | 8000 22.2 0.0254 71.3 0.0214178 121.657 1226 | 200 22.2 0.0254 39.6 0.0229028 116.066 1227 | 250 22.2 0.0254 39.6 0.0229028 117.386 1228 | 315 22.2 0.0254 39.6 0.0229028 120.716 1229 | 400 22.2 0.0254 39.6 0.0229028 123.416 1230 | 500 22.2 0.0254 39.6 0.0229028 129.776 1231 | 630 22.2 0.0254 39.6 0.0229028 137.026 1232 | 800 22.2 0.0254 39.6 0.0229028 137.076 1233 | 1000 22.2 0.0254 39.6 0.0229028 128.416 1234 | 1250 22.2 0.0254 39.6 0.0229028 126.446 1235 | 1600 22.2 0.0254 39.6 0.0229028 122.216 1236 | 2000 22.2 0.0254 39.6 0.0229028 121.256 1237 | 2500 22.2 0.0254 39.6 0.0229028 121.306 1238 | 3150 22.2 0.0254 39.6 0.0229028 120.856 1239 | 4000 22.2 0.0254 39.6 0.0229028 119.646 1240 | 5000 22.2 0.0254 39.6 0.0229028 118.816 1241 | 630 0 0.1016 71.3 0.00121072 124.155 1242 | 800 0 0.1016 71.3 0.00121072 126.805 1243 | 1000 0 0.1016 71.3 0.00121072 128.825 1244 | 1250 0 0.1016 71.3 0.00121072 130.335 1245 | 1600 0 0.1016 71.3 0.00121072 131.725 1246 | 2000 0 0.1016 71.3 0.00121072 132.095 1247 | 2500 0 0.1016 71.3 0.00121072 132.595 1248 | 3150 0 0.1016 71.3 0.00121072 131.955 1249 | 4000 0 0.1016 71.3 0.00121072 130.935 1250 | 5000 0 0.1016 71.3 0.00121072 130.795 1251 | 6300 0 0.1016 71.3 0.00121072 129.395 1252 | 8000 0 0.1016 71.3 0.00121072 125.465 1253 | 10000 0 0.1016 71.3 0.00121072 123.305 1254 | 12500 0 0.1016 71.3 0.00121072 119.375 1255 | 630 0 0.1016 55.5 0.00131983 126.170 1256 | 800 0 0.1016 55.5 0.00131983 127.920 1257 | 1000 0 0.1016 55.5 0.00131983 129.800 1258 | 1250 0 0.1016 55.5 0.00131983 131.430 1259 | 1600 0 0.1016 55.5 0.00131983 132.050 1260 | 2000 0 0.1016 55.5 0.00131983 132.540 1261 | 2500 0 0.1016 55.5 0.00131983 133.040 1262 | 3150 0 0.1016 55.5 0.00131983 131.780 1263 | 4000 0 0.1016 55.5 0.00131983 129.500 1264 | 5000 0 0.1016 55.5 0.00131983 128.360 1265 | 6300 0 0.1016 55.5 0.00131983 127.730 1266 | 8000 0 0.1016 55.5 0.00131983 124.450 1267 | 10000 0 0.1016 55.5 0.00131983 121.930 1268 | 12500 0 0.1016 55.5 0.00131983 119.910 1269 | 630 0 0.1016 39.6 0.00146332 125.401 1270 | 800 0 0.1016 39.6 0.00146332 128.401 1271 | 1000 0 0.1016 39.6 0.00146332 130.781 1272 | 1250 0 0.1016 39.6 0.00146332 132.271 1273 | 1600 0 0.1016 39.6 0.00146332 133.261 1274 | 2000 0 0.1016 39.6 0.00146332 133.251 1275 | 2500 0 0.1016 39.6 0.00146332 132.611 1276 | 3150 0 0.1016 39.6 0.00146332 130.961 1277 | 4000 0 0.1016 39.6 0.00146332 127.801 1278 | 5000 0 0.1016 39.6 0.00146332 126.021 1279 | 6300 0 0.1016 39.6 0.00146332 125.631 1280 | 8000 0 0.1016 39.6 0.00146332 122.341 1281 | 10000 0 0.1016 39.6 0.00146332 119.561 1282 | 630 0 0.1016 31.7 0.00150092 126.413 1283 | 800 0 0.1016 31.7 0.00150092 129.053 1284 | 1000 0 0.1016 31.7 0.00150092 131.313 1285 | 1250 0 0.1016 31.7 0.00150092 133.063 1286 | 1600 0 0.1016 31.7 0.00150092 133.553 1287 | 2000 0 0.1016 31.7 0.00150092 133.153 1288 | 2500 0 0.1016 31.7 0.00150092 132.003 1289 | 3150 0 0.1016 31.7 0.00150092 129.973 1290 | 4000 0 0.1016 31.7 0.00150092 126.933 1291 | 5000 0 0.1016 31.7 0.00150092 124.393 1292 | 6300 0 0.1016 31.7 0.00150092 124.253 1293 | 8000 0 0.1016 31.7 0.00150092 120.193 1294 | 10000 0 0.1016 31.7 0.00150092 115.893 1295 | 800 3.3 0.1016 71.3 0.00202822 131.074 1296 | 1000 3.3 0.1016 71.3 0.00202822 131.434 1297 | 1250 3.3 0.1016 71.3 0.00202822 132.304 1298 | 1600 3.3 0.1016 71.3 0.00202822 133.664 1299 | 2000 3.3 0.1016 71.3 0.00202822 134.034 1300 | 2500 3.3 0.1016 71.3 0.00202822 133.894 1301 | 3150 3.3 0.1016 71.3 0.00202822 132.114 1302 | 4000 3.3 0.1016 71.3 0.00202822 128.704 1303 | 5000 3.3 0.1016 71.3 0.00202822 127.054 1304 | 6300 3.3 0.1016 71.3 0.00202822 124.904 1305 | 8000 3.3 0.1016 71.3 0.00202822 121.234 1306 | 10000 3.3 0.1016 71.3 0.00202822 116.694 1307 | 630 3.3 0.1016 55.5 0.002211 126.599 1308 | 800 3.3 0.1016 55.5 0.002211 129.119 1309 | 1000 3.3 0.1016 55.5 0.002211 131.129 1310 | 1250 3.3 0.1016 55.5 0.002211 132.769 1311 | 1600 3.3 0.1016 55.5 0.002211 133.649 1312 | 2000 3.3 0.1016 55.5 0.002211 133.649 1313 | 2500 3.3 0.1016 55.5 0.002211 132.889 1314 | 3150 3.3 0.1016 55.5 0.002211 130.629 1315 | 4000 3.3 0.1016 55.5 0.002211 127.229 1316 | 5000 3.3 0.1016 55.5 0.002211 124.839 1317 | 6300 3.3 0.1016 55.5 0.002211 123.839 1318 | 8000 3.3 0.1016 55.5 0.002211 120.569 1319 | 10000 3.3 0.1016 55.5 0.002211 115.659 1320 | 630 3.3 0.1016 39.6 0.00245138 127.251 1321 | 800 3.3 0.1016 39.6 0.00245138 129.991 1322 | 1000 3.3 0.1016 39.6 0.00245138 131.971 1323 | 1250 3.3 0.1016 39.6 0.00245138 133.211 1324 | 1600 3.3 0.1016 39.6 0.00245138 133.071 1325 | 2000 3.3 0.1016 39.6 0.00245138 132.301 1326 | 2500 3.3 0.1016 39.6 0.00245138 130.791 1327 | 3150 3.3 0.1016 39.6 0.00245138 128.401 1328 | 4000 3.3 0.1016 39.6 0.00245138 124.881 1329 | 5000 3.3 0.1016 39.6 0.00245138 122.371 1330 | 6300 3.3 0.1016 39.6 0.00245138 120.851 1331 | 8000 3.3 0.1016 39.6 0.00245138 118.091 1332 | 10000 3.3 0.1016 39.6 0.00245138 115.321 1333 | 630 3.3 0.1016 31.7 0.00251435 128.952 1334 | 800 3.3 0.1016 31.7 0.00251435 131.362 1335 | 1000 3.3 0.1016 31.7 0.00251435 133.012 1336 | 1250 3.3 0.1016 31.7 0.00251435 134.022 1337 | 1600 3.3 0.1016 31.7 0.00251435 133.402 1338 | 2000 3.3 0.1016 31.7 0.00251435 131.642 1339 | 2500 3.3 0.1016 31.7 0.00251435 130.392 1340 | 3150 3.3 0.1016 31.7 0.00251435 128.252 1341 | 4000 3.3 0.1016 31.7 0.00251435 124.852 1342 | 5000 3.3 0.1016 31.7 0.00251435 122.082 1343 | 6300 3.3 0.1016 31.7 0.00251435 120.702 1344 | 8000 3.3 0.1016 31.7 0.00251435 117.432 1345 | 630 6.7 0.1016 71.3 0.00478288 131.448 1346 | 800 6.7 0.1016 71.3 0.00478288 134.478 1347 | 1000 6.7 0.1016 71.3 0.00478288 136.758 1348 | 1250 6.7 0.1016 71.3 0.00478288 137.658 1349 | 1600 6.7 0.1016 71.3 0.00478288 136.678 1350 | 2000 6.7 0.1016 71.3 0.00478288 134.568 1351 | 2500 6.7 0.1016 71.3 0.00478288 131.458 1352 | 3150 6.7 0.1016 71.3 0.00478288 124.458 1353 | 500 6.7 0.1016 55.5 0.0052139 129.343 1354 | 630 6.7 0.1016 55.5 0.0052139 133.023 1355 | 800 6.7 0.1016 55.5 0.0052139 135.953 1356 | 1000 6.7 0.1016 55.5 0.0052139 137.233 1357 | 1250 6.7 0.1016 55.5 0.0052139 136.883 1358 | 1600 6.7 0.1016 55.5 0.0052139 133.653 1359 | 2000 6.7 0.1016 55.5 0.0052139 129.653 1360 | 2500 6.7 0.1016 55.5 0.0052139 124.273 1361 | 400 6.7 0.1016 39.6 0.00578076 128.295 1362 | 500 6.7 0.1016 39.6 0.00578076 130.955 1363 | 630 6.7 0.1016 39.6 0.00578076 133.355 1364 | 800 6.7 0.1016 39.6 0.00578076 134.625 1365 | 1000 6.7 0.1016 39.6 0.00578076 134.515 1366 | 1250 6.7 0.1016 39.6 0.00578076 132.395 1367 | 1600 6.7 0.1016 39.6 0.00578076 127.375 1368 | 2000 6.7 0.1016 39.6 0.00578076 122.235 1369 | 315 6.7 0.1016 31.7 0.00592927 126.266 1370 | 400 6.7 0.1016 31.7 0.00592927 128.296 1371 | 500 6.7 0.1016 31.7 0.00592927 130.206 1372 | 630 6.7 0.1016 31.7 0.00592927 132.116 1373 | 800 6.7 0.1016 31.7 0.00592927 132.886 1374 | 1000 6.7 0.1016 31.7 0.00592927 131.636 1375 | 1250 6.7 0.1016 31.7 0.00592927 129.256 1376 | 1600 6.7 0.1016 31.7 0.00592927 124.346 1377 | 2000 6.7 0.1016 31.7 0.00592927 120.446 1378 | 200 8.9 0.1016 71.3 0.0103088 133.503 1379 | 250 8.9 0.1016 71.3 0.0103088 134.533 1380 | 315 8.9 0.1016 71.3 0.0103088 136.583 1381 | 400 8.9 0.1016 71.3 0.0103088 138.123 1382 | 500 8.9 0.1016 71.3 0.0103088 138.523 1383 | 630 8.9 0.1016 71.3 0.0103088 138.423 1384 | 800 8.9 0.1016 71.3 0.0103088 137.813 1385 | 1000 8.9 0.1016 71.3 0.0103088 135.433 1386 | 1250 8.9 0.1016 71.3 0.0103088 132.793 1387 | 1600 8.9 0.1016 71.3 0.0103088 128.763 1388 | 2000 8.9 0.1016 71.3 0.0103088 124.233 1389 | 2500 8.9 0.1016 71.3 0.0103088 123.623 1390 | 3150 8.9 0.1016 71.3 0.0103088 123.263 1391 | 4000 8.9 0.1016 71.3 0.0103088 120.243 1392 | 5000 8.9 0.1016 71.3 0.0103088 116.723 1393 | 6300 8.9 0.1016 71.3 0.0103088 117.253 1394 | 200 8.9 0.1016 39.6 0.0124596 133.420 1395 | 250 8.9 0.1016 39.6 0.0124596 134.340 1396 | 315 8.9 0.1016 39.6 0.0124596 135.380 1397 | 400 8.9 0.1016 39.6 0.0124596 135.540 1398 | 500 8.9 0.1016 39.6 0.0124596 133.790 1399 | 630 8.9 0.1016 39.6 0.0124596 131.920 1400 | 800 8.9 0.1016 39.6 0.0124596 130.940 1401 | 1000 8.9 0.1016 39.6 0.0124596 129.580 1402 | 1250 8.9 0.1016 39.6 0.0124596 127.710 1403 | 1600 8.9 0.1016 39.6 0.0124596 123.820 1404 | 2000 8.9 0.1016 39.6 0.0124596 119.040 1405 | 2500 8.9 0.1016 39.6 0.0124596 119.190 1406 | 3150 8.9 0.1016 39.6 0.0124596 119.350 1407 | 4000 8.9 0.1016 39.6 0.0124596 116.220 1408 | 5000 8.9 0.1016 39.6 0.0124596 113.080 1409 | 6300 8.9 0.1016 39.6 0.0124596 113.110 1410 | 200 12.3 0.1016 71.3 0.0337792 130.588 1411 | 250 12.3 0.1016 71.3 0.0337792 131.568 1412 | 315 12.3 0.1016 71.3 0.0337792 137.068 1413 | 400 12.3 0.1016 71.3 0.0337792 139.428 1414 | 500 12.3 0.1016 71.3 0.0337792 140.158 1415 | 630 12.3 0.1016 71.3 0.0337792 135.368 1416 | 800 12.3 0.1016 71.3 0.0337792 127.318 1417 | 1000 12.3 0.1016 71.3 0.0337792 127.928 1418 | 1250 12.3 0.1016 71.3 0.0337792 126.648 1419 | 1600 12.3 0.1016 71.3 0.0337792 124.748 1420 | 2000 12.3 0.1016 71.3 0.0337792 122.218 1421 | 2500 12.3 0.1016 71.3 0.0337792 121.318 1422 | 3150 12.3 0.1016 71.3 0.0337792 120.798 1423 | 4000 12.3 0.1016 71.3 0.0337792 118.018 1424 | 5000 12.3 0.1016 71.3 0.0337792 116.108 1425 | 6300 12.3 0.1016 71.3 0.0337792 113.958 1426 | 200 12.3 0.1016 55.5 0.0368233 132.304 1427 | 250 12.3 0.1016 55.5 0.0368233 133.294 1428 | 315 12.3 0.1016 55.5 0.0368233 135.674 1429 | 400 12.3 0.1016 55.5 0.0368233 136.414 1430 | 500 12.3 0.1016 55.5 0.0368233 133.774 1431 | 630 12.3 0.1016 55.5 0.0368233 124.244 1432 | 800 12.3 0.1016 55.5 0.0368233 125.114 1433 | 1000 12.3 0.1016 55.5 0.0368233 125.484 1434 | 1250 12.3 0.1016 55.5 0.0368233 124.214 1435 | 1600 12.3 0.1016 55.5 0.0368233 121.824 1436 | 2000 12.3 0.1016 55.5 0.0368233 118.564 1437 | 2500 12.3 0.1016 55.5 0.0368233 117.054 1438 | 3150 12.3 0.1016 55.5 0.0368233 116.914 1439 | 4000 12.3 0.1016 55.5 0.0368233 114.404 1440 | 5000 12.3 0.1016 55.5 0.0368233 112.014 1441 | 6300 12.3 0.1016 55.5 0.0368233 110.124 1442 | 200 12.3 0.1016 39.6 0.0408268 128.545 1443 | 250 12.3 0.1016 39.6 0.0408268 129.675 1444 | 315 12.3 0.1016 39.6 0.0408268 129.415 1445 | 400 12.3 0.1016 39.6 0.0408268 128.265 1446 | 500 12.3 0.1016 39.6 0.0408268 122.205 1447 | 630 12.3 0.1016 39.6 0.0408268 121.315 1448 | 800 12.3 0.1016 39.6 0.0408268 122.315 1449 | 1000 12.3 0.1016 39.6 0.0408268 122.435 1450 | 1250 12.3 0.1016 39.6 0.0408268 121.165 1451 | 1600 12.3 0.1016 39.6 0.0408268 117.875 1452 | 2000 12.3 0.1016 39.6 0.0408268 114.085 1453 | 2500 12.3 0.1016 39.6 0.0408268 113.315 1454 | 3150 12.3 0.1016 39.6 0.0408268 113.055 1455 | 4000 12.3 0.1016 39.6 0.0408268 110.905 1456 | 5000 12.3 0.1016 39.6 0.0408268 108.625 1457 | 6300 12.3 0.1016 39.6 0.0408268 107.985 1458 | 200 12.3 0.1016 31.7 0.0418756 124.987 1459 | 250 12.3 0.1016 31.7 0.0418756 125.857 1460 | 315 12.3 0.1016 31.7 0.0418756 124.717 1461 | 400 12.3 0.1016 31.7 0.0418756 123.207 1462 | 500 12.3 0.1016 31.7 0.0418756 118.667 1463 | 630 12.3 0.1016 31.7 0.0418756 119.287 1464 | 800 12.3 0.1016 31.7 0.0418756 120.037 1465 | 1000 12.3 0.1016 31.7 0.0418756 119.777 1466 | 1250 12.3 0.1016 31.7 0.0418756 118.767 1467 | 1600 12.3 0.1016 31.7 0.0418756 114.477 1468 | 2000 12.3 0.1016 31.7 0.0418756 110.447 1469 | 2500 12.3 0.1016 31.7 0.0418756 110.317 1470 | 3150 12.3 0.1016 31.7 0.0418756 110.307 1471 | 4000 12.3 0.1016 31.7 0.0418756 108.407 1472 | 5000 12.3 0.1016 31.7 0.0418756 107.147 1473 | 6300 12.3 0.1016 31.7 0.0418756 107.267 1474 | 200 15.6 0.1016 71.3 0.0437259 130.898 1475 | 250 15.6 0.1016 71.3 0.0437259 132.158 1476 | 315 15.6 0.1016 71.3 0.0437259 133.808 1477 | 400 15.6 0.1016 71.3 0.0437259 134.058 1478 | 500 15.6 0.1016 71.3 0.0437259 130.638 1479 | 630 15.6 0.1016 71.3 0.0437259 122.288 1480 | 800 15.6 0.1016 71.3 0.0437259 124.188 1481 | 1000 15.6 0.1016 71.3 0.0437259 124.438 1482 | 1250 15.6 0.1016 71.3 0.0437259 123.178 1483 | 1600 15.6 0.1016 71.3 0.0437259 121.528 1484 | 2000 15.6 0.1016 71.3 0.0437259 119.888 1485 | 2500 15.6 0.1016 71.3 0.0437259 118.998 1486 | 3150 15.6 0.1016 71.3 0.0437259 116.468 1487 | 4000 15.6 0.1016 71.3 0.0437259 113.298 1488 | 200 15.6 0.1016 39.6 0.0528487 123.514 1489 | 250 15.6 0.1016 39.6 0.0528487 124.644 1490 | 315 15.6 0.1016 39.6 0.0528487 122.754 1491 | 400 15.6 0.1016 39.6 0.0528487 120.484 1492 | 500 15.6 0.1016 39.6 0.0528487 115.304 1493 | 630 15.6 0.1016 39.6 0.0528487 118.084 1494 | 800 15.6 0.1016 39.6 0.0528487 118.964 1495 | 1000 15.6 0.1016 39.6 0.0528487 119.224 1496 | 1250 15.6 0.1016 39.6 0.0528487 118.214 1497 | 1600 15.6 0.1016 39.6 0.0528487 114.554 1498 | 2000 15.6 0.1016 39.6 0.0528487 110.894 1499 | 2500 15.6 0.1016 39.6 0.0528487 110.264 1500 | 3150 15.6 0.1016 39.6 0.0528487 109.254 1501 | 4000 15.6 0.1016 39.6 0.0528487 106.604 1502 | 5000 15.6 0.1016 39.6 0.0528487 106.224 1503 | 6300 15.6 0.1016 39.6 0.0528487 104.204 1504 | -------------------------------------------------------------------------------- /src/MultiValidPrediction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from scipy.special import logsumexp 4 | 5 | class MultiValidPrediction: 6 | def __init__(self, delta=0.1, n_buckets=50, groups=[(lambda x : True)], eta=0.5, r=1000, normalize_by_counts=True): 7 | # coverage parameter, want to be 1 - delta covered 8 | self.delta = delta 9 | # how many buckets do you want? 10 | self.n_buckets = n_buckets 11 | # groups, given as a collection of True/False outputting functions 12 | self.groups = groups 13 | self.n_groups = len(groups) 14 | # eta, should be set externally based on n_groups, n_buckets, and T 15 | self.eta = eta 16 | # nuisance parameter 17 | self.r = r 18 | # do you normalize computation by bucket-group counts? 19 | self.normalize_by_counts = normalize_by_counts 20 | 21 | # actual thresholds played 22 | self.thresholds = [] 23 | # scores encountered 24 | self.scores = [] 25 | # feature vectors encountered 26 | self.xs = [] 27 | 28 | # for each round: 1 = miscovered, 0 = covered 29 | self.err_seq = [] 30 | # vvals[i][g] = v_value on bucket i and group g so far 31 | self.vvals = np.zeros((self.n_buckets, self.n_groups), dtype=np.float64) 32 | # bg_miscoverage[i][g] = how many times was (i, g) miscovered so far? 33 | self.bg_miscoverage = np.zeros((self.n_buckets, self.n_groups), dtype=int) 34 | # bg_counts[i][g] = how many times did (i, g) come up so far? 35 | self.bg_counts = np.zeros((self.n_buckets, self.n_groups), dtype=int) 36 | 37 | 38 | def predict(self, x): 39 | curr_groups = [i for i in range(self.n_groups) if (self.groups[i])(x)] 40 | if len(curr_groups) == 0: # arbitrarily return threshold 0 for points with zero groups 41 | return 0 42 | 43 | all_c_neg = True # are all c values nonpositive? 44 | cmps_prev = 0.0 45 | cmps_curr = 0.0 46 | overcalibr_log_prev = 0.0 47 | overcalibr_log_curr = 0.0 48 | 49 | for i in range(self.n_buckets): 50 | # compute normalized bucket-group counts 51 | norm_fn = lambda x: np.sqrt((x+1)*(np.log2(x+2)**2)) 52 | bg_counts_norm = 1./norm_fn(self.bg_counts[i, curr_groups]) 53 | 54 | # compute sign of cvalue for bucket i 55 | a = self.eta * self.vvals[i, curr_groups] 56 | if self.normalize_by_counts: 57 | a *= bg_counts_norm 58 | mx = np.max(a) 59 | mn = np.min(a) 60 | 61 | if self.normalize_by_counts: 62 | overcalibr_log_curr = mx + logsumexp( a - mx, b=bg_counts_norm) 63 | undercalibr_log_curr = -mn + logsumexp(-a + mn, b=bg_counts_norm) 64 | else: 65 | overcalibr_log_curr = mx + logsumexp( a - mx) 66 | undercalibr_log_curr = -mn + logsumexp(-a + mn) 67 | cmps_curr = overcalibr_log_curr - undercalibr_log_curr 68 | 69 | if cmps_curr > 0: 70 | all_c_neg = False 71 | 72 | if (i != 0) and ((cmps_curr >= 0 and cmps_prev <= 0) or (cmps_curr <= 0 and cmps_prev >= 0)): 73 | cvalue_prev = np.exp(overcalibr_log_prev) - np.exp(undercalibr_log_prev) 74 | cvalue_curr = np.exp(overcalibr_log_curr) - np.exp(undercalibr_log_curr) 75 | 76 | Z = np.abs(cvalue_prev) + np.abs(cvalue_curr) 77 | prob_prev = 1 if Z == 0 else np.abs(cvalue_curr)/Z 78 | if np.random.random_sample() <= prob_prev: 79 | return (1.0 * i) / self.n_buckets - 1.0 /(self.r * self.n_buckets) 80 | else: 81 | return 1.0 * i / self.n_buckets 82 | 83 | cmps_prev = cmps_curr 84 | overcalibr_log_prev = overcalibr_log_curr 85 | undercalibr_log_prev = undercalibr_log_curr 86 | 87 | return (1.0 if all_c_neg else 0.0) 88 | 89 | def update(self, x, threshold, score): 90 | curr_groups = [i for i in range(self.n_groups) if (self.groups[i])(x)] 91 | if len(curr_groups) == 0: # don't update on points with zero groups 92 | return 93 | 94 | self.thresholds.append(threshold) 95 | self.scores.append(score) 96 | self.xs.append(x) 97 | 98 | bucket = min(int(threshold * self.n_buckets + 0.5/self.r), self.n_buckets - 1) 99 | err_ind = int(score > threshold) 100 | 101 | # update vvals 102 | self.vvals[bucket, curr_groups] += self.delta - err_ind # (1-err_ind) - (1-delta) 103 | # update stats 104 | self.bg_counts[bucket, curr_groups] += 1 105 | self.bg_miscoverage[bucket, curr_groups] += err_ind 106 | self.err_seq.append(err_ind) 107 | 108 | def coverage_stats(self, plot_thresholds=True, print_per_group_stats=True): 109 | if plot_thresholds: 110 | plt.plot(self.thresholds) 111 | plt.title('Thresholds') 112 | plt.show() 113 | 114 | if print_per_group_stats: 115 | print('Per-Group Coverage Statistics:') 116 | for group in range(self.n_groups): 117 | miscoverages = np.sum(self.bg_miscoverage[:, group]) 118 | counts = np.sum(self.bg_counts[:, group]) 119 | miscoverage_rate = miscoverages/counts 120 | 121 | spacing = int(np.ceil(np.log10(len(self.thresholds)))) 122 | group_spacing = int(np.ceil(np.log10(self.n_groups))) 123 | print( 'Group ', '{:{x}d}'.format(group, x=group_spacing), 124 | ': Count: ', '{:{x}d}'.format(counts, x=spacing), 125 | ', Miscoverages: ', '{:{x}d}'.format(miscoverages, x=spacing), 126 | ', Rate: ', f'{miscoverage_rate:.6f}') 127 | 128 | return self.thresholds, self.bg_miscoverage, self.bg_counts 129 | -------------------------------------------------------------------------------- /src/calibrationScorers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProgBelarus/MultiValidPrediction/be3baaf66d04e8c767987e5c656f4131c06917ee/src/calibrationScorers/__init__.py -------------------------------------------------------------------------------- /src/calibrationScorers/calibrationScorer.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class CalibrationScorer(ABC): 5 | @abstractmethod 6 | def calc_score(self, x, y): 7 | pass 8 | 9 | @abstractmethod 10 | def get_prediction_set(self, x, calibration_score): 11 | pass 12 | -------------------------------------------------------------------------------- /src/calibrationScorers/customResidualCalibrationScorer.py: -------------------------------------------------------------------------------- 1 | from calibrationScorers import calibrationScorer, predictionSet 2 | import numpy as np 3 | 4 | 5 | class customResidualCalibrationPredictionSet(predictionSet.PredictionSet): 6 | def __init__(self, interval_center, interval_width): 7 | self.interval_center = interval_center 8 | self.interval_width = interval_width 9 | # The width calculated using our interval will always be bounded above by one 10 | # To allow for larger intervals, we introduce this factor which stretches the interval 11 | # by the desired amount. 12 | 13 | def cover(self, y): 14 | return self.interval_center - self.interval_width <= y < self.interval_center + self.interval_width 15 | 16 | 17 | class customResidualCalibrationScorer(calibrationScorer.CalibrationScorer): 18 | def __init__(self, width_mult_factor): 19 | self.f_pred = None 20 | self.width_mult_factor = width_mult_factor 21 | 22 | def calc_score(self, x, y): 23 | return (1 / self.width_mult_factor) * np.abs(y - self.f_pred(x)) 24 | 25 | def get_prediction_set(self, x, calibration_threshold, width_mult_factor): 26 | interval_center = self.f_pred(x) 27 | interval_width = width_mult_factor * calibration_threshold 28 | new_prediction_set = customResidualCalibrationPredictionSet(interval_center, interval_width) 29 | return new_prediction_set 30 | 31 | def update(self, f_pred): 32 | self.f_pred = f_pred 33 | -------------------------------------------------------------------------------- /src/calibrationScorers/predictionSet.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | class PredictionSet(ABC): 4 | @abstractmethod 5 | def cover(self, y): 6 | pass 7 | 8 | -------------------------------------------------------------------------------- /src/calibrationScorers/rapsScorer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.insert(0, '../') 4 | 5 | from calibrationScorers import calibrationScorer, predictionSet 6 | 7 | import numpy as np 8 | from utils_raps import validate, get_logits_targets, sort_sum 9 | from scipy.special import softmax 10 | import conformal 11 | 12 | import pdb 13 | 14 | 15 | class RapsPredictionSet(predictionSet.PredictionSet): 16 | def __init__(self, ys): 17 | self.ys = ys 18 | 19 | def cover(self, y): 20 | return y in self.ys 21 | 22 | 23 | class RapsScorer(calibrationScorer.CalibrationScorer): 24 | def __init__(self, model, num_classes, delta, lamda, kreg, T=1.3, randomized=True, allow_zero_sets=True): 25 | self.model = model 26 | self.delta = delta 27 | 28 | self.num_classes = num_classes 29 | 30 | self.penalties = np.zeros((1, self.num_classes)) 31 | self.penalties[:, int(kreg):] += lamda 32 | 33 | self.T = T 34 | self.randomized = randomized 35 | self.allow_zero_sets = allow_zero_sets 36 | 37 | # Returns the softmax score and conformal score 38 | def calc_score(self, x, y): 39 | logit = self.model(x) 40 | logit_numpy = logit.detach().cpu().numpy() 41 | score = softmax(logit_numpy / self.T, axis=1) 42 | I, ordered, cumsum = sort_sum(score) 43 | 44 | E = conformal.giq(score, y, I=I, ordered=ordered, cumsum=cumsum, penalties=self.penalties, 45 | randomized=self.randomized, allow_zero_sets=self.allow_zero_sets) 46 | 47 | return score, E 48 | 49 | def get_prediction_set_from_softmax_score(self, soft_max_score, q): 50 | score = soft_max_score 51 | I, ordered, cumsum = sort_sum(score) 52 | 53 | S = conformal.gcq(score, q, I=I, ordered=ordered, cumsum=cumsum, penalties=self.penalties, randomized=self.randomized, 54 | allow_zero_sets=self.allow_zero_sets) 55 | 56 | return RapsPredictionSet(S[0]) 57 | 58 | def get_prediction_set(self, x, q): 59 | logit = self.model(x) 60 | logit_numpy = logit.detach().cpu().numpy() 61 | score = softmax(logit_numpy / self.T, axis=1) 62 | I, ordered, cumsum = sort_sum(score) 63 | 64 | S = conformal.gcq(score, q, I=I, ordered=ordered, cumsum=cumsum, penalties=self.penalties, randomized=self.randomized, 65 | allow_zero_sets=self.allow_zero_sets) 66 | 67 | return RapsPredictionSet(S) 68 | 69 | 70 | -------------------------------------------------------------------------------- /src/calibrationScorers/residualCalibrationScorer.py: -------------------------------------------------------------------------------- 1 | from calibrationScorers import calibrationScorer, predictionSet 2 | import numpy as np 3 | 4 | 5 | class ResidualCalibrationPredictionSet(predictionSet.PredictionSet): 6 | def __init__(self, iterval_center, interval_width): 7 | self.interval_center = iterval_center 8 | self.interval_width = interval_width 9 | 10 | def cover(self, y): 11 | return self.interval_center - self.interval_width <= y < self.interval_center + self.interval_width 12 | 13 | 14 | class ResidualCalibrationScorer(calibrationScorer.CalibrationScorer): 15 | def __init__(self): 16 | self.f_pred = None 17 | 18 | def calc_score(self, x, y): 19 | return np.abs(y - self.f_pred(x)) 20 | 21 | def get_prediction_set(self, x, calibration_threshold): 22 | interval_center = self.f_pred(x) 23 | interval_with = calibration_threshold 24 | new_prediction_set = ResidualCalibrationPredictionSet(interval_center, interval_with) 25 | return new_prediction_set 26 | 27 | def update(self, f_pred): 28 | self.f_pred = f_pred 29 | -------------------------------------------------------------------------------- /src/calibrationScorers/topSoftMaxScorer.py: -------------------------------------------------------------------------------- 1 | from calibrationScorers import calibrationScorer, predictionSet 2 | import numpy as np 3 | 4 | import pdb 5 | 6 | class TopSoftMaxScorePredictionSet(predictionSet.PredictionSet): 7 | def __init__(self, ys): 8 | self.ys = ys 9 | 10 | def cover(self, y): 11 | return y in self.ys 12 | 13 | 14 | class TopSoftMaxScorer(calibrationScorer.CalibrationScorer): 15 | def __init__(self, use_randomization=True): 16 | self.f_pred = None 17 | self.use_randomization = use_randomization 18 | 19 | def calc_score(self, x, y): 20 | y = np.array(y).reshape(-1, 1) 21 | x = np.array(x).reshape(len(y), -1) 22 | soft_maxes = np.array(self.f_pred(x)).reshape(len(y), -1) 23 | 24 | pi = (-soft_maxes).argsort(1) 25 | soft_maxes_sorted = np.take_along_axis(soft_maxes, pi, 1) 26 | soft_maxes_cumsum = soft_maxes_sorted.cumsum(1) 27 | y_to_cumulative_sum = np.take_along_axis(soft_maxes_cumsum, pi, 1) 28 | 29 | scores = np.take_along_axis(y_to_cumulative_sum, y, -1).ravel() 30 | 31 | return scores 32 | 33 | def get_prediction_set(self, x, calibration_threshold): 34 | x = x.reshape(1, -1) 35 | 36 | soft_maxes = np.array(self.f_pred(x)) 37 | 38 | pi = (-soft_maxes).argsort() 39 | soft_maxes_sorted = np.take_along_axis(soft_maxes, pi, -1) 40 | 41 | soft_maxes_cumsum = soft_maxes_sorted.cumsum() 42 | 43 | size = np.sum(soft_maxes_cumsum < calibration_threshold, -1) + 1 44 | 45 | # pdb.set_trace() 46 | if self.use_randomization: 47 | prob_exceeds = (soft_maxes_cumsum[size-1] - calibration_threshold) / soft_maxes[0][pi[0][size-1]] 48 | if np.random.random_sample() <= prob_exceeds: 49 | size = size - 1 50 | 51 | return TopSoftMaxScorePredictionSet(pi[0][:size]) 52 | 53 | def update(self, f_pred): 54 | self.f_pred = f_pred -------------------------------------------------------------------------------- /src/conformal.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.special import softmax 3 | import torch 4 | import torch.nn as nn 5 | import torch.optim as optim 6 | import torch.utils.data as tdata 7 | import pandas as pd 8 | import time 9 | from tqdm import tqdm 10 | from utils_raps import validate, get_logits_targets, sort_sum 11 | 12 | import pdb 13 | 14 | # Conformalize a model with a calibration set. 15 | # Save it to a file in .cache/modelname 16 | # The only difference is that the forward method of ConformalModel also outputs a set. 17 | class ConformalModel(nn.Module): 18 | def __init__(self, model, calib_loader, alpha, kreg=None, lamda=None, randomized=True, allow_zero_sets=False, pct_paramtune = 0.3, batch_size=32, lamda_criterion='size'): 19 | super(ConformalModel, self).__init__() 20 | self.model = model 21 | self.alpha = alpha 22 | self.T = torch.Tensor([1.3]) #initialize (1.3 is usually a good value) 23 | 24 | self.T, calib_logits = platt(self, calib_loader) 25 | self.randomized=randomized 26 | self.allow_zero_sets=allow_zero_sets 27 | self.num_classes = len(calib_loader.dataset.dataset.classes) 28 | 29 | if kreg == None or lamda == None: 30 | kreg, lamda, calib_logits = pick_parameters(model, calib_logits, alpha, kreg, lamda, randomized, allow_zero_sets, pct_paramtune, batch_size, lamda_criterion) 31 | 32 | self.kreg = kreg 33 | self.lamda = lamda 34 | 35 | self.penalties = np.zeros((1, self.num_classes)) 36 | self.penalties[:, kreg:] += lamda 37 | 38 | calib_loader = tdata.DataLoader(calib_logits, batch_size = batch_size, shuffle=False, pin_memory=True) 39 | 40 | self.Qhat = conformal_calibration_logits(self, calib_loader) 41 | 42 | def forward(self, *args, randomized=None, allow_zero_sets=None, **kwargs): 43 | # TODO: return only the scores 44 | # TODO: create a method that takes in x, q and then returns the prediction set 45 | if randomized == None: 46 | randomized = self.randomized 47 | if allow_zero_sets == None: 48 | allow_zero_sets = self.allow_zero_sets 49 | logits = self.model(*args, **kwargs) 50 | 51 | with torch.no_grad(): 52 | logits_numpy = logits.detach().cpu().numpy() 53 | scores = softmax(logits_numpy/self.T.item(), axis=1) 54 | 55 | I, ordered, cumsum = sort_sum(scores) 56 | 57 | S = gcq(scores, self.Qhat, I=I, ordered=ordered, cumsum=cumsum, penalties=self.penalties, randomized=randomized, allow_zero_sets=allow_zero_sets) 58 | 59 | return logits, S 60 | 61 | 62 | # Computes the conformal calibration 63 | def conformal_calibration(cmodel, calib_loader): 64 | print("Conformal calibration") 65 | with torch.no_grad(): 66 | E = np.array([]) 67 | for x, targets in tqdm(calib_loader): 68 | logits = cmodel.model(x.cuda()).detach().cpu().numpy() 69 | scores = softmax(logits/cmodel.T.item(), axis=1) 70 | 71 | I, ordered, cumsum = sort_sum(scores) 72 | 73 | E = np.concatenate((E,giq(scores,targets,I=I,ordered=ordered,cumsum=cumsum,penalties=cmodel.penalties,randomized=True, allow_zero_sets=True))) 74 | 75 | Qhat = np.quantile(E,1-cmodel.alpha,interpolation='higher') 76 | 77 | return Qhat 78 | 79 | # Temperature scaling 80 | def platt(cmodel, calib_loader, max_iters=10, lr=0.01, epsilon=0.01): 81 | print("Begin Platt scaling.") 82 | # Save logits so don't need to double compute them 83 | logits_dataset = get_logits_targets(cmodel.model, calib_loader) 84 | logits_loader = torch.utils.data.DataLoader(logits_dataset, batch_size = calib_loader.batch_size, shuffle=False, pin_memory=True) 85 | 86 | T = platt_logits(cmodel, logits_loader, max_iters=max_iters, lr=lr, epsilon=epsilon) 87 | 88 | print(f"Optimal T={T.item()}") 89 | return T, logits_dataset 90 | 91 | """ 92 | 93 | 94 | INTERNAL FUNCTIONS 95 | 96 | 97 | """ 98 | 99 | ### Precomputed-logit versions of the above functions. 100 | 101 | class ConformalModelLogits(nn.Module): 102 | def __init__(self, model, calib_loader, alpha, kreg=None, lamda=None, randomized=True, allow_zero_sets=False, naive=False, LAC=False, pct_paramtune = 0.3, batch_size=32, lamda_criterion='size'): 103 | super(ConformalModelLogits, self).__init__() 104 | self.model = model 105 | self.alpha = alpha 106 | self.randomized = randomized 107 | self.LAC = LAC 108 | self.allow_zero_sets = allow_zero_sets 109 | self.T = platt_logits(self, calib_loader) 110 | 111 | if (kreg == None or lamda == None) and not naive and not LAC: 112 | kreg, lamda, calib_logits = pick_parameters(model, calib_loader.dataset, alpha, kreg, lamda, randomized, allow_zero_sets, pct_paramtune, batch_size, lamda_criterion) 113 | calib_loader = tdata.DataLoader(calib_logits, batch_size=batch_size, shuffle=False, pin_memory=True) 114 | 115 | self.penalties = np.zeros((1, calib_loader.dataset[0][0].shape[0])) 116 | if not (kreg == None) and not naive and not LAC: 117 | self.penalties[:, kreg:] += lamda 118 | self.Qhat = 1-alpha 119 | if not naive and not LAC: 120 | self.Qhat = conformal_calibration_logits(self, calib_loader) 121 | elif not naive and LAC: 122 | gt_locs_cal = np.array([np.where(np.argsort(x[0]).flip(dims=(0,)) == x[1])[0][0] for x in calib_loader.dataset]) 123 | scores_cal = 1-np.array([np.sort(torch.softmax(calib_loader.dataset[i][0]/self.T.item(), dim=0))[::-1][gt_locs_cal[i]] for i in range(len(calib_loader.dataset))]) 124 | self.Qhat = np.quantile( scores_cal , np.ceil((scores_cal.shape[0]+1) * (1-alpha)) / scores_cal.shape[0] ) 125 | 126 | def forward(self, logits, randomized=None, allow_zero_sets=None): 127 | if randomized == None: 128 | randomized = self.randomized 129 | if allow_zero_sets == None: 130 | allow_zero_sets = self.allow_zero_sets 131 | 132 | with torch.no_grad(): 133 | logits_numpy = logits.detach().cpu().numpy() 134 | scores = softmax(logits_numpy/self.T.item(), axis=1) 135 | 136 | if not self.LAC: 137 | I, ordered, cumsum = sort_sum(scores) 138 | 139 | S = gcq(scores, self.Qhat, I=I, ordered=ordered, cumsum=cumsum, penalties=self.penalties, randomized=randomized, allow_zero_sets=allow_zero_sets) 140 | else: 141 | S = [ np.where( (1-scores[i,:]) < self.Qhat )[0] for i in range(scores.shape[0]) ] 142 | 143 | return logits, S 144 | 145 | def conformal_calibration_logits(cmodel, calib_loader): 146 | with torch.no_grad(): 147 | E = np.array([]) 148 | for logits, targets in calib_loader: 149 | logits = logits.detach().cpu().numpy() 150 | 151 | scores = softmax(logits/cmodel.T.item(), axis=1) 152 | 153 | I, ordered, cumsum = sort_sum(scores) 154 | 155 | E = np.concatenate((E,giq(scores,targets,I=I,ordered=ordered,cumsum=cumsum,penalties=cmodel.penalties,randomized=True,allow_zero_sets=True))) 156 | 157 | Qhat = np.quantile(E, 1-cmodel.alpha, interpolation='higher') 158 | 159 | return Qhat 160 | 161 | def platt_logits(cmodel, calib_loader, max_iters=10, lr=0.01, epsilon=0.01): 162 | nll_criterion = nn.CrossEntropyLoss().cuda() 163 | 164 | T = nn.Parameter(torch.Tensor([1.3]).cuda()) 165 | 166 | optimizer = optim.SGD([T], lr=lr) 167 | for iter in range(max_iters): 168 | T_old = T.item() 169 | for x, targets in calib_loader: 170 | optimizer.zero_grad() 171 | x = x.cuda() 172 | x.requires_grad = True 173 | out = x/T 174 | loss = nll_criterion(out, targets.long().cuda()) 175 | loss.backward() 176 | optimizer.step() 177 | if abs(T_old - T.item()) < epsilon: 178 | break 179 | return T 180 | 181 | ### CORE CONFORMAL INFERENCE FUNCTIONS 182 | 183 | 184 | 185 | # Generalized conditional quantile function. 186 | def gcq(scores, tau, I, ordered, cumsum, penalties, randomized, allow_zero_sets): 187 | penalties_cumsum = np.cumsum(penalties, axis=1) 188 | sizes_base = ((cumsum + penalties_cumsum) <= tau).sum(axis=1) + 1 # 1 - 1001 189 | sizes_base = np.minimum(sizes_base, scores.shape[1]) # 1-1000 190 | 191 | if randomized: 192 | V = np.zeros(sizes_base.shape) 193 | for i in range(sizes_base.shape[0]): 194 | V[i] = 1/ordered[i,sizes_base[i]-1] * \ 195 | (tau-(cumsum[i,sizes_base[i]-1]-ordered[i,sizes_base[i]-1])-penalties_cumsum[0,sizes_base[i]-1]) # -1 since sizes_base \in {1,...,1000}. 196 | 197 | sizes = sizes_base - (np.random.random(V.shape) >= V).astype(int) 198 | else: 199 | sizes = sizes_base 200 | 201 | if tau == 1.0: 202 | sizes[:] = cumsum.shape[1] # always predict max size if alpha==0. (Avoids numerical error.) 203 | 204 | if not allow_zero_sets: 205 | sizes[sizes == 0] = 1 # allow the user the option to never have empty sets (will lead to incorrect coverage if 1-alpha < model's top-1 accuracy 206 | 207 | S = list() 208 | 209 | # Construct S from equation (5) 210 | for i in range(I.shape[0]): 211 | S = S + [I[i,0:sizes[i]],] 212 | 213 | return S 214 | 215 | # Get the 'p-value' 216 | def get_tau(score, target, I, ordered, cumsum, penalty, randomized, allow_zero_sets): # For one example 217 | idx = np.where(I==target) 218 | tau_nonrandom = cumsum[idx] 219 | 220 | if not randomized: 221 | return tau_nonrandom + penalty[0] 222 | 223 | U = np.random.random() 224 | 225 | if idx == (0,0): 226 | if not allow_zero_sets: 227 | return tau_nonrandom + penalty[0] 228 | else: 229 | return U * tau_nonrandom + penalty[0] 230 | else: 231 | return U * ordered[idx] + cumsum[(idx[0],idx[1]-1)] + (penalty[0:(idx[1][0]+1)]).sum() 232 | 233 | # Gets the histogram of Taus. 234 | def giq(scores, targets, I, ordered, cumsum, penalties, randomized, allow_zero_sets): 235 | """ 236 | Generalized inverse quantile conformity score function. 237 | E from equation (7) in Romano, Sesia, Candes. Find the minimum tau in [0, 1] such that the correct label enters. 238 | """ 239 | E = -np.ones((scores.shape[0],)) 240 | for i in range(scores.shape[0]): 241 | E[i] = get_tau(scores[i:i+1,:],targets[i].item(),I[i:i+1,:],ordered[i:i+1,:],cumsum[i:i+1,:],penalties[0,:],randomized=randomized, allow_zero_sets=allow_zero_sets) 242 | 243 | return E 244 | 245 | ### AUTOMATIC PARAMETER TUNING FUNCTIONS 246 | def pick_kreg(paramtune_logits, alpha): 247 | gt_locs_kstar = np.array([np.where(np.argsort(x[0]).flip(dims=(0,)) == x[1])[0][0] for x in paramtune_logits]) 248 | kstar = np.quantile(gt_locs_kstar, 1-alpha, interpolation='higher') + 1 249 | return kstar 250 | 251 | def pick_lamda_size(model, paramtune_loader, alpha, kreg, randomized, allow_zero_sets): 252 | # Calculate lamda_star 253 | best_size = iter(paramtune_loader).__next__()[0][1].shape[0] # number of classes 254 | # Use the paramtune data to pick lamda. Does not violate exchangeability. 255 | for temp_lam in [0.001, 0.01, 0.1, 0.2, 0.5]: # predefined grid, change if more precision desired. 256 | conformal_model = ConformalModelLogits(model, paramtune_loader, alpha=alpha, kreg=kreg, lamda=temp_lam, randomized=randomized, allow_zero_sets=allow_zero_sets, naive=False) 257 | top1_avg, top5_avg, cvg_avg, sz_avg = validate(paramtune_loader, conformal_model, print_bool=False) 258 | if sz_avg < best_size: 259 | best_size = sz_avg 260 | lamda_star = temp_lam 261 | return lamda_star 262 | 263 | def pick_lamda_adaptiveness(model, paramtune_loader, alpha, kreg, randomized, allow_zero_sets, strata=[[0,1],[2,3],[4,6],[7,10],[11,100],[101,1000]]): 264 | # Calculate lamda_star 265 | lamda_star = 0 266 | best_violation = 1 267 | # Use the paramtune data to pick lamda. Does not violate exchangeability. 268 | for temp_lam in [0, 1e-5, 1e-4, 8e-4, 9e-4, 1e-3, 1.5e-3, 2e-3]: # predefined grid, change if more precision desired. 269 | conformal_model = ConformalModelLogits(model, paramtune_loader, alpha=alpha, kreg=kreg, lamda=temp_lam, randomized=randomized, allow_zero_sets=allow_zero_sets, naive=False) 270 | curr_violation = get_violation(conformal_model, paramtune_loader, strata, alpha) 271 | if curr_violation < best_violation: 272 | best_violation = curr_violation 273 | lamda_star = temp_lam 274 | return lamda_star 275 | 276 | def pick_parameters(model, calib_logits, alpha, kreg, lamda, randomized, allow_zero_sets, pct_paramtune, batch_size, lamda_criterion): 277 | num_paramtune = int(np.ceil(pct_paramtune * len(calib_logits))) 278 | paramtune_logits, calib_logits = tdata.random_split(calib_logits, [num_paramtune, len(calib_logits)-num_paramtune]) 279 | calib_loader = tdata.DataLoader(calib_logits, batch_size=batch_size, shuffle=False, pin_memory=True) 280 | paramtune_loader = tdata.DataLoader(paramtune_logits, batch_size=batch_size, shuffle=False, pin_memory=True) 281 | 282 | if kreg == None: 283 | kreg = pick_kreg(paramtune_logits, alpha) 284 | if lamda == None: 285 | if lamda_criterion == "size": 286 | lamda = pick_lamda_size(model, paramtune_loader, alpha, kreg, randomized, allow_zero_sets) 287 | elif lamda_criterion == "adaptiveness": 288 | lamda = pick_lamda_adaptiveness(model, paramtune_loader, alpha, kreg, randomized, allow_zero_sets) 289 | return kreg, lamda, calib_logits 290 | 291 | def get_violation(cmodel, loader_paramtune, strata, alpha): 292 | df = pd.DataFrame(columns=['size', 'correct']) 293 | for logit, target in loader_paramtune: 294 | # compute output 295 | output, S = cmodel(logit) # This is a 'dummy model' which takes logits, for efficiency. 296 | # measure accuracy and record loss 297 | size = np.array([x.size for x in S]) 298 | I, _, _ = sort_sum(logit.numpy()) 299 | correct = np.zeros_like(size) 300 | for j in range(correct.shape[0]): 301 | correct[j] = int( target[j] in list(S[j]) ) 302 | batch_df = pd.DataFrame({'size': size, 'correct': correct}) 303 | df = df.append(batch_df, ignore_index=True) 304 | wc_violation = 0 305 | for stratum in strata: 306 | temp_df = df[ (df['size'] >= stratum[0]) & (df['size'] <= stratum[1]) ] 307 | if len(temp_df) == 0: 308 | continue 309 | stratum_violation = abs(temp_df.correct.mean()-(1-alpha)) 310 | wc_violation = max(wc_violation, stratum_violation) 311 | return wc_violation # the violation 312 | 313 | -------------------------------------------------------------------------------- /src/recursiveLeastSquares.py: -------------------------------------------------------------------------------- 1 | # https://github.com/craig-m-k/Recursive-least-squares/blob/master/RLS.ipynb 2 | 3 | import numpy as np 4 | import math 5 | 6 | import pdb 7 | 8 | class RLS: 9 | def __init__(self, num_vars, lam, delta): 10 | ''' 11 | num_vars: number of variables including constant 12 | lam: forgetting factor, usually very close to 1. 13 | ''' 14 | self.num_vars = num_vars 15 | 16 | # delta controls the initial state. 17 | self.A = delta * np.matrix(np.identity(self.num_vars)) 18 | self.w = np.matrix(np.zeros(self.num_vars)) 19 | self.w = self.w.reshape(self.w.shape[1], 1) 20 | 21 | # Variables needed for add_obs 22 | self.lam_inv = lam ** (-1) 23 | self.sqrt_lam_inv = math.sqrt(self.lam_inv) 24 | 25 | # A priori error 26 | self.a_priori_error = 0 27 | 28 | # Count of number of observations added 29 | self.num_obs = 0 30 | 31 | def add_obs(self, x, t): 32 | ''' 33 | Add the observation x with label t. 34 | x is a column vector as a numpy matrix 35 | t is a real scalar 36 | ''' 37 | x = np.matrix(x).T 38 | z = self.lam_inv * self.A * x 39 | alpha = float((1 + x.T * z) ** (-1)) 40 | self.a_priori_error = float(t - self.w.T * x) 41 | self.w = self.w + (t - alpha * float(x.T * (self.w + t * z))) * z 42 | self.A -= alpha * z * z.T 43 | self.num_obs += 1 44 | 45 | def fit(self, X, y): 46 | ''' 47 | Fit a model to X,y. 48 | X and y are numpy arrays. 49 | Individual observations in X should have a prepended 1 for constant coefficient. 50 | ''' 51 | for i in range(len(X)): 52 | x = np.transpose(np.matrix(X[i])) 53 | self.add_obs(x, y[i]) 54 | 55 | def get_error(self): 56 | ''' 57 | Finds the a priori (instantaneous) error. 58 | Does not calculate the cumulative effect 59 | of round-off errors. 60 | ''' 61 | return self.a_priori_error 62 | 63 | def predict(self, x): 64 | ''' 65 | Predict the value of observation x. x should be a numpy matrix (row vector) 66 | ''' 67 | x = np.matrix(x) 68 | predicted_ys = (self.w.T * x.T).T 69 | return np.squeeze(np.asarray(predicted_ys)) -------------------------------------------------------------------------------- /src/splitConformalPrediction.py: -------------------------------------------------------------------------------- 1 | # https://github.com/craig-m-k/Recursive-least-squares/blob/master/RLS.ipynb 2 | 3 | import numpy as np 4 | import math 5 | 6 | import pdb 7 | 8 | class splitConformal: 9 | def __init__(self, num_groups, groups, delta): 10 | 11 | # num_groups: number of groups we are considering 12 | self.num_groups = num_groups 13 | 14 | # groups: list of functions defining the groups 15 | self.groups = groups 16 | 17 | # calibration_sets_xs: calibration data (input) corresponding to each separate group 18 | # calibration_sets_ys: calibration data (labels) corresponding to each separate group 19 | self.calibration_sets_xs = [None] * num_groups 20 | self.calibration_sets_ys = [None] * num_groups 21 | 22 | # delta: determines desired coverage probability (1 - delta) 23 | self.delta = delta 24 | 25 | def update_calibration_data(self, new_x, new_y): 26 | for i, group in enumerate(self.groups): 27 | if group(new_x): 28 | if self.calibration_sets_xs[i] is None: 29 | self.calibration_sets_xs[i] = new_x 30 | else: 31 | self.calibration_sets_xs[i] = np.vstack((self.calibration_sets_xs[i], new_x)) 32 | if self.calibration_sets_ys[i] is None: 33 | self.calibration_sets_ys[i] = np.array(new_y) 34 | else: 35 | self.calibration_sets_ys[i] = np.hstack((self.calibration_sets_ys[i], new_y)) 36 | 37 | def select_best_width(self, scorer, new_x): 38 | # Select the most conservative width from all generated by 39 | # calibration datasets for relevant groups 40 | curr_width = -1 * float('inf') 41 | for i, group in enumerate(self.groups): 42 | if group(new_x): 43 | calibration_set_x = self.calibration_sets_xs[i] 44 | calibration_set_y = self.calibration_sets_ys[i] 45 | residuals = scorer.calc_score(calibration_set_x, calibration_set_y) 46 | calibration_size = len(calibration_set_x) 47 | desired_quantile = np.ceil((1 - self.delta) * (calibration_size + 1)) / calibration_size 48 | chosen_quantile = np.minimum(1.0, desired_quantile) 49 | w_t = np.quantile(residuals, chosen_quantile) 50 | curr_width = np.maximum(curr_width, w_t) 51 | 52 | return curr_width 53 | 54 | def all_groups_covered(self): 55 | # Check to see if we have calibration data for all groups 56 | for i, _ in enumerate(self.groups): 57 | if self.calibration_sets_xs[i] is None: 58 | return False 59 | return True -------------------------------------------------------------------------------- /src/utils_raps.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | import torch.optim as optim 5 | import torchvision 6 | import torchvision.transforms as transforms 7 | import time 8 | import pathlib 9 | import os 10 | import pickle 11 | from tqdm import tqdm 12 | import pdb 13 | 14 | def sort_sum(scores): 15 | I = scores.argsort(axis=1)[:,::-1] 16 | ordered = np.sort(scores,axis=1)[:,::-1] 17 | cumsum = np.cumsum(ordered,axis=1) 18 | return I, ordered, cumsum 19 | 20 | class AverageMeter(object): 21 | """Computes and stores the average and current value""" 22 | 23 | def __init__(self, name, fmt=':f'): 24 | self.name = name 25 | self.fmt = fmt 26 | self.reset() 27 | 28 | def reset(self): 29 | self.val = 0 30 | self.avg = 0 31 | self.sum = 0 32 | self.count = 0 33 | 34 | def update(self, val, n=1): 35 | self.val = val 36 | self.sum += val * n 37 | self.count += n 38 | self.avg = self.sum / self.count 39 | 40 | def __str__(self): 41 | fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' 42 | return fmtstr.format(**self.__dict__) 43 | 44 | def validate(val_loader, model, print_bool): 45 | with torch.no_grad(): 46 | batch_time = AverageMeter('batch_time') 47 | top1 = AverageMeter('top1') 48 | top5 = AverageMeter('top5') 49 | coverage = AverageMeter('RAPS coverage') 50 | size = AverageMeter('RAPS size') 51 | # switch to evaluate mode 52 | model.eval() 53 | end = time.time() 54 | N = 0 55 | for i, (x, target) in enumerate(val_loader): 56 | target = target.cuda() 57 | # compute output 58 | output, S = model(x.cuda()) 59 | # measure accuracy and record loss 60 | prec1, prec5 = accuracy(output, target, topk=(1, 5)) 61 | cvg, sz = coverage_size(S, target) 62 | 63 | # Update meters 64 | top1.update(prec1.item()/100.0, n=x.shape[0]) 65 | top5.update(prec5.item()/100.0, n=x.shape[0]) 66 | coverage.update(cvg, n=x.shape[0]) 67 | size.update(sz, n=x.shape[0]) 68 | 69 | # measure elapsed time 70 | batch_time.update(time.time() - end) 71 | end = time.time() 72 | N = N + x.shape[0] 73 | if print_bool: 74 | print(f'\rN: {N} | Time: {batch_time.val:.3f} ({batch_time.avg:.3f}) | Cvg@1: {top1.val:.3f} ({top1.avg:.3f}) | Cvg@5: {top5.val:.3f} ({top5.avg:.3f}) | Cvg@RAPS: {coverage.val:.3f} ({coverage.avg:.3f}) | Size@RAPS: {size.val:.3f} ({size.avg:.3f})', end='') 75 | if print_bool: 76 | print('') #Endline 77 | 78 | return top1.avg, top5.avg, coverage.avg, size.avg 79 | 80 | def validate_with_sets(val_loader, model, print_bool): 81 | with torch.no_grad(): 82 | batch_time = AverageMeter('batch_time') 83 | top1 = AverageMeter('top1') 84 | top5 = AverageMeter('top5') 85 | coverage = AverageMeter('RAPS coverage') 86 | covered_list = list() 87 | size = AverageMeter('RAPS size') 88 | size_list = list() 89 | # switch to evaluate mode 90 | model.eval() 91 | end = time.time() 92 | N = 0 93 | for i, (x, target) in enumerate(val_loader): 94 | target = target.cuda() 95 | # compute output 96 | output, S = model(x.cuda()) 97 | # measure accuracy and record loss 98 | prec1, prec5 = accuracy(output, target, topk=(1, 5)) 99 | cvg, sz = coverage_size(S, target) 100 | to_append_cvg, to_append_sz = to_append_coverage_size(S, target) 101 | covered_list += to_append_cvg 102 | size_list += to_append_sz 103 | 104 | # Update meters 105 | top1.update(prec1.item()/100.0, n=x.shape[0]) 106 | top5.update(prec5.item()/100.0, n=x.shape[0]) 107 | coverage.update(cvg, n=x.shape[0]) 108 | size.update(sz, n=x.shape[0]) 109 | 110 | # measure elapsed time 111 | batch_time.update(time.time() - end) 112 | end = time.time() 113 | N = N + x.shape[0] 114 | if print_bool: 115 | print(f'\rN: {N} | Time: {batch_time.val:.3f} ({batch_time.avg:.3f}) | Cvg@1: {top1.val:.3f} ({top1.avg:.3f}) | Cvg@5: {top5.val:.3f} ({top5.avg:.3f}) | Cvg@RAPS: {coverage.val:.3f} ({coverage.avg:.3f}) | Size@RAPS: {size.val:.3f} ({size.avg:.3f})', end='') 116 | 117 | # for i in range(target.shape[0]): 118 | # covered = target[i].item() in S[i] 119 | # actual_size = S[i].shape[0] 120 | # print(covered, actual_size) 121 | # covered_list.append(covered) 122 | # size_list.append(actual_size) 123 | 124 | if print_bool: 125 | print('') #Endline 126 | 127 | return top1.avg, top5.avg, coverage.avg, size.avg, covered_list, size_list 128 | 129 | def to_append_coverage_size(S, targets): 130 | to_append_cvg = list() 131 | to_append_sz = list() 132 | for i in range(targets.shape[0]): 133 | to_append_cvg.append(targets[i].item() in S[i]) 134 | to_append_sz.append(S[i].shape[0]) 135 | return to_append_cvg, to_append_sz 136 | 137 | def coverage_size(S,targets): 138 | covered = 0 139 | size = 0 140 | for i in range(targets.shape[0]): 141 | if (targets[i].item() in S[i]): 142 | covered += 1 143 | size = size + S[i].shape[0] 144 | return float(covered)/targets.shape[0], size/targets.shape[0] 145 | 146 | def accuracy(output, target, topk=(1,)): 147 | """Computes the precision@k for the specified values of k""" 148 | maxk = max(topk) 149 | batch_size = target.size(0) 150 | 151 | _, pred = output.topk(maxk, 1, True, True) 152 | pred = pred.t() 153 | correct = pred.eq(target.view(1, -1).expand_as(pred)) 154 | 155 | res = [] 156 | for k in topk: 157 | correct_k = correct[:k].float().sum() 158 | res.append(correct_k.mul_(100.0 / batch_size)) 159 | return res 160 | 161 | def data2tensor(data): 162 | imgs = torch.cat([x[0].unsqueeze(0) for x in data], dim=0).cuda() 163 | targets = torch.cat([torch.Tensor([int(x[1])]) for x in data], dim=0).long() 164 | return imgs, targets 165 | 166 | def split2ImageFolder(path, transform, n1, n2): 167 | dataset = torchvision.datasets.ImageFolder(path, transform) 168 | data1, data2 = torch.utils.data.random_split(dataset, [n1, len(dataset)-n1]) 169 | data2, _ = torch.utils.data.random_split(data2, [n2, len(dataset)-n1-n2]) 170 | return data1, data2 171 | 172 | def split2(dataset, n1, n2): 173 | data1, temp = torch.utils.data.random_split(dataset, [n1, dataset.tensors[0].shape[0]-n1]) 174 | data2, _ = torch.utils.data.random_split(temp, [n2, dataset.tensors[0].shape[0]-n1-n2]) 175 | return data1, data2 176 | 177 | def get_model(modelname): 178 | if modelname == 'ResNet18': 179 | model = torchvision.models.resnet18(pretrained=True, progress=True) 180 | 181 | elif modelname == 'ResNet50': 182 | model = torchvision.models.resnet50(pretrained=True, progress=True) 183 | 184 | elif modelname == 'ResNet101': 185 | model = torchvision.models.resnet101(pretrained=True, progress=True) 186 | 187 | elif modelname == 'ResNet152': 188 | model = torchvision.models.resnet152(pretrained=True, progress=True) 189 | 190 | elif modelname == 'ResNeXt101': 191 | model = torchvision.models.resnext101_32x8d(pretrained=True, progress=True) 192 | 193 | elif modelname == 'VGG16': 194 | model = torchvision.models.vgg16(pretrained=True, progress=True) 195 | 196 | elif modelname == 'ShuffleNet': 197 | model = torchvision.models.shufflenet_v2_x1_0(pretrained=True, progress=True) 198 | 199 | elif modelname == 'Inception': 200 | model = torchvision.models.inception_v3(pretrained=True, progress=True) 201 | 202 | elif modelname == 'DenseNet161': 203 | model = torchvision.models.densenet161(pretrained=True, progress=True) 204 | 205 | else: 206 | raise NotImplementedError 207 | 208 | model.eval() 209 | model = torch.nn.DataParallel(model).cuda() 210 | 211 | return model 212 | 213 | # Computes logits and targets from a model and loader 214 | def get_logits_targets(model, loader): 215 | logits = torch.zeros((len(loader.dataset), 1000)) # 1000 classes in Imagenet. 216 | labels = torch.zeros((len(loader.dataset),)) 217 | i = 0 218 | print(f'Computing logits for model (only happens once).') 219 | with torch.no_grad(): 220 | for x, targets in tqdm(loader): 221 | batch_logits = model(x.cuda()).detach().cpu() 222 | logits[i:(i+x.shape[0]), :] = batch_logits 223 | labels[i:(i+x.shape[0])] = targets.cpu() 224 | i = i + x.shape[0] 225 | 226 | # Construct the dataset 227 | dataset_logits = torch.utils.data.TensorDataset(logits, labels.long()) 228 | return dataset_logits 229 | 230 | def get_logits_dataset(modelname, datasetname, datasetpath, cache=str(pathlib.Path(__file__).parent.absolute()) + '/experiments/.cache/'): 231 | fname = cache + datasetname + '/' + modelname + '.pkl' 232 | 233 | # If the file exists, load and return it. 234 | if os.path.exists(fname): 235 | with open(fname, 'rb') as handle: 236 | return pickle.load(handle) 237 | 238 | # Else we will load our model, run it on the dataset, and save/return the output. 239 | model = get_model(modelname) 240 | 241 | transform = transforms.Compose([ 242 | transforms.Resize(256), 243 | transforms.CenterCrop(224), 244 | transforms.ToTensor(), 245 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 246 | std =[0.229, 0.224, 0.225]) 247 | ]) 248 | 249 | dataset = torchvision.datasets.ImageFolder(datasetpath, transform) 250 | loader = torch.utils.data.DataLoader(dataset, batch_size = 32, shuffle=False, pin_memory=True) 251 | 252 | # Get the logits and targets 253 | dataset_logits = get_logits_targets(model, loader) 254 | 255 | # Save the dataset 256 | os.makedirs(os.path.dirname(fname), exist_ok=True) 257 | with open(fname, 'wb') as handle: 258 | pickle.dump(dataset_logits, handle, protocol=pickle.HIGHEST_PROTOCOL) 259 | 260 | return dataset_logits 261 | --------------------------------------------------------------------------------