├── .flake8 ├── .gitignore ├── .pre-commit-config.yaml ├── .pylintrc ├── LICENSE ├── NaTGenPD ├── __init__.py ├── analysis.py ├── ccmap.py ├── cems_linear_piecewise.py ├── clean.py ├── cli.py ├── cluster.py ├── filter.py ├── handler.py ├── piecewiseconvex.py ├── poly_fit.py └── version.py ├── README.md ├── SMOKE_Data.md ├── bin ├── 3_1_Generator_Y2017.xlsx ├── 6_1_EnviroAssoc_Y2017.xlsx ├── CEMS_CC_mapping.pdf ├── analyze_CEMS.py ├── analyze_CEMS.sh ├── cems_cc_mapping.csv ├── emission_01-17-2017.csv ├── filter_CEMS.py ├── filter_CEMS.sh ├── launch_CEMS.sh ├── launch_analysis.sh ├── launch_filter.sh ├── plot_ccs.py ├── run_CEMS.py └── run_CEMS.sh ├── data ├── piecewise_fits │ ├── Boiler (Coal)_piecewise_fits.csv │ ├── Boiler (NG)_piecewise_fits.csv │ ├── Boiler (Oil)_piecewise_fits.csv │ ├── Boiler (Other Solid Fuel)_piecewise_fits.csv │ ├── CC (Coal)_piecewise_fits.csv │ ├── CC (NG)_piecewise_fits.csv │ ├── CC (Oil)_piecewise_fits.csv │ ├── CT (NG)_piecewise_fits.csv │ ├── CT (Oil)_piecewise_fits.csv │ ├── Stoker (Other Solid Fuel)_piecewise_fits.csv │ └── all_fits │ │ ├── Boiler (Coal)_linpiecewise_fits.csv │ │ ├── Boiler (NG)_linpiecewise_fits.csv │ │ ├── Boiler (Oil)_linpiecewise_fits.csv │ │ ├── Boiler (Other Solid Fuel)_linpiecewise_fits.csv │ │ ├── CC (Coal)_linpiecewise_fits.csv │ │ ├── CC (NG)_linpiecewise_fits.csv │ │ ├── CC (Oil)_linpiecewise_fits.csv │ │ ├── CT (NG)_linpiecewise_fits.csv │ │ ├── CT (Oil)_linpiecewise_fits.csv │ │ └── Stoker (Other Solid Fuel)_linpiecewise_fits.csv └── poly_fits │ ├── Boiler (Coal)_fits.csv │ ├── Boiler (NG)_fits.csv │ ├── Boiler (Oil)_fits.csv │ ├── Boiler (Other Solid Fuel)_fits.csv │ ├── CC (Coal)_fits.csv │ ├── CC (NG)_fits.csv │ ├── CC (Oil)_fits.csv │ ├── CT (NG)_fits.csv │ ├── CT (Oil)_fits.csv │ ├── Cement Kiln (Coal)_fits.csv │ ├── Process Heater (NG)_fits.csv │ ├── Stoker (Coal)_fits.csv │ ├── Stoker (NG)_fits.csv │ ├── Stoker (Other Solid Fuel)_fits.csv │ └── generic_fits │ ├── Boiler (Coal)_generic_fit.csv │ ├── Boiler (NG)_generic_fit.csv │ ├── Boiler (Oil)_generic_fit.csv │ ├── Boiler (Other Solid Fuel)_generic_fit.csv │ ├── CC (Coal)_generic_fit.csv │ ├── CC (NG)_generic_fit.csv │ ├── CC (Oil)_generic_fit.csv │ ├── CT (NG)_generic_fit.csv │ ├── CT (Oil)_generic_fit.csv │ └── Stoker (Other Solid Fuel)_generic_fit.csv ├── docs ├── Makefile ├── README.md ├── make.bat └── source │ ├── NaTGenPD.rst │ ├── NaTGenPD │ ├── NaTGenPD.rst │ └── modules.rst │ ├── conf.py │ └── index.rst ├── notebooks ├── CEMS Analysis.ipynb ├── CEMS Figures Dev.ipynb ├── CEMS Figures.ipynb ├── CEMS Heat Rate Dev.ipynb ├── CEMS Heat Rate Extraction.ipynb ├── Extract Data.ipynb └── MS_Final_Figures.ipynb └── setup.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E731,E402,F,W503 3 | exclude = .git,__pycache__,docs/source/conf.py,old,build,dist 4 | max-complexity = 10 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | .pytest_cache 4 | *.py[cod] 5 | *$py.class 6 | 7 | # Distribution / packaging 8 | .Python 9 | env/ 10 | build/ 11 | develop-eggs/ 12 | dist/ 13 | downloads/ 14 | eggs/ 15 | .eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | wheels/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *.cover 45 | .hypothesis/ 46 | 47 | # Translations 48 | *.mo 49 | *.pot 50 | 51 | # Dask stuff: 52 | dask-worker-space/ 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # SageMath parsed files 81 | *.sage.py 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | .venv 88 | venv/ 89 | ENV/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | .spyproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | 98 | # mkdocs documentation 99 | /site 100 | 101 | # mypy 102 | .mypy_cache/ 103 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v1.2.3 4 | hooks: 5 | - id: check-json 6 | - id: check-merge-conflict 7 | - id: check-symlinks 8 | - id: flake8 9 | - id: mixed-line-ending 10 | - id: pretty-format-json 11 | args: [--autofix] 12 | - repo: https://github.com/pre-commit/mirrors-pylint 13 | rev: v2.3.1 14 | hooks: 15 | - id: pylint 16 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code 6 | extension-pkg-whitelist= 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=CVS 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. 21 | jobs=1 22 | 23 | # List of plugins (as comma separated values of python modules names) to load, 24 | # usually to register additional checkers. 25 | load-plugins= 26 | 27 | # Pickle collected data for later comparisons. 28 | persistent=yes 29 | 30 | # Specify a configuration file. 31 | #rcfile= 32 | 33 | # When enabled, pylint would attempt to guess common misconfiguration and emit 34 | # user-friendly hints instead of false-positive error messages 35 | suggestion-mode=yes 36 | 37 | # Allow loading of arbitrary C extensions. Extensions are imported into the 38 | # active Python interpreter and may run arbitrary code. 39 | unsafe-load-any-extension=no 40 | 41 | 42 | [MESSAGES CONTROL] 43 | 44 | # Only show warnings with the listed confidence levels. Leave empty to show 45 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 46 | confidence= 47 | 48 | # Disable the message, report, category or checker with the given id(s). You 49 | # can either give multiple identifiers separated by comma (,) or put this 50 | # option multiple times (only on the command line, not in the configuration 51 | # file where it should appear only once).You can also use "--disable=all" to 52 | # disable everything first and then reenable specific checks. For example, if 53 | # you want to run only the similarities checker, you can use "--disable=all 54 | # --enable=similarities". If you want to run only the classes checker, but have 55 | # no Warning level messages displayed, use"--disable=all --enable=classes 56 | # --disable=W" 57 | disable=all 58 | 59 | # Enable the message, report, category or checker with the given id(s). You can 60 | # either give multiple identifier separated by comma (,) or put this option 61 | # multiple time (only on the command line, not in the configuration file where 62 | # it should appear only once). See also the "--disable" option for examples. 63 | enable=unused-import, 64 | unused-argument, 65 | unused-variable, 66 | unused-wildcard-import, 67 | used-before-assignment, 68 | undefined-variable, 69 | undefined-all-variable, 70 | missing-docstring, 71 | empty-docstring, 72 | unneeded-not, 73 | singleton-comparison, 74 | unidiomatic-typecheck, 75 | consider-using-enumerate, 76 | consider-iterating-dictionary, 77 | no-member, 78 | no-self-use, 79 | duplicate-code, 80 | len-as-condition, 81 | missing-format-argument-key, 82 | import-self, 83 | reimported, 84 | wildcard-import, 85 | relative-import, 86 | deprecated-module, 87 | unpacking-non-sequence, 88 | invalid-all-object, 89 | undefined-all-variable, 90 | used-before-assignment, 91 | cell-var-from-loop, 92 | global-variable-undefined, 93 | redefine-in-handler, 94 | global-variable-not-assigned, 95 | undefined-loop-variable, 96 | global-statement, 97 | global-at-module-level, 98 | bad-open-mode, 99 | redundant-unittest-assert, 100 | attribute-defined-outside-init, 101 | unreachable 102 | 103 | 104 | [REPORTS] 105 | 106 | # Python expression which should return a note less than 10 (10 is the highest 107 | # note). You have access to the variables errors warning, statement which 108 | # respectively contain the number of errors / warnings messages and the total 109 | # number of statements analyzed. This is used by the global evaluation report 110 | # (RP0004). 111 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 112 | 113 | # Template used to display messages. This is a python new-style format string 114 | # used to format the message information. See doc for all details 115 | #msg-template= 116 | 117 | # Set the output format. Available formats are text, parseable, colorized, json 118 | # and msvs (visual studio).You can also give a reporter class, eg 119 | # mypackage.mymodule.MyReporterClass. 120 | output-format=text 121 | 122 | # Tells whether to display a full report or only the messages 123 | reports=no 124 | 125 | # Activate the evaluation score. 126 | score=yes 127 | 128 | 129 | [REFACTORING] 130 | 131 | # Maximum number of nested blocks for function / method body 132 | max-nested-blocks=5 133 | 134 | 135 | [BASIC] 136 | 137 | # Naming style matching correct argument names 138 | argument-naming-style=snake_case 139 | 140 | # Regular expression matching correct argument names. Overrides argument- 141 | # naming-style 142 | #argument-rgx= 143 | 144 | # Naming style matching correct attribute names 145 | attr-naming-style=snake_case 146 | 147 | # Regular expression matching correct attribute names. Overrides attr-naming- 148 | # style 149 | #attr-rgx= 150 | 151 | # Bad variable names which should always be refused, separated by a comma 152 | bad-names=foo, 153 | bar, 154 | baz, 155 | toto, 156 | tutu, 157 | tata 158 | 159 | # Naming style matching correct class attribute names 160 | class-attribute-naming-style=any 161 | 162 | # Regular expression matching correct class attribute names. Overrides class- 163 | # attribute-naming-style 164 | #class-attribute-rgx= 165 | 166 | # Naming style matching correct class names 167 | class-naming-style=PascalCase 168 | 169 | # Regular expression matching correct class names. Overrides class-naming-style 170 | #class-rgx= 171 | 172 | # Naming style matching correct constant names 173 | const-naming-style=UPPER_CASE 174 | 175 | # Regular expression matching correct constant names. Overrides const-naming- 176 | # style 177 | #const-rgx= 178 | 179 | # Minimum line length for functions/classes that require docstrings, shorter 180 | # ones are exempt. 181 | docstring-min-length=-1 182 | 183 | # Naming style matching correct function names 184 | function-naming-style=snake_case 185 | 186 | # Regular expression matching correct function names. Overrides function- 187 | # naming-style 188 | #function-rgx= 189 | 190 | # Good variable names which should always be accepted, separated by a comma 191 | good-names=i, 192 | j, 193 | k, 194 | ex, 195 | Run, 196 | _ 197 | 198 | # Include a hint for the correct naming format with invalid-name 199 | include-naming-hint=no 200 | 201 | # Naming style matching correct inline iteration names 202 | inlinevar-naming-style=any 203 | 204 | # Regular expression matching correct inline iteration names. Overrides 205 | # inlinevar-naming-style 206 | #inlinevar-rgx= 207 | 208 | # Naming style matching correct method names 209 | method-naming-style=snake_case 210 | 211 | # Regular expression matching correct method names. Overrides method-naming- 212 | # style 213 | #method-rgx= 214 | 215 | # Naming style matching correct module names 216 | module-naming-style=snake_case 217 | 218 | # Regular expression matching correct module names. Overrides module-naming- 219 | # style 220 | #module-rgx= 221 | 222 | # Colon-delimited sets of names that determine each other's naming style when 223 | # the name regexes allow several styles. 224 | name-group= 225 | 226 | # Regular expression which should only match function or class names that do 227 | # not require a docstring. 228 | no-docstring-rgx=^_ 229 | 230 | # List of decorators that produce properties, such as abc.abstractproperty. Add 231 | # to this list to register other decorators that produce valid properties. 232 | property-classes=abc.abstractproperty 233 | 234 | # Naming style matching correct variable names 235 | variable-naming-style=snake_case 236 | 237 | # Regular expression matching correct variable names. Overrides variable- 238 | # naming-style 239 | #variable-rgx= 240 | 241 | 242 | [FORMAT] 243 | 244 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 245 | expected-line-ending-format= 246 | 247 | # Regexp for a line that is allowed to be longer than the limit. 248 | ignore-long-lines=^\s*(# )??$ 249 | 250 | # Number of spaces of indent required inside a hanging or continued line. 251 | indent-after-paren=4 252 | 253 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 254 | # tab). 255 | indent-string=' ' 256 | 257 | # Maximum number of characters on a single line. 258 | max-line-length=100 259 | 260 | # Maximum number of lines in a module 261 | max-module-lines=1000 262 | 263 | # List of optional constructs for which whitespace checking is disabled. `dict- 264 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 265 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 266 | # `empty-line` allows space-only lines. 267 | no-space-check=trailing-comma, 268 | dict-separator 269 | 270 | # Allow the body of a class to be on the same line as the declaration if body 271 | # contains single statement. 272 | single-line-class-stmt=no 273 | 274 | # Allow the body of an if to be on the same line as the test if there is no 275 | # else. 276 | single-line-if-stmt=no 277 | 278 | 279 | [LOGGING] 280 | 281 | # Logging modules to check that the string format arguments are in logging 282 | # function parameter format 283 | logging-modules=logging 284 | 285 | 286 | [MISCELLANEOUS] 287 | 288 | # List of note tags to take in consideration, separated by a comma. 289 | notes=FIXME, 290 | XXX, 291 | TODO 292 | 293 | 294 | [SIMILARITIES] 295 | 296 | # Ignore comments when computing similarities. 297 | ignore-comments=yes 298 | 299 | # Ignore docstrings when computing similarities. 300 | ignore-docstrings=yes 301 | 302 | # Ignore imports when computing similarities. 303 | ignore-imports=yes 304 | 305 | # Minimum lines number of a similarity. 306 | min-similarity-lines=4 307 | 308 | 309 | [SPELLING] 310 | 311 | # Limits count of emitted suggestions for spelling mistakes 312 | max-spelling-suggestions=4 313 | 314 | # Spelling dictionary name. Available dictionaries: none. To make it working 315 | # install python-enchant package. 316 | spelling-dict= 317 | 318 | # List of comma separated words that should not be checked. 319 | spelling-ignore-words= 320 | 321 | # A path to a file that contains private dictionary; one word per line. 322 | spelling-private-dict-file= 323 | 324 | # Tells whether to store unknown words to indicated private dictionary in 325 | # --spelling-private-dict-file option instead of raising a message. 326 | spelling-store-unknown-words=no 327 | 328 | 329 | [TYPECHECK] 330 | 331 | # List of decorators that produce context managers, such as 332 | # contextlib.contextmanager. Add to this list to register other decorators that 333 | # produce valid context managers. 334 | contextmanager-decorators=contextlib.contextmanager 335 | 336 | # List of members which are set dynamically and missed by pylint inference 337 | # system, and so shouldn't trigger E1101 when accessed. Python regular 338 | # expressions are accepted. 339 | generated-members= 340 | 341 | # Tells whether missing members accessed in mixin class should be ignored. A 342 | # mixin class is detected if its name ends with "mixin" (case insensitive). 343 | ignore-mixin-members=yes 344 | 345 | # This flag controls whether pylint should warn about no-member and similar 346 | # checks whenever an opaque object is returned when inferring. The inference 347 | # can return multiple potential results while evaluating a Python object, but 348 | # some branches might not be evaluated, which results in partial inference. In 349 | # that case, it might be useful to still emit no-member and other checks for 350 | # the rest of the inferred objects. 351 | ignore-on-opaque-inference=yes 352 | 353 | # List of class names for which member attributes should not be checked (useful 354 | # for classes with dynamically set attributes). This supports the use of 355 | # qualified names. 356 | ignored-classes=optparse.Values,thread._local,_thread._local 357 | 358 | # List of module names for which member attributes should not be checked 359 | # (useful for modules/projects where namespaces are manipulated during runtime 360 | # and thus existing member attributes cannot be deduced by static analysis. It 361 | # supports qualified module names, as well as Unix pattern matching. 362 | ignored-modules= 363 | 364 | # Show a hint with possible names when a member name was not found. The aspect 365 | # of finding the hint is based on edit distance. 366 | missing-member-hint=yes 367 | 368 | # The minimum edit distance a name should have in order to be considered a 369 | # similar match for a missing member name. 370 | missing-member-hint-distance=1 371 | 372 | # The total number of similar names that should be taken in consideration when 373 | # showing a hint for a missing member. 374 | missing-member-max-choices=1 375 | 376 | 377 | [VARIABLES] 378 | 379 | # List of additional names supposed to be defined in builtins. Remember that 380 | # you should avoid to define new builtins when possible. 381 | additional-builtins= 382 | 383 | # Tells whether unused global variables should be treated as a violation. 384 | allow-global-unused-variables=yes 385 | 386 | # List of strings which can identify a callback function by name. A callback 387 | # name must start or end with one of those strings. 388 | callbacks=cb_, 389 | _cb 390 | 391 | # A regular expression matching the name of dummy variables (i.e. expectedly 392 | # not used). 393 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 394 | 395 | # Argument names that match this expression will be ignored. Default to name 396 | # with leading underscore 397 | ignored-argument-names=_.*|^ignored_|^unused_ 398 | 399 | # Tells whether we should check for unused import in __init__ files. 400 | init-import=no 401 | 402 | # List of qualified module names which can have objects that can redefine 403 | # builtins. 404 | redefining-builtins-modules=six.moves,past.builtins,future.builtins 405 | 406 | 407 | [CLASSES] 408 | 409 | # List of method names used to declare (i.e. assign) instance attributes. 410 | defining-attr-methods=__init__, 411 | __new__, 412 | setUp 413 | 414 | # List of member names, which should be excluded from the protected access 415 | # warning. 416 | exclude-protected=_asdict, 417 | _fields, 418 | _replace, 419 | _source, 420 | _make 421 | 422 | # List of valid names for the first argument in a class method. 423 | valid-classmethod-first-arg=cls 424 | 425 | # List of valid names for the first argument in a metaclass class method. 426 | valid-metaclass-classmethod-first-arg=mcs 427 | 428 | 429 | [DESIGN] 430 | 431 | # Maximum number of arguments for function / method 432 | max-args=5 433 | 434 | # Maximum number of attributes for a class (see R0902). 435 | max-attributes=7 436 | 437 | # Maximum number of boolean expressions in a if statement 438 | max-bool-expr=5 439 | 440 | # Maximum number of branch for function / method body 441 | max-branches=12 442 | 443 | # Maximum number of locals for function / method body 444 | max-locals=15 445 | 446 | # Maximum number of parents for a class (see R0901). 447 | max-parents=7 448 | 449 | # Maximum number of public methods for a class (see R0904). 450 | max-public-methods=20 451 | 452 | # Maximum number of return / yield for function / method body 453 | max-returns=6 454 | 455 | # Maximum number of statements in function / method body 456 | max-statements=50 457 | 458 | # Minimum number of public methods for a class (see R0903). 459 | min-public-methods=2 460 | 461 | 462 | [IMPORTS] 463 | 464 | # Allow wildcard imports from modules that define __all__. 465 | allow-wildcard-with-all=no 466 | 467 | # Analyse import fallback blocks. This can be used to support both Python 2 and 468 | # 3 compatible code, which means that the block might have code that exists 469 | # only in one or another interpreter, leading to false positives when analysed. 470 | analyse-fallback-blocks=no 471 | 472 | # Deprecated modules which should not be used, separated by a comma 473 | deprecated-modules=optparse,tkinter.tix 474 | 475 | # Create a graph of external dependencies in the given file (report RP0402 must 476 | # not be disabled) 477 | ext-import-graph= 478 | 479 | # Create a graph of every (i.e. internal and external) dependencies in the 480 | # given file (report RP0402 must not be disabled) 481 | import-graph= 482 | 483 | # Create a graph of internal dependencies in the given file (report RP0402 must 484 | # not be disabled) 485 | int-import-graph= 486 | 487 | # Force import order to recognize a module as part of the standard 488 | # compatibility libraries. 489 | known-standard-library= 490 | 491 | # Force import order to recognize a module as part of a third party library. 492 | known-third-party=enchant 493 | 494 | 495 | [EXCEPTIONS] 496 | 497 | # Exceptions that will emit a warning when being caught. Defaults to 498 | # "Exception" 499 | overgeneral-exceptions=Exception 500 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2019 Alliance for Sustainable Energy, LLC 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /NaTGenPD/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | The National Thermal Generator Performance Database. 4 | """ 5 | from __future__ import print_function, division, absolute_import 6 | from NaTGenPD.handler import CEMS, Fits 7 | from NaTGenPD.version import __version__ 8 | 9 | __author__ = """Michael Rossol""" 10 | __email__ = "michael.rossol@nrel.gov" 11 | -------------------------------------------------------------------------------- /NaTGenPD/ccmap.py: -------------------------------------------------------------------------------- 1 | """ 2 | Map CEMS CC generators to EIA CC units 3 | """ 4 | import logging 5 | import numpy as np 6 | import pandas as pd 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | def method_1(boilers, eia_plants): 12 | """ 13 | Method 1 to map boilers to eia plants 14 | """ 15 | # Create boiler-specific unit (Method 1) 16 | no_eia_plant = boilers.loc[~np.in1d(boilers["Plant Code"], eia_plants), :] 17 | no_eia_plant = no_eia_plant.reset_index() 18 | no_eia_plant["Unit Code"] = no_eia_plant["Boiler ID"] 19 | no_eia_plant["Unit Code Method"] = 1 20 | 21 | return no_eia_plant 22 | 23 | 24 | def method_2_3(boilers23, boilers_generators, generators): 25 | """ 26 | Method 2 and 3 27 | """ 28 | # Get boiler -> generator matches (Methods 2 + 3) 29 | boilers_units = boilers23.join(boilers_generators, 30 | on=["Plant Code", "Boiler ID"], how="inner") 31 | boilers_units = boilers_units.join(generators[["Unit Code"]], 32 | on=["Plant Code", "Generator ID"], 33 | how="inner") 34 | boilers_units = boilers_units.reset_index().drop_duplicates(["CEMSUnit", 35 | "Unit Code"]) 36 | 37 | gen_missing_unit_code = boilers_units["Unit Code"].isna() 38 | 39 | # Assign unit code directly (Method 2) 40 | direct_result = boilers_units.loc[~gen_missing_unit_code, :].copy() 41 | direct_result["Unit Code Method"] = 2 42 | 43 | # Create generator-specific unit (Method 3) 44 | direct_nounit_result = boilers_units.loc[gen_missing_unit_code, :].copy() 45 | direct_nounit_result["Unit Code"] = direct_nounit_result["Generator ID"] 46 | direct_nounit_result["Unit Code Method"] = 3 47 | 48 | return direct_result, direct_nounit_result 49 | 50 | 51 | def method_4(boilers4567, generators_cc): 52 | """ 53 | Method 4 54 | """ 55 | # Check for no CA/CTs 56 | boilers_plants = boilers4567.loc[~np.in1d(boilers4567["Plant Code"], 57 | generators_cc["Plant Code"]), :].copy() 58 | 59 | # Create boiler-specific unit (Method 4) 60 | boilers_plants["Unit Code"] = boilers_plants["Boiler ID"].astype(str) 61 | boilers_plants["Unit Code Method"] = 4 62 | 63 | return boilers_plants.reset_index() 64 | 65 | 66 | def method_5(boilers4567, generators_cc): 67 | """ 68 | Method 5 69 | """ 70 | # Check for single unit code among all CA/CTs in plant 71 | pos = np.in1d(generators_cc["Plant Code"], boilers4567["Plant Code"]) 72 | plants_units = generators_cc.loc[pos, ["Plant Code", "Unit Code"]] 73 | plants_units = plants_units.drop_duplicates().set_index("Plant Code") 74 | plants_units = plants_units["Unit Code"] 75 | unit_code_count = plants_units.groupby(level="Plant Code").nunique() 76 | pos = unit_code_count == 1 77 | single_unit_plants = unit_code_count.loc[pos].index.get_values() 78 | 79 | # Assign all boilers in plant to same unit code if single unit code exists 80 | # (Method 5) 81 | single_unit_plants = plants_units.loc[single_unit_plants] 82 | result = boilers4567.join(single_unit_plants, on="Plant Code", 83 | how="right").reset_index() 84 | result["Unit Code Method"] = 5 85 | 86 | return result 87 | 88 | 89 | def method_6_7(boilers4567, generators_cc): 90 | """ 91 | Method 6 and 7 92 | """ 93 | # Check for nonsingle unit code among all CA/CTs in plant 94 | pos = np.in1d(generators_cc["Plant Code"], boilers4567["Plant Code"]) 95 | plants_units = generators_cc.loc[pos, ["Plant Code", "Unit Code"]] 96 | plants_units = plants_units.drop_duplicates().set_index("Plant Code") 97 | plants_units = plants_units["Unit Code"] 98 | unit_code_count = plants_units.groupby(level="Plant Code").nunique() 99 | pos = unit_code_count != 1 100 | nonsingle_unit_plants = unit_code_count.loc[pos].index.get_values() 101 | 102 | # Group boilers and generators by plant 103 | boiler_groups = boilers4567.loc[ 104 | np.in1d(boilers4567["Plant Code"], nonsingle_unit_plants), 105 | :].reset_index().groupby("Plant Code") 106 | gen_groups = generators_cc.loc[ 107 | generators_cc["Prime Mover"] == "CT", :].groupby("Plant Code") 108 | 109 | colnames = ["Plant Code", "Boiler ID", "Generator ID", "Unit Code"] 110 | result6 = pd.DataFrame(columns=colnames) 111 | result7 = pd.DataFrame(columns=colnames) 112 | 113 | # Match boilers and generators by sorting 114 | for plant in nonsingle_unit_plants: 115 | 116 | bs = boiler_groups.get_group(plant).sort_values("Boiler ID") 117 | gs = gen_groups.get_group(plant).sort_values("Generator ID") 118 | 119 | n_bs = len(bs.index) 120 | n_gs = len(gs.index) 121 | 122 | # Match boilers to generator unit codes (Method 6) 123 | if n_bs <= n_gs: 124 | gs = gs.head(n_bs) 125 | result6 = result6.append(pd.DataFrame({ 126 | "CEMSUnit": np.array(bs["CEMSUnit"]), 127 | "Plant Code": plant, 128 | "Boiler ID": np.array(bs["Boiler ID"]), 129 | "Generator ID": np.array(gs["Generator ID"]), 130 | "Unit Code": np.array(gs["Unit Code"])}), sort=True) 131 | 132 | # Match boilers to generator unit codes, 133 | # creating new units for extra boilers (Method 7) 134 | else: 135 | bs_rem = bs.tail(n_bs - n_gs) 136 | bs = bs.head(n_gs) 137 | df = pd.DataFrame({"CEMSUnit": np.array(bs["CEMSUnit"]), 138 | "Plant Code": plant, 139 | "Boiler ID": np.array(bs["Boiler ID"]), 140 | "Generator ID": np.array(gs["Generator ID"]), 141 | "Unit Code": np.array(gs["Unit Code"])}) 142 | result7 = result7.append(df, sort=True) 143 | df = pd.DataFrame({"CEMSUnit": np.array(bs_rem["CEMSUnit"]), 144 | "Plant Code": plant, 145 | "Boiler ID": np.array(bs_rem["Boiler ID"]), 146 | "Unit Code": np.array(bs_rem["Boiler ID"])}) 147 | result7 = result7.append(df, sort=True) 148 | 149 | result6["Unit Code Method"] = 6 150 | result7["Unit Code Method"] = 7 151 | 152 | return result6, result7 153 | 154 | 155 | if __name__ == "__main__": 156 | 157 | # Load CEMS boilers 158 | boilers = pd.read_csv("../bin/emission_01-17-2017.csv", 159 | usecols=[2, 3, 25], header=0, 160 | names=["Plant Code", "Boiler ID", "Unit Type"]) 161 | boilers = boilers.loc[["combined cycle" in ut.lower() 162 | for ut in boilers["Unit Type"]], :] 163 | boilers.drop("Unit Type", axis=1, inplace=True) 164 | index = boilers["Plant Code"].astype(str) + "_" + boilers["Boiler ID"] 165 | boilers.index = index 166 | boilers.index.name = "CEMSUnit" 167 | 168 | # Load boiler-generator mapping 169 | boilers_generators = pd.read_excel( 170 | "../bin/6_1_EnviroAssoc_Y2017.xlsx", "Boiler Generator", 171 | header=1, usecols=[2, 4, 5], 172 | index_col=[0, 1], skipfooter=1) 173 | 174 | def read_generators(f, sheet): 175 | """ 176 | Read generator from excel sheet 177 | """ 178 | return f.parse(sheet, header=1, usecols=[2, 6, 8, 9], 179 | index_col=[0, 1], skipfooter=1) 180 | 181 | # Load generator-unit mapping 182 | with pd.ExcelFile("../bin/3_1_Generator_Y2017.xlsx") as f: 183 | generators = read_generators(f, "Operable") 184 | generators_retired = read_generators(f, "Retired and Canceled") 185 | generators_proposed = read_generators(f, "Proposed") 186 | 187 | generators = generators.append(generators_retired, sort=True) 188 | generators = generators.append(generators_proposed, sort=True) 189 | pos = np.in1d(generators["Prime Mover"], ["CA", "CT"]) 190 | generators_cc = generators.loc[pos, :].reset_index() 191 | 192 | # Any CC CA/CTs without a unit code are assigned to a plant-wide unit 193 | gcc_nounitcode = generators_cc["Unit Code"].isna() 194 | generators_cc.loc[gcc_nounitcode, "Unit Code"] = "" 195 | 196 | eia_plants = [p for (p, g) in generators.index] 197 | eia_plants_boilers = list(boilers_generators.index) 198 | 199 | boilers_234567 = boilers.loc[np.in1d(boilers["Plant Code"], eia_plants), :] 200 | boilers_23 = np.array([(p, b) in eia_plants_boilers 201 | for (_, p, b) in boilers_234567.itertuples()]) 202 | boilers_4567 = boilers_234567.loc[~boilers_23, :] 203 | boilers_23 = boilers_234567.loc[boilers_23, :] 204 | 205 | # Process CEMS boiler + EIA860 plant/generator data 206 | result1 = method_1(boilers, eia_plants) 207 | result2, result3 = method_2_3(boilers_23, boilers_generators, generators) 208 | result4 = method_4(boilers_4567, generators_cc) 209 | result5 = method_5(boilers_4567, generators_cc) 210 | result6, result7 = method_6_7(boilers_4567, generators_cc) 211 | 212 | # Recombine results from all methods 213 | result = pd.concat([result1, result2, result3, result4, result5, result6, 214 | result7], sort=True).set_index("CEMSUnit") 215 | results = results[["Plant Code", "Boiler ID", "Generator ID", "Unit Code", 216 | "Unit Code Method"]] 217 | plant_code = result["Plant Code"].astype(str) 218 | unit_code = result["Unit Code"].astype(str) 219 | result["CCUnit"] = plant_code + "_" + unit_code 220 | 221 | for i in range(4, 8): 222 | unit_info = results[results['CEMSUnit'] == '2393_0{}'.format(i)] 223 | unit_info['CEMSUnit'] = '2393_{}'.format(i) 224 | results = results.append(unit_info) 225 | 226 | logger.info("{} input boilers".format(len(boilers.index))) 227 | logger.info("{} output mappings".format(len(result.index))) 228 | outputs = len(result.drop_duplicates(["Plant Code", "Boiler ID"]).index) 229 | logger.info("{} unique output mappings".format(outputs)) 230 | logger.info(result.groupby("Unit Code Method").count()) 231 | 232 | result.to_csv("../bin/cems_cc_mapping.csv") 233 | -------------------------------------------------------------------------------- /NaTGenPD/cems_linear_piecewise.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import os 4 | import sys 5 | import warnings 6 | 7 | from handler import CEMS 8 | from piecewiseconvex import fit_simple, fit_piecewise_convex, generate_plexos_parameters 9 | 10 | fit_columns = [ 11 | "min_aicc", "1_b", "1_m1", "1_x1", 12 | "2_b", "2_m1", "2_m2", "2_x1", "2_x2", 13 | "3_b", "3_m1", "3_m2", "3_m3", "3_x1", "3_x2", "2_x3"] 14 | 15 | def linear_fits(filtered_h5, max_segments): 16 | 17 | results = [] 18 | 19 | print("Opening", filtered_h5) 20 | 21 | with CEMS(filtered_h5, mode='r') as f: 22 | 23 | for group_type in f.dsets: 24 | 25 | print("Processing", group_type, "group...") 26 | 27 | result = f[group_type].unit_dfs.apply( 28 | lambda unit_df: unit_df.groupby("cluster").apply( 29 | unit_linear_fit, max_segments=max_segments)) 30 | 31 | result["min_aicc"].astype(int, copy=False) 32 | result.reset_index(2, drop=True, inplace=True) 33 | result["group_type"] = group_type 34 | result.set_index("group_type", drop=True, append=True, inplace=True) 35 | result = result.reorder_levels(["group_type", "unit_id", "cluster"]) 36 | 37 | results.append(result) 38 | print(result) 39 | 40 | print("Closed", filtered_h5) 41 | return pd.concat(results, join="outer", sort=False) 42 | 43 | 44 | def unit_linear_fit(unit_cluster_df, max_segments): 45 | 46 | load = unit_cluster_df["load"] 47 | heatinput = unit_cluster_df["HTINPUT"] 48 | minload = min(load) 49 | maxload = max(load) 50 | 51 | if ((len(np.unique(load)) < 6) or # Require a minimum amount of unique data 52 | (maxload - minload < 1) or # Require at least 1 MW data spread 53 | ("unit_id" not in unit_cluster_df.columns) or # Drop data missing certain columns 54 | ("cluster" not in unit_cluster_df.columns) or 55 | (unit_cluster_df["cluster"].iloc[0] == -1)): # Ignore the -1 (noise) clusters 56 | return pd.DataFrame(index=[], columns=fit_columns) 57 | 58 | unitname = unit_cluster_df["unit_id"].iloc[0] 59 | unitcluster = unit_cluster_df["cluster"].iloc[0] 60 | print(unitname, unitcluster, unit_cluster_df.shape, 61 | len(np.unique(load)), "...", end="") 62 | 63 | aicc1, (ms1, bs1) = fit_simple(load, heatinput) 64 | b1, ms1, xs1 = generate_plexos_parameters(ms1, bs1, minload, maxload) 65 | results1 = np.concatenate([[b1], ms1, xs1]) 66 | 67 | aicc2, (ms2, bs2) = fit_piecewise_convex(load, heatinput, 2) 68 | b2, ms2, xs2 = generate_plexos_parameters(ms2, bs2, minload, maxload) 69 | results2 = np.concatenate([[b2], ms2, xs2]) 70 | 71 | aicc3, (ms3, bs3) = fit_piecewise_convex(load, heatinput, 3) 72 | b3, ms3, xs3 = generate_plexos_parameters(ms3, bs3, minload, maxload) 73 | results3 = np.concatenate([[b3], ms3, xs3]) 74 | 75 | min_aicc = np.argmin([aicc1, aicc2, aicc3]) + 1 76 | 77 | x = pd.DataFrame( 78 | np.concatenate([[min_aicc], results1, results2, results3]).reshape(1, -1), 79 | columns=fit_columns) 80 | print(" done.") 81 | return x 82 | 83 | 84 | def write_group_results(df, out_dir="."): 85 | 86 | group_type = df.index.get_level_values("group_type")[0] 87 | out_path = os.path.join(out_dir, group_type + "_linpiecewise_fits.csv") 88 | df.to_csv(out_path) 89 | 90 | 91 | if __name__ == "__main__": 92 | 93 | h5file = sys.argv[1] 94 | outputdir = sys.argv[2] 95 | 96 | results = linear_fits(h5file, max_segments=3) 97 | results.groupby(level="group_type").apply(write_group_results, out_dir=outputdir) 98 | -------------------------------------------------------------------------------- /NaTGenPD/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Command Line Interface and Entry point 4 | @author: mrossol 5 | """ 6 | import click 7 | import logging 8 | import os 9 | 10 | from NaTGenPD.clean import ParseSmoke, CleanSmoke 11 | from NaTGenPD.handler import CEMS 12 | 13 | FORMAT = '%(levelname)s - %(asctime)s [%(filename)s:%(lineno)d] : %(message)s' 14 | LOG_LEVEL = {'INFO': logging.INFO, 15 | 'DEBUG': logging.DEBUG, 16 | 'WARNING': logging.WARNING, 17 | 'ERROR': logging.ERROR, 18 | 'CRITICAL': logging.CRITICAL} 19 | 20 | 21 | def get_handler(log_level="INFO", log_file=None, log_format=FORMAT): 22 | """ 23 | get logger handler 24 | 25 | Parameters 26 | ---------- 27 | log_level : str 28 | handler-specific logging level, must be key in LOG_LEVEL. 29 | log_file : str 30 | path to the log file 31 | log_format : str 32 | format string to use with the logging package 33 | 34 | Returns 35 | ------- 36 | handler : logging.FileHandler | logging.StreamHandler 37 | handler to add to logger 38 | """ 39 | if log_file: 40 | # file handler with mode "a" 41 | handler = logging.FileHandler(log_file, mode='a') 42 | else: 43 | # stream handler to system stdout 44 | handler = logging.StreamHandler() 45 | 46 | if log_format: 47 | logformat = logging.Formatter(log_format) 48 | handler.setFormatter(logformat) 49 | 50 | # Set a handler-specific logging level (root logger should be at debug) 51 | handler.setLevel(LOG_LEVEL[log_level]) 52 | 53 | return handler 54 | 55 | 56 | def setup_logger(logger_name, log_level="INFO", log_file=None, 57 | log_format=FORMAT): 58 | """ 59 | Setup logging instance with given name and attributes 60 | 61 | Parameters 62 | ---------- 63 | logger_name : str 64 | Name of logger 65 | log_level : str 66 | Level of logging to capture, must be key in LOG_LEVEL. If multiple 67 | handlers/log_files are requested in a single call of this function, 68 | the specified logging level will be applied to all requested handlers. 69 | log_file : str | list 70 | Path to file to use for logging, if None use a StreamHandler 71 | list of multiple handlers is permitted 72 | log_format : str 73 | Format for loggings, default is FORMAT 74 | 75 | Returns 76 | ------- 77 | logger : logging.logger 78 | instance of logger for given name, with given level and added handler 79 | handler : logging.FileHandler | logging.StreamHandler | list 80 | handler(s) added to logger 81 | """ 82 | logger = logging.getLogger(logger_name) 83 | current_handlers = [str(h) for h in logger.handlers] 84 | 85 | # Set root logger to debug, handlers will control levels above debug 86 | logger.setLevel(LOG_LEVEL["DEBUG"]) 87 | 88 | handlers = [] 89 | if isinstance(log_file, list): 90 | for h in log_file: 91 | handlers.append(get_handler(log_level=log_level, log_file=h, 92 | log_format=log_format)) 93 | else: 94 | handlers.append(get_handler(log_level=log_level, log_file=log_file, 95 | log_format=log_format)) 96 | for handler in handlers: 97 | if str(handler) not in current_handlers: 98 | logger.addHandler(handler) 99 | return logger 100 | 101 | 102 | @click.group() 103 | @click.option('--log_file', default=None, type=click.Path(), 104 | help='Path to .log file') 105 | @click.option('--verbose', '-v', is_flag=True, 106 | help='If used upgrade logging to DEBUG') 107 | @click.pass_context 108 | def main(ctx, log_file, verbose): 109 | """ 110 | CLI entry point 111 | """ 112 | ctx.ensure_object(dict) 113 | 114 | if verbose: 115 | log_level = 'DEBUG' 116 | else: 117 | log_level = 'INFO' 118 | 119 | ctx.obj['LOG_LEVEL'] = log_level 120 | if log_file is not None: 121 | log_dir = os.path.dirname(log_file) 122 | if not os.path.exists(log_dir): 123 | os.makedirs(log_dir) 124 | 125 | ctx.obj['LOG_FILE'] = log_file 126 | 127 | 128 | @main.command() 129 | @click.argument('smoke_dir', type=click.Path(exists=True)) 130 | @click.argument('year', type=int) 131 | @click.pass_context 132 | def import_smoke_data(ctx, smoke_dir, year): 133 | """ 134 | Parse Smoke data from .txt files in 'smoke_dir', 135 | extract performance variables, 136 | and save to disc as a .h5 file 137 | 138 | Parameters 139 | ---------- 140 | smoke_dir : str 141 | Path to root directory containing SMOKE .txt files 142 | year : int | str 143 | Year to parse 144 | """ 145 | setup_logger("NaTGenPD.precleaning", log_file=ctx.obj['LOG_FILE'], 146 | log_level=ctx.obj['LOG_LEVEL']) 147 | ParseSmoke.performance_vars(smoke_dir, year, save=True) 148 | 149 | 150 | @main.command() 151 | @click.argument('smoke_file', type=click.Path(exists=True)) 152 | @click.argument('unit_attrs', type=click.Path(exists=True)) 153 | @click.argument('out_file', type=click.Path()) 154 | @click.option('--cc_map', '-cc', type=click.Path(exists=True), 155 | help='Path to .csv with CEMS to EIA CC unit mapping') 156 | @click.pass_context 157 | def clean_smoke_data(ctx, smoke_file, unit_attrs, out_file, cc_map): 158 | """ 159 | Clean-up SMOKE data for heat rate analysis: 160 | - Convert gross load to net load 161 | - Remove null/unrealistic values 162 | - Remove start-up and shut-down 163 | 164 | Parameters 165 | ---------- 166 | smoke_file : str 167 | Path to .h5 file with SMOKE data 168 | unit_attrs : str 169 | Path to .csv containing facility (unit) attributes 170 | out_file : str 171 | Path to output .h5 file to write clean-data too 172 | """ 173 | setup_logger("NaTGenPD.precleaning", log_file=ctx.obj['LOG_FILE'], 174 | log_level=ctx.obj['LOG_LEVEL']) 175 | CleanSmoke.clean(smoke_file, unit_attrs_path=unit_attrs, cc_map=cc_map, 176 | out_file=out_file) 177 | 178 | 179 | @main.command() 180 | @click.argument('comp_file', type=click.Path()) 181 | @click.argument('year_files', type=click.Path(exists=True), nargs=-1) 182 | @click.pass_context 183 | def combine_clean_files(ctx, comb_file, year_files): 184 | """ 185 | Combine multiple years of CEMS data into a single file 186 | 187 | Parameters 188 | ---------- 189 | comb_file : str 190 | Path to .h5 file to combine data into 191 | year_files : list | tuple 192 | List of file paths to combine. 193 | Each file should correspond to a years worth of Clean SMOKE data 194 | """ 195 | setup_logger("NaTGenPD.handler", log_file=ctx.obj['LOG_FILE'], 196 | log_level=ctx.obj['LOG_LEVEL']) 197 | CEMS.combine_years(comb_file, year_files) 198 | 199 | 200 | if __name__ == '__main__': 201 | try: 202 | main(obj={}) 203 | except Exception: 204 | raise 205 | -------------------------------------------------------------------------------- /NaTGenPD/filter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Data filtering utilities 4 | @author: mrossol 5 | """ 6 | import concurrent.futures as cf 7 | import logging 8 | import numpy as np 9 | import pandas as pd 10 | 11 | from NaTGenPD.cluster import SingleCluster, ClusterCC 12 | from NaTGenPD.handler import CEMS, Fits 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | class Filter: 18 | """ 19 | Run Cluster filters on all units 20 | """ 21 | FILTERS = {'Boiler': SingleCluster.filter, 22 | 'CT': SingleCluster.filter, 23 | 'CC': ClusterCC.filter} 24 | 25 | def __init__(self, clean_h5, years=1): 26 | """ 27 | Parameters 28 | ---------- 29 | clean_h5 : str 30 | Path to .h5 file with pre-cleaned CEMS data 31 | years : int 32 | Number of years of data being filtered 33 | """ 34 | self._clean_h5 = clean_h5 35 | self._years = years 36 | 37 | @property 38 | def total_points(self): 39 | """ 40 | Number of total points possible in dataset 41 | 42 | Returns 43 | ------- 44 | points : int 45 | total number of possible timesteps 46 | """ 47 | points = self._years * 8760 48 | return points 49 | 50 | def filter_group(self, group_type, parallel=True, **kwargs): 51 | """ 52 | Filter all units of given group_type 53 | 54 | Parameters 55 | ---------- 56 | group_type : str 57 | Group type (generator type + fuel type) to filter 58 | parallel : bool 59 | For each group filter units in parallel 60 | 61 | Returns 62 | ------- 63 | group_df : pd.DataFrame 64 | Updated group DataFrame with cluster labels post filtering 65 | """ 66 | logger.info('Filtering all {} units'.format(group_type)) 67 | total_points = self.total_points 68 | min_samples = int(total_points / 1000) 69 | logger.debug('\t- Using min_samples = {}'.format(min_samples)) 70 | threshold = int(total_points / 100) 71 | logger.debug('\t- Skipping units with < {} points'.format(threshold)) 72 | filter = self.FILTERS.get(group_type.split(' (')[0], 73 | SingleCluster.filter) 74 | 75 | with CEMS(self._clean_h5, mode='r') as f: 76 | group = f[group_type] 77 | 78 | if parallel: 79 | with cf.ProcessPoolExecutor() as executor: 80 | futures = [] 81 | for unit_id, unit_df in group.unit_dfs: 82 | logger.debug('- Filtering unit {}'.format(unit_id)) 83 | futures.append(executor.submit(filter, unit_df, 84 | min_samples, 85 | threshold=threshold, 86 | **kwargs)) 87 | 88 | group_df = [future.result() for future in futures] 89 | else: 90 | group_df = [] 91 | for unit_id, unit_df in group.unit_dfs: 92 | logger.debug('- Filtering unit {}'.format(unit_id)) 93 | group_df.append(filter(unit_df, min_samples, 94 | threshold=threshold, **kwargs)) 95 | 96 | group_df = pd.concat(group_df).sort_values(['unit_id', 'time']) 97 | return group_df.reset_index(drop=True) 98 | 99 | def filter_groups(self, out_h5, group_types, parallel=True, **kwargs): 100 | """ 101 | Filter given group_types from clean_h5 and save to out_h5 102 | 103 | Parameters 104 | ---------- 105 | out_h5 : str 106 | Path to .h5 file into which filtered data should be saved 107 | group_types : list 108 | Group types to filter 109 | parallel : bool 110 | For each group filter units in parallel 111 | kwargs : dict 112 | Internal kwargs 113 | """ 114 | with CEMS(out_h5, mode='a') as f_out: 115 | for g_type in group_types: 116 | f_out[g_type] = self.filter_group(g_type, parallel=parallel, 117 | **kwargs) 118 | 119 | def filter_all(self, out_h5, parallel=True, **kwargs): 120 | """ 121 | Filter all groups in clean_h5 and save to out_h5 122 | 123 | Parameters 124 | ---------- 125 | out_h5 : str 126 | Path to .h5 file into which filtered data should be saved 127 | parallel : bool 128 | For each group filter units in parallel 129 | kwargs : dict 130 | Internal kwargs 131 | """ 132 | with CEMS(self._clean_h5, mode='r') as f_in: 133 | group_types = f_in.dsets 134 | 135 | with CEMS(out_h5, mode='w') as f_out: 136 | for g_type in group_types: 137 | f_out[g_type] = self.filter_group(g_type, parallel=parallel, 138 | **kwargs) 139 | 140 | @classmethod 141 | def run(cls, clean_h5, out_h5, group_types=None, years=1, parallel=True, 142 | **kwargs): 143 | """ 144 | Filter all groups in clean_h5 and save to out_h5 145 | 146 | Parameters 147 | ---------- 148 | clean_h5 : str 149 | Path to .h5 file with pre-cleaned CEMS data 150 | out_h5 : str 151 | Path to .h5 file into which filtered data should be saved 152 | group_types : list 153 | Group types to filter, if None, filter all 154 | years : int 155 | Number of years worth of data being filtered 156 | parallel : bool 157 | For each group filter units in parallel 158 | kwargs : dict 159 | Internal kwargs 160 | """ 161 | f = cls(clean_h5, years=years) 162 | if group_types is not None: 163 | f.filter_groups(out_h5, group_types, parallel=parallel, **kwargs) 164 | else: 165 | f.filter_all(out_h5, parallel=parallel, **kwargs) 166 | 167 | 168 | class FitFilter: 169 | """ 170 | Filter Heat Rate fits 171 | """ 172 | def __init__(self, fit_dir): 173 | """ 174 | Parameters 175 | ---------- 176 | fit_dir : str 177 | Path to directory containing heat rate fit files 178 | """ 179 | self._hr_fits = Fits(fit_dir) 180 | 181 | @property 182 | def hr_fits(self): 183 | """ 184 | Returns 185 | ------- 186 | _hr_fits : Fits 187 | Instance of Fits handler class 188 | """ 189 | return self._hr_fits 190 | 191 | @staticmethod 192 | def _get_hr_min(unit, points=100): 193 | """ 194 | Extract minimum heat rate from unit fits 195 | 196 | Parameters 197 | ---------- 198 | unit : pandas.Series 199 | Row containing the fit stats and parameters for a single unit 200 | 201 | Returns 202 | ------- 203 | hr_min: float 204 | Minimum heat rate from heat rate curve fit 205 | """ 206 | params = [i for i in unit.index if i.startswith('a')] 207 | fit_params = unit[params] 208 | if not fit_params.isnull().any(): 209 | fit_params = fit_params.values 210 | poly_fit = np.poly1d(fit_params) 211 | load_min, load_max = unit[['load_min', 'load_max']].values 212 | x = np.linspace(load_min, load_max, points) 213 | hr_min = poly_fit(x).min() 214 | if hr_min < 4.5: 215 | logger.warning('\t- Minimun heat rate value is < 4.5!') 216 | hr_min = None 217 | else: 218 | hr_min = None 219 | 220 | return hr_min 221 | 222 | @staticmethod 223 | def _min_hr_filter(min_hr, stdev_multiplier=2, threshold=(None, None)): 224 | """ 225 | Filter out the most and least efficient units based on multiples of the 226 | standard deviation from the mean 227 | 228 | Parameters 229 | ---------- 230 | min_hr : pd.Series 231 | Series of minimum heat_rate values for all generators with a 232 | valid fit 233 | stdev_multiplier : float 234 | Multiple of the stdev from the mean to use as the filter thresholds 235 | threshold : tuple 236 | Pre-impossed limits for the filter threshold 237 | 238 | Returns 239 | ------- 240 | failed_units : ndarray 241 | Array of units that failed the filter 242 | """ 243 | min_hr_values = min_hr.values 244 | if threshold[0] is not None: 245 | min_hr_values = min_hr_values[min_hr_values > threshold[0]] 246 | 247 | if threshold[1] is not None: 248 | min_hr_values = min_hr_values[min_hr_values < threshold[1]] 249 | 250 | mean = min_hr_values.mean() 251 | stdev = min_hr_values.std() 252 | thresh = np.array([-stdev_multiplier, stdev_multiplier]) * stdev + mean 253 | 254 | if threshold[0] is not None: 255 | thresh[0] = threshold[0] 256 | 257 | if threshold[1] is not None: 258 | thresh[1] = threshold[1] 259 | 260 | pos = np.logical_or(min_hr.values < thresh[0], 261 | min_hr.values > thresh[1]) 262 | failed_units = min_hr[pos].index 263 | 264 | return failed_units 265 | 266 | @staticmethod 267 | def _filter(group_df, **kwargs): 268 | """ 269 | Filter units of given group 270 | 271 | Parameters 272 | ---------- 273 | group_df : pd.DataFrame 274 | DataFrame of fits for a given group types 275 | kwargs : dict 276 | Internal kwargs 277 | 278 | Returns 279 | ------- 280 | group_df : pd.DataFrame 281 | Updated DataFrame of fits with filtered units fit values 282 | (load, heat_rate, and params) set to None 283 | """ 284 | group_df = group_df.set_index('unit_id') 285 | min_hr = group_df.apply(FitFilter._get_hr_min, axis=1).dropna() 286 | 287 | failed_units = FitFilter._min_hr_filter(min_hr, **kwargs) 288 | filter_cols = [c for c in group_df.columns 289 | if c.startswith(('a', 'heat_rate', 'load')) 290 | and c not in ['load_min', 'load_max']] 291 | logger.debug('\t- {} units being filtered'.format(len(failed_units))) 292 | group_df.loc[failed_units, filter_cols] = None 293 | 294 | return group_df.reset_index() 295 | 296 | @staticmethod 297 | def _filer_CCs(cc_df, cut_off=9, **kwargs): 298 | """ 299 | Filter CC units 300 | 301 | Parameters 302 | ---------- 303 | cc_df : pd.DataFrame 304 | DataFrame of CC fits for each operating mode 305 | cut_off : float 306 | Threshold above which units should not be included in filter. 307 | These units did not report the steam generation and are thus 308 | appear overly in-efficient for a CC 309 | kwargs : dict 310 | Internal kwargs 311 | 312 | Returns 313 | ------- 314 | cc_df : pd.DataFrame 315 | Updated DataFrame of CC fits with filtered units fit values 316 | (load, heat_rate, and params) set to None 317 | """ 318 | cc_df['cc_id'] = cc_df['unit_id'].str.split('-').str[0] 319 | cc_df = cc_df.set_index('cc_id') 320 | cc_min_hr = cc_df.apply(FitFilter._get_hr_min, axis=1) 321 | cc_min_hr = cc_min_hr.dropna().to_frame().reset_index() 322 | cc_min_hr = cc_min_hr.groupby('cc_id').min() 323 | 324 | failed_units = FitFilter._min_hr_filter(cc_min_hr, 325 | threshold=(None, cut_off), 326 | **kwargs) 327 | filter_cols = [c for c in cc_df.columns 328 | if c.startswith(('a', 'heat_rate', 'load')) 329 | and c not in ['load_min', 'load_max']] 330 | logger.debug('\t- {} units being filtered'.format(len(failed_units))) 331 | cc_df.loc[failed_units, filter_cols] = None 332 | 333 | return cc_df.reset_index(drop=True) 334 | 335 | def filter(self, out_dir=None, min_units=100, **kwargs): 336 | """ 337 | Filter all group types 338 | 339 | Parameters 340 | ---------- 341 | out_dir : str | NoneType 342 | Output directory to save filtered fits to, if None update in-place 343 | min_units : int 344 | Minimum number of units with fits needed for filtering 345 | kwargs : dict 346 | internal kwargs 347 | """ 348 | logger.info('Filtering out the most and least efficient heat ' 349 | 'rate fits') 350 | out_fits = None 351 | if out_dir is not None: 352 | out_fits = Fits(out_dir) 353 | 354 | for g_type in self._hr_fits.group_types: 355 | group_df = self._hr_fits[g_type] 356 | fit_units = np.sum(~group_df['a0'].isnull()) 357 | if fit_units > min_units: 358 | logger.info('- Filtering {}'.format(g_type)) 359 | if 'CC' in g_type: 360 | group_df = self._filer_CCs(group_df, **kwargs) 361 | else: 362 | group_df = self._filter(group_df, **kwargs) 363 | 364 | if out_fits is not None: 365 | out_fits[g_type] = group_df 366 | else: 367 | self._hr_fits[g_type] = group_df 368 | else: 369 | logger.warning('- Skipping {} as it only has {} unique fits' 370 | .format(g_type, fit_units)) 371 | if out_fits is not None: 372 | out_fits[g_type] = group_df 373 | 374 | @classmethod 375 | def run(cls, fit_dir, out_dir=None, **kwargs): 376 | """ 377 | Filter all group types in fit_dir and save to disk 378 | 379 | Parameters 380 | ---------- 381 | fit_dir : str 382 | Path to directory containing heat rate fit files 383 | out_dir : str | NoneType 384 | Output directory to save filtered fits to, if None update in-place 385 | """ 386 | ff = cls(fit_dir) 387 | ff.filter(out_dir=out_dir, **kwargs) 388 | -------------------------------------------------------------------------------- /NaTGenPD/handler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Wrapper on .h5 to handle CEMS data 4 | @author: mrossol 5 | """ 6 | import h5py 7 | import logging 8 | import numpy as np 9 | import os 10 | import pandas as pd 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class CEMSGroup: 16 | """ 17 | Class to handle DataFrame of CEMS data for a single fuel/technology type 18 | """ 19 | def __init__(self, group_data): 20 | self._df = self._parse_group(group_data) 21 | self._unit_dfs = self._df.groupby('unit_id') 22 | self._i = 0 23 | 24 | def __repr__(self): 25 | msg = "{} for {}".format(self.__class__.__name__, self.group_type) 26 | return msg 27 | 28 | def __getitem__(self, unit_id): 29 | if unit_id not in self.units: 30 | raise KeyError('{} is not a valid unit id' 31 | .format(unit_id)) 32 | 33 | unit_df = self._unit_dfs.get_group(unit_id) 34 | 35 | return unit_df.reset_index(drop=True) 36 | 37 | def __len__(self): 38 | return len(self.units) 39 | 40 | def __iter__(self): 41 | return self 42 | 43 | def __next__(self): 44 | if self._i < len(self): 45 | unit_id = self.units[self._i] 46 | unit_df = self[unit_id] 47 | self._i += 1 48 | return unit_id, unit_df 49 | else: 50 | raise StopIteration 51 | 52 | @property 53 | def df(self): 54 | """ 55 | DataFrame of group data 56 | 57 | Returns 58 | ------- 59 | _df : pd.DataFrame 60 | DataFrame of group data 61 | """ 62 | return self._df 63 | 64 | @property 65 | def unit_dfs(self): 66 | """ 67 | Group of unit DataFrames 68 | 69 | Returns 70 | ------- 71 | _unit_dfs : pd.Groupby 72 | Group of unit DataFrames 73 | """ 74 | return self._unit_dfs 75 | 76 | @property 77 | def units(self): 78 | """ 79 | Units in group 80 | 81 | Returns 82 | ------- 83 | units : ndarray 84 | List of units present in the group 85 | """ 86 | return self.df['unit_id'].unique() 87 | 88 | @property 89 | def group_type(self): 90 | """ 91 | Group type for data 92 | 93 | Returns 94 | ------- 95 | group : str 96 | Fuel/technology type for units 97 | """ 98 | return self.df['group_type'].unique()[0] 99 | 100 | @staticmethod 101 | def _parse_rec_arrary(arr): 102 | """ 103 | Convert records array to DataFrame, decoding byte 104 | 105 | Parameters 106 | ---------- 107 | arr : np.rec.array 108 | Records array of group data 109 | 110 | Returns 111 | ------- 112 | df : pd.DataFrame 113 | DataFrame of group data 114 | """ 115 | df = pd.DataFrame() 116 | for col in arr.dtype.names: 117 | data = arr[col] 118 | if np.issubdtype(data.dtype, np.bytes_): 119 | data = np.char.decode(data, encoding='utf-8') 120 | 121 | df[col] = data 122 | 123 | return df 124 | 125 | @staticmethod 126 | def _parse_group(group_data): 127 | """ 128 | Parse group data 129 | 130 | Parameters 131 | ---------- 132 | group_data : pd.DataFrame | np.rec.array | CEMSGroup 133 | Records array of group data 134 | 135 | Returns 136 | ------- 137 | group_data : pd.DataFrame 138 | DataFrame of group data 139 | """ 140 | if isinstance(group_data, np.ndarray): 141 | group_data = CEMSGroup._parse_rec_arrary(group_data) 142 | elif isinstance(group_data, CEMSGroup): 143 | group_data = group_data.df 144 | elif isinstance(group_data, pd.DataFrame): 145 | group_data = group_data 146 | else: 147 | raise ValueError('Cannot parse DataFrame from group_data of type' 148 | ' {}'.format(type(group_data))) 149 | 150 | return group_data 151 | 152 | 153 | class CEMS: 154 | """ 155 | Class to handle CEMS data .h5 files 156 | """ 157 | def __init__(self, h5_path, mode='r'): 158 | """ 159 | Parameters 160 | ---------- 161 | h5_path : str 162 | Path to CEMS .h5 file 163 | NOTE: CEMS class cannot handle Raw SMOKE .h5 produced by ParseSmoke 164 | mode : str 165 | Mode with which to open h5py File instance 166 | """ 167 | self._h5_file = os.path.basename(h5_path) 168 | self._h5 = h5py.File(h5_path, mode=mode) 169 | self._mode = mode 170 | 171 | def __repr__(self): 172 | msg = "{} for {}".format(self.__class__.__name__, self._h5_file) 173 | return msg 174 | 175 | def __enter__(self): 176 | return self 177 | 178 | def __exit__(self, type, value, traceback): 179 | self.close() 180 | 181 | if type is not None: 182 | raise 183 | 184 | def __getitem__(self, dset): 185 | if dset in self.dsets: 186 | group_df = self._h5[dset][...] 187 | else: 188 | raise KeyError('{} is not a valid group_type' 189 | .format(dset)) 190 | 191 | return CEMSGroup(group_df) 192 | 193 | def __setitem__(self, dset, arr): 194 | if self.writable: 195 | if isinstance(arr, CEMSGroup): 196 | arr = CEMSGroup.df 197 | 198 | if isinstance(arr, pd.DataFrame): 199 | arr = self.to_records_array(arr) 200 | 201 | self.update_dset(dset, arr) 202 | 203 | @property 204 | def writable(self): 205 | """ 206 | Check to see if h5py.File instance is writable 207 | Returns 208 | ------- 209 | bool 210 | Flag if mode is writable 211 | """ 212 | mode = ['a', 'w', 'w-', 'x'] 213 | if self._mode not in mode: 214 | msg = 'mode must be writable: {}'.format(mode) 215 | raise RuntimeError(msg) 216 | 217 | return True 218 | 219 | @property 220 | def dsets(self): 221 | """ 222 | Datasets available in .h5 file 223 | 224 | Returns 225 | ------- 226 | list 227 | List of group_types stored as individual DataSets 228 | """ 229 | return list(self._h5) 230 | 231 | @staticmethod 232 | def get_dtype(col): 233 | """ 234 | Get column dtype for converstion to records array 235 | 236 | Parameters 237 | ---------- 238 | col : pandas.Series 239 | Column from pandas DataFrame 240 | 241 | Returns 242 | ------- 243 | str 244 | converted dtype for column 245 | - float = float32 246 | - int = int16 or int32 depending on data range 247 | - object/str = U* max length of strings in col 248 | """ 249 | dtype = col.dtype 250 | if np.issubdtype(dtype, np.floating): 251 | out = 'float32' 252 | elif np.issubdtype(dtype, np.integer): 253 | if col.max() < 32767: 254 | out = 'int16' 255 | else: 256 | out = 'int32' 257 | elif np.issubdtype(dtype, np.object_): 258 | col = col.astype(str) 259 | size = int(col.str.len().max()) 260 | out = 'S{:}'.format(size) 261 | elif np.issubdtype(dtype, np.datetime64): 262 | col = col.astype(str) 263 | size = int(col.str.len().max()) 264 | out = 'S{:}'.format(size) 265 | else: 266 | out = dtype 267 | 268 | return out 269 | 270 | @staticmethod 271 | def to_records_array(df): 272 | """ 273 | Convert pandas DataFrame to numpy Records Array 274 | 275 | Parameters 276 | ---------- 277 | df : pandas.DataFrame 278 | Pandas DataFrame to be converted 279 | 280 | Returns 281 | ------- 282 | numpy.rec.array 283 | Records array of input df 284 | """ 285 | meta_arrays = [] 286 | dtypes = [] 287 | for c_name, c_data in df.iteritems(): 288 | dtype = CEMS.get_dtype(c_data) 289 | if np.issubdtype(dtype, np.bytes_): 290 | c_data = c_data.astype(str).fillna('None') 291 | c_data = c_data.str.encode('utf-8').values 292 | else: 293 | c_data = c_data.fillna(0) 294 | c_data = c_data.values 295 | 296 | arr = np.array(c_data, dtype=dtype) 297 | meta_arrays.append(arr) 298 | dtypes.append((c_name, dtype)) 299 | 300 | return np.core.records.fromarrays(meta_arrays, dtype=dtypes) 301 | 302 | def update_dset(self, dset, arr): 303 | """ 304 | Save dset to disk, if needed create dataset 305 | 306 | Parameters 307 | ---------- 308 | 309 | """ 310 | if dset in self.dsets: 311 | self._h5[dset][...] = arr 312 | else: 313 | self._h5.create_dataset(dset, shape=arr.shape, dtype=arr.dtype, 314 | data=arr) 315 | 316 | def close(self): 317 | """ 318 | Close h5 instance 319 | """ 320 | self._h5.close() 321 | 322 | @classmethod 323 | def combine_years(cls, comb_file, year_files): 324 | """ 325 | Combine multiple years of CEMS data into a single file 326 | 327 | Parameters 328 | ---------- 329 | comb_file : str 330 | Path to .h5 file to combine data into 331 | year_files : list | tuple 332 | List of file paths to combine. 333 | Each file should correspond to a years worth of Clean SMOKE data 334 | """ 335 | years = [os.path.basename(f) for f in year_files] 336 | logger.info('Combining data from {} into {}' 337 | .format(years, comb_file)) 338 | with cls(comb_file, mode='w') as f_comb: 339 | dsets = [] 340 | for file in year_files: 341 | with h5py.File(file, mode='r') as f: 342 | dsets.extend(list(f)) 343 | 344 | dsets = list(set(dsets)) 345 | for ds in dsets: 346 | comb_arr = [] 347 | for file in year_files: 348 | with h5py.File(file, mode='r') as f: 349 | if ds in f: 350 | comb_arr.append(f[ds][...]) 351 | logger.debug('- {} extracted from {}' 352 | .format(ds, os.path.basename(file))) 353 | 354 | comb_arr = np.hstack(comb_arr) 355 | f_comb.update_dset(ds, comb_arr) 356 | logger.info('{} combined'.format(ds)) 357 | 358 | 359 | class Fits: 360 | """ 361 | Class to handle heat reate fits 362 | """ 363 | def __init__(self, fit_dir, suffix='fits.csv'): 364 | """ 365 | Parameters 366 | ---------- 367 | fit_dir : str 368 | Path to directory containing heat rate fit files 369 | """ 370 | self._fit_dir = fit_dir 371 | self._suffix = suffix 372 | self._group_fits = self._find_group_fits(fit_dir, suffix=suffix) 373 | self._i = 0 374 | 375 | def __repr__(self): 376 | msg = "{} w/ {} fits".format(self.__class__.__name__, 377 | len(self._group_fits)) 378 | return msg 379 | 380 | def __getitem__(self, g_type): 381 | if g_type in self.group_types: 382 | fit_df = pd.read_csv(self._group_fits[g_type]) 383 | else: 384 | raise KeyError('{} is not a valid group_type' 385 | .format(g_type)) 386 | 387 | return fit_df 388 | 389 | def __setitem__(self, g_type, df): 390 | if g_type in self.group_types: 391 | path = self._group_fits[g_type] 392 | else: 393 | path = os.path.join(self._fit_dir, 394 | '{}_{}'.format(g_type, self._suffix)) 395 | 396 | df.to_csv(path, index=False) 397 | 398 | def __len__(self): 399 | return len(self.group_types) 400 | 401 | def __iter__(self): 402 | return self 403 | 404 | def __next__(self): 405 | if self._i < len(self): 406 | group_name = self.group_types[self._i] 407 | group_fits = self[group_name] 408 | self._i += 1 409 | return group_name, group_fits 410 | else: 411 | raise StopIteration 412 | 413 | @property 414 | def group_types(self): 415 | """ 416 | Datasets available in .h5 file 417 | 418 | Returns 419 | ------- 420 | list 421 | List of group_types stored as individual DataSets 422 | """ 423 | return sorted(list(self._group_fits)) 424 | 425 | @staticmethod 426 | def _find_group_fits(fit_dir, suffix='fits.csv'): 427 | """ 428 | Identify all heat rate fit files 429 | 430 | Parameters 431 | ---------- 432 | fit_dir : str 433 | Path to directory containing heat rate fit files 434 | 435 | Returns 436 | ------- 437 | group_fits : dict 438 | Dictionary of group types and their corresponding fit file 439 | """ 440 | group_fits = {} 441 | for f_name in os.listdir(fit_dir): 442 | if f_name.endswith(suffix): 443 | group_type = f_name.split('_')[0] 444 | group_fits[group_type] = os.path.join(fit_dir, f_name) 445 | 446 | return group_fits 447 | -------------------------------------------------------------------------------- /NaTGenPD/piecewiseconvex.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def fit_simple(loads, heat_inputs): 4 | 5 | if len(np.unique(loads)) < 2: 6 | print("Unique load values:", len(np.unique(loads))) 7 | print(np.linalg.matrix_rank([loads, heat_inputs])) 8 | ms, bs = np.polyfit(loads, heat_inputs, 1) 9 | ms = np.array([ms]) 10 | bs = np.array([bs]) 11 | aicc = _fit_aicc(loads, heat_inputs, 12 | np.zeros(len(loads), dtype=int), ms, bs) 13 | 14 | return aicc, (ms, bs) 15 | 16 | 17 | def fit_piecewise_convex(loads, heat_rates, n_partitions, 18 | n_trials=10, max_iterations=10, max_starts=10): 19 | 20 | best_aicc = np.inf 21 | best_fit = (np.full(n_partitions, np.nan), np.full(n_partitions, np.nan)) 22 | 23 | for i in range(n_trials): 24 | 25 | aicc, ms, bs = _fit_piecewise_convex( 26 | loads, heat_rates, n_partitions, max_iterations, max_starts) 27 | 28 | if aicc < best_aicc: 29 | best_aicc = aicc 30 | best_fit = (ms, bs) 31 | 32 | return best_aicc, best_fit 33 | 34 | 35 | def _fit_piecewise_convex(loads, heat_rates, n_partitions, 36 | max_iterations, max_starts): 37 | 38 | max_load = max(loads) 39 | complete = False 40 | ms = np.zeros(n_partitions) 41 | bs = np.zeros(n_partitions) 42 | starts = 0 43 | 44 | while not complete and starts <= max_starts: 45 | 46 | partition_assignments = _assign_random_partitions(loads, n_partitions) 47 | prev_partition_assignments = np.copy(partition_assignments) 48 | 49 | iterations = 0 50 | starts += 1 51 | 52 | while iterations < max_iterations: 53 | 54 | iterations += 1 55 | 56 | ms, bs = _fit_partitions(loads, heat_rates, 57 | prev_partition_assignments, ms, bs) 58 | partition_assignments = _assign_partitions(loads, ms, bs, max_load) 59 | 60 | # Lost a partition, break and start over 61 | if len(np.unique(partition_assignments)) < n_partitions: 62 | break 63 | 64 | # Some partition spans a single load value, break and start over 65 | if np.any([len(np.unique(loads[partition_assignments == j])) <= 1 for j in range(n_partitions)]): 66 | break 67 | 68 | # Converged on a local optima, break and return 69 | if np.array_equal(prev_partition_assignments, partition_assignments): 70 | complete = True 71 | break 72 | 73 | prev_partition_assignments = partition_assignments 74 | 75 | # Timed out, return 76 | if iterations == max_iterations: 77 | complete = True 78 | 79 | if complete: 80 | aicc = _fit_aicc(loads, heat_rates, partition_assignments, ms, bs) 81 | else: 82 | aicc = np.inf 83 | 84 | return aicc, ms, bs 85 | 86 | 87 | def _assign_partitions(loads, ms, bs, max_load): 88 | 89 | load_points = (bs[1:] - bs[:-1]) / (ms[:-1] - ms[1:]) 90 | load_points = np.append(load_points, max_load) 91 | 92 | if (load_points <= 0).any() or \ 93 | (np.diff(load_points) <= 0).any(): 94 | return np.zeros_like(loads) 95 | 96 | return np.searchsorted(load_points, loads) 97 | 98 | """ 99 | Randomly partition load values such that each partition is guaranteed to 100 | contain at least two distinct points (prevents rank deficiency issues in 101 | linear fits). 102 | """ 103 | def _assign_random_partitions(loads, n_partitions): 104 | 105 | assignments = np.zeros(len(loads), dtype=int) 106 | 107 | if n_partitions == 1: 108 | return assignments 109 | 110 | n_bounds = n_partitions-1 111 | unique_loads = np.sort(np.unique(np.array(loads))) 112 | n_unique_loads = len(unique_loads) 113 | bounds = [] 114 | 115 | while len(bounds) < n_bounds: 116 | 117 | validbounds = set(range(1, n_unique_loads-2)) 118 | 119 | for i in range(n_bounds): 120 | 121 | if not validbounds: 122 | print("Infeasible partition scheme, retrying") 123 | bounds = [] 124 | break 125 | 126 | bound = np.random.choice(list(validbounds)) 127 | validbounds -= set(range(bound-1, bound+2)) 128 | bounds.append(bound) 129 | 130 | bounds = sorted(bounds) 131 | bounds = unique_loads[bounds] 132 | 133 | for i, bound in enumerate(bounds): 134 | assignments[loads > bound] = i + 1 135 | 136 | return assignments 137 | 138 | 139 | def _fit_partitions(loads, heat_rates, partition_assignments, ms, bs): 140 | 141 | for j in range(len(ms)): 142 | partition_idx = partition_assignments == j 143 | partition_loads = loads[partition_idx] 144 | partition_heat_rates = heat_rates[partition_idx] 145 | if len(np.unique(partition_loads)) < 2: 146 | print("Unique load values:", len(np.unique(partition_loads))) 147 | print("Rank:", np.linalg.matrix_rank([partition_loads, partition_heat_rates])) 148 | ms[j], bs[j] = np.polyfit(partition_loads, partition_heat_rates, 1) 149 | 150 | ordering = np.argsort(bs)[::-1] 151 | ms = ms[ordering] 152 | bs = bs[ordering] 153 | 154 | return ms, bs 155 | 156 | 157 | def _fit_aicc(loads, heat_rates, partition_assignments, ms, bs): 158 | 159 | rss = 0. 160 | n = len(loads) 161 | k = 3*len(ms) 162 | 163 | for i, m, b in zip(range(len(ms)), ms, bs): 164 | partition_idx = partition_assignments == i 165 | partition_loads = loads[partition_idx] 166 | partition_heat_rates = heat_rates[partition_idx] 167 | rss += np.sum(((m * partition_loads + b) - partition_heat_rates)**2) 168 | 169 | return 2*k + n*np.log(rss) + (2*k)*(k+1) / (n - k - 1) 170 | 171 | 172 | def generate_plexos_parameters(ms, bs, min_load, max_load): 173 | 174 | if np.any(np.isnan(ms) | np.isnan(bs)): 175 | nans = np.full(ms.shape[0], np.nan) 176 | return np.nan, nans, nans.copy() 177 | 178 | # Sort lines into active partition order, 179 | # which is also descending y-intercept order 180 | ordering = np.argsort(bs)[::-1] 181 | ms = ms[ordering] 182 | bs = bs[ordering] 183 | 184 | load_points = (bs[1:] - bs[:-1]) / (ms[:-1] - ms[1:]) 185 | load_points = np.append(load_points, max_load) 186 | 187 | # Return Base Heat Rate and lists of Heat Rate Incr, Load Point 188 | return bs[0], ms, load_points 189 | -------------------------------------------------------------------------------- /NaTGenPD/poly_fit.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Polynomial fitting utilities 4 | @author: mrossol 5 | """ 6 | import logging 7 | import numpy as np 8 | import os 9 | import pandas as pd 10 | import warnings 11 | 12 | from NaTGenPD.handler import CEMS, Fits 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | class PolyFit: 18 | """ 19 | Fit filtered units to a polynomial 20 | """ 21 | META_COLS = ['latitude', 'longitude', 'state', 'EPA_region', 'NERC_region', 22 | 'unit_type', 'fuel_type', 'group_type'] 23 | 24 | def __init__(self, filtered_h5, order=4): 25 | """ 26 | Parameters 27 | ---------- 28 | filtered_h5 : str 29 | Path to .h5 file containing filtered CEMS data 30 | order : int 31 | Order of the polynomial fit 32 | """ 33 | self._h5 = filtered_h5 34 | self._order = order 35 | 36 | @staticmethod 37 | def extract_fit(load, heat_rate, order=4, points=5): 38 | """ 39 | Fit unit to a polynomial of given order 40 | 41 | Parameters 42 | ---------- 43 | load : ndarray 44 | Load data for unit 45 | heat_rate : ndarray 46 | Heat Rate data for unit 47 | order : int 48 | Order/degree of the polynomial fit 49 | points : int 50 | Number of load/heat-rate points to save 51 | 52 | Returns 53 | ------- 54 | unit_fit : pandas.Series 55 | Heat rate fit and meta-data for unit 56 | Fit parameters p = [p_n, p_n-1, ...] 57 | Such that y = p_n x^n + p_n-1 x^n-1 ... 58 | """ 59 | fit_params = None 60 | load_range = None 61 | hr_range = None 62 | if load.size: 63 | load_min = load.min() 64 | load_max = load.max() 65 | try: 66 | fit = np.polyfit(load, heat_rate, order, full=True) 67 | if fit[2] > 1: 68 | fit_params = fit[0] 69 | load_range = np.linspace(load_min, load_max, points) 70 | hr_range = np.poly1d(fit_params)(load_range) 71 | except Exception: 72 | logger.exception('Cannot fit unit') 73 | else: 74 | load_min = np.nan 75 | load_max = np.nan 76 | 77 | unit_fit = pd.Series(fit_params, index=['a4', 'a3', 'a2', 'a1', 'a0']) 78 | load_index = ['load_{:}'.format(i) for i in range(1, points + 1)] 79 | load_index[0] = 'load_min' 80 | load_index[-1] = 'load_max' 81 | 82 | unit_load = pd.Series(load_range, index=load_index) 83 | unit_load['load_min'] = load_min 84 | unit_load['load_max'] = load_max 85 | unit_load['total_load'] = load.sum() 86 | unit_load['min_gen_perc'] = load_min / load_max 87 | 88 | hr_index = ['heat_rate({:})'.format(load_i) for load_i in load_index] 89 | unit_hr = pd.Series(hr_range, index=hr_index) 90 | 91 | return pd.concat([unit_fit, unit_load, unit_hr]) 92 | 93 | def fit_unit(self, unit_df, **kwargs): 94 | """ 95 | Extract meta data and heat-rate fit(s) for given unit 96 | 97 | Parameters 98 | ---------- 99 | unit_df : pandas.DataFrame 100 | DataFrame for unit to be fit 101 | kwargs : dict 102 | internal kwargs 103 | 104 | Returns 105 | ------- 106 | unit_fit : pandas.DataFrame 107 | DataFrame of heat-rate fit(s) for given unit 108 | """ 109 | unit_meta = unit_df.iloc[0] 110 | unit_id = unit_meta['unit_id'] 111 | clusters = sorted([label for label in unit_df['cluster'].unique() 112 | if label >= 0]) 113 | 114 | if len(clusters) > 1: 115 | unit_fit = [] 116 | for label in clusters: 117 | fit_id = '{}-{}'.format(unit_id, label) 118 | 119 | pos = unit_df['cluster'] == label 120 | cluster_df = unit_df.loc[pos, ['load', 'heat_rate']] 121 | cluster_df = cluster_df.sort_values(['load', 'heat_rate']) 122 | load = cluster_df['load'].values 123 | heat_rate = cluster_df['heat_rate'].values 124 | cluster_fit = self.extract_fit(load, heat_rate, 125 | order=self._order, **kwargs) 126 | cluster_fit.name = fit_id 127 | unit_fit.append(cluster_fit.to_frame().T) 128 | 129 | unit_fit = pd.concat(unit_fit) 130 | else: 131 | fit_id = '{}'.format(unit_id) 132 | pos = unit_df['cluster'] >= 0 133 | cluster_df = unit_df.loc[pos, ['load', 'heat_rate']] 134 | cluster_df = cluster_df.sort_values(['load', 'heat_rate']) 135 | load = cluster_df['load'].values 136 | heat_rate = cluster_df['heat_rate'].values 137 | cluster_fit = self.extract_fit(load, heat_rate, 138 | order=self._order, **kwargs) 139 | cluster_fit.name = fit_id 140 | unit_fit = cluster_fit.to_frame().T 141 | 142 | for col in self.META_COLS: 143 | unit_fit.loc[:, col] = unit_meta[col] 144 | 145 | return unit_fit 146 | 147 | def fit_group(self, group_type, out_file=None, **kwargs): 148 | """ 149 | Extract polynomial fits for all units in given group 150 | 151 | Parameters 152 | ---------- 153 | group_type : str 154 | Group type (generator type + fuel type) to filter 155 | out_file : str 156 | Path to file inwhich to save fit information (.json or .csv) 157 | kwargs : dict 158 | internal kwargs 159 | 160 | Returns 161 | ------- 162 | group_fits : pandas.DataFrame 163 | DataFrame of fit information 164 | """ 165 | logger.info('Fitting all {} units'.format(group_type)) 166 | with CEMS(self._h5, mode='r') as f: 167 | group = f[group_type] 168 | 169 | group_fits = [] 170 | for unit_id, unit_df in group.unit_dfs: 171 | logger.debug('- Fitting unit {}'.format(unit_id)) 172 | group_fits.append(self.fit_unit(unit_df, **kwargs)) 173 | 174 | group_fits = pd.concat(group_fits) 175 | group_fits.index.name = 'unit_id' 176 | if out_file: 177 | logger.debug('- Saving fits to {}' 178 | .format(out_file)) 179 | if out_file.endswith('.csv'): 180 | group_fits.to_csv(out_file) 181 | elif out_file.endswith('.json'): 182 | group_fits.to_json(out_file) 183 | else: 184 | raise ValueError('Invalid file type, cannot save to .{}' 185 | .format(os.path.splitext(out_file)[-1])) 186 | 187 | return group_fits 188 | 189 | def fit_all(self, out_dir, **kwargs): 190 | """ 191 | Extract unit_fits for all units 192 | 193 | Parameters 194 | ---------- 195 | out_dir : str 196 | Directory into which fit files (.csvs) should be saved 197 | kwargs: dict 198 | Internal kwargs 199 | """ 200 | with CEMS(self._h5, mode='r') as f: 201 | group_types = f.dsets 202 | 203 | for g_type in group_types: 204 | out_path = "{}_fits.csv".format(g_type) 205 | out_path = os.path.join(out_dir, out_path) 206 | _ = self.fit_group(g_type, out_file=out_path, **kwargs) 207 | 208 | @classmethod 209 | def run(cls, filtered_h5, out_dir, order=4, **kwargs): 210 | """ 211 | Extract unit_fits for all units 212 | 213 | Parameters 214 | ---------- 215 | filtered_h5 : str 216 | Path to .h5 file containing filtered CEMS data 217 | out_dir : str 218 | Directory into which fit files (.csvs) should be saved 219 | order : int 220 | Order of the polynomial fit 221 | kwargs: dict 222 | Internal kwargs 223 | """ 224 | fit = cls(filtered_h5, order=order) 225 | fit.fit_all(out_dir, **kwargs) 226 | 227 | 228 | class GenericFit: 229 | """ 230 | Create generic fits from polynomial fits 231 | """ 232 | def __init__(self, fit_dir): 233 | """ 234 | Parameters 235 | ---------- 236 | fit_dir : str 237 | Path to directory containing polynomial fit .csvs 238 | """ 239 | self._fit_dir = fit_dir 240 | self._fits = Fits(fit_dir) 241 | 242 | @staticmethod 243 | def _create_generic_fit(group_fits, method='median', points=20): 244 | """ 245 | Calculate generic heat rate curve for given group type using 246 | given aggregation method with given number of data points 247 | 248 | Parameters 249 | ---------- 250 | group_fits : pd.DataFrame 251 | DataFrame group type polynomial fits 252 | method : str 253 | Option to take the median or mean 254 | points : int 255 | Number of data points for generic fit 256 | 257 | Returns 258 | ------- 259 | generic_curves: pd.DataFrame | NoneType 260 | DataFrame containing the generic heat_rate values as a function of 261 | normalized load 262 | """ 263 | group_fits = group_fits[~group_fits['a0'].isnull()] 264 | if group_fits.shape[0] > 0: 265 | min_gen = group_fits['load_min'] / group_fits['load_max'] 266 | if method.lower() == 'median': 267 | min_gen = min_gen.median() 268 | else: 269 | min_gen = min_gen.mean() 270 | 271 | load_norm = np.linspace(min_gen, 1, points) 272 | 273 | load_ranges = np.tile(load_norm, (group_fits.shape[0], 1)) 274 | load_maxs = group_fits.as_matrix(['load_max']) 275 | load_ranges = load_maxs * load_ranges 276 | fit_params = group_fits.as_matrix(['a4', 'a3', 'a2', 'a1', 'a0']) 277 | poly_fits = [np.poly1d(params) for params in fit_params] 278 | 279 | hr_curves = np.array([poly_fit(load_range) 280 | for poly_fit, load_range 281 | in zip(poly_fits, load_ranges)]) 282 | 283 | if method.lower() == 'median': 284 | hr_curve = np.median(hr_curves, axis=0) 285 | else: 286 | hr_curve = np.mean(hr_curves, axis=0) 287 | 288 | generic_curve = pd.DataFrame({'Normalized Load (%)': load_norm, 289 | 'Heat Rate (mmBTU/MWh)': hr_curve}) 290 | else: 291 | msg = '- Cannot create a generic curve, no valid units present!' 292 | logger.warning(msg) 293 | warnings.warn(msg) 294 | generic_curve = None 295 | 296 | return generic_curve 297 | 298 | def fit_all(self, out_dir, **kwargs): 299 | """ 300 | Extract generic fits for all groups 301 | 302 | Parameters 303 | ---------- 304 | out_dir : str 305 | Directory into which generic fit files (.csvs) should be saved 306 | kwargs: dict 307 | Internal kwargs 308 | """ 309 | generic_fits = Fits(out_dir, suffix='generic_fit.csv') 310 | for g_name, group_fits in self._fits: 311 | logger.info('Creating Generic Fit for {}'.format(g_name)) 312 | generic_fit = self._create_generic_fit(group_fits, **kwargs) 313 | if generic_fit is not None: 314 | generic_fits[g_name] = generic_fit 315 | 316 | @classmethod 317 | def run(cls, fit_dir, out_dir, method='median', points=20): 318 | """ 319 | Create generic fits for all group types 320 | 321 | Parameters 322 | ---------- 323 | fit_dir : str 324 | Path to directory containing polynomial fit .csvs 325 | out_dir : str 326 | Directory into which generic fit files (.csvs) should be saved 327 | method : str 328 | Option to take the median or mean 329 | points : int 330 | Number of data points for generic fit 331 | """ 332 | generic = cls(fit_dir) 333 | generic.fit_all(out_dir, method=method, points=points) 334 | -------------------------------------------------------------------------------- /NaTGenPD/version.py: -------------------------------------------------------------------------------- 1 | """ 2 | NaTGenPD version number 3 | """ 4 | __version__ = "0.1.0" 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NaTGenPD 2 | 3 | The National Thermal Generation Performance Database (NaTGenPD) was developed by the National Renewable Energy Laboratory (NREL) in support of the 4 | Environmental Science and Technology (ES&T) publication ["An Analysis of Thermal Plant Flexibility Using a National Generator Performance Database"](https://pubs.acs.org/doi/abs/10.1021/acs.est.9b04522). 5 | 6 | ## Installation 7 | 8 | ``` 9 | pip install git+ssh://git@github.com/NREL/NaTGenPD.git 10 | ``` 11 | or 12 | ``` 13 | pip install git+https://github.com/NREL/NaTGenPD.git 14 | ``` 15 | -------------------------------------------------------------------------------- /SMOKE_Data.md: -------------------------------------------------------------------------------- 1 | # EPA CEMS SMOKE Data 2 | 3 | Hourly Continuous Emissions Monitoring (CEM) data files formatted for use with the Sparse Matrix Operator Kernel Emissions (SMOKE) modeling system. [Pre-packaged data](https://ampd.epa.gov/ampd/) 4 | 5 | ## SMOKE Data Format 6 | 7 | | Position | Name | Type | Description | 8 | |-----|-----|-----|-----| 9 | | A | ORISID | Char (6) | DOE Plant Identification Code (required) | 10 | | B | BLRID | Char (6) | Boiler Identification Code (required) | 11 | | C | YYMMDD | Int | Date of data in YYMMDD format (required) | 12 | | D | HOUR | Integer | Hour value from 0 to 23 | 13 | | E | NOXMASS | Real | Nitrogen oxide emissions (lb/hr) (required) | 14 | | F | SO2MASS | Real | Sulfur dioxide emissions (lb/hr) (required) | 15 | | G | NOXRATE | Real| Nitrogen oxide emissions rate (lb/MMBtu) | 16 | | H | OPTIME | Real | Fraction of hour unit was operating (optional) | 17 | | I | GLOAD | Real | Gross load (MW) (optional) | 18 | | J | SLOAD | Real | Steam load (1000 lbs/hr) (optional) | 19 | | K | HTINPUT | Real | Heat input (mmBtu) (required) | 20 | | L | HTINPUTMEASURE | Character(2) | Code number indicating measured or substituted | 21 | | M | SO2MEASURE | Character(2) | Code number indicating measured or substituted | 22 | | N | NOXMTMEASURE | Character(2) | Code number indicating measured or substituted | 23 | | O | NOXRMEASURE | Character(2) | Code number indicating measured or substituted | 24 | | P | UNITFLOW | Real |Flow rate (ft3/sec) for the Boiler Unit (optional) | 25 | 26 | Code Numbers used for HTINPUTMEASURE, SO2MEASURE, NOXMMEASURE, NOXRMEASURE: 27 | • 01 = 'Measured' 28 | • 02 = 'Calculated' 29 | • 03 = 'Substitute' 30 | • 04 = 'Measured and Substitute' 31 | • 97 = 'Not Applicable' 32 | • 98 = 'Undetermined' 33 | • 99 = 'Unknown Code' 34 | -------------------------------------------------------------------------------- /bin/3_1_Generator_Y2017.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NREL/NaTGenPD/5045fb284aa4e9dc9f21da83f432ca3c7e2e233d/bin/3_1_Generator_Y2017.xlsx -------------------------------------------------------------------------------- /bin/6_1_EnviroAssoc_Y2017.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NREL/NaTGenPD/5045fb284aa4e9dc9f21da83f432ca3c7e2e233d/bin/6_1_EnviroAssoc_Y2017.xlsx -------------------------------------------------------------------------------- /bin/CEMS_CC_mapping.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NREL/NaTGenPD/5045fb284aa4e9dc9f21da83f432ca3c7e2e233d/bin/CEMS_CC_mapping.pdf -------------------------------------------------------------------------------- /bin/analyze_CEMS.py: -------------------------------------------------------------------------------- 1 | """ 2 | HPC Script to run CEMS procedure analysis 3 | """ 4 | import os 5 | import sys 6 | PKG_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 7 | sys.path.append(PKG_DIR) 8 | import NaTGenPD as npd 9 | from NaTGenPD.analysis import ProcedureAnalysis, QuartileAnalysis 10 | 11 | 12 | def process(data_dir, out_dir, log_file): 13 | """ 14 | Parse, clean, filter, and fit all units for the given years 15 | 16 | Parameters 17 | ---------- 18 | data_dir : str 19 | Directory containing SMOKE data 20 | out_dir : str 21 | Directory into which output analysis is to be saved 22 | log_file : str 23 | Path to file to use for logging 24 | """ 25 | npd.setup_logger('NaTGenPD', log_level='INFO', log_file=log_file) 26 | fits_dir = os.path.join(data_dir, 'Final_Fits') 27 | raw_paths = [os.path.join(data_dir, '{y}/SMOKE_{y}.h5'.format(y=y)) 28 | for y in (2016, 2017)] 29 | clean_path = os.path.join(data_dir, 'SMOKE_Clean_2016-2017.h5') 30 | filter_path = os.path.join(data_dir, 'SMOKE_Filtered_2016-2017.h5') 31 | process_dir = os.path.join(out_dir, 'process') 32 | if not os.path.exists(process_dir): 33 | os.makedirs(process_dir) 34 | 35 | out_path = os.path.join(process_dir, 'process_stats_2016-2017.csv') 36 | 37 | ProcedureAnalysis.stats(fits_dir, raw_paths, clean_path, filter_path, 38 | out_path) 39 | 40 | 41 | def quartiles(data_dir, out_dir, log_file): 42 | """ 43 | Parse, clean, filter, and fit all units for the given years 44 | 45 | Parameters 46 | ---------- 47 | data_dir : str 48 | Directory containing SMOKE data 49 | out_dir : str 50 | Directory into which output analysis is to be saved 51 | log_file : str 52 | Path to file to use for logging 53 | """ 54 | logger = npd.setup_logger('NaTGenPD', log_level='INFO', log_file=log_file) 55 | 56 | logger.info('Running Quartile Analysis for Filtered Units') 57 | fits_dir = os.path.join(data_dir, 'CEMS_Fits') 58 | filter_path = os.path.join(data_dir, 'SMOKE_Filtered_2016-2017.h5') 59 | quartile_dir = os.path.join(out_dir, 'filtered_fits') 60 | if not os.path.exists(quartile_dir): 61 | os.makedirs(quartile_dir) 62 | 63 | out_path = os.path.join(quartile_dir, 'filtered_quartile_stats.csv') 64 | 65 | QuartileAnalysis.stats(fits_dir, filter_path, out_path) 66 | 67 | logger.info('Running Quartile Analysis for Final Units') 68 | fits_dir = os.path.join(data_dir, 'Final_Fits') 69 | filter_path = os.path.join(data_dir, 'SMOKE_Filtered_2016-2017.h5') 70 | quartile_dir = os.path.join(out_dir, 'final_fits') 71 | if not os.path.exists(quartile_dir): 72 | os.makedirs(quartile_dir) 73 | 74 | out_path = os.path.join(quartile_dir, 'final_quartile_stats.csv') 75 | 76 | QuartileAnalysis.stats(fits_dir, filter_path, out_path) 77 | 78 | 79 | if __name__ == '__main__': 80 | data_dir = '/scratch/mrossol/CEMS' 81 | out_dir = os.path.join(data_dir, 'analysis') 82 | if not os.path.exists(out_dir): 83 | os.makedirs(out_dir) 84 | analysis = sys.argv[1] 85 | if analysis == 'process': 86 | log_file = os.path.join(out_dir, 'Process_Analysis.log') 87 | process(data_dir, out_dir, log_file) 88 | elif analysis == 'quartiles': 89 | log_file = os.path.join(out_dir, 'Quartile_Analysis.log') 90 | quartiles(data_dir, out_dir, log_file) 91 | else: 92 | print("ERROR: 'analysis' should be 'process' or 'quartiles'") 93 | -------------------------------------------------------------------------------- /bin/analyze_CEMS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #PBS -l nodes=1,walltime=48:00:00,qos=high 3 | #PBS -A naris 4 | #PBS -q bigmem 5 | #PBS -e $PBS_JOBNAME-$PBS_JOBID.err 6 | #PBS -o $PBS_JOBNAME-$PBS_JOBID.out 7 | 8 | python analyze_CEMS.py ${analysis} 9 | -------------------------------------------------------------------------------- /bin/filter_CEMS.py: -------------------------------------------------------------------------------- 1 | """ 2 | HPC Script to filter, and fit CEMS data 3 | """ 4 | import os 5 | import sys 6 | PKG_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 7 | sys.path.append(PKG_DIR) 8 | import NaTGenPD as npd 9 | 10 | 11 | def filter(data_dir, log_file): 12 | """ 13 | Parse, clean, filter, and fit all units for the given years 14 | 15 | Parameters 16 | ---------- 17 | data_dir : str 18 | Directory containing SMOKE data 19 | log_file : str 20 | Path to file to use for logging 21 | years : tuple 22 | Years to parse, clean, combine, filter, and fit 23 | """ 24 | logger = npd.setup_logger('NaTGenPD', log_level='INFO', 25 | log_file=log_file) 26 | comb_file = os.path.join(data_dir, 'SMOKE_Clean_2016-2017.h5') 27 | 28 | logger.info('Filter Clean SMOKE Data') 29 | filtered_file = os.path.join(data_dir, 'SMOKE_Filtered_2016-2017.h5') 30 | npd.Filter.run(comb_file, filtered_file, years=2) 31 | 32 | fit_dir = os.path.join(data_dir, 'CEMS_Fits') 33 | if not os.path.exists(fit_dir): 34 | os.makedirs(fit_dir) 35 | 36 | npd.PolyFit.run(filtered_file, fit_dir, order=4) 37 | 38 | 39 | if __name__ == '__main__': 40 | data_dir = '/scratch/mrossol/CEMS' 41 | log_file = os.path.join(data_dir, 'Filter_CEMS_2016-2017.log') 42 | filter(data_dir, log_file) 43 | -------------------------------------------------------------------------------- /bin/filter_CEMS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #PBS -l nodes=1:ppn=16,walltime=48:00:00,qos=high 3 | #PBS -A naris 4 | #PBS -q bigmem 5 | #PBS -e $PBS_JOBNAME-$PBS_JOBID.err 6 | #PBS -o $PBS_JOBNAME-$PBS_JOBID.out 7 | 8 | python filter_CEMS.py 9 | -------------------------------------------------------------------------------- /bin/launch_CEMS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /home/mrossol/NaTGenPD/bin 4 | 5 | qstat -u mrossol | grep CEMS | grep ' [RQ] ' || qsub -N CEMS run_CEMS.sh 6 | -------------------------------------------------------------------------------- /bin/launch_analysis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /home/mrossol/NaTGenPD/bin 4 | 5 | # for analysis in process quartiles 6 | # do 7 | # qstat -u mrossol | grep CEMS_${analysis} | grep ' [RQ] ' || qsub -N CEMS_${analysis} -v analysis="${analysis}" analyze_CEMS.sh 8 | # done 9 | 10 | analysis=quartiles 11 | qstat -u mrossol | grep CEMS_${analysis} | grep ' [RQ] ' || qsub -N CEMS_${analysis} -v analysis="${analysis}" analyze_CEMS.sh 12 | -------------------------------------------------------------------------------- /bin/launch_filter.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /home/mrossol/NaTGenPD/bin 4 | 5 | qstat -u mrossol | grep CEMS | grep ' [RQ] ' || qsub -N CEMS filter_CEMS.sh 6 | -------------------------------------------------------------------------------- /bin/plot_ccs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to plot final fits for all CC units 3 | """ 4 | import numpy as np 5 | import os 6 | import plotting as mplt 7 | import seaborn as sns 8 | import sys 9 | 10 | pkg_dir = '/home/mrossol/NaTGenPD' 11 | sys.path.append(pkg_dir) 12 | from NaTGenPD.analysis import ProcedureAnalysis 13 | 14 | 15 | def get_hr_fit(group_fits, unit_id): 16 | """ 17 | Get heat rate fit 18 | """ 19 | unit_fit = group_fits.loc[group_fits['unit_id'] == unit_id] 20 | cols = ['load_min', 'load_2', 'load_3', 'load_4', 'load_max'] 21 | load = unit_fit[cols].values 22 | hr = unit_fit[['heat_rate({})'.format(c) for c in cols]].values 23 | try: 24 | fit = np.dstack((load, hr))[0] 25 | except IndexError: 26 | fit = np.array([[0, 0], [0, 0]]) 27 | 28 | return fit 29 | 30 | 31 | def get_raw_cc(raw_group, unit_id): 32 | """ 33 | Get Raw CC unit 34 | """ 35 | pos = raw_group.df['cc_unit'] == unit_id 36 | cc_unit = raw_group.df.loc[pos] 37 | cc_unit = cc_unit.groupby('time')[['load', 'HTINPUT']].sum() 38 | cc_unit = cc_unit.reset_index() 39 | 40 | cc_unit['heat_rate'] = cc_unit['HTINPUT'] / cc_unit['load'] 41 | return cc_unit[['load', 'heat_rate']].values 42 | 43 | 44 | def lighten(color, perc): 45 | """ 46 | lighten color by given perc 47 | """ 48 | color = [min(c + (1 * perc), 1) for c in color] 49 | return color 50 | 51 | 52 | def darken(color, perc): 53 | """ 54 | darken color by given perc 55 | """ 56 | color = [max(c - (1 * perc), 0) for c in color] 57 | return color 58 | 59 | 60 | def plot_unit(unit_id, cc_filtered, cc_fits, out_dir): 61 | """ 62 | Plot cc unit with fits 63 | """ 64 | title = unit_id 65 | try: 66 | filtered_unit = cc_filtered[unit_id] 67 | pos = filtered_unit['cluster'] >= 0 68 | clusters = len(filtered_unit.loc[pos, 'cluster'].unique()) 69 | colors = sns.color_palette('colorblind', clusters) 70 | cluster_units = [] 71 | cluster_fits = [] 72 | legend = [] 73 | for c, cluster_df in filtered_unit.loc[pos].groupby('cluster'): 74 | cts = int(cluster_df.iloc[0]['cts']) 75 | legend.append('CTs = {}'.format(cts)) 76 | cluster_units.append(cluster_df[['load', 'heat_rate']].values) 77 | colors.append(lighten(colors[c], 0.5)) 78 | cluster_id = '{}-{}'.format(unit_id, c) 79 | cluster_fits.append(get_hr_fit(cc_fits, cluster_id)) 80 | 81 | plt_data = cluster_units + cluster_fits 82 | linestyles = ('', ) * len(legend) + ('--',) * len(legend) 83 | f_path = os.path.join(out_dir, '{}.png'.format(unit_id)) 84 | x = filtered_unit.loc[pos, 'load'].values 85 | x_lim = np.nanmax(x[x != np.inf]) * 1.1 86 | y = filtered_unit.loc[pos, 'heat_rate'].values 87 | y_lim = np.nanmax(y[y != np.inf]) * 1.1 88 | mplt.line_plot(*plt_data, despine=True, title=title, 89 | linestyles=linestyles, markers=('o', ), 90 | colors=colors, 91 | xlabel='Load (MWh)', ylabel='Heat Rate (mmBTU/MWh)', 92 | xlim=(0, x_lim), ylim=(0, y_lim), 93 | legend=legend, legend_loc=0, 94 | filename=f_path, showplot=False 95 | ) 96 | except Exception as ex: 97 | print("{} failed due to {}".format(unit_id, ex)) 98 | 99 | 100 | if __name__ == '__main__': 101 | data_dir = '/scratch/mrossol/CEMS' 102 | out_dir = os.path.join(data_dir, 'analysis', 'Figures', 'All_CCs') 103 | if not os.path.exists(out_dir): 104 | os.makedirs(out_dir) 105 | 106 | fits_dir = os.path.join(data_dir, 'Final_Fits') 107 | raw_paths = [os.path.join(data_dir, '{y}/SMOKE_{y}.h5'.format(y=y)) 108 | for y in (2016, 2017)] 109 | clean_path = os.path.join(data_dir, 'SMOKE_Clean_2016-2017.h5') 110 | filter_path = os.path.join(data_dir, 'SMOKE_Filtered_2016-2017.h5') 111 | analysis = ProcedureAnalysis(fits_dir, raw_paths, clean_path, filter_path) 112 | 113 | group_type = 'CC (NG)' 114 | cc_raw = analysis._get_raw(group_type) 115 | cc_filtered = analysis._get_filtered(group_type) 116 | cc_fits = analysis._fits[group_type] 117 | cc_fits = cc_fits.loc[~cc_fits['a0'].isnull()] 118 | 119 | cc_ids = (cc_fits['unit_id'].str.split('-').str[0]).unique() 120 | for unit_id in cc_ids: 121 | plot_unit(unit_id, cc_filtered, cc_fits, out_dir) 122 | print('{} plotted'.format(unit_id)) 123 | -------------------------------------------------------------------------------- /bin/run_CEMS.py: -------------------------------------------------------------------------------- 1 | """ 2 | HPC Script to parse, clean, filter, and fit CEMS data 3 | """ 4 | import os 5 | import sys 6 | PKG_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 7 | sys.path.append(PKG_DIR) 8 | import NaTGenPD as npd 9 | 10 | 11 | def run(data_dir, log_file, years=(2016, 2017)): 12 | """ 13 | Parse, clean, filter, and fit all units for the given years 14 | 15 | Parameters 16 | ---------- 17 | data_dir : str 18 | Directory containing SMOKE data 19 | log_file : str 20 | Path to file to use for logging 21 | years : tuple 22 | Years to parse, clean, combine, filter, and fit 23 | """ 24 | logger = npd.setup_logger('NaTGenPD', log_level='INFO', 25 | log_file=log_file) 26 | for year in years: 27 | logger.info('Parsing {} SMOKE Data'.format(year)) 28 | dir_path = os.path.join(data_dir, str(year)) 29 | npd.ParseSmoke.performance_vars(dir_path, year, save=True) 30 | 31 | unit_attrs_path = os.path.join(PKG_DIR, 'bin', 'emission_01-17-2017.csv') 32 | cc_map_path = os.path.join(PKG_DIR, 'bin', 'cems_cc_mapping.csv') 33 | 34 | for year in years: 35 | logger.info('Cleaning {} SMOKE Data'.format(year)) 36 | smoke_path = os.path.join(data_dir, '{y}/SMOKE_{y}.h5'.format(y=year)) 37 | print('Cleaning {}'.format(os.path.basename(smoke_path))) 38 | out_path = os.path.join(data_dir, 'SMOKE_Clean_{}.h5'.format(year)) 39 | _ = npd.CleanSmoke.clean(smoke_path, unit_attrs_path=unit_attrs_path, 40 | cc_map=cc_map_path, out_file=out_path) 41 | 42 | logger.info('Combining Clean SMOKE Data') 43 | year_files = [] 44 | for year in years: 45 | year_files.append(os.path.join(data_dir, 46 | 'SMOKE_Clean_{}.h5'.format(year))) 47 | 48 | comb_file = os.path.join(data_dir, 'SMOKE_Clean_2016-2017.h5') 49 | npd.CEMS.combine_years(comb_file, year_files) 50 | 51 | logger.info('Filter Clean SMOKE Data') 52 | filtered_file = os.path.join(data_dir, 'SMOKE_Filtered_2016-2017.h5') 53 | npd.Filter.run(comb_file, filtered_file, years=2) 54 | 55 | fit_dir = os.path.join(data_dir, 'CEMS_Fits') 56 | if not os.path.exists(fit_dir): 57 | os.makedirs(fit_dir) 58 | 59 | npd.PolyFit.run(filtered_file, fit_dir, order=4) 60 | 61 | 62 | if __name__ == '__main__': 63 | data_dir = '/scratch/mrossol/CEMS' 64 | log_file = os.path.join(data_dir, 'CEMS_2016-2017.log') 65 | run(data_dir, log_file) 66 | -------------------------------------------------------------------------------- /bin/run_CEMS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #PBS -l nodes=1:ppn=16,walltime=48:00:00,qos=high 3 | #PBS -A naris 4 | #PBS -q bigmem 5 | #PBS -e $PBS_JOBNAME-$PBS_JOBID.err 6 | #PBS -o $PBS_JOBNAME-$PBS_JOBID.out 7 | 8 | python run_CEMS.py 9 | -------------------------------------------------------------------------------- /data/piecewise_fits/Boiler (Oil)_piecewise_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,group_type,cluster,heat_rate_base,heat_rate_incr1,load_point1,heat_rate_incr2,load_point2,heat_rate_incr3,load_point3,latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type 2 | 1507_1,Boiler (Oil),0,52.260522142760976,10.396452226920658,52.96500015258789,,,,,43.75,-70.15689849853516,ME,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil 3 | 1507_3,Boiler (Oil),0,103.93350360336628,8.68695747941135,97.2723749870112,9.421260133047282,120.375,,,43.75,-70.15689849853516,ME,1.0,NPCC,Tangentially-fired,Residual Oil 4 | 1507_4,Boiler (Oil),0,442.66845382235033,7.764083411580978,181.12437592876645,9.444054339524175,503.85093044993073,13.054595827422569,605.7269897460938,43.75,-70.15689849853516,ME,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil 5 | 1554_1,Boiler (Oil),0,125.06721985915381,10.097514359072015,127.11599731445312,,,,,39.1781,-76.52680206298828,MD,3.0,MAAC,Dry bottom wall-fired boiler,Other Oil 6 | 1554_4,Boiler (Oil),0,524.4290146458654,9.51914346388925,398.6820068359375,,,,,39.1781,-76.52680206298828,MD,3.0,MAAC,Dry bottom wall-fired boiler,Other Oil 7 | 1599_2,Boiler (Oil),0,436.33095866583307,8.888974816654537,266.11293739935,9.2393705712705,572.02197265625,,,41.7694,-70.50969696044922,MA,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil 8 | 1619_4,Boiler (Oil),0,343.6418364941674,11.439625166969938,395.7929992675781,,,,,41.7125,-71.19139862060547,MA,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil 9 | 1642_3,Boiler (Oil),0,111.79803849448679,11.57179501367974,97.26300048828124,,,,,42.0956,-72.5958023071289,MA,1.0,NPCC,Tangentially-fired,Residual Oil 10 | 2516_1,Boiler (Oil),0,176.46229915031398,9.542058638762503,235.983061983788,9.714206104642262,397.718994140625,,,40.9231,-73.3416976928711,NY,2.0,,Tangentially-fired,Residual Oil 11 | 2516_2,Boiler (Oil),0,1106.4602784347194,0.6327550814743235,106.59160013141693,9.401737405549383,191.53793443738303,10.081267152976444,398.6820068359375,40.9231,-73.3416976928711,NY,2.0,,Tangentially-fired,Residual Oil 12 | 2516_3,Boiler (Oil),0,156.8712428306079,9.47146329246206,241.73478263710967,9.849873036724311,367.8659973144531,,,40.9231,-73.3416976928711,NY,2.0,,Tangentially-fired,Residual Oil 13 | 2516_4,Boiler (Oil),0,184.26493244301602,9.648976079205832,362.0880126953125,,,,,40.9231,-73.3416976928711,NY,2.0,,Tangentially-fired,Residual Oil 14 | 2517_3,Boiler (Oil),0,209.82168674515748,7.524811827669443,49.034873687105176,10.082032049509234,178.06567473704368,20.87508855694273,198.3780059814453,40.9503,-73.07859802246094,NY,2.0,,Tangentially-fired,Residual Oil 15 | 2517_4,Boiler (Oil),0,145.56798818867492,9.419239946865208,180.20260852989742,20.614997039820693,200.3040008544922,,,40.9503,-73.07859802246094,NY,2.0,,Tangentially-fired,Residual Oil 16 | 3148_3,Boiler (Oil),0,558.8692427632375,9.432708429717259,439.60782822837984,9.571117564424382,686.6190185546875,,,40.796,-75.10700225830078,PA,3.0,MAAC,Tangentially-fired,Residual Oil 17 | 3148_4,Boiler (Oil),0,930.5453424662196,7.296988235768142,283.2471071348823,10.460632030977823,713.5830078125,,,40.796,-75.10700225830078,PA,3.0,MAAC,Tangentially-fired,Residual Oil 18 | 3161_3,Boiler (Oil),0,375.612461845877,8.884315680290527,132.92263186882423,10.300653042931618,202.66131160749043,11.174139315423545,262.89898681640625,39.858,-75.322998046875,PA,3.0,,Tangentially-fired,Diesel Oil 19 | 3161_4,Boiler (Oil),0,417.7421432793159,8.249358168002082,151.07480298890508,11.488739530072634,193.56300354003903,,,39.858,-75.322998046875,PA,3.0,,Tangentially-fired,Diesel Oil 20 | 3804_5,Boiler (Oil),0,722.186181479618,8.63533733576565,784.844970703125,,,,,38.5367,-77.28060150146484,VA,3.0,SERC,Tangentially-fired,Residual Oil 21 | 546_5,Boiler (Oil),0,-56.684761891456006,12.336390093933801,84.74400329589844,,,,,41.4281,-72.10189819335938,CT,1.0,NPCC,Tangentially-fired,Residual Oil 22 | 546_6,Boiler (Oil),0,58.24255834183362,13.496294045679894,286.5105111094178,14.837664003196206,401.5710144042969,,,41.4281,-72.10189819335938,CT,1.0,NPCC,Tangentially-fired,Residual Oil 23 | 593_5,Boiler (Oil),0,385.63197061723406,7.761241696621218,78.61556238382781,9.01408012188729,272.18007831600863,13.975385298696933,310.08599853515625,39.7372,-75.50330352783203,DE,3.0,MAAC,Dry bottom wall-fired boiler,Residual Oil 24 | 6156_NHB1,Boiler (Oil),0,706.9373303797013,7.482973828141415,199.9049599683292,11.387027686761945,453.572998046875,,,41.2836,-72.9041976928711,CT,1.0,,Tangentially-fired,Residual Oil 25 | 676_2,Boiler (Oil),0,123.53872440600365,4.685484927901504,103.04100036621094,,,,,28.0809,-81.92559814453125,FL,4.0,,Dry bottom wall-fired boiler,Residual Oil 26 | 8002_1,Boiler (Oil),0,285.5945243682796,12.017257006166716,415.0530090332031,,,,,43.0974,-70.78340148925781,NH,1.0,NPCC,Tangentially-fired,Residual Oil 27 | 8006_1,Boiler (Oil),0,448.32749596666963,9.424089623573467,613.4310302734375,,,,,41.5711,-73.9738998413086,NY,2.0,NPCC,Tangentially-fired,Residual Oil 28 | 8006_2,Boiler (Oil),0,563.8609990704119,8.343978734333513,317.1533073686897,9.599969190356443,610.5419921875,,,41.5711,-73.9738998413086,NY,2.0,NPCC,Tangentially-fired,Residual Oil 29 | -------------------------------------------------------------------------------- /data/piecewise_fits/Boiler (Other Solid Fuel)_piecewise_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,group_type,cluster,heat_rate_base,heat_rate_incr1,load_point1,heat_rate_incr2,load_point2,latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type 2 | 2367_5,Boiler (Other Solid Fuel),0,377.0741645857077,6.565269321585995,45.32500076293945,,,43.0978,-70.7842025756836,NH,1.0,,Circulating fluidized bed boiler,Wood 3 | 55708_BFB-1,Boiler (Other Solid Fuel),0,182.62656670785626,11.298415103791282,101.75,,,31.8326,-94.90059661865234,TX,6.0,,Bubbling fluidized bed boiler,Wood 4 | 57241_BFB1,Boiler (Other Solid Fuel),0,490.6572597940531,9.435641728895185,106.375,,,29.7675,-82.39640045166016,FL,4.0,,Bubbling fluidized bed boiler,Wood 5 | 58054_ST01,Boiler (Other Solid Fuel),0,2688.1914571824427,-42.935166588995365,47.22497239577861,13.342058144576725,70.30000305175781,44.4719,-71.17530059814453,NH,1.0,,Bubbling fluidized bed boiler,Wood 6 | 589_1,Boiler (Other Solid Fuel),0,79.17659734323634,12.841015141574912,52.41591826255065,51.46402699687714,54.57500076293945,44.4917,-73.20809936523438,VT,1.0,NPCC,Other boiler,Wood 7 | -------------------------------------------------------------------------------- /data/piecewise_fits/CC (Coal)_piecewise_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,group_type,cluster,heat_rate_base,heat_rate_incr1,load_point1,heat_rate_incr2,load_point2,latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type 2 | 1004_1,CC (Coal),0,2366.0040898481084,3.5063832843727054,411.57379472988805,5.478978143773165,445.8500061035156,,,,,,, 3 | 1004_1,CC (Coal),2,924.4948361226303,4.685440762404118,222.9250030517578,,,,,,,,, 4 | 1004_1,CC (Coal),3,478.3236178830526,15.484537783884235,37.92499923706055,,,,,,,,, 5 | -------------------------------------------------------------------------------- /data/piecewise_fits/CC (Oil)_piecewise_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,group_type,cluster,heat_rate_base,heat_rate_incr1,load_point1,heat_rate_incr2,load_point2,heat_rate_incr3,load_point3,latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type 2 | 6081_CC1,CC (Oil),0,312.4696475488729,10.298613707191453,143.48699951171875,,,,,,,,,,, 3 | 6081_CC1,CC (Oil),1,386.027538107217,9.471162215300213,167.56199645996094,,,,,,,,,,, 4 | 6081_CC1,CC (Oil),2,200.43984179103512,9.620993562868753,83.78099822998047,,,,,,,,,,, 5 | 7153_CC2,CC (Oil),0,296.2891385706118,8.77609044031386,112.6709976196289,,,,,,,,,,, 6 | 7153_CC2,CC (Oil),1,536.248720084898,9.191090936056746,220.5269927978516,,,,,,,,,,, 7 | 7153_CC2,CC (Oil),2,784.9391615625111,9.2885678586128,292.67782687537033,13.290637207093631,326.99163617263537,28.29346921182736,341.864990234375,,,,,,, 8 | 7153_CC2,CC (Oil),3,766.6748407160499,10.513876666469102,293.09566167682084,16.325857538812937,313.93798828125,,,,,,,,, 9 | -------------------------------------------------------------------------------- /data/piecewise_fits/CT (Oil)_piecewise_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,group_type,cluster,heat_rate_base,heat_rate_incr1,load_point1,heat_rate_incr2,load_point2,heat_rate_incr3,load_point3,latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type 2 | 1068_2A,CT (Oil),0,103.01125879755524,7.3943762256588865,18.92948480554116,9.074413846143074,28.88999938964844,,,42.0474,-92.8593978881836,IA,7.0,,Combustion turbine,Diesel Oil 3 | 1068_2B,CT (Oil),0,60.498192960291604,9.355675258332553,26.964000701904297,,,,,42.0474,-92.8593978881836,IA,7.0,,Combustion turbine,Diesel Oil 4 | 10823_S42,CT (Oil),0,59.64406698904525,7.695757545064932,15.407999992370605,,,,,42.3497,-70.96009826660156,MA,1.0,,Combustion turbine,Diesel Oil 5 | 10823_S43,CT (Oil),0,54.07258121547776,7.759214258979545,16.371000289916992,,,,,42.3497,-70.96009826660156,MA,1.0,,Combustion turbine,Diesel Oil 6 | 1355_8,CT (Oil),0,319.6438234874727,9.400262742801187,115.55999755859376,,,,,37.7889,-84.71389770507812,KY,4.0,SERC,Combustion turbine,Diesel Oil 7 | 1355_9,CT (Oil),0,317.1124873429112,9.707123409093633,115.55999755859376,,,,,37.7889,-84.71389770507812,KY,4.0,SERC,Combustion turbine,Diesel Oil 8 | 3403_GCT5,CT (Oil),0,262.1704205955137,9.008752715170404,82.48046770290205,10.554669604107305,92.447998046875,,,36.3156,-86.40059661865234,TN,4.0,,Combustion turbine,Diesel Oil 9 | 3403_GCT6,CT (Oil),0,254.0361029767632,9.044507259596132,60.50813083162405,9.783373420351728,90.52200317382812,,,36.3156,-86.40059661865234,TN,4.0,,Combustion turbine,Diesel Oil 10 | 3403_GCT7,CT (Oil),0,302.24051525491274,8.24121482875211,64.34202780133178,9.692009309843986,92.447998046875,,,36.3156,-86.40059661865234,TN,4.0,,Combustion turbine,Diesel Oil 11 | 3403_GCT8,CT (Oil),0,424.21429628780817,5.6786538284074775,54.126484576432155,9.895340898780457,94.3740005493164,,,36.3156,-86.40059661865234,TN,4.0,,Combustion turbine,Diesel Oil 12 | 3406_JCT17,CT (Oil),0,294.20225122157143,7.674704658805947,59.1666365966222,8.918146756232417,86.48354853908229,9.942677186181148,95.33699798583984,36.0278,-87.98609924316406,TN,4.0,,Combustion turbine,Diesel Oil 13 | 3406_JCT18,CT (Oil),0,303.5692532199529,7.528245715454342,54.51668581250106,8.997843673170475,92.447998046875,,,36.0278,-87.98609924316406,TN,4.0,,Combustion turbine,Diesel Oil 14 | 3406_JCT19,CT (Oil),0,268.39073571688493,8.377683030694822,64.85392672922723,9.055627342757603,89.21861629045199,10.360333803783892,94.3740005493164,36.0278,-87.98609924316406,TN,4.0,,Combustion turbine,Diesel Oil 15 | 3406_JCT20,CT (Oil),0,627.6400382843323,2.6260829363790736,132.19741200547122,7.843562386769197,160.8209991455078,,,36.0278,-87.98609924316406,TN,4.0,,Combustion turbine,Diesel Oil 16 | 55381_CT-001,CT (Oil),0,80.87122481122505,8.065381189635291,48.1500015258789,,,,,37.9892,-75.54000091552734,VA,3.0,MAAC,Combustion turbine,Diesel Oil 17 | 55381_CT-002,CT (Oil),0,173.74696721777988,6.149845167009113,47.1870002746582,,,,,37.9892,-75.54000091552734,VA,3.0,MAAC,Combustion turbine,Diesel Oil 18 | 55381_CT-003,CT (Oil),0,110.70570552444158,7.5547327045883295,48.1500015258789,,,,,37.9892,-75.54000091552734,VA,3.0,MAAC,Combustion turbine,Diesel Oil 19 | 55381_CT-004,CT (Oil),0,97.48358699178357,7.693299415345797,48.1500015258789,,,,,37.9892,-75.54000091552734,VA,3.0,MAAC,Combustion turbine,Diesel Oil 20 | 55381_CT-006,CT (Oil),0,104.9473117948881,7.558725331575228,48.1500015258789,,,,,37.9892,-75.54000091552734,VA,3.0,MAAC,Combustion turbine,Diesel Oil 21 | 55699_2,CT (Oil),0,151.28188028869326,8.08608405677913,52.00199890136719,,,,,40.6106,-73.76139831542969,NY,2.0,,Combustion turbine,Diesel Oil 22 | 55969_U-01,CT (Oil),0,119.3242162213992,7.884930902895213,53.928001403808594,,,,,41.1056,-72.37670135498047,NY,2.0,,Combustion turbine,Diesel Oil 23 | 6046_**7,CT (Oil),0,174.35355618599675,10.845694233165077,51.066778002589004,11.093628896109447,76.0770034790039,,,28.9047,-81.33190155029297,FL,4.0,FRCC,Combustion turbine,Diesel Oil 24 | 6046_**8,CT (Oil),0,190.08910306691536,10.81191136587858,47.506374571598435,11.300930008362567,73.18800354003906,,,28.9047,-81.33190155029297,FL,4.0,FRCC,Combustion turbine,Diesel Oil 25 | 6046_**9,CT (Oil),0,203.6006704069558,10.498632368586273,53.71878526797406,11.172055489204435,74.1510009765625,,,28.9047,-81.33190155029297,FL,4.0,FRCC,Combustion turbine,Diesel Oil 26 | 630_P2,CT (Oil),0,172.91277052951665,10.66711265536766,27.92700004577637,,,,,28.0037,-82.66259765625,FL,4.0,FRCC,Combustion turbine,Diesel Oil 27 | 630_P3,CT (Oil),0,205.03606565320683,10.939054118569855,33.705001831054695,,,,,28.0037,-82.66259765625,FL,4.0,FRCC,Combustion turbine,Diesel Oil 28 | 630_P4,CT (Oil),0,221.50278382872102,10.702575132733855,34.667999267578125,,,,,28.0037,-82.66259765625,FL,4.0,FRCC,Combustion turbine,Diesel Oil 29 | 634_P2,CT (Oil),0,252.46741012508343,9.698005661889956,26.827923602432197,10.979518731920082,50.07600021362305,,,27.8613,-82.6011962890625,FL,4.0,FRCC,Combustion turbine,Diesel Oil 30 | 634_P4,CT (Oil),0,240.45084491508868,10.304930405848074,52.96500015258789,,,,,27.8613,-82.6011962890625,FL,4.0,FRCC,Combustion turbine,Diesel Oil 31 | 638_1A,CT (Oil),0,118.91214865024789,8.940877149822889,26.964000701904297,,,,,30.3764,-83.18060302734375,FL,4.0,FRCC,Combustion turbine,Diesel Oil 32 | 638_1B,CT (Oil),0,94.64395533919011,9.897299009866575,25.038000106811523,,,,,30.3764,-83.18060302734375,FL,4.0,FRCC,Combustion turbine,Diesel Oil 33 | 638_3A,CT (Oil),0,106.77224272706802,9.763937476342383,26.964000701904297,,,,,30.3764,-83.18060302734375,FL,4.0,FRCC,Combustion turbine,Diesel Oil 34 | 638_3B,CT (Oil),0,97.5213444460864,9.185124962135175,26.964000701904297,,,,,30.3764,-83.18060302734375,FL,4.0,FRCC,Combustion turbine,Diesel Oil 35 | 7288_1,CT (Oil),0,341.78483153621005,8.63884734243177,70.62472449537812,10.819726637486337,80.89199829101562,,,39.451,-75.05780029296875,NJ,2.0,MAAC,Combustion turbine,Diesel Oil 36 | 7845_LCT1,CT (Oil),0,224.2752297465596,9.324092587501019,85.23228134589337,10.779601279652137,91.48500061035156,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 37 | 7845_LCT10,CT (Oil),0,263.71028170644325,8.696773678738671,87.39228044675656,16.469176670729528,90.52200317382812,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 38 | 7845_LCT11,CT (Oil),0,194.50299478370792,9.558387512067661,90.52200317382812,,,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 39 | 7845_LCT12,CT (Oil),0,208.91425441762462,9.34705619768089,90.52200317382812,,,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 40 | 7845_LCT2,CT (Oil),0,220.85425184082646,9.351639608496015,84.12738503629588,10.334987920951509,91.48500061035156,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 41 | 7845_LCT3,CT (Oil),0,224.22250696568642,9.33382263690089,80.70889137295829,10.155616867587181,91.48500061035156,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 42 | 7845_LCT4,CT (Oil),0,221.78633144573945,9.293339697816084,81.98063873722784,10.089632576621645,92.447998046875,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 43 | 7845_LCT5,CT (Oil),0,232.76077749581464,9.204213636458576,78.81586278838121,10.724206107040619,91.48500061035156,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 44 | 7845_LCT6,CT (Oil),0,229.6744203117437,9.430217516985573,84.06777953268518,10.512161537955864,92.447998046875,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 45 | 7845_LCT7,CT (Oil),0,201.5725184967692,9.377745558990616,84.13451354357655,11.428337777495509,91.48500061035156,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 46 | 7845_LCT8,CT (Oil),0,202.50154288168403,9.453732508173811,82.42089568010383,11.142419481963396,90.52200317382812,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 47 | 7845_LCT9,CT (Oil),0,254.91319178893662,8.683101169970097,81.28876108097828,10.447512660010576,89.55899810791016,,,35.6578,-89.39640045166016,TN,4.0,,Combustion turbine,Diesel Oil 48 | 7960_KCT1,CT (Oil),0,237.4590471637393,9.218413030645703,88.59600067138672,,,,,32.7976,-88.60569763183594,MS,4.0,,Combustion turbine,Diesel Oil 49 | 7960_KCT2,CT (Oil),0,174.4758478826948,9.921871605015168,89.55899810791016,,,,,32.7976,-88.60569763183594,MS,4.0,,Combustion turbine,Diesel Oil 50 | 7960_KCT3,CT (Oil),0,159.78369052490973,10.105197578413467,85.70700073242188,,,,,32.7976,-88.60569763183594,MS,4.0,,Combustion turbine,Diesel Oil 51 | 7960_KCT4,CT (Oil),0,201.6742783765958,9.526595666211382,88.59600067138672,,,,,32.7976,-88.60569763183594,MS,4.0,,Combustion turbine,Diesel Oil 52 | 8049_**10,CT (Oil),0,180.09983186700248,11.336657061720786,55.08662823703537,11.61704745811457,72.2249984741211,,,28.2628,-81.54859924316406,FL,4.0,FRCC,Combustion turbine,Diesel Oil 53 | 8049_**7,CT (Oil),0,166.64795222034374,11.58249151839387,73.18800354003906,,,,,28.2628,-81.54859924316406,FL,4.0,FRCC,Combustion turbine,Diesel Oil 54 | 8049_**8,CT (Oil),0,194.78442663314843,11.222971159612644,74.1510009765625,,,,,28.2628,-81.54859924316406,FL,4.0,FRCC,Combustion turbine,Diesel Oil 55 | 8049_**9,CT (Oil),0,173.07708794312717,11.277648777686874,72.2249984741211,,,,,28.2628,-81.54859924316406,FL,4.0,FRCC,Combustion turbine,Diesel Oil 56 | -------------------------------------------------------------------------------- /data/piecewise_fits/Stoker (Other Solid Fuel)_piecewise_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,group_type,cluster,heat_rate_base,heat_rate_incr1,load_point1,heat_rate_incr2,load_point2,heat_rate_incr3,load_point3,latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type 2 | 10333_1,Stoker (Other Solid Fuel),0,318.245949583784,9.292631303676936,76.7750015258789,,,,,28.5814,-82.4302978515625,FL,4.0,FRCC,Stoker,Wood 3 | 10525_ES5A,Stoker (Other Solid Fuel),0,719.4559638949464,-0.4293813491897569,48.099998474121094,,,,,35.1285,-77.16999816894531,NC,4.0,SERC,Stoker,Wood 4 | 3982_2,Stoker (Other Solid Fuel),0,53.36411771390053,13.253786222801395,13.875,,,,,46.5869,-90.90170288085938,WI,5.0,MAPP,Stoker,Wood 5 | 54238_BIOMS1,Stoker (Other Solid Fuel),0,159.9127392018307,12.502972771896223,46.25,,,,,37.9416,-121.32730102539062,CA,9.0,,Stoker,Wood 6 | 54415_EUBLR,Stoker (Other Solid Fuel),0,97.98016264771529,12.848327425159294,36.07500076293945,,,,,44.2634,-85.43479919433594,MI,5.0,,Stoker,Wood 7 | 54751_01,Stoker (Other Solid Fuel),0,21.522713084617415,13.016021811131141,32.375,,,,,43.0841,-83.67430114746094,MI,5.0,,Stoker,Wood 8 | 55867_BLR-1,Stoker (Other Solid Fuel),0,258.1493623380842,8.397508299259187,45.168121029575225,14.362277831679833,58.17656221119059,45.09384528651357,61.974998474121094,45.2996,-95.5604019165039,MN,5.0,MAPP,Stoker,Other Solid Fuel 9 | -------------------------------------------------------------------------------- /data/piecewise_fits/all_fits/Boiler (Oil)_linpiecewise_fits.csv: -------------------------------------------------------------------------------- 1 | group_type,unit_id,cluster,min_aicc,1_b,1_m1,1_x1,2_b,2_m1,2_m2,2_x1,2_x2,3_b,3_m1,3_m2,3_m3,3_x1,3_x2,2_x3 2 | Boiler (Oil),1507_1,0,1.0,52.260522142760976,10.396452226920658,52.96500015258789,53.297562680288046,10.335423705767637,16.588562697217675,49.91641323973698,52.96500015258789,,,,,,, 3 | Boiler (Oil),1507_3,0,2.0,93.58820548871093,8.8750293852994,120.375,103.93350360336629,8.68695747941135,9.421260133047285,97.2723749870112,120.375,418.7667090817986,-1.9037742289230815,8.743725536542495,9.495735177846315,29.989777556736485,100.54907788214003,120.375 4 | Boiler (Oil),1507_4,0,3.0,240.6625033008544,9.349845439788648,605.7269897460938,290.7741280254455,9.048508475024239,13.87814557066082,501.44374302051983,605.7269897460938,442.6684538223503,7.764083411580977,9.444054339524177,13.054595827422567,181.12437592876643,503.8509304499307,605.7269897460938 5 | Boiler (Oil),1554_1,0,1.0,125.0672198591538,10.097514359072017,127.11599731445312,126.48894600767795,10.083847058523817,10.211684621095568,47.45163615155065,127.11599731445312,,,,,,, 6 | Boiler (Oil),1554_4,0,1.0,524.4290146458655,9.51914346388925,398.6820068359375,653.4768825350336,9.13365608012289,13.485113305283319,94.69047447379234,398.6820068359375,,,,,,, 7 | Boiler (Oil),1599_2,0,2.0,399.0076405114629,9.106952594833837,572.02197265625,436.33095866583307,8.888974816654537,9.239370571270499,266.11293739935,572.02197265625,,,,,,, 8 | Boiler (Oil),1619_4,0,1.0,343.64183649416736,11.439625166969938,395.7929992675781,364.8098407449758,11.21247215669363,22.893804475043286,379.5705254936242,395.7929992675781,,,,,,, 9 | Boiler (Oil),1642_3,0,1.0,111.79803849448678,11.57179501367974,97.26300048828125,225.83073885280692,8.15597092017858,11.53529696749259,32.71948618666024,97.26300048828125,,,,,,, 10 | Boiler (Oil),2516_1,0,2.0,164.99766427897848,9.62362456444719,397.718994140625,176.46229915031395,9.542058638762503,9.714206104642262,235.983061983788,397.718994140625,,,,,,, 11 | Boiler (Oil),2516_2,0,3.0,115.17672386134315,9.850180915741054,398.6820068359375,186.53952046037446,9.294112346347825,10.074654746863457,182.8791958887902,398.6820068359375,1106.4602784347194,0.6327550814743235,9.401737405549383,10.081267152976443,106.59160013141693,191.53793443738303,398.6820068359375 12 | Boiler (Oil),2516_3,0,2.0,134.475958960333,9.634757037633577,367.8659973144531,156.8712428306079,9.47146329246206,9.84987303672431,241.73478263710967,367.8659973144531,156.95225327805505,9.470809812045697,9.851238836126573,16.810216101723537,241.4966242572614,360.9552235078914,367.8659973144531 13 | Boiler (Oil),2516_4,0,1.0,184.26493244301602,9.648976079205832,362.0880126953125,204.91997584948447,9.580876045378869,12.792546505598756,115.29731179852845,362.0880126953125,,,,,,, 14 | Boiler (Oil),2517_3,0,3.0,88.16326634020558,10.138941952883131,198.3780059814453,97.28233197936406,9.991284797079718,20.87508855694273,177.7619799070843,198.3780059814453,209.82168674515748,7.524811827669444,10.082032049509234,20.87508855694273,49.034873687105176,178.06567473704368,198.3780059814453 15 | Boiler (Oil),2517_4,0,2.0,139.60721276941936,9.515637159461312,200.3040008544922,145.56798818867492,9.419239946865208,20.614997039820693,180.20260852989742,200.3040008544922,264.20001745136574,6.820675160287649,9.438582202129398,20.614997039820693,46.36407461969483,180.26888351087572,200.3040008544922 16 | Boiler (Oil),3148_3,0,2.0,532.2418809341808,9.513351515525262,686.6190185546875,558.8692427632375,9.43270842971726,9.57111756442438,439.60782822837984,686.6190185546875,,,,,,, 17 | Boiler (Oil),3148_4,0,2.0,106.34389133856087,10.326353599891815,713.5830078125,930.5453424662195,7.296988235768141,10.460632030977822,283.2471071348823,713.5830078125,,,,,,, 18 | Boiler (Oil),3161_3,0,3.0,287.6960230008469,9.763556067651951,262.89898681640625,376.0389891930091,8.87632537001215,10.616223668569715,134.02506362121517,262.89898681640625,375.612461845877,8.884315680290527,10.300653042931618,11.174139315423547,132.92263186882425,202.66131160749043,262.89898681640625 19 | Boiler (Oil),3161_4,0,2.0,401.21028106294835,8.41860013543895,193.56300354003906,417.7421432793159,8.249358168002082,11.488739530072635,151.07480298890508,193.56300354003906,,,,,,, 20 | Boiler (Oil),3804_5,0,1.0,722.1861814796179,8.63533733576565,784.844970703125,787.7990519459229,8.518614610539021,10.467131371189485,315.85878325797285,784.844970703125,,,,,,, 21 | Boiler (Oil),546_5,0,1.0,-56.68476189145601,12.336390093933801,84.74400329589844,1938.1030355912244,-40.08315507597661,12.364351253258382,38.06921379865569,84.74400329589844,,,,,,, 22 | Boiler (Oil),546_6,0,2.0,-4.960251649239943,13.890902726718446,401.5710144042969,58.242558341833615,13.496294045679894,14.837664003196203,286.5105111094178,401.5710144042969,,,,,,, 23 | Boiler (Oil),593_5,0,3.0,298.6206292183763,8.982775121661746,310.08599853515625,301.8744329880124,8.949792655841014,13.724437895638099,270.16635433469474,310.08599853515625,385.63197061723406,7.761241696621218,9.01408012188729,13.975385298696931,78.6155623838278,272.18007831600863,310.08599853515625 24 | Boiler (Oil),6156_NHB1,0,2.0,372.03472417256006,10.232726311988573,453.572998046875,706.9373303797012,7.482973828141415,11.387027686761945,199.90495996832922,453.572998046875,742.0197331162486,7.2202813967543324,9.420544975587024,11.387027686761945,101.82919398171828,300.7761334067533,453.572998046875 25 | Boiler (Oil),676_2,0,1.0,123.53872440600364,4.685484927901504,103.04100036621094,406.1344405939935,1.1676329406730142,5.799733206170892,69.17924759696527,103.04100036621094,,,,,,, 26 | Boiler (Oil),8002_1,0,1.0,285.59452436827956,12.017257006166714,415.0530090332031,314.00086626608487,11.830626831485894,13.1115891574095,318.7844485131228,415.0530090332031,391.2358626972268,11.081736641133308,12.074128436486408,13.683569108528364,137.94403762174332,360.1363613398082,415.0530090332031 27 | Boiler (Oil),8006_1,0,1.0,448.3274959666696,9.424089623573469,613.4310302734375,470.33411326405763,9.328816806314245,36.49045726724909,599.8073930113501,613.4310302734375,,,,,,, 28 | Boiler (Oil),8006_2,0,2.0,383.15682302930765,9.151656882337157,610.5419921875,563.860999070412,8.343978734333513,9.599969190356445,317.1533073686897,610.5419921875,,,,,,, 29 | -------------------------------------------------------------------------------- /data/piecewise_fits/all_fits/Boiler (Other Solid Fuel)_linpiecewise_fits.csv: -------------------------------------------------------------------------------- 1 | group_type,unit_id,cluster,min_aicc,1_b,1_m1,1_x1,2_b,2_m1,2_m2,2_x1,2_x2,3_b,3_m1,3_m2,3_m3,3_x1,3_x2,2_x3 2 | Boiler (Other Solid Fuel),2367_5,0,1.0,377.0741645857077,6.565269321585995,45.32500076293945,377.7557437295917,6.548655302341554,9.678387350865533,43.80204222749434,45.32500076293945,,,,,,, 3 | Boiler (Other Solid Fuel),55708_BFB-1,0,1.0,182.62656670785623,11.298415103791282,101.75,,,,,,,,,,,, 4 | Boiler (Other Solid Fuel),57241_BFB1,0,1.0,490.6572597940531,9.435641728895183,106.375,492.7643848807191,9.40882664253767,58.076936160538274,105.3895149227537,106.375,,,,,,, 5 | Boiler (Other Solid Fuel),58054_ST01,0,2.0,32.187430394978676,13.31758836906746,70.30000305175781,2688.1914571824423,-42.93516658899537,13.342058144576727,47.22497239577861,70.30000305175781,,,,,,, 6 | Boiler (Other Solid Fuel),589_1,0,2.0,86.27838222395619,12.636436087118124,54.57500076293945,79.17659734323635,12.841015141574912,51.46402699687714,52.41591826255065,54.57500076293945,115.47946639195715,11.317052618692273,12.582995681035323,13.284397021737009,21.787625365951516,48.818293555856975,54.57500076293945 7 | -------------------------------------------------------------------------------- /data/piecewise_fits/all_fits/CC (Coal)_linpiecewise_fits.csv: -------------------------------------------------------------------------------- 1 | group_type,unit_id,cluster,min_aicc,1_b,1_m1,1_x1,2_b,2_m1,2_m2,2_x1,2_x2,3_b,3_m1,3_m2,3_m3,3_x1,3_x2,2_x3 2 | CC (Coal),1004_1,0,2.0,2161.6594889379903,4.065185023461277,445.8500061035156,2366.0040898481084,3.5063832843727054,5.478978143773165,411.57379472988805,445.8500061035156,,,,,,, 3 | CC (Coal),1004_1,2,1.0,924.4948361226304,4.685440762404118,222.9250030517578,954.1113146578436,4.540744677887433,5.40162747627055,160.40347790276485,222.9250030517578,,,,,,, 4 | CC (Coal),1004_1,3,1.0,478.3236178830526,15.484537783884237,37.92499923706055,,,,,,,,,,,, 5 | -------------------------------------------------------------------------------- /data/piecewise_fits/all_fits/CC (Oil)_linpiecewise_fits.csv: -------------------------------------------------------------------------------- 1 | group_type,unit_id,cluster,min_aicc,1_b,1_m1,1_x1,2_b,2_m1,2_m2,2_x1,2_x2,3_b,3_m1,3_m2,3_m3,3_x1,3_x2,2_x3 2 | CC (Oil),6081_CC1,0,1.0,312.4696475488729,10.298613707191453,143.48699951171875,526.5474522685038,8.721110399866676,10.443844326003704,134.38369486210686,143.48699951171875,,,,,,, 3 | CC (Oil),6081_CC1,1,1.0,386.027538107217,9.471162215300211,167.56199645996094,401.6712601413778,9.372895008793353,9.555672065588777,161.92269798369523,167.56199645996094,,,,,,, 4 | CC (Oil),6081_CC1,2,1.0,200.43984179103512,9.620993562868753,83.78099822998047,268.7076388670615,8.653826683547155,9.477328573325483,79.91089944170118,83.78099822998047,,,,,,, 5 | CC (Oil),7153_CC2,0,1.0,296.2891385706118,8.77609044031386,112.6709976196289,341.5344300600447,8.341137534431523,9.20036675861178,91.8300602936743,112.6709976196289,,,,,,, 6 | CC (Oil),7153_CC2,1,1.0,536.248720084898,9.191090936056746,220.52699279785156,547.2433855527455,9.133543153668017,9.659437641044583,160.8865417809811,220.52699279785156,,,,,,, 7 | CC (Oil),7153_CC2,2,3.0,530.9039148054852,10.280284254704519,341.864990234375,800.8305342517017,9.22442275659539,14.168664077574505,293.82042209444865,341.864990234375,784.9391615625111,9.288567858612799,13.290637207093631,28.293469211827357,292.6778268753703,326.9916361726354,341.864990234375 8 | CC (Oil),7153_CC2,3,2.0,472.8900300997647,11.643981070019102,313.93798828125,766.6748407160499,10.513876666469102,16.325857538812937,293.09566167682084,313.93798828125,782.5751317823274,10.451916548173074,11.310905147606293,16.325857538812937,284.2630276038115,294.15792124637335,313.93798828125 9 | -------------------------------------------------------------------------------- /data/piecewise_fits/all_fits/CT (Oil)_linpiecewise_fits.csv: -------------------------------------------------------------------------------- 1 | group_type,unit_id,cluster,min_aicc,1_b,1_m1,1_x1,2_b,2_m1,2_m2,2_x1,2_x2,3_b,3_m1,3_m2,3_m3,3_x1,3_x2,2_x3 2 | CT (Oil),1068_2A,0,2.0,84.43943918207192,8.534342521134981,28.889999389648438,103.01125879755523,7.3943762256588865,9.074413846143074,18.929484805541158,28.889999389648438,,,,,,, 3 | CT (Oil),1068_2B,0,1.0,60.4981929602916,9.355675258332552,26.964000701904297,61.01880028096216,9.330128341290328,9.404782725976126,23.70471670698235,26.964000701904297,,,,,,, 4 | CT (Oil),10823_S42,0,1.0,59.64406698904525,7.695757545064931,15.407999992370605,60.33748496481438,7.632366750509345,8.279967970989455,14.056075113478535,15.407999992370605,,,,,,, 5 | CT (Oil),10823_S43,0,1.0,54.072581215477754,7.759214258979546,16.371000289916992,55.43841472977519,7.643470829440037,9.335053619560455,9.04915297852564,16.371000289916992,,,,,,, 6 | CT (Oil),1355_8,0,1.0,319.6438234874727,9.400262742801187,115.55999755859375,393.25336978443323,8.658686671042688,9.604941917183819,93.33964259405906,115.55999755859375,,,,,,, 7 | CT (Oil),1355_9,0,1.0,317.11248734291115,9.707123409093631,115.55999755859375,396.08633611715493,8.904788641924895,9.980023074790394,92.03025361167424,115.55999755859375,,,,,,, 8 | CT (Oil),3403_GCT5,0,2.0,252.1463101725646,9.173769941107835,92.447998046875,262.1704205955137,9.008752715170404,10.554669604107303,82.48046770290205,92.447998046875,259.9169621443036,9.05290560369005,9.475384597099938,12.277506479593303,74.08315250460784,89.35266068312028,92.447998046875 9 | CT (Oil),3403_GCT6,0,2.0,226.8942068193514,9.549141235543882,90.52200317382812,254.0361029767632,9.044507259596132,9.783373420351726,60.508130831624044,90.52200317382812,,,,,,, 10 | CT (Oil),3403_GCT7,0,2.0,243.28612902531447,9.268868945274498,92.447998046875,302.24051525491274,8.24121482875211,9.692009309843986,64.34202780133178,92.447998046875,,,,,,, 11 | CT (Oil),3403_GCT8,0,2.0,204.74259189438797,9.78315611543019,94.3740005493164,424.2142962878081,5.678653828407476,9.895340898780455,54.12648457643215,94.3740005493164,210.45464284339187,9.720528322779185,9.986746980662641,12.231477732878416,75.39626140153025,43.5805790591427,94.3740005493164 12 | CT (Oil),3406_JCT17,0,3.0,237.55591270121826,8.714765613414345,95.33699798583984,281.93474673921725,7.914905119493791,9.098793103634716,62.92336921390689,95.33699798583984,294.20225122157143,7.674704658805948,8.918146756232417,9.942677186181148,59.1666365966222,86.4835485390823,95.33699798583984 13 | CT (Oil),3406_JCT18,0,2.0,237.47074170889456,8.813169200093885,92.447998046875,303.5692532199529,7.528245715454343,8.997843673170474,54.51668581250106,92.447998046875,303.5692532199529,7.528245715454343,8.867611753405257,9.916623282467823,53.438422784990266,82.52058489229607,92.447998046875 14 | CT (Oil),3406_JCT19,0,3.0,245.07458369767474,8.79992144047874,94.3740005493164,269.09214100615054,8.363989573800753,9.148613769291332,65.42358461637258,94.3740005493164,268.3907357168849,8.377683030694822,9.055627342757605,10.360333803783893,64.85392672922723,89.21861629045198,94.3740005493164 15 | CT (Oil),3406_JCT20,0,2.0,559.1534742986885,3.5215733256739337,160.8209991455078,627.6400382843323,2.6260829363790736,7.8435623867691975,132.19741200547125,160.8209991455078,625.9370969572959,2.6490501745267587,7.278284219712964,10.558953356677732,130.39551087883083,142.06251703539297,160.8209991455078 16 | CT (Oil),55381_CT-001,0,1.0,80.87122481122505,8.065381189635291,48.150001525878906,470.3273833948173,-0.20271476096914237,8.303076980011133,46.96478722981778,48.150001525878906,,,,,,, 17 | CT (Oil),55381_CT-002,0,1.0,173.74696721777988,6.149845167009113,47.1870002746582,,,,,,,,,,,, 18 | CT (Oil),55381_CT-003,0,1.0,110.7057055244416,7.5547327045883295,48.150001525878906,211.66697632097666,5.406708523967592,7.780584746955156,46.81458432407629,48.150001525878906,,,,,,, 19 | CT (Oil),55381_CT-004,0,1.0,97.48358699178357,7.693299415345798,48.150001525878906,524.2440877026806,-1.28719204376683,9.259082620966856,47.10302010351836,48.150001525878906,,,,,,, 20 | CT (Oil),55381_CT-006,0,1.0,104.9473117948881,7.558725331575229,48.150001525878906,262.51122225712686,4.1687941003651,7.810536098361508,46.16410414067615,48.150001525878906,,,,,,, 21 | CT (Oil),55699_2,0,1.0,151.28188028869323,8.08608405677913,52.00199890136719,357.5188139161918,4.033195949538371,11.429345028034655,49.985251650971556,52.00199890136719,,,,,,, 22 | CT (Oil),55969_U-01,0,1.0,119.32421622139918,7.8849309028952135,53.928001403808594,119.49721331708871,7.879469202990621,19.0377045825161,52.57664431822186,53.928001403808594,,,,,,, 23 | CT (Oil),6046_**7,0,2.0,166.11349128310894,11.025798699768163,76.0770034790039,174.35355618599675,10.845694233165075,11.093628896109449,51.066778002589004,76.0770034790039,,,,,,, 24 | CT (Oil),6046_**8,0,2.0,171.45812688456522,11.230075322524602,73.18800354003906,190.08910306691536,10.811911365878581,11.300930008362569,47.50637457159844,73.18800354003906,,,,,,, 25 | CT (Oil),6046_**9,0,2.0,178.28614131269293,11.009300171970597,74.1510009765625,203.6006704069558,10.498632368586273,11.172055489204435,53.71878526797406,74.1510009765625,209.52817469191046,10.36623672307175,10.829754328219193,11.170948637321821,49.804562589871665,55.517917876738274,74.1510009765625 26 | CT (Oil),630_P2,0,1.0,172.91277052951665,10.66711265536766,27.927000045776367,208.37203722831973,9.309795433509215,12.994922065550746,25.06095912466788,27.927000045776367,,,,,,, 27 | CT (Oil),630_P3,0,1.0,205.03606565320683,10.939054118569855,33.70500183105469,237.96689160455762,9.893138579377911,13.654099743096664,30.238205170023573,33.70500183105469,,,,,,, 28 | CT (Oil),630_P4,0,1.0,221.502783828721,10.702575132733855,34.667999267578125,,,,,,,,,,,, 29 | CT (Oil),634_P2,0,2.0,225.14104109363552,10.798599996671117,50.07600021362305,252.46741012508346,9.698005661889956,10.979518731920082,26.827923602432197,50.07600021362305,247.40862621013406,9.939817294456104,10.751439642818294,11.061613025069043,24.723692776860858,38.33085071175315,50.07600021362305 30 | CT (Oil),634_P4,0,1.0,240.45084491508868,10.304930405848074,52.96500015258789,278.8604964743108,9.33775710035066,10.69231864081994,36.75127043271125,52.96500015258789,,,,,,, 31 | CT (Oil),638_1A,0,1.0,118.91214865024787,8.940877149822889,26.964000701904297,131.03798953805523,8.406724397838987,9.358168326344629,20.944308152907748,26.964000701904297,,,,,,, 32 | CT (Oil),638_1B,0,1.0,94.64395533919013,9.897299009866575,25.038000106811523,136.30824199727965,8.05871606327843,10.779246256825854,21.455741795069244,25.038000106811523,,,,,,, 33 | CT (Oil),638_3A,0,1.0,106.77224272706802,9.763937476342385,26.964000701904297,139.03600277413858,8.454734723531429,10.040818671184052,24.117204491508826,26.964000701904297,,,,,,, 34 | CT (Oil),638_3B,0,1.0,97.5213444460864,9.185124962135175,26.964000701904297,116.14584748060737,8.405047908785575,9.901149001554758,22.237586941966327,26.964000701904297,111.4559243748862,8.591248841104735,8.835618263747497,9.528556058527418,21.136149104972777,23.75196077680657,26.964000701904297 35 | CT (Oil),7288_1,0,2.0,253.5037806174548,9.936785435432549,80.89199829101562,341.78483153621005,8.63884734243177,10.819726637486337,70.62472449537812,80.89199829101562,332.3235878944936,8.778544108783146,8.920985745642799,10.97187553323188,67.31844818345962,71.5311807950491,80.89199829101562 36 | CT (Oil),7845_LCT1,0,2.0,206.09514796176362,9.580689253003277,91.48500061035156,224.27522974655957,9.324092587501019,10.779601279652137,85.23228134589336,91.48500061035156,,,,,,, 37 | CT (Oil),7845_LCT10,0,2.0,243.93665459625444,8.980615557325242,90.52200317382812,263.71028170644325,8.69677367873867,16.469176670729528,87.39228044675654,90.52200317382812,270.47928519153254,8.59863059206744,9.115632833135932,15.621732207105747,79.3323333983687,87.49478531650561,90.52200317382812 38 | CT (Oil),7845_LCT11,0,1.0,194.50299478370792,9.558387512067661,90.52200317382812,195.12667864856473,9.549329296706706,14.226377126197837,88.22376854655568,90.52200317382812,,,,,,, 39 | CT (Oil),7845_LCT12,0,1.0,208.91425441762462,9.34705619768089,90.52200317382812,210.34900365981034,9.326468753626367,11.92019927441523,88.44473100316925,90.52200317382812,,,,,,, 40 | CT (Oil),7845_LCT2,0,2.0,209.50704807036766,9.514864524089901,91.48500061035156,220.85425184082644,9.351639608496017,10.334987920951507,84.12738503629588,91.48500061035156,,,,,,, 41 | CT (Oil),7845_LCT3,0,2.0,209.9227930636719,9.540763844194622,91.48500061035156,224.22250696568642,9.33382263690089,10.155616867587181,80.7088913729583,91.48500061035156,,,,,,, 42 | CT (Oil),7845_LCT4,0,2.0,207.69967246320104,9.496330327664934,92.447998046875,221.78633144573945,9.293339697816084,10.089632576621643,81.98063873722784,92.447998046875,,,,,,, 43 | CT (Oil),7845_LCT5,0,2.0,202.7753438551147,9.642517593295054,91.48500061035156,232.76077749581466,9.204213636458576,10.724206107040619,78.81586278838122,91.48500061035156,,,,,,, 44 | CT (Oil),7845_LCT6,0,2.0,220.28942850089103,9.564381785503867,92.447998046875,229.67442031174372,9.430217516985572,10.512161537955862,84.06777953268517,92.447998046875,,,,,,, 45 | CT (Oil),7845_LCT7,0,2.0,183.7950929249424,9.635091952697175,91.48500061035156,201.5725184967692,9.377745558990615,11.428337777495507,84.13451354357655,91.48500061035156,,,,,,, 46 | CT (Oil),7845_LCT8,0,2.0,185.5849145657354,9.701184026318211,90.52200317382812,202.50154288168403,9.45373250817381,11.142419481963396,82.42089568010383,90.52200317382812,448.894992880941,6.454863823345252,9.53170457060244,11.142419481963396,81.79362365853478,83.13736206165314,90.52200317382812 47 | CT (Oil),7845_LCT9,0,2.0,231.02315568424473,9.024422666631384,89.55899810791016,254.91319178893664,8.683101169970097,10.447512660010576,81.28876108097826,89.55899810791016,,,,,,, 48 | CT (Oil),7960_KCT1,0,1.0,237.45904716373934,9.218413030645703,88.59600067138672,300.96406596188416,8.271611118460285,9.285872076725951,67.66864203051102,88.59600067138672,,,,,,, 49 | CT (Oil),7960_KCT2,0,1.0,174.4758478826948,9.921871605015168,89.55899810791016,456.652969281078,6.548186174221383,11.443741770977455,79.62479802702012,89.55899810791016,,,,,,, 50 | CT (Oil),7960_KCT3,0,1.0,159.78369052490973,10.105197578413467,85.70700073242188,256.498098395307,8.664171841254449,10.164615291767785,67.47028809782711,85.70700073242188,,,,,,, 51 | CT (Oil),7960_KCT4,0,1.0,201.6742783765958,9.52659566621138,88.59600067138672,353.9083574087008,7.753874781371822,9.57269017512459,85.48882399463761,88.59600067138672,,,,,,, 52 | CT (Oil),8049_**10,0,2.0,172.60433789597428,11.49563284285065,72.2249984741211,180.09983186700248,11.336657061720787,11.617047458114568,55.08662823703537,72.2249984741211,188.9044938105328,11.121493920280468,11.401544104428545,11.619306315006403,43.129879487124605,56.60852817491637,72.2249984741211 53 | CT (Oil),8049_**7,0,1.0,166.64795222034377,11.58249151839387,73.18800354003906,171.489059628695,11.482289073355663,11.615547532808925,52.50482252322739,73.18800354003906,,,,,,, 54 | CT (Oil),8049_**8,0,1.0,194.78442663314837,11.222971159612644,74.1510009765625,197.8388442945615,11.177019646240996,14.503335393196185,40.9885108580189,74.1510009765625,,,,,,, 55 | CT (Oil),8049_**9,0,1.0,173.0770879431272,11.277648777686876,72.2249984741211,177.45342021046733,11.185983649690195,11.323127956103107,53.78539476722069,72.2249984741211,,,,,,, 56 | -------------------------------------------------------------------------------- /data/piecewise_fits/all_fits/Stoker (Other Solid Fuel)_linpiecewise_fits.csv: -------------------------------------------------------------------------------- 1 | group_type,unit_id,cluster,min_aicc,1_b,1_m1,1_x1,2_b,2_m1,2_m2,2_x1,2_x2,3_b,3_m1,3_m2,3_m3,3_x1,3_x2,2_x3 2 | Stoker (Other Solid Fuel),10333_1,0,1.0,318.245949583784,9.292631303676936,76.7750015258789,640.748938511751,4.592834111917296,12.218276233594292,59.48820159376113,76.7750015258789,,,,,,, 3 | Stoker (Other Solid Fuel),10525_ES5A,0,1.0,719.4559638949464,-0.4293813491897569,48.099998474121094,895.4374861842763,-4.1057286471782914,9.63152534576774,45.63098048876031,48.099998474121094,,,,,,, 4 | Stoker (Other Solid Fuel),3982_2,0,1.0,53.364117713900534,13.253786222801397,13.875,61.394108877245586,12.55209301838167,15.305060900673396,10.552290081177313,13.875,,,,,,, 5 | Stoker (Other Solid Fuel),54238_BIOMS1,0,1.0,159.91273920183073,12.502972771896223,46.25,1133.225253387893,-8.937697229212484,13.261542900172994,45.30223491442937,46.25,,,,,,, 6 | Stoker (Other Solid Fuel),54415_EUBLR,0,1.0,97.98016264771529,12.848327425159297,36.07500076293945,396.15247037237475,3.7621731201688684,13.178939051119954,32.14030166736607,36.07500076293945,,,,,,, 7 | Stoker (Other Solid Fuel),54751_01,0,1.0,21.522713084617415,13.01602181113114,32.375,551.148741618441,-4.325874029537624,20.867056520152996,24.036725272587795,32.375,,,,,,, 8 | Stoker (Other Solid Fuel),55867_BLR-1,0,3.0,-27.40841297286313,14.755000416450404,61.974998474121094,51.64665051888929,13.191759472710824,34.07746751188315,57.346001702334874,61.974998474121094,258.1493623380842,8.397508299259188,14.362277831679833,45.09384528651357,45.16812102957523,58.17656221119059,61.974998474121094 9 | -------------------------------------------------------------------------------- /data/poly_fits/Boiler (Oil)_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,a4,a3,a2,a1,a0,load_min,load_2,load_3,load_4,load_max,total_load,min_gen_perc,heat_rate(load_min),heat_rate(load_2),heat_rate(load_3),heat_rate(load_4),heat_rate(load_max),latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type,group_type 2 | 1353_AUX2,,,,,,,,,,,0.0,,,,,,,38.1707,-82.61759948730469,KY,4.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 3 | 1507_1,1.0047961590737182e-05,-0.0013737808094856033,0.06863416297957413,-1.5343672551578966,25.507949837756986,8.666999816894531,19.74149990081787,30.81599998474121,41.890500068664544,52.96500015258789,4122.603515625,0.16363635659217834,16.527486233737473,12.92238373348143,12.260939404244569,11.627464709485094,11.733589948231435,43.75,-70.15689849853516,ME,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 4 | 1507_2,,,,,,,,,,,0.0,,,,,,,43.75,-70.15689849853516,ME,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 5 | 1507_3,1.907318553283255e-07,-6.730401699185237e-05,0.008831981908078388,-0.5196759521091618,21.644658797010997,27.92700004577637,51.03900003433228,74.15100002288818,97.26300001144408,120.375,17185.697265625,0.23199999332427976,12.669968826321625,10.473898083582684,9.997399575361332,9.792438692972883,9.717105368704622,43.75,-70.15689849853516,ME,1.0,NPCC,Tangentially-fired,Residual Oil,Boiler (Oil) 6 | 1507_4,1.1991755193994295e-09,-1.7239490234634164e-06,0.0008769493020373811,-0.18517487524242915,23.51860863111813,16.371000289916992,163.70999765396118,311.04899501800537,458.3879923820496,605.7269897460938,174041.0625,0.027027027681469917,20.71466373184948,10.004100085860491,10.110293157419084,9.800377731723726,11.404771998273576,43.75,-70.15689849853516,ME,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 7 | 1554_1,1.6107397116044954e-08,-1.0128374345174374e-05,0.0021175826416398318,-0.19561226086131334,18.272657141621394,33.705001831054695,57.0577507019043,80.4104995727539,103.76324844360352,127.11599731445312,68228.546875,0.2651515305042267,13.718151027336747,12.294744660463504,11.642785362859195,11.326731485652022,11.026012468705163,39.1781,-76.52680206298828,MD,3.0,MAAC,Dry bottom wall-fired boiler,Other Oil,Boiler (Oil) 8 | 1554_4,-3.7323224928232663e-10,-1.9244005932056176e-07,0.00031744083021327773,-0.10407690874698644,22.810295784146227,26.000999450683594,119.17125129699707,212.34150314331055,305.51175498962397,398.6820068359375,20777.6875,0.06521739065647125,20.315245335425058,14.514575774442058,12.422257054961367,11.90357590936859,10.148825739821948,39.1781,-76.52680206298828,MD,3.0,MAAC,Dry bottom wall-fired boiler,Other Oil,Boiler (Oil) 9 | 1564_8,,,,,,,,,,,0.0,,,,,,,38.4878,-75.82080078125,MD,3.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 10 | 1599_1,,,,,,,,,,,0.0,,,,,,,41.7694,-70.50969696044922,MA,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 11 | 1599_2,3.1391862277607887e-10,-5.135681191409929e-07,0.00031032076284748537,-0.08437853179173258,19.095228374508306,67.41000366210939,193.56299591064453,319.7159881591797,445.86898040771484,572.02197265625,71763.7265625,0.11784513294696808,14.666569211427666,11.105529067374377,10.334700679271156,10.049528583232396,9.85363383045758,41.7694,-70.50969696044922,MA,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 12 | 1619_4,1.1528356688235662e-09,-9.951532998995351e-07,0.00036728614343919816,-0.08109584653717747,20.591329872729627,42.37200164794922,130.72725105285645,219.08250045776367,307.43774986267084,395.7929992675781,20680.42578125,0.10705596208572388,17.7425677747955,14.38011220968212,12.64478209445939,11.756017795758446,12.619455760162491,41.7125,-71.19139862060547,MA,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 13 | 1642_3,1.081701057634863e-06,-0.00030627865108047013,0.03144590270936876,-1.4053372481880102,36.837397404571355,25.038000106811523,43.09425020217896,61.15050029754639,79.2067503929138,97.26300048828124,14154.173828125,0.2574257552623749,16.981700079837243,13.892959496319072,13.578768778481972,13.186762455312172,12.624064530941494,42.0956,-72.5958023071289,MA,1.0,NPCC,Tangentially-fired,Residual Oil,Boiler (Oil) 14 | 1682_8,,,,,,,,,,,0.0,,,,,,,41.8653,-71.1061019897461,MA,1.0,NPCC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 15 | 2378_3,,,,,,,,,,,0.0,,,,,,,39.29,-74.63390350341797,NJ,2.0,MAAC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 16 | 2440_001,,,,,,,,,,,0.0,,,,,,,34.8456,-80.89420318603516,SC,4.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 17 | 2493_60,,,,,,,,,,,0.0,,,,,,,40.7281,-73.97419738769531,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 18 | 2493_70,,,,,,,,,,,0.0,,,,,,,40.7281,-73.97419738769531,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 19 | 2503_BLR116,,,,,,,,,,,0.0,,,,,,,40.7711,-73.99109649658203,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 20 | 2503_BLR117,,,,,,,,,,,0.0,,,,,,,40.7711,-73.99109649658203,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 21 | 2503_BLR118,,,,,,,,,,,0.0,,,,,,,40.7711,-73.99109649658203,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 22 | 2516_1,1.979512233477155e-10,-2.735848088346873e-07,0.00014486320947501978,-0.035876238208199386,13.673005449331589,101.11499786376952,175.2659969329834,249.41699600219727,323.5679950714112,397.718994140625,681066.375,0.2542372941970825,11.264350563728067,10.548901648368137,10.25776718181059,10.132984913963547,10.060220047773178,40.9231,-73.3416976928711,NY,2.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 23 | 2516_2,-3.171843270467512e-10,1.614786371770667e-07,2.0628558520172334e-05,-0.02103146427267679,12.885885851165273,89.55899810791016,166.839750289917,244.12050247192383,321.4012546539306,398.6820068359375,759474.75,0.22463767230510712,11.263377182016391,10.455366469476036,10.203778278167396,10.233841045595335,9.999259146648898,40.9231,-73.3416976928711,NY,2.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 24 | 2516_3,-2.2457284134045375e-10,1.2517300361110676e-07,1.0576115812815594e-05,-0.016192035116671943,12.41575451521763,95.33699798583984,163.46924781799316,231.60149765014648,299.7337474822998,367.8659973144531,741213.375,0.2591623067855835,11.058096163957813,10.437897596505737,10.141831401825034,10.070706441224305,10.009192224555726,40.9231,-73.3416976928711,NY,2.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 25 | 2516_4,-1.263751754458064e-10,8.539215755418536e-08,7.773196051323659e-07,-0.011922735299804246,12.483159998391221,95.33699798583984,162.024751663208,228.71250534057617,295.4002590179444,362.0880126953125,1630753.75,0.2632978558540344,11.417102209217546,10.847907766333575,10.472761393031824,10.267877295079387,10.149482647106328,40.9231,-73.3416976928711,NY,2.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 26 | 2517_3,4.3733725194852725e-08,-2.1299323348112666e-05,0.003790051330681896,-0.2971169062963108,19.664362588946094,36.59400177001953,77.04000282287599,117.48600387573242,157.93200492858887,198.3780059814453,303916.0625,0.18446601927280426,12.901680799108467,11.070607777682559,10.863292199344077,10.578785067918409,11.324997226828424,40.9503,-73.07859802246094,NY,2.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 27 | 2517_4,4.7979868348813886e-08,-2.292891178981535e-05,0.004012216323607596,-0.31671047553143505,20.657260657981116,30.81599998474121,73.18800020217896,115.5600004196167,157.93200063705444,200.3040008544922,257693.984375,0.1538461595773697,14.079898620262586,11.357030291658294,10.810225319183676,10.240925093182156,11.162384590080379,40.9503,-73.07859802246094,NY,2.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 28 | 2594_5,,,,,,,,,,,0.0,,,,,,,43.46,-76.52999877929688,NY,2.0,NPCC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 29 | 2594_6,,,,,,,,,,,0.0,,,,,,,43.46,-76.52999877929688,NY,2.0,NPCC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 30 | 2828_B008,,,,,,,,,,,0.0,,,,,,,40.2522,-80.64859771728516,OH,5.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 31 | 2828_B010,,,,,,,,,,,0.0,,,,,,,40.2522,-80.64859771728516,OH,5.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 32 | 2872_B001,,,,,,,,,,,0.0,,,,,,,39.5908,-81.67970275878906,OH,5.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 33 | 3148_3,-1.911734563036943e-11,2.3287800625677657e-08,5.716789539408174e-07,-0.0104827959981825,13.933297473468425,245.56500244140625,355.82850646972656,466.0920104980469,576.3555145263673,686.6190185546875,2608563.0,0.3576437532901764,11.668893878533463,11.018312182879686,10.627316960034165,10.430449356904269,10.29442923877705,40.796,-75.10700225830078,PA,3.0,MAAC,Tangentially-fired,Residual Oil,Boiler (Oil) 34 | 3148_4,-4.897603236420868e-11,4.697715144651199e-08,7.166126759312257e-06,-0.01669359275759431,14.226428174735842,167.56199645996094,304.0672492980957,440.57250213623047,577.0777549743652,713.5830078125,2943460.5,0.2348178029060364,11.812821879236633,10.715024427017076,10.434766150786317,10.5758283469653,10.333868160362973,40.796,-75.10700225830078,PA,3.0,MAAC,Tangentially-fired,Residual Oil,Boiler (Oil) 35 | 3161_3,2.2267731745012115e-08,-1.5403802323522332e-05,0.003931917003635266,-0.4461465444949531,30.778115223079986,31.77899932861328,89.55899620056152,147.33899307250974,205.118989944458,262.89898681640625,118113.875,0.12087912112474443,20.099231369759195,12.726306551866895,11.624761314668458,11.177219474931764,11.722888957811033,39.858000000000004,-75.322998046875,PA,3.0,,Tangentially-fired,Diesel Oil,Boiler (Oil) 36 | 3161_4,3.2722819827942743e-08,-1.84860770033807e-05,0.004126818015558885,-0.4443615367825034,30.731517351849824,28.88999938964844,70.05825042724611,111.22650146484376,152.3947525024414,193.56300354003903,81518.90625,0.1492537260055542,20.915336633891254,14.287128749085356,11.931993168896765,11.077801893323072,11.20828426184972,39.858000000000004,-75.322998046875,PA,3.0,,Tangentially-fired,Diesel Oil,Boiler (Oil) 37 | 3319_1,,,,,,,,,,,0.0,,,,,,,33.2422,-79.98750305175781,SC,4.0,SERC,Dry bottom wall-fired boiler,Other Oil,Boiler (Oil) 38 | 3319_2,,,,,,,,,,,0.0,,,,,,,33.2422,-79.98750305175781,SC,4.0,SERC,Dry bottom wall-fired boiler,Other Oil,Boiler (Oil) 39 | 3399_A1,,,,,,,,,,,0.0,,,,,,,36.3903,-87.65390014648438,TN,4.0,SERC,Dry bottom wall-fired boiler,Diesel Oil,Boiler (Oil) 40 | 3399_B1,,,,,,,,,,,0.0,,,,,,,36.3903,-87.65390014648438,TN,4.0,SERC,Dry bottom wall-fired boiler,Diesel Oil,Boiler (Oil) 41 | 3804_5,-2.021667287984845e-10,4.0358251578005893e-07,-0.00028304737212392014,0.07822011333740661,3.840333556003156,193.56300354003903,341.3834953308105,489.20398712158203,637.0244789123535,784.844970703125,103538.875,0.24662578105926514,11.01907218535899,10.867251705378878,10.037966232036402,9.844180271168543,9.282213281467053,38.5367,-77.28060150146484,VA,3.0,SERC,Tangentially-fired,Residual Oil,Boiler (Oil) 42 | 3809_3,,,,,,,,,,,0.0,,,,,,,37.2144,-76.46109771728516,VA,3.0,SERC,Tangentially-fired,Residual Oil,Boiler (Oil) 43 | 3935_AUX1,,,,,,,,,,,0.0,,,,,,,38.4731,-81.82330322265625,WV,3.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 44 | 3935_AUX3,,,,,,,,,,,0.0,,,,,,,38.4731,-81.82330322265625,WV,3.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 45 | 3948_AUX1,,,,,,,,,,,0.0,,,,,,,39.8297,-80.81529998779297,WV,3.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 46 | 50296_017,,,,,,,,,,,0.0,,,,,,,35.0442,-88.26519775390625,TN,4.0,,Tangentially-fired,Other Oil,Boiler (Oil) 47 | 50607_23,,,,,,,,,,,0.0,,,,,,,39.9422,-75.1875,PA,3.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 48 | 50607_24,,,,,,,,,,,0.0,,,,,,,39.9422,-75.1875,PA,3.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 49 | 50607_26,,,,,,,,,,,0.0,,,,,,,39.9422,-75.1875,PA,3.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 50 | 54099_000044,,,,,,,,,,,0.0,,,,,,,43.8914,-73.39610290527344,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 51 | 546_5,4.162871266144616e-06,-0.0011990399788220625,0.12235881743713595,-5.251982686861892,91.48451808828469,36.59400177001953,48.63150215148926,60.66900253295898,72.7065029144287,84.74400329589844,19860.91015625,0.4318181872367859,11.85425071796314,10.831328025877935,11.866119970857016,11.933759457294386,10.107108853278378,41.4281,-72.10189819335938,CT,1.0,NPCC,Tangentially-fired,Residual Oil,Boiler (Oil) 52 | 546_6,-1.8446534344788098e-09,1.6700975623784646e-06,-0.0005037735193716602,0.056739076364252475,12.018361306984522,82.81800079345703,162.506254196167,242.19450759887692,321.88276100158686,401.5710144042969,36560.29296875,0.20623500645160675,14.12397880475785,13.815808301760347,13.589322566296776,13.982031653915206,13.746178284399488,41.4281,-72.10189819335938,CT,1.0,NPCC,Tangentially-fired,Residual Oil,Boiler (Oil) 53 | 562_4,,,,,,,,,,,0.0,,,,,,,41.5544,-72.57929992675781,CT,1.0,NPCC,Tangentially-fired,Residual Oil,Boiler (Oil) 54 | 593_5,8.751524651398146e-09,-6.906439925924597e-06,0.002006532720476812,-0.2618976750982225,23.94014059024739,49.112998962402344,114.35624885559082,179.5994987487793,244.84274864196775,310.08599853515625,484103.0,0.15838509798049927,15.150238862522556,11.398878233855902,10.721540525057053,10.1833090494901,10.654991269150166,39.7372,-75.50330352783203,DE,3.0,MAAC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 55 | 6156_NHB1,5.73433576916446e-10,-9.628942306201746e-07,0.000590478571298173,-0.15312357814841646,24.839013272918,52.00199890136719,152.3947486877441,252.7874984741211,353.18024826049805,453.572998046875,71873.5,0.1146496832370758,18.341845066336,12.118526863464652,10.651208936155593,10.915308140576755,11.284230352195575,41.2836,-72.9041976928711,CT,1.0,,Tangentially-fired,Residual Oil,Boiler (Oil) 56 | 6166_AB1,,,,,,,,,,,0.0,,,,,,,37.9256,-87.03720092773438,IN,5.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 57 | 6166_AB2,,,,,,,,,,,0.0,,,,,,,37.9256,-87.03720092773438,IN,5.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 58 | 6264_AUX1,,,,,,,,,,,0.0,,,,,,,38.9794,-81.93440246582031,WV,3.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 59 | 6264_AUX2,,,,,,,,,,,0.0,,,,,,,38.9794,-81.93440246582031,WV,3.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 60 | 676_1,,,,,,,,,,,0.0,,,,,,,28.0809,-81.92559814453125,FL,4.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 61 | 676_2,1.2706936566354916e-06,-0.0003471331944458873,0.033719180461206186,-1.4086374170538745,28.97518936656678,18.297000885009766,39.483000755310066,60.66900062561035,81.85500049591064,103.04100036621094,50428.45703125,0.17757010459899902,12.505925749892054,7.644898937936817,7.3237834953097405,6.258766378077593,5.309988674077153,28.0809,-81.92559814453125,FL,4.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 62 | 715_1,,,,,,,,,,,0.0,,,,,,,31.2125,-81.5457992553711,GA,4.0,SERC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 63 | 715_2,,,,,,,,,,,0.0,,,,,,,31.2125,-81.5457992553711,GA,4.0,SERC,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 64 | 733_4,,,,,,,,,,,0.0,,,,,,,32.1486,-81.14579772949219,GA,4.0,SERC,Tangentially-fired,Residual Oil,Boiler (Oil) 65 | 8002_1,2.086567764108509e-09,-2.2852286878397003e-06,0.000922173119518144,-0.16580199360724732,24.501544141509356,51.03900146484375,142.0425033569336,233.04600524902344,324.04950714111334,415.0530090332031,43870.4296875,0.1229698359966278,18.15174359327,13.856687947667586,13.176516730937088,12.855728060219992,13.073419960578756,43.0974,-70.78340148925781,NH,1.0,NPCC,Tangentially-fired,Residual Oil,Boiler (Oil) 66 | 8006_1,-4.926819752552598e-10,7.429018196864844e-07,-0.00037773512228103575,0.06925024058388612,8.053212885065616,119.41200256347656,242.9167594909668,366.421516418457,489.92627334594727,613.4310302734375,108846.9296875,0.194662481546402,12.101093200433487,11.519009309685538,10.378802546381742,10.29097706093984,10.114897223076113,41.5711,-73.9738998413086,NY,2.0,NPCC,Tangentially-fired,Residual Oil,Boiler (Oil) 67 | 8006_2,9.515173431706954e-11,-2.2743233728120874e-07,0.00018984926931664165,-0.06736277471868465,18.70570738536087,104.96700286865234,231.36075019836426,357.7544975280762,484.14824485778814,610.5419921875,176957.03125,0.17192429304122925,13.47512967686135,10.738874869746521,10.04977489124092,10.010692998976054,9.807306899825631,41.5711,-73.9738998413086,NY,2.0,NPCC,Tangentially-fired,Residual Oil,Boiler (Oil) 68 | 8102_B001,,,,,,,,,,,0.0,,,,,,,38.9347,-82.11579895019531,OH,5.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 69 | 8102_B002,,,,,,,,,,,0.0,,,,,,,38.9347,-82.11579895019531,OH,5.0,ECAR,Other boiler,Diesel Oil,Boiler (Oil) 70 | 880006_1,,,,,,,,,,,0.0,,,,,,,39.949,-75.15570068359375,PA,3.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 71 | 880006_2,,,,,,,,,,,0.0,,,,,,,39.949,-75.15570068359375,PA,3.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 72 | 880006_3,,,,,,,,,,,0.0,,,,,,,39.949,-75.15570068359375,PA,3.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 73 | 880006_4,,,,,,,,,,,0.0,,,,,,,39.949,-75.15570068359375,PA,3.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 74 | 880100_BLR001,,,,,,,,,,,0.0,,,,,,,40.76,-73.75,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 75 | 880100_BLR002,,,,,,,,,,,0.0,,,,,,,40.76,-73.75,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 76 | 880100_BLR003,,,,,,,,,,,0.0,,,,,,,40.76,-73.75,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 77 | 880100_BLR004,,,,,,,,,,,0.0,,,,,,,40.76,-73.75,NY,2.0,,Dry bottom wall-fired boiler,Residual Oil,Boiler (Oil) 78 | -------------------------------------------------------------------------------- /data/poly_fits/Boiler (Other Solid Fuel)_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,a4,a3,a2,a1,a0,load_min,load_2,load_3,load_4,load_max,total_load,min_gen_perc,heat_rate(load_min),heat_rate(load_2),heat_rate(load_3),heat_rate(load_4),heat_rate(load_max),latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type,group_type 2 | 10464_E0001,,,,,,,,,,,0.0,,,,,,,44.0361,-75.77120208740234,NY,2.0,NPCC,Circulating fluidized bed boiler,Wood,Boiler (Other Solid Fuel) 3 | 10464_E0002,,,,,,,,,,,0.0,,,,,,,44.0361,-75.77120208740234,NY,2.0,NPCC,Circulating fluidized bed boiler,Wood,Boiler (Other Solid Fuel) 4 | 10464_E0003,,,,,,,,,,,0.0,,,,,,,44.0361,-75.77120208740234,NY,2.0,NPCC,Circulating fluidized bed boiler,Wood,Boiler (Other Solid Fuel) 5 | 2367_5,-9.888908177498086e-07,-2.728739771846437e-05,0.0005585318976593939,0.1459373849514545,13.564525379940221,35.150001525878906,37.693751335144036,40.23750114440918,42.78125095367432,45.32500076293945,304310.1875,0.7755102515220642,16.689691634665426,16.401321924368883,15.971066508973047,15.381008781818284,14.612238431961144,43.0978,-70.7842025756836,NH,1.0,,Circulating fluidized bed boiler,Wood,Boiler (Other Solid Fuel) 6 | 4146_B1,,,,,,,,,,,0.0,,,,,,,42.7083,-90.98529815673828,WI,5.0,,Dry bottom wall-fired boiler,Wood,Boiler (Other Solid Fuel) 7 | 4146_B2,,,,,,,,,,,0.0,,,,,,,42.7083,-90.98529815673828,WI,5.0,,Dry bottom wall-fired boiler,Wood,Boiler (Other Solid Fuel) 8 | 50202_1,,,,,,,,,,,0.0,,,,,,,43.0804,-79.0082015991211,NY,2.0,NPCC,Circulating fluidized bed boiler,Other Solid Fuel,Boiler (Other Solid Fuel) 9 | 55708_BFB-1,3.1516014848324085e-07,-2.1781207707846044e-05,-0.0021751557094261288,0.005467848441205941,24.431195856216522,70.30000305175781,78.16250228881836,86.0250015258789,93.88750076293944,101.75,192736.71875,0.6909091472625732,14.195899514527635,12.931867635141758,12.19817987387351,12.233126164849233,13.303902242855434,31.8326,-94.90059661865234,TX,6.0,,Bubbling fluidized bed boiler,Wood,Boiler (Other Solid Fuel) 10 | 57241_BFB1,4.232644214226513e-07,-7.089643219182071e-05,-0.0011786586502541276,0.6093490492084089,-6.425089543114213,57.349998474121094,69.60624885559082,81.86249923706055,94.11874961853027,106.375,363554.625,0.5391304492950441,15.850307134653391,16.305196214351938,15.673963126971776,14.589861257345753,13.91536451594448,29.7675,-82.39640045166016,FL,4.0,,Bubbling fluidized bed boiler,Wood,Boiler (Other Solid Fuel) 11 | 58054_ST01,-6.079325425557167e-09,-4.222893512621283e-06,-8.969101866879315e-05,0.053813472741297545,12.030929250635538,46.25,52.26250076293945,58.2750015258789,64.28750228881836,70.30000305175781,526079.875,0.6578947305679321,13.882353396142273,13.95021225063254,13.95649746832467,13.893949183707404,13.755116859142426,44.4719,-71.17530059814453,NH,1.0,,Bubbling fluidized bed boiler,Wood,Boiler (Other Solid Fuel) 12 | 58124_1,,,,,,,,,,,0.0,,,,,,,44.8878,-89.62979888916016,WI,5.0,,Circulating fluidized bed boiler,Wood,Boiler (Other Solid Fuel) 13 | 589_1,-8.673848005600466e-07,8.27317241201496e-05,0.0007247130739273025,-0.2876353768246691,22.017055225122125,17.575000762939453,26.82500076293945,36.07500076293945,45.32500076293945,54.57500076293945,578865.0,0.3220339119434357,17.552073891917555,15.970546814115892,14.998808812535493,14.511563582747184,14.231113061084065,44.4917,-73.20809936523438,VT,1.0,NPCC,Other boiler,Wood,Boiler (Other Solid Fuel) 14 | -------------------------------------------------------------------------------- /data/poly_fits/CC (Coal)_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,a4,a3,a2,a1,a0,load_min,load_2,load_3,load_4,load_max,total_load,min_gen_perc,heat_rate(load_min),heat_rate(load_2),heat_rate(load_3),heat_rate(load_4),heat_rate(load_max),latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type,group_type 2 | 1004_1-0,1.6308414614092316e-10,-5.153549356876276e-08,-3.677658996960901e-05,-0.0017928459353682665,15.172558384400384,237.7250061035156,289.7562561035156,341.7875061035156,393.8187561035156,445.8500061035156,5109046.0,0.5331950187683105,12.496481812521813,11.461213398794605,10.431471623192726,9.537796407700284,8.939414336800347,38.8067,-87.2472,IN,5.0,ECAR,Integrated gasification combined cycle,Coal,CC (Coal) 3 | 1004_1-1,-0.00022008074773262906,0.02388694856317741,-0.04087503167406729,-65.74230544575323,1812.0479922312866,49.95000076293945,51.56875038146973,53.1875,54.80624961853027,56.42499923706055,881.5249633789062,0.8852459192276001,33.145545578008516,32.49651052772242,32.593733380111416,32.87164267298249,32.728399912521574,38.8067,-87.2472,IN,5.0,ECAR,Integrated gasification combined cycle,Coal,CC (Coal) 4 | 1004_1-2,-1.4610800235909674e-08,4.693115030255573e-06,0.0003434299327179581,-0.2576839625463561,33.10699763639672,62.9000015258789,102.90625190734863,142.9125022888184,182.9187526702881,222.9250030517578,503821.625,0.28215768933296204,19.19664036774304,13.702349374831517,10.898672883239453,9.828963258088072,8.638324077504263,38.8067,-87.2472,IN,5.0,ECAR,Integrated gasification combined cycle,Coal,CC (Coal) 5 | 1004_1-3,0.007457548218745823,-0.8999596275916174,40.29488436639656,-794.0254060911979,5848.698698139242,23.125,26.824999809265137,30.524999618530273,34.22499942779541,37.92499923706055,562.4000244140625,0.6097561120986938,38.536833342886894,34.18001054281649,34.493406645305186,25.92870715920799,28.481541141896738,38.8067,-87.2472,IN,5.0,ECAR,Integrated gasification combined cycle,Coal,CC (Coal) 6 | -------------------------------------------------------------------------------- /data/poly_fits/CC (Oil)_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,a4,a3,a2,a1,a0,load_min,load_2,load_3,load_4,load_max,total_load,min_gen_perc,heat_rate(load_min),heat_rate(load_2),heat_rate(load_3),heat_rate(load_4),heat_rate(load_max),latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type,group_type 2 | 6081_CC1-0,-5.0539949456045115e-08,1.2403268616171567e-05,6.220230265910249e-05,-0.2200841632148241,27.462874831671066,96.3000030517578,108.09675216674805,119.89350128173828,131.69025039672852,143.48699951171875,91882.7265625,0.6711409687995911,13.575929734362745,13.165318945788648,12.903362453267365,12.685236425012489,12.382626358477307,42.1977,-72.5103,MA,1.0,NPCC,Combined cycle,Diesel Oil,CC (Oil) 3 | 6081_CC1-1,1.3924679593404668e-09,-4.845123075924609e-07,-5.6820776968558754e-05,0.018156531094270217,11.506584589885271,156.00599670410156,158.8949966430664,161.78399658203125,164.6729965209961,167.56199645996094,8191.27783203125,0.931034505367279,11.941396454378127,11.900867200777796,11.859055453409667,11.816020320471143,11.77182323817896,42.1977,-72.5103,MA,1.0,NPCC,Combined cycle,Diesel Oil,CC (Oil) 4 | 6081_CC1-2,-5.157112517960797e-07,6.754828694924687e-05,-2.509739252502667e-05,-0.3222123533660886,24.456318883024306,44.29800033569336,54.16874980926514,64.0394992828369,73.91024875640869,83.78099822998047,27519.65234375,0.5287356376647949,14.019610809765775,13.225095080141866,12.785680780622359,12.387609189567046,11.599626861157274,42.1977,-72.5103,MA,1.0,NPCC,Combined cycle,Diesel Oil,CC (Oil) 5 | 7153_CC2-0,-6.183547008656446e-08,1.2704344898776589e-05,-0.00012813943992977806,-0.13990826711534002,20.5189558286414,66.44699859619139,78.00299835205078,89.55899810791016,101.11499786376952,112.6709976196289,58053.4921875,0.5897436141967773,13.178450563438926,12.566403428863287,12.109028551687135,11.732083709485796,11.334861246799582,39.7436,-75.5072,DE,3.0,,Combined cycle,Diesel Oil,CC (Oil) 6 | 7153_CC2-1,-7.429495810323373e-10,5.143797944087797e-07,-7.984902169172823e-05,-0.02069883573399137,16.277812396408752,139.63499450683594,159.85799407958984,180.08099365234375,200.30399322509768,220.5269927978516,221631.5625,0.6331877708435059,12.94864028810483,12.544551550461216,12.18349928572517,11.865943243202214,11.589360861581099,39.7436,-75.5072,DE,3.0,,Combined cycle,Diesel Oil,CC (Oil) 7 | 7153_CC2-2,6.673233010344841e-10,-1.4743937102482113e-07,-6.158443093766939e-05,0.0036993714459156576,15.032333616266255,208.9709930419922,242.1944923400879,275.4179916381836,308.6414909362793,341.864990234375,2007229.125,0.611267626285553,13.043175903324403,12.517359414842868,12.13920707275306,12.028282566687755,12.323662809975573,39.7436,-75.5072,DE,3.0,,Combined cycle,Diesel Oil,CC (Oil) 8 | 7153_CC2-3,6.858417236556338e-09,-3.413859005715702e-06,-0.00012608507283342067,0.2627767688604657,-17.67174120365155,234.00900268554688,253.99124908447263,273.97349548339844,293.95574188232416,313.93798828125,35917.01171875,0.7453988194465637,13.73565163507591,13.542924406504572,13.294348809241562,13.173186949369825,13.388943866689718,39.7436,-75.5072,DE,3.0,,Combined cycle,Diesel Oil,CC (Oil) 9 | -------------------------------------------------------------------------------- /data/poly_fits/Cement Kiln (Coal)_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,a4,a3,a2,a1,a0,load_min,load_2,load_3,load_4,load_max,total_load,min_gen_perc,heat_rate(load_min),heat_rate(load_2),heat_rate(load_3),heat_rate(load_4),heat_rate(load_max),latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type,group_type 2 | 880044_41000,,,,,,,,,,,0.0,,,,,,,42.4833,-73.81670379638672,NY,2.0,,Cement Kiln,Coal,Cement Kiln (Coal) 3 | 880052_01070,,,,,,,,,,,0.0,,,,,,,43.3064,-73.62079620361328,NY,2.0,,Cement Kiln,Coal,Cement Kiln (Coal) 4 | -------------------------------------------------------------------------------- /data/poly_fits/Process Heater (NG)_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,a4,a3,a2,a1,a0,load_min,load_2,load_3,load_4,load_max,total_load,min_gen_perc,heat_rate(load_min),heat_rate(load_2),heat_rate(load_3),heat_rate(load_4),heat_rate(load_max),latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type,group_type 2 | 50628_780001,,,,,,,,,,,0.0,,,,,,,39.84,-75.25830078125,NJ,2.0,,Process Heater,Process Gas,Process Heater (NG) 3 | -------------------------------------------------------------------------------- /data/poly_fits/Stoker (Coal)_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,a4,a3,a2,a1,a0,load_min,load_2,load_3,load_4,load_max,total_load,min_gen_perc,heat_rate(load_min),heat_rate(load_2),heat_rate(load_3),heat_rate(load_4),heat_rate(load_max),latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type,group_type 2 | 10071_BLR01A,,,,,,,,,,,0.0,,,,,,,36.8703,-76.35189819335938,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 3 | 10071_BLR01B,,,,,,,,,,,0.0,,,,,,,36.8703,-76.35189819335938,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 4 | 10071_BLR01C,,,,,,,,,,,0.0,,,,,,,36.8703,-76.35189819335938,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 5 | 10071_BLR02A,,,,,,,,,,,0.0,,,,,,,36.8703,-76.35189819335938,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 6 | 10071_BLR02B,,,,,,,,,,,0.0,,,,,,,36.8703,-76.35189819335938,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 7 | 10071_BLR02C,,,,,,,,,,,0.0,,,,,,,36.8703,-76.35189819335938,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 8 | 10377_BLR01A,,,,,,,,,,,0.0,,,,,,,37.2939,-77.26969909667969,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 9 | 10377_BLR01B,,,,,,,,,,,0.0,,,,,,,37.2939,-77.26969909667969,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 10 | 10377_BLR01C,,,,,,,,,,,0.0,,,,,,,37.2939,-77.26969909667969,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 11 | 10377_BLR02A,,,,,,,,,,,0.0,,,,,,,37.2939,-77.26969909667969,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 12 | 10377_BLR02B,,,,,,,,,,,0.0,,,,,,,37.2939,-77.26969909667969,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 13 | 10377_BLR02C,,,,,,,,,,,0.0,,,,,,,37.2939,-77.26969909667969,VA,3.0,SERC,Stoker,Coal,Stoker (Coal) 14 | 10380_UNIT1,,,,,,,,,,,0.0,,,,,,,34.6474,-78.64070129394531,NC,4.0,,Stoker,Coal,Stoker (Coal) 15 | 10380_UNIT2,,,,,,,,,,,0.0,,,,,,,34.6474,-78.64070129394531,NC,4.0,,Stoker,Coal,Stoker (Coal) 16 | 10382_UNIT1,,,,,,,,,,,0.0,,,,,,,34.59,-78.9968032836914,NC,4.0,,Stoker,"Coal, Wood",Stoker (Coal) 17 | 10382_UNIT2,,,,,,,,,,,0.0,,,,,,,34.59,-78.9968032836914,NC,4.0,,Stoker,"Coal, Wood",Stoker (Coal) 18 | 10384_BLR01A,,,,,,,,,,,0.0,,,,,,,36.0378,-77.75420379638672,NC,4.0,,Stoker,Coal,Stoker (Coal) 19 | 10384_BLR01B,,,,,,,,,,,0.0,,,,,,,36.0378,-77.75420379638672,NC,4.0,,Stoker,Coal,Stoker (Coal) 20 | 10384_BLR02A,,,,,,,,,,,0.0,,,,,,,36.0378,-77.75420379638672,NC,4.0,,Stoker,Coal,Stoker (Coal) 21 | 10384_BLR02B,,,,,,,,,,,0.0,,,,,,,36.0378,-77.75420379638672,NC,4.0,,Stoker,Coal,Stoker (Coal) 22 | 2123_6,,,,,,,,,,,0.0,,,,,,,38.9658,-92.31749725341795,MO,7.0,SERC,Stoker,Coal,Stoker (Coal) 23 | 2123_7,,,,,,,,,,,0.0,,,,,,,38.9658,-92.31749725341795,MO,7.0,SERC,Stoker,Coal,Stoker (Coal) 24 | 50244_RB,,,,,,,,,,,0.0,,,,,,,35.535,-82.84110260009766,NC,4.0,,Stoker,Coal,Stoker (Coal) 25 | 50481_83-23,,,,,,,,,,,0.0,,,,,,,36.5192,-82.53500366210938,TN,4.0,,Stoker,Coal,Stoker (Coal) 26 | 50481_83-24,,,,,,,,,,,0.0,,,,,,,36.5192,-82.53500366210938,TN,4.0,,Stoker,Coal,Stoker (Coal) 27 | 50806_16,,,,,,,,,,,0.0,,,,,,,34.1519,-79.55870056152344,SC,4.0,,Stoker,Coal,Stoker (Coal) 28 | 50835_1,,,,,,,,,,,0.0,,,,,,,44.217,-86.29060363769531,MI,5.0,,Stoker,Coal,Stoker (Coal) 29 | 50835_2,,,,,,,,,,,0.0,,,,,,,44.217,-86.29060363769531,MI,5.0,,Stoker,Coal,Stoker (Coal) 30 | 50900_002,,,,,,,,,,,0.0,,,,,,,37.7997,-79.99459838867188,VA,3.0,,Stoker,Coal,Stoker (Coal) 31 | 50900_003,,,,,,,,,,,0.0,,,,,,,37.7997,-79.99459838867188,VA,3.0,,Stoker,Coal,Stoker (Coal) 32 | 54081_BLR01A,,,,,,,,,,,0.0,,,,,,,37.4556,-77.43080139160156,VA,3.0,,Stoker,Coal,Stoker (Coal) 33 | 54081_BLR01B,,,,,,,,,,,0.0,,,,,,,37.4556,-77.43080139160156,VA,3.0,,Stoker,Coal,Stoker (Coal) 34 | 54081_BLR02A,,,,,,,,,,,0.0,,,,,,,37.4556,-77.43080139160156,VA,3.0,,Stoker,Coal,Stoker (Coal) 35 | 54081_BLR02B,,,,,,,,,,,0.0,,,,,,,37.4556,-77.43080139160156,VA,3.0,,Stoker,Coal,Stoker (Coal) 36 | 54081_BLR03A,,,,,,,,,,,0.0,,,,,,,37.4556,-77.43080139160156,VA,3.0,,Stoker,Coal,Stoker (Coal) 37 | 54081_BLR03B,,,,,,,,,,,0.0,,,,,,,37.4556,-77.43080139160156,VA,3.0,,Stoker,Coal,Stoker (Coal) 38 | 54081_BLR04A,,,,,,,,,,,0.0,,,,,,,37.4556,-77.43080139160156,VA,3.0,,Stoker,Coal,Stoker (Coal) 39 | 54081_BLR04B,,,,,,,,,,,0.0,,,,,,,37.4556,-77.43080139160156,VA,3.0,,Stoker,Coal,Stoker (Coal) 40 | 880041_Z005,,,,,,,,,,,0.0,,,,,,,34.6339,-87.02439880371094,AL,4.0,,Stoker,Coal,Stoker (Coal) 41 | 880041_Z006,,,,,,,,,,,0.0,,,,,,,34.6339,-87.02439880371094,AL,4.0,,Stoker,Coal,Stoker (Coal) 42 | 880079_34,,,,,,,,,,,0.0,,,,,,,35.7433,-84.32080078125,TN,4.0,,Stoker,Coal,Stoker (Coal) 43 | 880079_35,,,,,,,,,,,0.0,,,,,,,35.7433,-84.32080078125,TN,4.0,,Stoker,Coal,Stoker (Coal) 44 | -------------------------------------------------------------------------------- /data/poly_fits/Stoker (NG)_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,a4,a3,a2,a1,a0,load_min,load_2,load_3,load_4,load_max,total_load,min_gen_perc,heat_rate(load_min),heat_rate(load_2),heat_rate(load_3),heat_rate(load_4),heat_rate(load_max),latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type,group_type 2 | 1866_5,,,,,,,,,,,0.0,,,,,,,42.2081,-83.14530181884766,MI,5.0,ECAR,Stoker,Pipeline Natural Gas,Stoker (NG) 3 | 880057_03,,,,,,,,,,,0.0,,,,,,,34.2337,-80.65799713134766,SC,4.0,,Stoker,Pipeline Natural Gas,Stoker (NG) 4 | 992_16,,,,,,,,,,,0.0,,,,,,,39.7622,-86.16670227050781,IN,5.0,ECAR,Stoker,Pipeline Natural Gas,Stoker (NG) 5 | -------------------------------------------------------------------------------- /data/poly_fits/Stoker (Other Solid Fuel)_fits.csv: -------------------------------------------------------------------------------- 1 | unit_id,a4,a3,a2,a1,a0,load_min,load_2,load_3,load_4,load_max,total_load,min_gen_perc,heat_rate(load_min),heat_rate(load_2),heat_rate(load_3),heat_rate(load_4),heat_rate(load_max),latitude,longitude,state,EPA_region,NERC_region,unit_type,fuel_type,group_type 2 | 10333_1,-5.1180387169518676e-08,4.440776517671093e-06,-0.0005148653250814158,-0.05787521534393315,20.05210938518217,23.125,36.53750038146973,49.95000076293945,63.36250114440918,76.7750015258789,365098.4375,0.3012048006057739,18.478693142265953,17.37554917847581,16.111485638143712,14.622627528139033,12.805348329997466,28.5814,-82.4302978515625,FL,4.0,FRCC,Stoker,Wood,Stoker (Other Solid Fuel) 3 | 10378_BLR01A,,,,,,,,,,,0.0,,,,,,,33.9444,-78.01249694824219,NC,4.0,,Stoker,Wood,Stoker (Other Solid Fuel) 4 | 10378_BLR01B,,,,,,,,,,,0.0,,,,,,,33.9444,-78.01249694824219,NC,4.0,,Stoker,Wood,Stoker (Other Solid Fuel) 5 | 10378_BLR01C,,,,,,,,,,,0.0,,,,,,,33.9444,-78.01249694824219,NC,4.0,,Stoker,Wood,Stoker (Other Solid Fuel) 6 | 10378_BLR02A,,,,,,,,,,,0.0,,,,,,,33.9444,-78.01249694824219,NC,4.0,,Stoker,Wood,Stoker (Other Solid Fuel) 7 | 10378_BLR02B,,,,,,,,,,,0.0,,,,,,,33.9444,-78.01249694824219,NC,4.0,,Stoker,Wood,Stoker (Other Solid Fuel) 8 | 10378_BLR02C,,,,,,,,,,,0.0,,,,,,,33.9444,-78.01249694824219,NC,4.0,,Stoker,Wood,Stoker (Other Solid Fuel) 9 | 10379_BLR01A,,,,,,,,,,,0.0,,,,,,,36.435,-78.96189880371094,NC,4.0,SERC,Stoker,Wood,Stoker (Other Solid Fuel) 10 | 10379_BLR01B,,,,,,,,,,,0.0,,,,,,,36.435,-78.96189880371094,NC,4.0,SERC,Stoker,Wood,Stoker (Other Solid Fuel) 11 | 10379_BLR01C,,,,,,,,,,,0.0,,,,,,,36.435,-78.96189880371094,NC,4.0,SERC,Stoker,Wood,Stoker (Other Solid Fuel) 12 | 10381_BLR01A,,,,,,,,,,,0.0,,,,,,,35.0225,-77.85749816894531,NC,4.0,SERC,Stoker,Wood,Stoker (Other Solid Fuel) 13 | 10381_BLR01B,,,,,,,,,,,0.0,,,,,,,35.0225,-77.85749816894531,NC,4.0,SERC,Stoker,Wood,Stoker (Other Solid Fuel) 14 | 10525_ES5A,-1.2165081089381935e-06,-1.7847415386184973e-05,0.0011765216288067866,0.15825263694681246,12.68015470259452,42.54999923706055,43.937499046325684,45.32499885559082,46.71249866485596,48.099998474121094,553278.5,0.8846154212951659,16.181375283872796,15.857077395443154,15.473988431451115,15.028341672904372,14.516262193157404,35.1285,-77.16999816894531,NC,4.0,SERC,Stoker,Wood,Stoker (Other Solid Fuel) 15 | 10771_1,,,,,,,,,,,0.0,,,,,,,37.2969,-77.2833023071289,VA,3.0,,Stoker,Wood,Stoker (Other Solid Fuel) 16 | 10771_2,,,,,,,,,,,0.0,,,,,,,37.2969,-77.2833023071289,VA,3.0,,Stoker,Wood,Stoker (Other Solid Fuel) 17 | 10773_1,,,,,,,,,,,0.0,,,,,,,37.1183,-79.27259826660156,VA,3.0,,Stoker,Wood,Stoker (Other Solid Fuel) 18 | 10773_2,,,,,,,,,,,0.0,,,,,,,37.1183,-79.27259826660156,VA,3.0,,Stoker,Wood,Stoker (Other Solid Fuel) 19 | 10774_1,,,,,,,,,,,0.0,,,,,,,36.6525,-76.99530029296875,VA,3.0,,Stoker,Wood,Stoker (Other Solid Fuel) 20 | 10774_2,,,,,,,,,,,0.0,,,,,,,36.6525,-76.99530029296875,VA,3.0,,Stoker,Wood,Stoker (Other Solid Fuel) 21 | 10822_1,,,,,,,,,,,0.0,,,,,,,44.6062,-84.69010162353516,MI,5.0,,Stoker,Wood,Stoker (Other Solid Fuel) 22 | 1897_3,,,,,,,,,,,0.0,,,,,,,46.7356,-92.1530990600586,MN,5.0,MAPP,Stoker,Wood,Stoker (Other Solid Fuel) 23 | 1897_4,,,,,,,,,,,0.0,,,,,,,46.7356,-92.1530990600586,MN,5.0,MAPP,Stoker,Wood,Stoker (Other Solid Fuel) 24 | 3982_1,0.0003775328273417524,-0.0047321995249500854,-0.061791426772643224,0.25554584715128875,24.554824114085864,10.175000190734863,11.100000143051147,12.025000095367432,12.950000047683716,13.875,159761.390625,0.7333333492279053,19.81930454340969,19.037372581759342,18.35819138134169,17.842205920288066,17.55649452506099,46.5869,-90.90170288085938,WI,5.0,MAPP,Stoker,Wood,Stoker (Other Solid Fuel) 25 | 3982_2,0.00014064768760715577,-0.002789221770905666,-0.02552890950259907,0.39278581870979135,18.549080388736343,9.25,10.40625,11.5625,12.71875,13.875,153243.828125,0.6666666865348816,18.820167085481,18.378163925294935,17.87992365881136,17.35689250567021,16.84654992855767,46.5869,-90.90170288085938,WI,5.0,MAPP,Stoker,Wood,Stoker (Other Solid Fuel) 26 | 50736_B1,,,,,,,,,,,0.0,,,,,,,41.7132,-71.82230377197266,CT,1.0,NPCC,Stoker,Tire Derived Fuel,Stoker (Other Solid Fuel) 27 | 50736_B2,,,,,,,,,,,0.0,,,,,,,41.7132,-71.82230377197266,CT,1.0,NPCC,Stoker,Tire Derived Fuel,Stoker (Other Solid Fuel) 28 | 54238_BIOMS1,-1.0638916969149055e-06,-1.0133155691289828e-05,0.0014089346879257952,0.15542871663648974,11.515518955115414,34.22499847412109,37.23124885559082,40.23749923706055,43.24374961853027,46.25,572846.125,0.7399999499320984,16.619464671976722,16.688161573117643,16.601758490552143,16.33173276792349,15.847476253377788,37.9416,-121.32730102539062,CA,9.0,,Stoker,Wood,Stoker (Other Solid Fuel) 29 | 54415_EUBLR,-4.6243887418178354e-05,0.0023883985441858963,0.004193454228791917,-1.7747034436377715,38.90076914033296,12.949999809265135,18.73125004768372,24.51250028610229,30.293750524520874,36.07500076293945,242971.59375,0.3589743375778198,20.50804170465293,17.133582854127084,16.400212382988194,16.44007011708616,14.145492979319208,44.2634,-85.43479919433594,MI,5.0,,Stoker,Wood,Stoker (Other Solid Fuel) 30 | 54751_01,0.00016350340420739494,-0.005634196824324665,-0.10773931990776736,4.759548625066387,-16.618626683059215,9.25,15.03125,20.8125,26.59375,32.375,168081.75,0.2857142984867096,14.926548697644426,19.79292608422285,15.655763965187344,9.572003270381627,12.982125610526133,43.0841,-83.67430114746094,MI,5.0,,Stoker,Wood,Stoker (Other Solid Fuel) 31 | 55867_BLR-1,8.593441972925693e-07,-2.1816356637509908e-05,-0.00261867613428557,-0.02796906282972844,19.350469891152148,33.299999237060554,40.46874904632568,47.63749885559082,54.80624866485596,61.974998474121094,695637.875,0.5373134613037109,15.766366289213874,14.788897651664374,14.142493129225526,14.11365152387878,15.043340974121804,45.2996,-95.5604019165039,MN,5.0,MAPP,Stoker,Other Solid Fuel,Stoker (Other Solid Fuel) 32 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/Boiler (Coal)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.3351648184815969,11.646182184330286 3 | 0.37015614382467077,11.553237231069657 4 | 0.4051474691677446,11.419210250065486 5 | 0.44013879451081844,11.31958362221781 6 | 0.4751301198538923,11.24758764812982 7 | 0.5101214451969661,11.166096087091592 8 | 0.54511277054004,11.089157624275522 9 | 0.5801040958831138,11.025098052637095 10 | 0.6150954212261877,10.955524322148062 11 | 0.6500867465692615,10.914726681158028 12 | 0.6850780719123353,10.87232697037363 13 | 0.7200693972554092,10.844173712669146 14 | 0.7550607225984831,10.81196900548535 15 | 0.7900520479415569,10.780379945368114 16 | 0.8250433732846307,10.752088636315335 17 | 0.8600346986277045,10.728085478743933 18 | 0.8950260239707784,10.705686580864725 19 | 0.9300173493138522,10.70155782914202 20 | 0.9650086746569262,10.68588817157717 21 | 1.0,10.656718087293896 22 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/Boiler (NG)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.16158402854010326,14.126198705733056 3 | 0.2057111849327294,13.216834476528215 4 | 0.24983834132535554,12.659570869294477 5 | 0.2939654977179817,12.090771550915205 6 | 0.33809265411060785,11.73748965530024 7 | 0.382219810503234,11.499734520644044 8 | 0.42634696689586016,11.325316432221964 9 | 0.47047412328848626,11.254498771721106 10 | 0.5146012796811125,11.178709539869395 11 | 0.5587284360737386,11.090938674939915 12 | 0.6028555924663648,11.051577787699511 13 | 0.6469827488589909,10.989463241923392 14 | 0.691109905251617,10.92065528113 15 | 0.7352370616442432,10.83295661208261 16 | 0.7793642180368693,10.756432701905668 17 | 0.8234913744294955,10.668831335664272 18 | 0.8676185308221216,10.625483174795331 19 | 0.9117456872147477,10.627287968979342 20 | 0.9558728436073739,10.744654907006169 21 | 1.0,10.908936935386148 22 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/Boiler (Oil)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.1844660227779598,13.365493354149798 3 | 0.22738886368438296,12.595485458547206 4 | 0.2703117045908061,12.133577587252038 5 | 0.3132345454972293,11.882366429864616 6 | 0.35615738640365247,11.488554201746695 7 | 0.39908022731007564,11.225308790181922 8 | 0.4420030682164988,11.044176597899693 9 | 0.484925909122922,10.925262825488804 10 | 0.5278487500293452,10.732301209134562 11 | 0.5707715909357683,10.73885033869141 12 | 0.6136944318421915,10.647044869610397 13 | 0.6566172727486147,10.59801631710583 14 | 0.6995401136550378,10.48130989700367 15 | 0.7424629545614609,10.52090878963869 16 | 0.7853857954678841,10.48185769165559 17 | 0.8283086363743073,10.440198041689465 18 | 0.8712314772807305,10.40407447420262 19 | 0.9141543181871536,10.387479175778886 20 | 0.9570771590935768,10.477500043675146 21 | 1.0,10.333868160362973 22 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/Boiler (Other Solid Fuel)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.6578947082825702,15.011712889699691 3 | 0.6759002499519087,14.939593830945416 4 | 0.6939057916212471,14.872877856793249 5 | 0.7119113332905854,14.811297647851568 6 | 0.7299168749599239,14.75456647485033 7 | 0.7479224166292623,14.702378198641071 8 | 0.7659279582986007,14.654407270196904 9 | 0.783933499967939,14.61030873061252 10 | 0.8019390416372775,14.569718211104185 11 | 0.8199445833066159,14.532251933009746 12 | 0.8379501249759543,14.497506707788627 13 | 0.8559556666452928,14.465059937021834 14 | 0.8739612083146311,14.434469612411945 15 | 0.8919667499839695,14.405274315783114 16 | 0.909972291653308,14.373075578583519 17 | 0.9279778333226463,14.235370932735306 18 | 0.9459833749919848,14.116553786111313 19 | 0.9639889166613231,14.02074950023795 20 | 0.9819944583306616,13.952220148460759 21 | 1.0,13.915364515944495 22 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/CC (Coal)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.571475568482372,31.4168799865492 3 | 0.5940294859306682,27.170067816598106 4 | 0.6165834033789644,24.521546994815328 5 | 0.6391373208272606,23.067729843778352 6 | 0.6616912382755569,22.452952232983144 7 | 0.684245155723853,22.369473578852812 8 | 0.7067990731721493,22.557476844726235 9 | 0.7293529906204455,22.805068540868053 10 | 0.7519069080687417,22.94827872446186 11 | 0.7744608255170379,22.87106099961612 12 | 0.7970147429653341,22.505292517359592 13 | 0.8195686604136303,21.830773975640007 14 | 0.8421225778619266,20.87522961932767 15 | 0.8646764953102227,19.714307240218655 16 | 0.8872304127585189,18.47968497268979 17 | 0.9097843302068152,17.334977769806585 18 | 0.9323382476551114,16.482857248392904 19 | 0.9548921651034076,16.207117473924807 20 | 0.9774460825517037,16.831345999975973 21 | 1.0,18.710477739346267 22 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/CC (NG)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.5609060404874759,8.462314469028176 3 | 0.584016248882872,8.374837764240109 4 | 0.607126457278268,8.32448494598197 5 | 0.630236665673664,8.32798723669211 6 | 0.65334687406906,8.319520186937625 7 | 0.676457082464456,8.28537624459053 8 | 0.699567290859852,8.23817262456415 9 | 0.7226774992552479,8.233664764792296 10 | 0.745787707650644,8.229740638831943 11 | 0.76889791604604,8.263278285811262 12 | 0.792008124441436,8.26197900108269 13 | 0.815118332836832,8.264432705632746 14 | 0.838228541232228,8.269981417544365 15 | 0.861338749627624,8.278426044953143 16 | 0.8844489580230199,8.285382933437571 17 | 0.9075591664184159,8.293069760324528 18 | 0.930669374813812,8.206653753025595 19 | 0.953779583209208,8.200930662743923 20 | 0.976889791604604,8.241631666532367 21 | 1.0,8.251851384436307 22 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/CC (Oil)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.6331877686956618,12.94864028810483 3 | 0.6524936756064165,12.860118420426279 4 | 0.6717995825171711,12.773405058255264 5 | 0.6911054894279258,12.688537444879087 6 | 0.7104113963386804,12.605546965192932 7 | 0.7297173032494351,12.524459145699883 8 | 0.7490232101601897,12.44529365451091 9 | 0.7683291170709443,12.368064301344877 10 | 0.7876350239816989,12.29277903752854 11 | 0.8069409308924536,12.219439955996549 12 | 0.8262468378032082,12.157019519584615 13 | 0.8455527447139629,12.121530117162138 14 | 0.8648586516247175,12.084051707270607 15 | 0.8841645585354722,12.044647997913842 16 | 0.9034704654462268,12.02874514607252 17 | 0.9227763723569815,12.050880969635946 18 | 0.9420822792677361,12.06744585919255 19 | 0.9613881861784908,11.932126959251512 20 | 0.9806940930892454,11.821235926486164 21 | 1.0,11.771823238178962 22 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/CT (NG)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.6067415550302355,11.955388059639992 3 | 0.6274393679233811,11.84512757724211 4 | 0.6481371808165265,11.725589980353067 5 | 0.668834993709672,11.650535502310477 6 | 0.6895328066028175,11.578344970968107 7 | 0.710230619495963,11.489879302015666 8 | 0.7309284323891085,11.420619378140026 9 | 0.751626245282254,11.345537130779107 10 | 0.7723240581753995,11.28840937727659 11 | 0.793021871068545,11.230868714829235 12 | 0.8137196839616905,11.160032389806636 13 | 0.834417496854836,11.096963419938072 14 | 0.8551153097479816,11.061187500185218 15 | 0.875813122641127,11.002987242207665 16 | 0.8965109355342725,10.923847387379693 17 | 0.9172087484274181,10.850223286785898 18 | 0.9379065613205635,10.782074026393904 19 | 0.9586043742137089,10.701889865417666 20 | 0.9793021871068545,10.642424375455782 21 | 1.0,10.562273115997325 22 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/CT (Oil)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.6428571125413475,13.008209565145721 3 | 0.6616541066181186,12.897584892580792 4 | 0.6804511006948899,12.851776922938694 5 | 0.699248094771661,12.797427849227414 6 | 0.7180450888484322,12.74306075870333 7 | 0.7368420829252034,12.680053989411185 8 | 0.7556390770019745,12.60668634577938 9 | 0.7744360710787458,12.528556641356113 10 | 0.793233065155517,12.456448770066013 11 | 0.8120300592322881,12.415371297967784 12 | 0.8308270533090594,12.325405625232904 13 | 0.8496240473858305,12.215291385856228 14 | 0.8684210414626017,12.149279974018224 15 | 0.8872180355393728,12.091243593360234 16 | 0.906015029616144,12.035774264403175 17 | 0.9248120236929153,11.983302270454052 18 | 0.9436090177696864,11.969179774605191 19 | 0.9624060118464576,11.91940893035689 20 | 0.9812030059232288,11.916086674093071 21 | 1.0,11.897360111130906 22 | -------------------------------------------------------------------------------- /data/poly_fits/generic_fits/Stoker (Other Solid Fuel)_generic_fit.csv: -------------------------------------------------------------------------------- 1 | Normalized Load (%),Heat Rate (mmBTU/MWh) 2 | 0.6019900502105886,16.73719757431391 3 | 0.6229379423047682,16.414762666234992 4 | 0.6438858343989478,16.350241391321003 5 | 0.6648337264931272,16.38959876197947 6 | 0.6857816185873068,16.433107753022433 7 | 0.7067295106814864,16.47763607427563 8 | 0.7276774027756658,16.519859210720263 9 | 0.7486252948698454,16.556260422493033 10 | 0.769573186964025,16.583130744886116 11 | 0.7905210790582046,16.596568988347162 12 | 0.8114689711523841,16.592481738479307 13 | 0.8324168632465636,16.545717733484224 14 | 0.8533647553407432,16.429605081911117 15 | 0.8743126474349228,16.278148240112802 16 | 0.8952605395291022,16.08630113684538 17 | 0.9162084316232818,15.848821677740702 18 | 0.9371563237174614,15.560271745306405 19 | 0.958104215811641,15.21501719892585 20 | 0.9790521079058205,14.84087369249815 21 | 1.0,14.779801583639603 22 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = reV 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | BRANCH := $(shell git rev-parse --abbrev-ref HEAD) 11 | 12 | # Put it first so that "make" without argument is like "make help". 13 | help: 14 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 15 | 16 | .PHONY: help Makefile 17 | 18 | # Catch-all target: route all unknown targets to Sphinx using the new 19 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 20 | %: Makefile 21 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 22 | 23 | github: html 24 | -git branch -D gh-pages 25 | -git push origin --delete gh-pages 26 | ghp-import -n -b gh-pages -m "Update documentation" ./build/html 27 | git checkout gh-pages 28 | git push --set-upstream origin gh-pages 29 | git checkout ${BRANCH} 30 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Sphinx Documentation 2 | 3 | The documentation is built with [Sphinx](http://sphinx-doc.org/index.html). See their documentation for (a lot) more details. 4 | 5 | ## Installation 6 | 7 | To generate the docs yourself, you'll need the appropriate package: 8 | 9 | ``` 10 | pip install sphinx 11 | ``` 12 | 13 | ## Refreshing the API Documentation 14 | 15 | - Make sure reV is in your PYTHONPATH 16 | - Delete the contents of `source/NaTGenPD`. 17 | - Run `sphinx-apidoc -o source/NaTGenPD ..` from the `docs` folder. 18 | - Compare `source/NaTGenPD/modules.rst` to `source/NaTGenPD.rst`. 19 | - `git push` changes to the documentation source code as needed. 20 | - Make the documentation per below 21 | 22 | ## Building HTML Docs 23 | 24 | ### Mac/Linux 25 | 26 | ``` 27 | make html 28 | ``` 29 | 30 | ### Windows 31 | 32 | ``` 33 | make.bat html 34 | ``` 35 | 36 | ## Building PDF Docs 37 | 38 | To build a PDF, you'll need a latex distribution for your system. 39 | 40 | ### Mac/Linux 41 | 42 | ``` 43 | make latexpdf 44 | ``` 45 | 46 | ### Windows 47 | 48 | ``` 49 | make.bat latexpdf 50 | ``` 51 | 52 | ## Pushing to GitHub Pages 53 | 54 | ### Mac/Linux 55 | 56 | ``` 57 | make github 58 | ``` 59 | 60 | ### Windows 61 | 62 | ``` 63 | make.bat html 64 | ``` 65 | 66 | Then run the github-related commands by hand: 67 | 68 | ``` 69 | git branch -D gh-pages 70 | git push origin --delete gh-pages 71 | ghp-import -n -b gh-pages -m "Update documentation" ./build/html 72 | git checkout gh-pages 73 | git push origin gh-pages 74 | git checkout master # or whatever branch you were on 75 | ``` 76 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=WindPACT 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/source/NaTGenPD.rst: -------------------------------------------------------------------------------- 1 | NaTGenPD 2 | ### 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | NaTGenPD/modules 8 | NaTGenPD/NaTGenPD 9 | -------------------------------------------------------------------------------- /docs/source/NaTGenPD/NaTGenPD.rst: -------------------------------------------------------------------------------- 1 | NaTGenPD package 2 | ================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | NaTGenPD.analysis module 8 | ------------------------ 9 | 10 | .. automodule:: NaTGenPD.analysis 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | NaTGenPD.ccmap module 15 | --------------------- 16 | 17 | .. automodule:: NaTGenPD.ccmap 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | NaTGenPD.clean module 22 | --------------------- 23 | 24 | .. automodule:: NaTGenPD.clean 25 | :members: 26 | :undoc-members: 27 | :show-inheritance: 28 | NaTGenPD.cli module 29 | ------------------- 30 | 31 | .. automodule:: NaTGenPD.cli 32 | :members: 33 | :undoc-members: 34 | :show-inheritance: 35 | NaTGenPD.cluster module 36 | ----------------------- 37 | 38 | .. automodule:: NaTGenPD.cluster 39 | :members: 40 | :undoc-members: 41 | :show-inheritance: 42 | NaTGenPD.filter module 43 | ---------------------- 44 | 45 | .. automodule:: NaTGenPD.filter 46 | :members: 47 | :undoc-members: 48 | :show-inheritance: 49 | NaTGenPD.handler module 50 | ----------------------- 51 | 52 | .. automodule:: NaTGenPD.handler 53 | :members: 54 | :undoc-members: 55 | :show-inheritance: 56 | NaTGenPD.version module 57 | ----------------------- 58 | 59 | .. automodule:: NaTGenPD.version 60 | :members: 61 | :undoc-members: 62 | :show-inheritance: 63 | 64 | 65 | Module contents 66 | --------------- 67 | 68 | .. automodule:: NaTGenPD 69 | :members: 70 | :undoc-members: 71 | :show-inheritance: 72 | -------------------------------------------------------------------------------- /docs/source/NaTGenPD/modules.rst: -------------------------------------------------------------------------------- 1 | NaTGenPD 2 | ======== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | NaTGenPD 8 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Documentation config file 4 | """ 5 | # 6 | # Configuration file for the Sphinx documentation builder. 7 | # 8 | # This file does only contain a selection of the most common options. For a 9 | # full list see the documentation: 10 | # http://www.sphinx-doc.org/en/stable/config 11 | 12 | # -- Path setup -------------------------------------------------------------- 13 | 14 | # If extensions (or modules to document with autodoc) are in another directory, 15 | # add these directories to sys.path here. If the directory is relative to the 16 | # documentation root, use os.path.abspath to make it absolute, like shown here. 17 | # 18 | # import os 19 | # import sys 20 | # sys.path.insert(0, os.path.abspath('.')) 21 | import sphinx_rtd_theme 22 | 23 | 24 | # -- Project information ----------------------------------------------------- 25 | 26 | project = 'NaTGenPD' 27 | copyright = '2019, Michael Rossol, Grant Buster' 28 | author = 'Michael Rossol, Grant Buster' 29 | 30 | # The short X.Y version 31 | version = '1.0.0' 32 | # The full version, including alpha/beta/rc tags 33 | release = '1.0.0' 34 | 35 | 36 | # -- General configuration --------------------------------------------------- 37 | 38 | # If your documentation needs a minimal Sphinx version, state it here. 39 | # 40 | # needs_sphinx = '1.0' 41 | 42 | # Add any Sphinx extension module names here, as strings. They can be 43 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 44 | # ones. 45 | extensions = [ 46 | 'sphinx.ext.autodoc', 47 | 'sphinx.ext.doctest', 48 | 'sphinx.ext.coverage', 49 | 'sphinx.ext.viewcode', 50 | 'sphinx.ext.githubpages', 51 | 'sphinx.ext.napoleon' # had numpydoc, but got lots of extraneous warnings 52 | ] 53 | 54 | # Add any paths that contain templates here, relative to this directory. 55 | templates_path = ['_templates'] 56 | 57 | # The suffix(es) of source filenames. 58 | # You can specify multiple suffix as a list of string: 59 | # 60 | # source_suffix = ['.rst', '.md'] 61 | source_suffix = '.rst' 62 | 63 | # source_parsers = {'.md': MarkdownParser} 64 | 65 | # The master toctree document. 66 | master_doc = 'index' 67 | 68 | # The language for content autogenerated by Sphinx. Refer to documentation 69 | # for a list of supported languages. 70 | # 71 | # This is also used if you do content translation via gettext catalogs. 72 | # Usually you set "language" from the command line for these cases. 73 | language = None 74 | 75 | # List of patterns, relative to source directory, that match files and 76 | # directories to ignore when looking for source files. 77 | # This pattern also affects html_static_path and html_extra_path . 78 | exclude_patterns = [] 79 | 80 | # The name of the Pygments (syntax highlighting) style to use. 81 | pygments_style = 'sphinx' 82 | 83 | 84 | # -- Options for HTML output ------------------------------------------------- 85 | 86 | # The theme to use for HTML and HTML Help pages. See the documentation for 87 | # a list of builtin themes. 88 | # 89 | html_theme = 'sphinx_rtd_theme' 90 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 91 | 92 | # Theme options are theme-specific and customize the look and feel of a theme 93 | # further. For a list of options available for each theme, see the 94 | # documentation. 95 | # 96 | html_theme_options = { 97 | 'collapse_navigation': False, 98 | 'sticky_navigation': True, 99 | 'titles_only': False 100 | } 101 | 102 | # Add any paths that contain custom static files (such as style sheets) here, 103 | # relative to this directory. They are copied after the builtin static files, 104 | # so a file named "default.css" will overwrite the builtin "default.css". 105 | html_static_path = ['_static'] 106 | 107 | # Custom sidebar templates, must be a dictionary that maps document names 108 | # to template names. 109 | # 110 | # The default sidebars (for documents that don't match any pattern) are 111 | # defined by theme itself. Builtin themes are using these templates by 112 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 113 | # 'searchbox.html']``. 114 | # 115 | # html_sidebars = {} 116 | 117 | 118 | # -- Options for HTMLHelp output --------------------------------------------- 119 | 120 | # Output file base name for HTML help builder. 121 | htmlhelp_basename = 'NaTGenPDdoc' 122 | 123 | 124 | # -- Options for LaTeX output ------------------------------------------------ 125 | 126 | latex_elements = { 127 | # The paper size ('letterpaper' or 'a4paper'). 128 | # 129 | # 'papersize': 'letterpaper', 130 | 131 | # The font size ('10pt', '11pt' or '12pt'). 132 | # 133 | # 'pointsize': '10pt', 134 | 135 | # Additional stuff for the LaTeX preamble. 136 | # 137 | # 'preamble': '', 138 | 139 | # Latex figure (float) alignment 140 | # 141 | # 'figure_align': 'htbp', 142 | } 143 | 144 | # Grouping the document tree into LaTeX files. List of tuples 145 | # (source start file, target name, title, 146 | # author, documentclass [howto, manual, or own class]). 147 | latex_documents = [ 148 | (master_doc, 'NaTGenPD.tex', 'NaTGenPD Documentation', 149 | 'Michael Rossol', 'Grant Buster', 'manual'), 150 | ] 151 | 152 | 153 | # -- Options for manual page output ------------------------------------------ 154 | 155 | # One entry per manual page. List of tuples 156 | # (source start file, name, description, authors, manual section). 157 | man_pages = [ 158 | (master_doc, 'NaTGenPD', 'NaTGenPD Documentation', 159 | [author], 1) 160 | ] 161 | 162 | 163 | # -- Options for Texinfo output ---------------------------------------------- 164 | 165 | # Grouping the document tree into Texinfo files. List of tuples 166 | # (source start file, target name, title, author, 167 | # dir menu entry, description, category) 168 | texinfo_documents = [ 169 | (master_doc, 'NaTGenPD', 'NaTGenPD Documentation', 170 | author, 'NaTGenPD', 'One line description of project.', 171 | 'Miscellaneous'), 172 | ] 173 | 174 | 175 | # -- Extension configuration ------------------------------------------------- 176 | 177 | autoclass_content = 'both' 178 | autodoc_member_order = 'bysource' 179 | # Uncomment when https://github.com/sphinx-doc/sphinx/pull/4076 is 180 | # released. 181 | # autodoc_special_members = ['__getitem__', '__setitem__','__iter__'] 182 | numpy_show_class_member = True 183 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. NaTGenPD documentation master file, created by 2 | sphinx-quickstart on Thu Jun 14 10:26:46 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to NaTGenPD's documentation! 7 | ==================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | NaTGenPD 14 | 15 | Indices and tables 16 | ================== 17 | 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /notebooks/Extract Data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "ExecuteTime": { 8 | "end_time": "2019-12-09T18:19:22.742991Z", 9 | "start_time": "2019-12-09T18:19:18.314581Z" 10 | } 11 | }, 12 | "outputs": [], 13 | "source": [ 14 | "import numpy as np\n", 15 | "import os\n", 16 | "import pandas as pd\n", 17 | "\n", 18 | "import sys\n", 19 | "cwd = os.getcwd()\n", 20 | "pkg_dir = '/home/mrossol/NaTGenPD'\n", 21 | "pkg_dir = os.path.dirname(cwd)\n", 22 | "sys.path.append(pkg_dir)\n", 23 | "import NaTGenPD as npd\n", 24 | "from NaTGenPD.poly_fit import GenericFit\n", 25 | "from NaTGenPD.cli import setup_logger\n", 26 | "data_dir = '/projects/naris/CEMS'\n", 27 | "filtered_dir = os.path.join(data_dir, 'Filtered')\n", 28 | "\n", 29 | "data_dir = os.path.join(pkg_dir, 'data')\n", 30 | "generic_dir = os.path.join(data_dir, 'poly_fits/generic_fits')\n", 31 | "if not os.path.exists(generic_dir):\n", 32 | " os.makedirs(generic_dir)\n", 33 | "\n", 34 | "logger = setup_logger('NaTGenPD')" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 2, 40 | "metadata": { 41 | "ExecuteTime": { 42 | "end_time": "2019-12-09T18:19:22.834996Z", 43 | "start_time": "2019-12-09T18:19:22.807766Z" 44 | } 45 | }, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "text/plain": [ 50 | "['poly_fits', 'piecewise_fits']" 51 | ] 52 | }, 53 | "execution_count": 2, 54 | "metadata": {}, 55 | "output_type": "execute_result" 56 | } 57 | ], 58 | "source": [ 59 | "os.listdir(data_dir)" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "# Generic Fits" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 2, 72 | "metadata": { 73 | "ExecuteTime": { 74 | "end_time": "2019-11-01T15:15:45.815530Z", 75 | "start_time": "2019-11-01T15:15:45.386464Z" 76 | } 77 | }, 78 | "outputs": [ 79 | { 80 | "name": "stderr", 81 | "output_type": "stream", 82 | "text": [ 83 | "INFO - 2019-11-01 09:15:45,412 [poly_fit.py:311] : Creating Generic Fit for Boiler (Coal)\n", 84 | "/Users/mrossol/Git_Repos/NaTGenPD/NaTGenPD/poly_fit.py:274: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n", 85 | " load_maxs = group_fits.as_matrix(['load_max'])\n", 86 | "/Users/mrossol/Git_Repos/NaTGenPD/NaTGenPD/poly_fit.py:276: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n", 87 | " fit_params = group_fits.as_matrix(['a4', 'a3', 'a2', 'a1', 'a0'])\n", 88 | "INFO - 2019-11-01 09:15:45,480 [poly_fit.py:311] : Creating Generic Fit for Boiler (NG)\n", 89 | "INFO - 2019-11-01 09:15:45,500 [poly_fit.py:311] : Creating Generic Fit for Boiler (Oil)\n", 90 | "INFO - 2019-11-01 09:15:45,513 [poly_fit.py:311] : Creating Generic Fit for Boiler (Other Solid Fuel)\n", 91 | "INFO - 2019-11-01 09:15:45,533 [poly_fit.py:311] : Creating Generic Fit for CC (Coal)\n", 92 | "INFO - 2019-11-01 09:15:45,566 [poly_fit.py:311] : Creating Generic Fit for CC (NG)\n", 93 | "INFO - 2019-11-01 09:15:45,634 [poly_fit.py:311] : Creating Generic Fit for CC (Oil)\n", 94 | "INFO - 2019-11-01 09:15:45,659 [poly_fit.py:311] : Creating Generic Fit for CT (NG)\n", 95 | "INFO - 2019-11-01 09:15:45,712 [poly_fit.py:311] : Creating Generic Fit for CT (Oil)\n", 96 | "INFO - 2019-11-01 09:15:45,736 [poly_fit.py:311] : Creating Generic Fit for Cement Kiln (Coal)\n", 97 | "WARNING - 2019-11-01 09:15:45,738 [poly_fit.py:292] : - Cannot create a generic curve, no valid units present!\n", 98 | "/Users/mrossol/Git_Repos/NaTGenPD/NaTGenPD/poly_fit.py:293: UserWarning: - Cannot create a generic curve, no valid units present!\n", 99 | " warnings.warn(msg)\n", 100 | "INFO - 2019-11-01 09:15:45,758 [poly_fit.py:311] : Creating Generic Fit for Process Heater (NG)\n", 101 | "WARNING - 2019-11-01 09:15:45,760 [poly_fit.py:292] : - Cannot create a generic curve, no valid units present!\n", 102 | "INFO - 2019-11-01 09:15:45,772 [poly_fit.py:311] : Creating Generic Fit for Stoker (Coal)\n", 103 | "WARNING - 2019-11-01 09:15:45,777 [poly_fit.py:292] : - Cannot create a generic curve, no valid units present!\n", 104 | "INFO - 2019-11-01 09:15:45,788 [poly_fit.py:311] : Creating Generic Fit for Stoker (NG)\n", 105 | "WARNING - 2019-11-01 09:15:45,790 [poly_fit.py:292] : - Cannot create a generic curve, no valid units present!\n", 106 | "INFO - 2019-11-01 09:15:45,802 [poly_fit.py:311] : Creating Generic Fit for Stoker (Other Solid Fuel)\n" 107 | ] 108 | } 109 | ], 110 | "source": [ 111 | "fit_dir = os.path.join(data_dir, 'poly_fits')\n", 112 | "GenericFit.run(fit_dir, generic_dir)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "# Filtered Data" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 26, 125 | "metadata": {}, 126 | "outputs": [ 127 | { 128 | "name": "stdout", 129 | "output_type": "stream", 130 | "text": [ 131 | "Boiler-Coal\n", 132 | "Boiler-NG\n", 133 | "Boiler-Oil\n", 134 | "Boiler-Other_Solid_Fuel\n", 135 | "CC-Coal\n", 136 | "CC-NG\n", 137 | "CC-Oil\n", 138 | "CT-NG\n", 139 | "CT-Oil\n", 140 | "Cement_Kiln-Coal\n", 141 | "Process_Heater-NG\n", 142 | "Stoker-Coal\n", 143 | "Stoker-NG\n", 144 | "Stoker-Other_Solid_Fuel\n" 145 | ] 146 | } 147 | ], 148 | "source": [ 149 | "h5_path = os.path.join(data_dir, 'SMOKE_Filtered_2016-2017.h5')\n", 150 | "with npd.CEMS(h5_path, 'r') as f:\n", 151 | " for group in f.dsets:\n", 152 | " f_name = group.strip(')').replace(' (', '-').replace(' ', '_')\n", 153 | " f_out = os.path.join(filtered_dir, f_name + '.csv')\n", 154 | " print(f_name)\n", 155 | " df = f[group].df\n", 156 | " if 'CC' not in group:\n", 157 | " df = df.drop(columns='cts')\n", 158 | " \n", 159 | " df.to_csv(f_out, index=False)" 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": {}, 165 | "source": [ 166 | "# Piecewise Fits" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 28, 172 | "metadata": { 173 | "ExecuteTime": { 174 | "end_time": "2019-12-09T19:55:05.575194Z", 175 | "start_time": "2019-12-09T19:55:05.554492Z" 176 | } 177 | }, 178 | "outputs": [], 179 | "source": [ 180 | "def extract_best_fit(row):\n", 181 | " best_fit = int(row['min_aicc'])\n", 182 | " fit_cols = ['group_type','unit_id', 'cluster']\n", 183 | " col = '{}_b'.format(best_fit)\n", 184 | " fit_cols.append(col)\n", 185 | " rename_cols = {col : 'heat_rate_base'}\n", 186 | " for i in range(1, best_fit + 1):\n", 187 | " col = \"{}_m{}\".format(best_fit, i)\n", 188 | " fit_cols.append(col)\n", 189 | " rename_cols[col] = 'heat_rate_incr{}'.format(i)\n", 190 | " \n", 191 | " col = \"{}_x{}\".format(best_fit, i)\n", 192 | " fit_cols.append(col)\n", 193 | " rename_cols[col] = 'load_point{}'.format(i)\n", 194 | " \n", 195 | " row = row[fit_cols]\n", 196 | " row = row.rename(rename_cols)\n", 197 | " return row\n", 198 | "\n", 199 | "\n", 200 | "def reformat_piecewise_fit(piecewise_fits, poly_fits, out_path):\n", 201 | " piecewise_fits = pd.read_csv(piecewise_fits)\n", 202 | " if '2_x3' in piecewise_fits:\n", 203 | " piecewise_fits = piecewise_fits.rename(columns={'2_x3': '3_x3'})\n", 204 | " \n", 205 | " piecewise_fits = piecewise_fits.apply(extract_best_fit, axis=1)\n", 206 | " col_order = ['unit_id', 'group_type', 'cluster', 'heat_rate_base',\n", 207 | " 'heat_rate_incr1', 'load_point1',\n", 208 | " 'heat_rate_incr2', 'load_point2',\n", 209 | " 'heat_rate_incr3', 'load_point3']\n", 210 | " col_order = [c for c in col_order if c in piecewise_fits]\n", 211 | "\n", 212 | " merge_cols = ['unit_id', 'latitude', 'longitude',\n", 213 | " 'state', 'EPA_region', 'NERC_region',\n", 214 | " 'unit_type', 'fuel_type']\n", 215 | " \n", 216 | " poly_fits = pd.read_csv(poly_fits)\n", 217 | " piecewise_fits = pd.merge(piecewise_fits[col_order], poly_fits[merge_cols],\n", 218 | " on='unit_id', how='left')\n", 219 | " \n", 220 | " piecewise_fits.to_csv(out_path, index=None)" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": 29, 226 | "metadata": { 227 | "ExecuteTime": { 228 | "end_time": "2019-12-09T19:55:14.086329Z", 229 | "start_time": "2019-12-09T19:55:07.736291Z" 230 | } 231 | }, 232 | "outputs": [ 233 | { 234 | "name": "stdout", 235 | "output_type": "stream", 236 | "text": [ 237 | "CT (Oil)_linpiecewise_fits.csv processed\n", 238 | "CC (Coal)_linpiecewise_fits.csv processed\n", 239 | "CC (NG)_linpiecewise_fits.csv processed\n", 240 | "CC (Oil)_linpiecewise_fits.csv processed\n", 241 | "CT (NG)_linpiecewise_fits.csv processed\n", 242 | "Boiler (Other Solid Fuel)_linpiecewise_fits.csv processed\n", 243 | "Boiler (Coal)_linpiecewise_fits.csv processed\n", 244 | "Boiler (Oil)_linpiecewise_fits.csv processed\n", 245 | "Stoker (Other Solid Fuel)_linpiecewise_fits.csv processed\n", 246 | "Boiler (NG)_linpiecewise_fits.csv processed\n" 247 | ] 248 | } 249 | ], 250 | "source": [ 251 | "out_dir = os.path.join(data_dir, 'piecewise_fits')\n", 252 | "piecewise_dir = os.path.join(out_dir, 'all_fits')\n", 253 | "poly_dir = os.path.join(data_dir, 'poly_fits')\n", 254 | "for fname in os.listdir(piecewise_dir):\n", 255 | " group_name = fname.split('_')[0]\n", 256 | " in_path = os.path.join(piecewise_dir, fname)\n", 257 | " poly_path = os.path.join(poly_dir, '{}_fits.csv'.format(group_name))\n", 258 | " out_path = os.path.join(out_dir, '{}_piecewise_fits.csv'.format(group_name))\n", 259 | " reformat_piecewise_fit(in_path, poly_path, out_path)\n", 260 | " print('{} processed'.format(fname))" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [] 269 | } 270 | ], 271 | "metadata": { 272 | "kernelspec": { 273 | "display_name": "Python [conda env:root] *", 274 | "language": "python", 275 | "name": "conda-root-py" 276 | }, 277 | "language_info": { 278 | "codemirror_mode": { 279 | "name": "ipython", 280 | "version": 3 281 | }, 282 | "file_extension": ".py", 283 | "mimetype": "text/x-python", 284 | "name": "python", 285 | "nbconvert_exporter": "python", 286 | "pygments_lexer": "ipython3", 287 | "version": "3.7.3" 288 | } 289 | }, 290 | "nbformat": 4, 291 | "nbformat_minor": 2 292 | } 293 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | setup.py 4 | """ 5 | import os 6 | from codecs import open 7 | from setuptools import setup, find_packages 8 | from setuptools.command.develop import develop 9 | from subprocess import check_call 10 | import shlex 11 | import sys 12 | from warnings import warn 13 | 14 | py_version = sys.version_info 15 | if py_version.major < 3: 16 | raise RuntimeError("reVX is not compatible with python 2!") 17 | elif py_version.minor < 6: 18 | warn("You will the get best results by running reVX with python >= 3.6") 19 | 20 | try: 21 | from pypandoc import convert_text 22 | except ImportError: 23 | convert_text = lambda string, *args, **kwargs: string 24 | 25 | here = os.path.abspath(os.path.dirname(__file__)) 26 | 27 | with open("README.md", encoding="utf-8") as readme_file: 28 | readme = convert_text(readme_file.read(), "rst", format="md") 29 | 30 | with open(os.path.join(here, "NaTGenPD", "version.py"), encoding="utf-8") as f: 31 | version = f.read() 32 | 33 | version = version.split('=')[-1].strip().strip('"').strip("'") 34 | 35 | 36 | class PostDevelopCommand(develop): 37 | """ 38 | Class to run post setup commands 39 | """ 40 | 41 | def run(self): 42 | """ 43 | Run method that tries to install pre-commit hooks 44 | """ 45 | try: 46 | check_call(shlex.split("pre-commit install")) 47 | except Exception as e: 48 | warn("Unable to run 'pre-commit install': {}" 49 | .format(e)) 50 | 51 | develop.run(self) 52 | 53 | 54 | test_requires = ["pytest", ] 55 | 56 | numpy_dependency = "numpy>=1.15.0" 57 | pandas_dependency = "pandas>=0.23.0" 58 | click_dependency = "click>=7.0" 59 | scipy_dependency = "scipy>=1.2.1" 60 | 61 | setup( 62 | name="NaTGenPD", 63 | version=version, 64 | description="National Thermal Generation Performace Database", 65 | long_description=readme, 66 | author="Michael Rossol", 67 | author_email="michael.rossol@nrel.gov", 68 | url="https://github.com/NREL/NaTGenPD", 69 | packages=find_packages(), 70 | package_dir={"NaTGenPD": "NaTGenPD"}, 71 | entry_points={ 72 | "console_scripts": ["NaTGenPD=NaTGenPD.cli:main", ], 73 | }, 74 | include_package_data=True, 75 | license="BSD license", 76 | zip_safe=False, 77 | keywords="NaTGenPD", 78 | classifiers=[ 79 | "Development Status :: Beta", 80 | "Intended Audience :: Modelers", 81 | "License :: OSI Approved :: BSD License", 82 | "Natural Language :: English", 83 | "Programming Language :: Python :: 3.6", 84 | "Programming Language :: Python :: 3.7", 85 | "Programming Language :: Python :: 3.8", 86 | ], 87 | test_suite="tests", 88 | install_requires=["click", "h5py", "numpy", "pandas", "scipy", 89 | "scikit-learn"], 90 | extras_require={ 91 | "test": test_requires, 92 | "dev": test_requires + ["pypandoc", "flake8", "pre-commit", "pylint"], 93 | }, 94 | cmdclass={"develop": PostDevelopCommand}, 95 | ) 96 | --------------------------------------------------------------------------------