├── .gitignore
├── .idea
    ├── 11785ESRNN.iml
    ├── misc.xml
    ├── modules.xml
    ├── vcs.xml
    └── workspace.xml
├── LICENSE
├── README.md
├── __init__.py
├── es_rnn
    ├── DRNN.py
    ├── __init__.py
    ├── config.py
    ├── data_loading.py
    ├── loss_modules.py
    ├── main.py
    ├── model.py
    └── trainer.py
├── m4_baseline.R
└── utils
    ├── __init__.py
    ├── helper_funcs.py
    └── logger.py


/.gitignore:
--------------------------------------------------------------------------------
1 | # Original Slawek's files
2 | slawek_github/
3 | .idea/
4 | data/*.csv
5 | data/Test/*.csv
6 | data/Train/*.csv


--------------------------------------------------------------------------------
/.idea/11785ESRNN.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="inheritedJdk" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="TestRunnerService">
 9 |     <option name="PROJECT_TEST_RUNNER" value="Unittests" />
10 |   </component>
11 | </module>


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="JavaScriptSettings">
4 |     <option name="languageLevel" value="ES6" />
5 |   </component>
6 |   <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (tf_gpu)" project-jdk-type="Python SDK" />
7 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/11785ESRNN.iml" filepath="$PROJECT_DIR$/.idea/11785ESRNN.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
   1 | <?xml version="1.0" encoding="UTF-8"?>
   2 | <project version="4">
   3 |   <component name="BookmarkManager">
   4 |     <bookmark url="file://$PROJECT_DIR$/es_rnn/main.py" line="17" />
   5 |   </component>
   6 |   <component name="CMakeRunConfigurationManager" shouldGenerate="true" shouldDeleteObsolete="true">
   7 |     <generated />
   8 |   </component>
   9 |   <component name="CMakeSettings">
  10 |     <configurations>
  11 |       <configuration PROFILE_NAME="Debug" CONFIG_NAME="Debug" />
  12 |     </configurations>
  13 |   </component>
  14 |   <component name="ChangeListManager">
  15 |     <list id="89211242-8b9c-4e07-9a48-c96746e62bba" name="adds data to the repository" comment="" />
  16 |     <list default="true" id="25b09a0b-845a-4c42-922d-18c337f24bbe" name="Default Changelist" comment="changes drnn res structure to match the papers">
  17 |       <change beforePath="$PROJECT_DIR$/.idea/11785ESRNN.iml" beforeDir="false" />
  18 |       <change beforePath="$PROJECT_DIR$/.idea/encodings.xml" beforeDir="false" />
  19 |       <change beforePath="$PROJECT_DIR$/.idea/misc.xml" beforeDir="false" />
  20 |       <change beforePath="$PROJECT_DIR$/.idea/modules.xml" beforeDir="false" />
  21 |       <change beforePath="$PROJECT_DIR$/.idea/vcs.xml" beforeDir="false" />
  22 |       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
  23 |       <change beforePath="$PROJECT_DIR$/data/Test/Daily-test.csv" beforeDir="false" />
  24 |       <change beforePath="$PROJECT_DIR$/data/Test/Hourly-test.csv" beforeDir="false" />
  25 |       <change beforePath="$PROJECT_DIR$/data/Test/Monthly-test.csv" beforeDir="false" />
  26 |       <change beforePath="$PROJECT_DIR$/data/Test/Quarterly-test-small.csv" beforeDir="false" />
  27 |       <change beforePath="$PROJECT_DIR$/data/Test/Quarterly-test.csv" beforeDir="false" />
  28 |       <change beforePath="$PROJECT_DIR$/data/Test/Weekly-test.csv" beforeDir="false" />
  29 |       <change beforePath="$PROJECT_DIR$/data/Test/Yearly-test.csv" beforeDir="false" />
  30 |       <change beforePath="$PROJECT_DIR$/data/Train/Daily-train.csv" beforeDir="false" />
  31 |       <change beforePath="$PROJECT_DIR$/data/Train/Hourly-train.csv" beforeDir="false" />
  32 |       <change beforePath="$PROJECT_DIR$/data/Train/Monthly-train.csv" beforeDir="false" />
  33 |       <change beforePath="$PROJECT_DIR$/data/Train/Quarterly-train-small.csv" beforeDir="false" />
  34 |       <change beforePath="$PROJECT_DIR$/data/Train/Quarterly-train.csv" beforeDir="false" />
  35 |       <change beforePath="$PROJECT_DIR$/data/Train/Weekly-train.csv" beforeDir="false" />
  36 |       <change beforePath="$PROJECT_DIR$/data/Train/Yearly-train.csv" beforeDir="false" />
  37 |       <change beforePath="$PROJECT_DIR$/data/info.csv" beforeDir="false" />
  38 |       <change beforePath="$PROJECT_DIR$/slawek_github/ES_RNN_SlawekSmyl.pdf" beforeDir="false" />
  39 |       <change beforePath="$PROJECT_DIR$/slawek_github/R/merge.R" beforeDir="false" />
  40 |       <change beforePath="$PROJECT_DIR$/slawek_github/R/merge_PI.R" beforeDir="false" />
  41 |       <change beforePath="$PROJECT_DIR$/slawek_github/R/readme.txt" beforeDir="false" />
  42 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/ES_RNN.cc" beforeDir="false" />
  43 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/ES_RNN_E.cc" beforeDir="false" />
  44 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/ES_RNN_E_PI.cc" beforeDir="false" />
  45 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/ES_RNN_PI.cc" beforeDir="false" />
  46 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/Working_ES_RNN.cc" beforeDir="false" />
  47 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/linux_example_scripts/build_mkl" beforeDir="false" />
  48 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/linux_example_scripts/readme.txt" beforeDir="false" />
  49 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/linux_example_scripts/run18" beforeDir="false" />
  50 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/readme.txt" beforeDir="false" />
  51 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/slstm.cpp" beforeDir="false" />
  52 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/slstm.h" beforeDir="false" />
  53 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M4.sln" beforeDir="false" />
  54 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M41/ES_RNN.cc" beforeDir="false" />
  55 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M41/M41.vcxproj" beforeDir="false" />
  56 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M41/M41.vcxproj.user" beforeDir="false" />
  57 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M41/slstm.cpp" beforeDir="false" />
  58 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M41/slstm.h" beforeDir="false" />
  59 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M42/ES_RNN_PI.cc" beforeDir="false" />
  60 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M42/M42.vcxproj" beforeDir="false" />
  61 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M42/M42.vcxproj.filters" beforeDir="false" />
  62 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M42/M42.vcxproj.user" beforeDir="false" />
  63 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M42/slstm.h" beforeDir="false" />
  64 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M43/ES_RNN_E.cc" beforeDir="false" />
  65 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M43/M43.filters" beforeDir="false" />
  66 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M43/M43.vcxproj" beforeDir="false" />
  67 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M43/M43.vcxproj.user" beforeDir="false" />
  68 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M43/slstm.h" beforeDir="false" />
  69 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M44/ES_RNN_E_PI.cc" beforeDir="false" />
  70 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M44/M44.filters" beforeDir="false" />
  71 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M44/M44.vcxproj" beforeDir="false" />
  72 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M44/M44.vcxproj.user" beforeDir="false" />
  73 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/M44/slstm.h" beforeDir="false" />
  74 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/readme.txt" beforeDir="false" />
  75 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/x64/RelWithDebug/readme.txt" beforeDir="false" />
  76 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/x64/RelWithDebug/run61.cmd" beforeDir="false" />
  77 |       <change beforePath="$PROJECT_DIR$/slawek_github/c++/windows_VisualStudio/x64/RelWithDebug/run61_e.cmd" beforeDir="false" />
  78 |       <change beforePath="$PROJECT_DIR$/slawek_github/readme.txt" beforeDir="false" />
  79 |       <change beforePath="$PROJECT_DIR$/slawek_github/sql/createM72nn_SQLServer.sql" beforeDir="false" />
  80 |       <change beforePath="$PROJECT_DIR$/slawek_github/sql/createM72nn_mysql.txt" beforeDir="false" />
  81 |       <change beforePath="$PROJECT_DIR$/slawek_github/sql/readme.txt" beforeDir="false" />
  82 |     </list>
  83 |     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
  84 |     <option name="TRACKING_ENABLED" value="true" />
  85 |     <option name="SHOW_DIALOG" value="false" />
  86 |     <option name="HIGHLIGHT_CONFLICTS" value="true" />
  87 |     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
  88 |     <option name="LAST_RESOLUTION" value="IGNORE" />
  89 |   </component>
  90 |   <component name="CoverageDataManager">
  91 |     <SUITE FILE_PATH="coverage/11785ESRNN$main.coverage" NAME="main Coverage Results" MODIFIED="1544651358355" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/es_rnn" />
  92 |     <SUITE FILE_PATH="coverage/11785ESRNN$config.coverage" NAME="config Coverage Results" MODIFIED="1543722718196" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/es_rnn" />
  93 |     <SUITE FILE_PATH="coverage/11785ESRNN$loss_modules.coverage" NAME="loss_modules Coverage Results" MODIFIED="1543721313964" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/es_rnn" />
  94 |     <SUITE FILE_PATH="coverage/11785ESRNN$data_loading.coverage" NAME="data_loading Coverage Results" MODIFIED="1544646872231" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/es_rnn" />
  95 |     <SUITE FILE_PATH="coverage/11785ESRNN$trainer.coverage" NAME="trainer Coverage Results" MODIFIED="1543721780237" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/es_rnn" />
  96 |   </component>
  97 |   <component name="FUSProjectUsageTrigger">
  98 |     <session id="283267010">
  99 |       <usages-collector id="statistics.file.extensions.open">
 100 |         <counts>
 101 |           <entry key="R" value="1" />
 102 |           <entry key="cc" value="1" />
 103 |           <entry key="cpp" value="2" />
 104 |           <entry key="py" value="28" />
 105 |           <entry key="sh" value="1" />
 106 |         </counts>
 107 |       </usages-collector>
 108 |       <usages-collector id="statistics.file.types.open">
 109 |         <counts>
 110 |           <entry key="C++" value="3" />
 111 |           <entry key="PLAIN_TEXT" value="2" />
 112 |           <entry key="Python" value="28" />
 113 |         </counts>
 114 |       </usages-collector>
 115 |       <usages-collector id="statistics.file.extensions.edit">
 116 |         <counts>
 117 |           <entry key="R" value="1" />
 118 |           <entry key="cc" value="2" />
 119 |           <entry key="py" value="7391" />
 120 |           <entry key="sh" value="32" />
 121 |           <entry key="txt" value="812" />
 122 |         </counts>
 123 |       </usages-collector>
 124 |       <usages-collector id="statistics.file.types.edit">
 125 |         <counts>
 126 |           <entry key="C++" value="2" />
 127 |           <entry key="PLAIN_TEXT" value="845" />
 128 |           <entry key="Python" value="7391" />
 129 |         </counts>
 130 |       </usages-collector>
 131 |       <usages-collector id="statistics.vcs.git.usages">
 132 |         <counts>
 133 |           <entry key="git.branch.compare" value="1" />
 134 |         </counts>
 135 |       </usages-collector>
 136 |       <usages-collector id="statistics.lifecycle.project">
 137 |         <counts>
 138 |           <entry key="project.closed" value="1" />
 139 |           <entry key="project.open.time.12" value="1" />
 140 |           <entry key="project.opened" value="1" />
 141 |         </counts>
 142 |       </usages-collector>
 143 |     </session>
 144 |   </component>
 145 |   <component name="FileEditorManager">
 146 |     <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
 147 |       <file leaf-file-name="data_loading.py" pinned="false" current-in-tab="false">
 148 |         <entry file="file://$PROJECT_DIR$/es_rnn/data_loading.py">
 149 |           <provider selected="true" editor-type-id="text-editor">
 150 |             <state relative-caret-position="234">
 151 |               <caret line="19" column="16" selection-start-line="19" selection-start-column="16" selection-end-line="19" selection-end-column="16" />
 152 |               <folding>
 153 |                 <element signature="e#0#18#0" expanded="true" />
 154 |               </folding>
 155 |             </state>
 156 |           </provider>
 157 |         </entry>
 158 |       </file>
 159 |       <file leaf-file-name="main.py" pinned="false" current-in-tab="true">
 160 |         <entry file="file://$PROJECT_DIR$/es_rnn/main.py">
 161 |           <provider selected="true" editor-type-id="text-editor">
 162 |             <state relative-caret-position="360">
 163 |               <caret line="20" column="78" selection-start-line="20" selection-start-column="78" selection-end-line="20" selection-end-column="78" />
 164 |               <folding>
 165 |                 <element signature="e#0#19#0" expanded="true" />
 166 |               </folding>
 167 |             </state>
 168 |           </provider>
 169 |         </entry>
 170 |       </file>
 171 |       <file leaf-file-name="config.py" pinned="false" current-in-tab="false">
 172 |         <entry file="file://$PROJECT_DIR$/es_rnn/config.py">
 173 |           <provider selected="true" editor-type-id="text-editor">
 174 |             <state relative-caret-position="432">
 175 |               <caret line="69" column="25" selection-start-line="69" selection-start-column="25" selection-end-line="69" selection-end-column="25" />
 176 |               <folding>
 177 |                 <element signature="e#0#21#0" expanded="true" />
 178 |               </folding>
 179 |             </state>
 180 |           </provider>
 181 |         </entry>
 182 |       </file>
 183 |       <file leaf-file-name="trainer.py" pinned="false" current-in-tab="false">
 184 |         <entry file="file://$PROJECT_DIR$/es_rnn/trainer.py">
 185 |           <provider selected="true" editor-type-id="text-editor">
 186 |             <state relative-caret-position="216">
 187 |               <caret line="12" column="70" selection-start-line="12" selection-start-column="70" selection-end-line="12" selection-end-column="70" />
 188 |               <folding>
 189 |                 <element signature="e#0#9#0" expanded="true" />
 190 |               </folding>
 191 |             </state>
 192 |           </provider>
 193 |         </entry>
 194 |       </file>
 195 |       <file leaf-file-name="model.py" pinned="false" current-in-tab="false">
 196 |         <entry file="file://$PROJECT_DIR$/es_rnn/model.py">
 197 |           <provider selected="true" editor-type-id="text-editor">
 198 |             <state relative-caret-position="2339">
 199 |               <caret line="154" column="59" selection-start-line="154" selection-start-column="59" selection-end-line="154" selection-end-column="59" />
 200 |               <folding>
 201 |                 <element signature="e#0#12#0" expanded="true" />
 202 |               </folding>
 203 |             </state>
 204 |           </provider>
 205 |         </entry>
 206 |       </file>
 207 |     </leaf>
 208 |   </component>
 209 |   <component name="FileTemplateManagerImpl">
 210 |     <option name="RECENT_TEMPLATES">
 211 |       <list>
 212 |         <option value="Python Script" />
 213 |       </list>
 214 |     </option>
 215 |   </component>
 216 |   <component name="FindInProjectRecents">
 217 |     <findStrings>
 218 |       <find>tau</find>
 219 |       <find>input_batch</find>
 220 |       <find>int i=(m4Obj.n - OUTPUT_SIZE_I); i&lt;m4Obj.n; i++</find>
 221 |       <find>Monthly</find>
 222 |       <find>epoch</find>
 223 |       <find>'num_of_chunks': 2,</find>
 224 |       <find>Month</find>
 225 |       <find>LEVEL</find>
 226 |       <find>lr_ratio</find>
 227 |       <find>rnnstack</find>
 228 |       <find>loss</find>
 229 |       <find>category</find>
 230 |       <find>test</find>
 231 |       <find>levelVarLoss_ex</find>
 232 |       <find>arx</find>
 233 |     </findStrings>
 234 |   </component>
 235 |   <component name="Git.Settings">
 236 |     <option name="PREVIOUS_COMMIT_AUTHORS">
 237 |       <list>
 238 |         <option value="Kaung M. Khin &lt;10078273+damitkwr@users.noreply.github.com&gt;" />
 239 |       </list>
 240 |     </option>
 241 |     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
 242 |   </component>
 243 |   <component name="IdeDocumentHistory">
 244 |     <option name="CHANGED_PATHS">
 245 |       <list>
 246 |         <option value="$PROJECT_DIR$/DRNN.py" />
 247 |         <option value="$PROJECT_DIR$/es_rnn/test_drnn.py" />
 248 |         <option value="$PROJECT_DIR$/es_rnn/DRNN.py" />
 249 |         <option value="$PROJECT_DIR$/es_rnn/ZDRNN.py" />
 250 |         <option value="$PROJECT_DIR$/es_rnn/Zmodel.py" />
 251 |         <option value="$PROJECT_DIR$/slawek_github/c++/ES_RNN.cc" />
 252 |         <option value="$PROJECT_DIR$/setup.sh" />
 253 |         <option value="$PROJECT_DIR$/__init__.py" />
 254 |         <option value="$PROJECT_DIR$/slawek_github/R/merge.R" />
 255 |         <option value="$PROJECT_DIR$/es_rnn/test.py" />
 256 |         <option value="$PROJECT_DIR$/es_rnn/loss_modules.py" />
 257 |         <option value="$PROJECT_DIR$/es_rnn/model.py" />
 258 |         <option value="$PROJECT_DIR$/es_rnn/trainer.py" />
 259 |         <option value="$PROJECT_DIR$/es_rnn/data_loading.py" />
 260 |         <option value="$PROJECT_DIR$/es_rnn/main.py" />
 261 |         <option value="$PROJECT_DIR$/es_rnn/config.py" />
 262 |       </list>
 263 |     </option>
 264 |   </component>
 265 |   <component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
 266 |   <component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
 267 |   <component name="JsGulpfileManager">
 268 |     <detection-done>true</detection-done>
 269 |     <sorting>DEFINITION_ORDER</sorting>
 270 |   </component>
 271 |   <component name="NodePackageJsonFileManager">
 272 |     <packageJsonPaths />
 273 |   </component>
 274 |   <component name="ProjectFrameBounds" extendedState="4">
 275 |     <option name="y" value="27" />
 276 |     <option name="width" value="1308" />
 277 |     <option name="height" value="1413" />
 278 |   </component>
 279 |   <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
 280 |   <component name="ProjectView">
 281 |     <navigator proportions="" version="1">
 282 |       <foldersAlwaysOnTop value="true" />
 283 |     </navigator>
 284 |     <panes>
 285 |       <pane id="ProjectPane">
 286 |         <subPane>
 287 |           <expand>
 288 |             <path>
 289 |               <item name="11785ESRNN" type="b2602c69:ProjectViewProjectNode" />
 290 |               <item name="11785ESRNN" type="462c0819:PsiDirectoryNode" />
 291 |             </path>
 292 |             <path>
 293 |               <item name="11785ESRNN" type="b2602c69:ProjectViewProjectNode" />
 294 |               <item name="11785ESRNN" type="462c0819:PsiDirectoryNode" />
 295 |               <item name="es_rnn" type="462c0819:PsiDirectoryNode" />
 296 |             </path>
 297 |           </expand>
 298 |           <select />
 299 |         </subPane>
 300 |       </pane>
 301 |       <pane id="Scope" />
 302 |       <pane id="AndroidView" />
 303 |       <pane id="PackagesPane" />
 304 |     </panes>
 305 |   </component>
 306 |   <component name="PropertiesComponent">
 307 |     <property name="WebServerToolWindowFactoryState" value="false" />
 308 |     <property name="last_opened_file_path" value="$PROJECT_DIR$" />
 309 |     <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
 310 |     <property name="nodejs_npm_path_reset_for_default_project" value="true" />
 311 |     <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
 312 |   </component>
 313 |   <component name="RecentsManager">
 314 |     <key name="CopyFile.RECENT_KEYS">
 315 |       <recent name="C:\Users\andre\OneDrive\School\CMU\Fall2018\11785\11785_project\11785ESRNN\es_rnn" />
 316 |       <recent name="$PROJECT_DIR$/es_rnn" />
 317 |     </key>
 318 |   </component>
 319 |   <component name="RunDashboard">
 320 |     <option name="ruleStates">
 321 |       <list>
 322 |         <RuleState>
 323 |           <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
 324 |         </RuleState>
 325 |         <RuleState>
 326 |           <option name="name" value="StatusDashboardGroupingRule" />
 327 |         </RuleState>
 328 |       </list>
 329 |     </option>
 330 |   </component>
 331 |   <component name="RunManager" selected="Python.main">
 332 |     <configuration name="config" type="PythonConfigurationType" factoryName="Python" temporary="true">
 333 |       <module name="11785ESRNN" />
 334 |       <option name="INTERPRETER_OPTIONS" value="" />
 335 |       <option name="PARENT_ENVS" value="true" />
 336 |       <envs>
 337 |         <env name="PYTHONUNBUFFERED" value="1" />
 338 |       </envs>
 339 |       <option name="SDK_HOME" value="" />
 340 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/es_rnn" />
 341 |       <option name="IS_MODULE_SDK" value="true" />
 342 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 343 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 344 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 345 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/es_rnn/config.py" />
 346 |       <option name="PARAMETERS" value="" />
 347 |       <option name="SHOW_COMMAND_LINE" value="false" />
 348 |       <option name="EMULATE_TERMINAL" value="false" />
 349 |       <option name="MODULE_MODE" value="false" />
 350 |     </configuration>
 351 |     <configuration name="data_loading" type="PythonConfigurationType" factoryName="Python" temporary="true">
 352 |       <module name="11785ESRNN" />
 353 |       <option name="INTERPRETER_OPTIONS" value="" />
 354 |       <option name="PARENT_ENVS" value="true" />
 355 |       <envs>
 356 |         <env name="PYTHONUNBUFFERED" value="1" />
 357 |       </envs>
 358 |       <option name="SDK_HOME" value="" />
 359 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/es_rnn" />
 360 |       <option name="IS_MODULE_SDK" value="true" />
 361 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 362 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 363 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 364 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/es_rnn/data_loading.py" />
 365 |       <option name="PARAMETERS" value="" />
 366 |       <option name="SHOW_COMMAND_LINE" value="false" />
 367 |       <option name="EMULATE_TERMINAL" value="false" />
 368 |       <option name="MODULE_MODE" value="false" />
 369 |     </configuration>
 370 |     <configuration name="loss_modules" type="PythonConfigurationType" factoryName="Python" temporary="true">
 371 |       <module name="11785ESRNN" />
 372 |       <option name="INTERPRETER_OPTIONS" value="" />
 373 |       <option name="PARENT_ENVS" value="true" />
 374 |       <envs>
 375 |         <env name="PYTHONUNBUFFERED" value="1" />
 376 |       </envs>
 377 |       <option name="SDK_HOME" value="" />
 378 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/es_rnn" />
 379 |       <option name="IS_MODULE_SDK" value="true" />
 380 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 381 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 382 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 383 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/es_rnn/loss_modules.py" />
 384 |       <option name="PARAMETERS" value="" />
 385 |       <option name="SHOW_COMMAND_LINE" value="false" />
 386 |       <option name="EMULATE_TERMINAL" value="false" />
 387 |       <option name="MODULE_MODE" value="false" />
 388 |     </configuration>
 389 |     <configuration name="main" type="PythonConfigurationType" factoryName="Python" temporary="true">
 390 |       <module name="11785ESRNN" />
 391 |       <option name="INTERPRETER_OPTIONS" value="" />
 392 |       <option name="PARENT_ENVS" value="true" />
 393 |       <envs>
 394 |         <env name="PYTHONUNBUFFERED" value="1" />
 395 |       </envs>
 396 |       <option name="SDK_HOME" value="" />
 397 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/es_rnn" />
 398 |       <option name="IS_MODULE_SDK" value="true" />
 399 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 400 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 401 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 402 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/es_rnn/main.py" />
 403 |       <option name="PARAMETERS" value="" />
 404 |       <option name="SHOW_COMMAND_LINE" value="false" />
 405 |       <option name="EMULATE_TERMINAL" value="false" />
 406 |       <option name="MODULE_MODE" value="false" />
 407 |     </configuration>
 408 |     <configuration name="test" type="PythonConfigurationType" factoryName="Python" temporary="true">
 409 |       <module name="11785ESRNN" />
 410 |       <option name="INTERPRETER_OPTIONS" value="" />
 411 |       <option name="PARENT_ENVS" value="true" />
 412 |       <envs>
 413 |         <env name="PYTHONUNBUFFERED" value="1" />
 414 |       </envs>
 415 |       <option name="SDK_HOME" value="" />
 416 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/es_rnn" />
 417 |       <option name="IS_MODULE_SDK" value="true" />
 418 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 419 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 420 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 421 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/es_rnn/test.py" />
 422 |       <option name="PARAMETERS" value="" />
 423 |       <option name="SHOW_COMMAND_LINE" value="false" />
 424 |       <option name="EMULATE_TERMINAL" value="false" />
 425 |       <option name="MODULE_MODE" value="false" />
 426 |     </configuration>
 427 |     <list>
 428 |       <item itemvalue="Python.main" />
 429 |       <item itemvalue="Python.loss_modules" />
 430 |       <item itemvalue="Python.config" />
 431 |       <item itemvalue="Python.test" />
 432 |       <item itemvalue="Python.data_loading" />
 433 |     </list>
 434 |     <recent_temporary>
 435 |       <list>
 436 |         <item itemvalue="Python.main" />
 437 |         <item itemvalue="Python.data_loading" />
 438 |         <item itemvalue="Python.test" />
 439 |         <item itemvalue="Python.config" />
 440 |         <item itemvalue="Python.loss_modules" />
 441 |       </list>
 442 |     </recent_temporary>
 443 |   </component>
 444 |   <component name="SvnConfiguration">
 445 |     <configuration />
 446 |   </component>
 447 |   <component name="TaskManager">
 448 |     <task active="true" id="Default" summary="Default task">
 449 |       <changelist id="25b09a0b-845a-4c42-922d-18c337f24bbe" name="Default Changelist" comment="" />
 450 |       <changelist id="89211242-8b9c-4e07-9a48-c96746e62bba" name="adds data to the repository" comment="" />
 451 |       <created>1543539477190</created>
 452 |       <option name="number" value="Default" />
 453 |       <option name="presentableId" value="Default" />
 454 |       <updated>1543539477190</updated>
 455 |       <workItem from="1543539478413" duration="63000" />
 456 |       <workItem from="1543539587836" duration="18000" />
 457 |       <workItem from="1543539620651" duration="72000" />
 458 |       <workItem from="1543539706985" duration="564000" />
 459 |       <workItem from="1543611359032" duration="625000" />
 460 |       <workItem from="1543617191506" duration="66000" />
 461 |       <workItem from="1543617280989" duration="2558000" />
 462 |     </task>
 463 |     <task id="LOCAL-00001" summary="Added first">
 464 |       <created>1543600238871</created>
 465 |       <option name="number" value="00001" />
 466 |       <option name="presentableId" value="LOCAL-00001" />
 467 |       <option name="project" value="LOCAL" />
 468 |       <updated>1543600238871</updated>
 469 |     </task>
 470 |     <task id="LOCAL-00002" summary="Finished stack, concat and padding the dilations">
 471 |       <created>1543602966477</created>
 472 |       <option name="number" value="00002" />
 473 |       <option name="presentableId" value="LOCAL-00002" />
 474 |       <option name="project" value="LOCAL" />
 475 |       <updated>1543602966477</updated>
 476 |     </task>
 477 |     <task id="LOCAL-00003" summary="Finished GPU and vectorizing loss">
 478 |       <created>1543719604560</created>
 479 |       <option name="number" value="00003" />
 480 |       <option name="presentableId" value="LOCAL-00003" />
 481 |       <option name="project" value="LOCAL" />
 482 |       <updated>1543719604560</updated>
 483 |     </task>
 484 |     <task id="LOCAL-00004" summary="Vectorized Pinball loss">
 485 |       <created>1543721406183</created>
 486 |       <option name="number" value="00004" />
 487 |       <option name="presentableId" value="LOCAL-00004" />
 488 |       <option name="project" value="LOCAL" />
 489 |       <updated>1543721406183</updated>
 490 |     </task>
 491 |     <task id="LOCAL-00005" summary="moves pinball device">
 492 |       <created>1543723379951</created>
 493 |       <option name="number" value="00005" />
 494 |       <option name="presentableId" value="LOCAL-00005" />
 495 |       <option name="project" value="LOCAL" />
 496 |       <updated>1543723379951</updated>
 497 |     </task>
 498 |     <task id="LOCAL-00006" summary="clean up">
 499 |       <created>1543725087499</created>
 500 |       <option name="number" value="00006" />
 501 |       <option name="presentableId" value="LOCAL-00006" />
 502 |       <option name="project" value="LOCAL" />
 503 |       <updated>1543725087499</updated>
 504 |     </task>
 505 |     <task id="LOCAL-00007" summary="adds logging and outputs the loss">
 506 |       <created>1543725119069</created>
 507 |       <option name="number" value="00007" />
 508 |       <option name="presentableId" value="LOCAL-00007" />
 509 |       <option name="project" value="LOCAL" />
 510 |       <updated>1543725119069</updated>
 511 |     </task>
 512 |     <task id="LOCAL-00008" summary="move seasonality to the gpu">
 513 |       <created>1543726712759</created>
 514 |       <option name="number" value="00008" />
 515 |       <option name="presentableId" value="LOCAL-00008" />
 516 |       <option name="project" value="LOCAL" />
 517 |       <updated>1543726712759</updated>
 518 |     </task>
 519 |     <task id="LOCAL-00009" summary="fixes parameter update issue&#10;it was a copy">
 520 |       <created>1543838298737</created>
 521 |       <option name="number" value="00009" />
 522 |       <option name="presentableId" value="LOCAL-00009" />
 523 |       <option name="project" value="LOCAL" />
 524 |       <updated>1543838298737</updated>
 525 |     </task>
 526 |     <task id="LOCAL-00010" summary="fixes parameter update issue&#10;it was a copy">
 527 |       <created>1543846966299</created>
 528 |       <option name="number" value="00010" />
 529 |       <option name="presentableId" value="LOCAL-00010" />
 530 |       <option name="project" value="LOCAL" />
 531 |       <updated>1543846966299</updated>
 532 |     </task>
 533 |     <task id="LOCAL-00011" summary="removes the skip connections">
 534 |       <created>1543868879271</created>
 535 |       <option name="number" value="00011" />
 536 |       <option name="presentableId" value="LOCAL-00011" />
 537 |       <option name="project" value="LOCAL" />
 538 |       <updated>1543868879271</updated>
 539 |     </task>
 540 |     <task id="LOCAL-00012" summary="ADDS VALIDATION STUFF: VECTORIZES SMAPE CALC, REVERTS BACK TO OLD RNN MODEL">
 541 |       <created>1543876626222</created>
 542 |       <option name="number" value="00012" />
 543 |       <option name="presentableId" value="LOCAL-00012" />
 544 |       <option name="project" value="LOCAL" />
 545 |       <updated>1543876626222</updated>
 546 |     </task>
 547 |     <task id="LOCAL-00013" summary="updates and improves the config file">
 548 |       <created>1543902881155</created>
 549 |       <option name="number" value="00013" />
 550 |       <option name="presentableId" value="LOCAL-00013" />
 551 |       <option name="project" value="LOCAL" />
 552 |       <updated>1543902881155</updated>
 553 |     </task>
 554 |     <task id="LOCAL-00014" summary="updates and improves the config file">
 555 |       <created>1543903082744</created>
 556 |       <option name="number" value="00014" />
 557 |       <option name="presentableId" value="LOCAL-00014" />
 558 |       <option name="project" value="LOCAL" />
 559 |       <updated>1543903082744</updated>
 560 |     </task>
 561 |     <task id="LOCAL-00015" summary="adds data to the repository">
 562 |       <created>1543903726870</created>
 563 |       <option name="number" value="00015" />
 564 |       <option name="presentableId" value="LOCAL-00015" />
 565 |       <option name="project" value="LOCAL" />
 566 |       <updated>1543903726870</updated>
 567 |     </task>
 568 |     <task id="LOCAL-00016" summary="removes all sorts of stuff">
 569 |       <created>1543904277264</created>
 570 |       <option name="number" value="00016" />
 571 |       <option name="presentableId" value="LOCAL-00016" />
 572 |       <option name="project" value="LOCAL" />
 573 |       <updated>1543904277264</updated>
 574 |     </task>
 575 |     <task id="LOCAL-00017" summary="removes all sorts of stuff">
 576 |       <created>1543904283610</created>
 577 |       <option name="number" value="00017" />
 578 |       <option name="presentableId" value="LOCAL-00017" />
 579 |       <option name="project" value="LOCAL" />
 580 |       <updated>1543904283610</updated>
 581 |     </task>
 582 |     <task id="LOCAL-00018" summary="changes drnn res structure to match the papers">
 583 |       <created>1543904952238</created>
 584 |       <option name="number" value="00018" />
 585 |       <option name="presentableId" value="LOCAL-00018" />
 586 |       <option name="project" value="LOCAL" />
 587 |       <updated>1543904952238</updated>
 588 |     </task>
 589 |     <task id="LOCAL-00019" summary="changes drnn res structure to match the papers">
 590 |       <created>1543905034444</created>
 591 |       <option name="number" value="00019" />
 592 |       <option name="presentableId" value="LOCAL-00019" />
 593 |       <option name="project" value="LOCAL" />
 594 |       <updated>1543905034444</updated>
 595 |     </task>
 596 |     <task id="LOCAL-00020" summary="changes drnn res structure to match the papers">
 597 |       <created>1543905950362</created>
 598 |       <option name="number" value="00020" />
 599 |       <option name="presentableId" value="LOCAL-00020" />
 600 |       <option name="project" value="LOCAL" />
 601 |       <updated>1543905950363</updated>
 602 |     </task>
 603 |     <task id="LOCAL-00021" summary="moves add_nl_layer to a model object attribute rather than a forward parameter also connects it to the config">
 604 |       <created>1543943694142</created>
 605 |       <option name="number" value="00021" />
 606 |       <option name="presentableId" value="LOCAL-00021" />
 607 |       <option name="project" value="LOCAL" />
 608 |       <updated>1543943694142</updated>
 609 |     </task>
 610 |     <task id="LOCAL-00022" summary="adds back dev set">
 611 |       <created>1543947937608</created>
 612 |       <option name="number" value="00022" />
 613 |       <option name="presentableId" value="LOCAL-00022" />
 614 |       <option name="project" value="LOCAL" />
 615 |       <updated>1543947937608</updated>
 616 |     </task>
 617 |     <task id="LOCAL-00023" summary="beginning of holdout">
 618 |       <created>1543954619902</created>
 619 |       <option name="number" value="00023" />
 620 |       <option name="presentableId" value="LOCAL-00023" />
 621 |       <option name="project" value="LOCAL" />
 622 |       <updated>1543954619902</updated>
 623 |     </task>
 624 |     <task id="LOCAL-00024" summary="fixes error in MAPE calculation and adds numpy vectorized version">
 625 |       <created>1543970855517</created>
 626 |       <option name="number" value="00024" />
 627 |       <option name="presentableId" value="LOCAL-00024" />
 628 |       <option name="project" value="LOCAL" />
 629 |       <updated>1543970855518</updated>
 630 |     </task>
 631 |     <task id="LOCAL-00025" summary="adds holdout mape calculation by domain">
 632 |       <created>1543970911713</created>
 633 |       <option name="number" value="00025" />
 634 |       <option name="presentableId" value="LOCAL-00025" />
 635 |       <option name="project" value="LOCAL" />
 636 |       <updated>1543970911713</updated>
 637 |     </task>
 638 |     <task id="LOCAL-00026" summary="builds a dynamic residual dilated lstm based on the dilations vector">
 639 |       <created>1543979186309</created>
 640 |       <option name="number" value="00026" />
 641 |       <option name="presentableId" value="LOCAL-00026" />
 642 |       <option name="project" value="LOCAL" />
 643 |       <updated>1543979186309</updated>
 644 |     </task>
 645 |     <task id="LOCAL-00027" summary="changes drnn res structure to match the papers">
 646 |       <created>1543980792285</created>
 647 |       <option name="number" value="00027" />
 648 |       <option name="presentableId" value="LOCAL-00027" />
 649 |       <option name="project" value="LOCAL" />
 650 |       <updated>1543980792285</updated>
 651 |     </task>
 652 |     <task id="LOCAL-00028" summary="comment the level penalty for now">
 653 |       <created>1543981524611</created>
 654 |       <option name="number" value="00028" />
 655 |       <option name="presentableId" value="LOCAL-00028" />
 656 |       <option name="project" value="LOCAL" />
 657 |       <updated>1543981524611</updated>
 658 |     </task>
 659 |     <task id="LOCAL-00029" summary="adds train mask to filter out test and validation values">
 660 |       <created>1544045549839</created>
 661 |       <option name="number" value="00029" />
 662 |       <option name="presentableId" value="LOCAL-00029" />
 663 |       <option name="project" value="LOCAL" />
 664 |       <updated>1544045549839</updated>
 665 |     </task>
 666 |     <task id="LOCAL-00030" summary="changes drnn res structure to match the papers">
 667 |       <created>1544045704126</created>
 668 |       <option name="number" value="00030" />
 669 |       <option name="presentableId" value="LOCAL-00030" />
 670 |       <option name="project" value="LOCAL" />
 671 |       <updated>1544045704126</updated>
 672 |     </task>
 673 |     <option name="localTasksCounter" value="31" />
 674 |     <servers />
 675 |   </component>
 676 |   <component name="TimeTrackingManager">
 677 |     <option name="totallyTimeSpent" value="3966000" />
 678 |   </component>
 679 |   <component name="TodoView" selected-index="4" />
 680 |   <component name="ToolWindowManager">
 681 |     <frame x="0" y="-3" width="1318" height="1443" extended-state="4" />
 682 |     <editor active="true" />
 683 |     <layout>
 684 |       <window_info anchor="bottom" id="TODO" order="6" />
 685 |       <window_info anchor="bottom" id="Event Log" order="10" side_tool="true" />
 686 |       <window_info anchor="bottom" id="Database Changes" order="7" />
 687 |       <window_info active="true" anchor="bottom" id="Version Control" order="8" visible="true" weight="0.32290077" />
 688 |       <window_info anchor="bottom" id="Python Console" order="12" />
 689 |       <window_info anchor="bottom" id="Run" order="2" weight="0.32900763" />
 690 |       <window_info anchor="bottom" id="Terminal" order="9" />
 691 |       <window_info content_ui="combo" id="Project" order="0" weight="0.18003145" />
 692 |       <window_info anchor="bottom" id="Docker" order="11" show_stripe_button="false" />
 693 |       <window_info anchor="right" id="Database" order="4" />
 694 |       <window_info anchor="right" id="SciView" order="3" weight="0.3294025" />
 695 |       <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
 696 |       <window_info id="Favorites" order="2" side_tool="true" />
 697 |       <window_info anchor="bottom" id="Debug" order="3" weight="0.32900763" />
 698 |       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
 699 |       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
 700 |       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
 701 |       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
 702 |       <window_info anchor="bottom" id="Message" order="0" />
 703 |       <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
 704 |       <window_info anchor="bottom" id="Find" order="1" weight="0.3291815" />
 705 |     </layout>
 706 |   </component>
 707 |   <component name="TypeScriptGeneratedFilesManager">
 708 |     <option name="version" value="1" />
 709 |   </component>
 710 |   <component name="UnknownFeatures">
 711 |     <option featureType="com.intellij.fileTypeFactory" implementationName="*.sh" />
 712 |   </component>
 713 |   <component name="Vcs.Log.Tabs.Properties">
 714 |     <option name="TAB_STATES">
 715 |       <map>
 716 |         <entry key="MAIN">
 717 |           <value>
 718 |             <State>
 719 |               <option name="RECENTLY_FILTERED_USER_GROUPS">
 720 |                 <collection />
 721 |               </option>
 722 |               <option name="RECENTLY_FILTERED_BRANCH_GROUPS">
 723 |                 <collection />
 724 |               </option>
 725 |               <option name="COLUMN_ORDER">
 726 |                 <list>
 727 |                   <option value="0" />
 728 |                   <option value="1" />
 729 |                   <option value="2" />
 730 |                   <option value="3" />
 731 |                 </list>
 732 |               </option>
 733 |             </State>
 734 |           </value>
 735 |         </entry>
 736 |       </map>
 737 |     </option>
 738 |   </component>
 739 |   <component name="VcsContentAnnotationSettings">
 740 |     <option name="myLimit" value="2678400000" />
 741 |   </component>
 742 |   <component name="VcsManagerConfiguration">
 743 |     <ignored-roots>
 744 |       <path value="$PROJECT_DIR$/pytorch-dilated-rnn" />
 745 |     </ignored-roots>
 746 |     <MESSAGE value="Added first" />
 747 |     <MESSAGE value="Finished stack, concat and padding the dilations" />
 748 |     <MESSAGE value="Finished GPU and vectorizing loss" />
 749 |     <MESSAGE value="Vectorized Pinball loss" />
 750 |     <MESSAGE value="moves pinball device" />
 751 |     <MESSAGE value="clean up" />
 752 |     <MESSAGE value="adds logging and outputs the loss" />
 753 |     <MESSAGE value="move seasonality to the gpu" />
 754 |     <MESSAGE value="fixes parameter update issue&#10;it was a copy" />
 755 |     <MESSAGE value="removes the skip connections" />
 756 |     <MESSAGE value="ADDS VALIDATION STUFF: VECTORIZES SMAPE CALC, REVERTS BACK TO OLD RNN MODEL" />
 757 |     <MESSAGE value="updates and improves the config file" />
 758 |     <MESSAGE value="adds data to the repository" />
 759 |     <MESSAGE value="removes all sorts of stuff" />
 760 |     <MESSAGE value="moves add_nl_layer to a model object attribute rather than a forward parameter also connects it to the config" />
 761 |     <MESSAGE value="adds back dev set" />
 762 |     <MESSAGE value="beginning of holdout" />
 763 |     <MESSAGE value="fixes error in MAPE calculation and adds numpy vectorized version" />
 764 |     <MESSAGE value="adds holdout mape calculation by domain" />
 765 |     <MESSAGE value="builds a dynamic residual dilated lstm based on the dilations vector" />
 766 |     <MESSAGE value="comment the level penalty for now" />
 767 |     <MESSAGE value="adds train mask to filter out test and validation values" />
 768 |     <MESSAGE value="changes drnn res structure to match the papers" />
 769 |     <option name="LAST_COMMIT_MESSAGE" value="changes drnn res structure to match the papers" />
 770 |     <option name="REFORMAT_BEFORE_PROJECT_COMMIT" value="true" />
 771 |   </component>
 772 |   <component name="XDebuggerManager">
 773 |     <breakpoint-manager>
 774 |       <breakpoints>
 775 |         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
 776 |           <url>file://$PROJECT_DIR$/es_rnn/data_loading.py</url>
 777 |           <line>53</line>
 778 |           <option name="timeStamp" value="3" />
 779 |         </line-breakpoint>
 780 |       </breakpoints>
 781 |       <option name="time" value="4" />
 782 |     </breakpoint-manager>
 783 |   </component>
 784 |   <component name="debuggerHistoryManager">
 785 |     <expressions id="evaluateCodeFragment">
 786 |       <expression>
 787 |         <expression-string>self.config['output_size']*self.config['batch_size']</expression-string>
 788 |         <language-id>Python</language-id>
 789 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
 790 |       </expression>
 791 |       <expression>
 792 |         <expression-string>self.config['output_size'])*self.config['batch_size']</expression-string>
 793 |         <language-id>Python</language-id>
 794 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
 795 |       </expression>
 796 |       <expression>
 797 |         <expression-string>self.config['output_size']) * self.config['batch_size']</expression-string>
 798 |         <language-id>Python</language-id>
 799 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
 800 |       </expression>
 801 |       <expression>
 802 |         <expression-string>self.config['output_size']) * self.config[
 803 |             'batch_size']</expression-string>
 804 |         <language-id>Python</language-id>
 805 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
 806 |       </expression>
 807 |       <expression>
 808 |         <expression-string>np.concatenate((info_cat.detach().cpu().numpy(), 
 809 |                                                     hold_out_pred.detach().cpu().numpy(), 
 810 |                                                     hold_out_act.detach().cpu().numpy()), axis=1)</expression-string>
 811 |         <language-id>Python</language-id>
 812 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
 813 |       </expression>
 814 |     </expressions>
 815 |     <expressions id="evaluateExpression">
 816 |       <expression>
 817 |         <expression-string>window_input[:-self.config['output_size']]</expression-string>
 818 |         <language-id>Python</language-id>
 819 |         <evaluation-mode>EXPRESSION</evaluation-mode>
 820 |       </expression>
 821 |       <expression>
 822 |         <expression-string>window_input[-self.config['output_size']:]</expression-string>
 823 |         <language-id>Python</language-id>
 824 |         <evaluation-mode>EXPRESSION</evaluation-mode>
 825 |       </expression>
 826 |       <expression>
 827 |         <expression-string>np.concatenate((train[0], vals[0], test[0]))</expression-string>
 828 |         <language-id>Python</language-id>
 829 |         <evaluation-mode>EXPRESSION</evaluation-mode>
 830 |       </expression>
 831 |       <expression>
 832 |         <expression-string>train[0]</expression-string>
 833 |         <language-id>Python</language-id>
 834 |         <evaluation-mode>EXPRESSION</evaluation-mode>
 835 |       </expression>
 836 |       <expression>
 837 |         <expression-string>train[0] + vals[0] + test[0]</expression-string>
 838 |         <language-id>Python</language-id>
 839 |         <evaluation-mode>EXPRESSION</evaluation-mode>
 840 |       </expression>
 841 |       <expression>
 842 |         <expression-string>train[0] + val[0] + test[0]</expression-string>
 843 |         <language-id>Python</language-id>
 844 |         <evaluation-mode>EXPRESSION</evaluation-mode>
 845 |       </expression>
 846 |       <expression>
 847 |         <expression-string>x.preds</expression-string>
 848 |         <language-id>Python</language-id>
 849 |         <evaluation-mode>EXPRESSION</evaluation-mode>
 850 |       </expression>
 851 |       <expression>
 852 |         <expression-string>x.shape[0]</expression-string>
 853 |         <language-id>Python</language-id>
 854 |         <evaluation-mode>EXPRESSION</evaluation-mode>
 855 |       </expression>
 856 |       <expression>
 857 |         <expression-string>predictions = torch.from_numpy(np.array(predictions))</expression-string>
 858 |         <language-id>Python</language-id>
 859 |         <evaluation-mode>EXPRESSION</evaluation-mode>
 860 |       </expression>
 861 |     </expressions>
 862 |   </component>
 863 |   <component name="editorHistoryManager">
 864 |     <entry file="file://$PROJECT_DIR$/slawek_github/c++/ES_RNN.cc">
 865 |       <provider selected="true" editor-type-id="text-editor">
 866 |         <state relative-caret-position="10872">
 867 |           <caret line="604" column="23" selection-start-line="604" selection-start-column="23" selection-end-line="604" selection-end-column="23" />
 868 |         </state>
 869 |       </provider>
 870 |     </entry>
 871 |     <entry file="file://$PROJECT_DIR$/es_rnn/main.py">
 872 |       <provider selected="true" editor-type-id="text-editor">
 873 |         <state relative-caret-position="144">
 874 |           <caret line="14" selection-start-line="14" selection-end-line="14" />
 875 |           <folding>
 876 |             <element signature="e#0#19#0" expanded="true" />
 877 |           </folding>
 878 |         </state>
 879 |       </provider>
 880 |     </entry>
 881 |     <entry file="file://$PROJECT_DIR$/es_rnn/data_loading.py">
 882 |       <provider selected="true" editor-type-id="text-editor">
 883 |         <state relative-caret-position="630">
 884 |           <caret line="38" selection-start-line="38" selection-end-line="38" />
 885 |           <folding>
 886 |             <element signature="e#0#18#0" expanded="true" />
 887 |           </folding>
 888 |         </state>
 889 |       </provider>
 890 |     </entry>
 891 |     <entry file="file://$PROJECT_DIR$/es_rnn/trainer.py">
 892 |       <provider selected="true" editor-type-id="text-editor">
 893 |         <state relative-caret-position="1134">
 894 |           <caret line="71" column="34" lean-forward="true" selection-start-line="71" selection-start-column="34" selection-end-line="71" selection-end-column="34" />
 895 |           <folding>
 896 |             <element signature="e#0#9#0" expanded="true" />
 897 |           </folding>
 898 |         </state>
 899 |       </provider>
 900 |     </entry>
 901 |     <entry file="file://$PROJECT_DIR$/slawek_github/c++/slstm.cpp">
 902 |       <provider selected="true" editor-type-id="text-editor">
 903 |         <state relative-caret-position="54">
 904 |           <caret line="3" column="83" selection-start-line="3" selection-start-column="83" selection-end-line="3" selection-end-column="115" />
 905 |         </state>
 906 |       </provider>
 907 |     </entry>
 908 |     <entry file="file://$PROJECT_DIR$/es_rnn/config.py">
 909 |       <provider selected="true" editor-type-id="text-editor">
 910 |         <state relative-caret-position="180">
 911 |           <caret line="12" column="32" selection-start-line="12" selection-start-column="32" selection-end-line="12" selection-end-column="32" />
 912 |           <folding>
 913 |             <element signature="e#0#21#0" expanded="true" />
 914 |           </folding>
 915 |         </state>
 916 |       </provider>
 917 |     </entry>
 918 |     <entry file="file://$PROJECT_DIR$/es_rnn/model.py">
 919 |       <provider selected="true" editor-type-id="text-editor">
 920 |         <state relative-caret-position="1854">
 921 |           <caret line="105" column="41" selection-start-line="105" selection-start-column="41" selection-end-line="105" selection-end-column="41" />
 922 |           <folding>
 923 |             <element signature="e#0#12#0" expanded="true" />
 924 |           </folding>
 925 |         </state>
 926 |       </provider>
 927 |     </entry>
 928 |     <entry file="file://$PROJECT_DIR$/utils/logger.py">
 929 |       <provider selected="true" editor-type-id="text-editor">
 930 |         <state relative-caret-position="68">
 931 |           <caret line="4" column="17" selection-start-line="4" selection-start-column="17" selection-end-line="4" selection-end-column="17" />
 932 |           <folding>
 933 |             <element signature="e#0#23#0" expanded="true" />
 934 |           </folding>
 935 |         </state>
 936 |       </provider>
 937 |     </entry>
 938 |     <entry file="file://$PROJECT_DIR$/DRNN.py">
 939 |       <provider selected="true" editor-type-id="text-editor">
 940 |         <state relative-caret-position="268">
 941 |           <caret line="90" selection-start-line="90" selection-end-line="101" selection-end-column="40" />
 942 |         </state>
 943 |       </provider>
 944 |     </entry>
 945 |     <entry file="file://$USER_HOME$/Anaconda3/envs/pytorch_p36/Lib/site-packages/torch/nn/modules/rnn.py" />
 946 |     <entry file="file://$PROJECT_DIR$/es_rnn/test_drnn.py" />
 947 |     <entry file="file://$PROJECT_DIR$/es_rnn/gpu_trainer.py" />
 948 |     <entry file="file://$USER_HOME$/anaconda3/envs/tensorflow18/lib/python3.6/site-packages/torch/nn/modules/module.py" />
 949 |     <entry file="file://$PROJECT_DIR$/es_rnn/DRNN.py">
 950 |       <provider selected="true" editor-type-id="text-editor">
 951 |         <state relative-caret-position="-355">
 952 |           <caret line="50" column="20" selection-start-line="50" selection-start-column="20" selection-end-line="50" selection-end-column="20" />
 953 |         </state>
 954 |       </provider>
 955 |     </entry>
 956 |     <entry file="file://$PROJECT_DIR$/es_rnn/Zmodel.py" />
 957 |     <entry file="file://$PROJECT_DIR$/es_rnn/ZDRNN.py" />
 958 |     <entry file="file://$PROJECT_DIR$/slawek_github/c++/Working_ES_RNN.cc">
 959 |       <provider selected="true" editor-type-id="text-editor">
 960 |         <state relative-caret-position="543">
 961 |           <caret line="58" column="37" selection-start-line="58" selection-start-column="12" selection-end-line="58" selection-end-column="37" />
 962 |         </state>
 963 |       </provider>
 964 |     </entry>
 965 |     <entry file="file://$PROJECT_DIR$/setup.sh">
 966 |       <provider selected="true" editor-type-id="text-editor">
 967 |         <state relative-caret-position="238">
 968 |           <caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
 969 |         </state>
 970 |       </provider>
 971 |     </entry>
 972 |     <entry file="file://$USER_HOME$/Anaconda3/envs/pytorch_p36/Lib/site-packages/torch/nn/modules/module.py" />
 973 |     <entry file="file://$PROJECT_DIR$/slawek_github/R/merge.R">
 974 |       <provider selected="true" editor-type-id="text-editor">
 975 |         <state relative-caret-position="-238">
 976 |           <caret column="31" selection-start-column="23" selection-end-column="31" />
 977 |         </state>
 978 |       </provider>
 979 |     </entry>
 980 |     <entry file="file://$USER_HOME$/Anaconda3/envs/pytorch_p36/Lib/site-packages/torch/tensor.py" />
 981 |     <entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/pydevd.py">
 982 |       <provider selected="true" editor-type-id="text-editor">
 983 |         <state relative-caret-position="110">
 984 |           <caret line="1067" selection-start-line="1067" selection-end-line="1067" />
 985 |         </state>
 986 |       </provider>
 987 |     </entry>
 988 |     <entry file="file://$USER_HOME$/Anaconda3/envs/pytorch_p36/Lib/site-packages/pandas/core/generic.py" />
 989 |     <entry file="file://$PROJECT_DIR$/__init__.py">
 990 |       <provider selected="true" editor-type-id="text-editor" />
 991 |     </entry>
 992 |     <entry file="file://$USER_HOME$/Anaconda3/envs/pytorch_p36/Lib/site-packages/pandas/core/series.py" />
 993 |     <entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/_pydev_imps/_pydev_execfile.py">
 994 |       <provider selected="true" editor-type-id="text-editor">
 995 |         <state relative-caret-position="289">
 996 |           <caret line="17" selection-start-line="17" selection-end-line="17" />
 997 |         </state>
 998 |       </provider>
 999 |     </entry>
1000 |     <entry file="file://$USER_HOME$/Anaconda3/envs/pytorch_p36/Lib/site-packages/pandas/core/groupby/groupby.py" />
1001 |     <entry file="file://$PROJECT_DIR$/es_rnn/loss_modules.py">
1002 |       <provider selected="true" editor-type-id="text-editor">
1003 |         <state relative-caret-position="107">
1004 |           <caret line="75" column="12" selection-start-line="75" selection-start-column="12" selection-end-line="75" selection-end-column="12" />
1005 |           <folding>
1006 |             <element signature="e#0#12#0" expanded="true" />
1007 |           </folding>
1008 |         </state>
1009 |       </provider>
1010 |     </entry>
1011 |     <entry file="file://$PROJECT_DIR$/es_rnn/test.py" />
1012 |     <entry file="file://$USER_HOME$/Anaconda3/envs/pytorch_p36/Lib/site-packages/torch/nn/functional.py" />
1013 |     <entry file="file://$PROJECT_DIR$/es_rnn/config.py">
1014 |       <provider selected="true" editor-type-id="text-editor">
1015 |         <state relative-caret-position="432">
1016 |           <caret line="69" column="25" selection-start-line="69" selection-start-column="25" selection-end-line="69" selection-end-column="25" />
1017 |           <folding>
1018 |             <element signature="e#0#21#0" expanded="true" />
1019 |           </folding>
1020 |         </state>
1021 |       </provider>
1022 |     </entry>
1023 |     <entry file="file://$PROJECT_DIR$/es_rnn/trainer.py">
1024 |       <provider selected="true" editor-type-id="text-editor">
1025 |         <state relative-caret-position="216">
1026 |           <caret line="12" column="70" selection-start-line="12" selection-start-column="70" selection-end-line="12" selection-end-column="70" />
1027 |           <folding>
1028 |             <element signature="e#0#9#0" expanded="true" />
1029 |           </folding>
1030 |         </state>
1031 |       </provider>
1032 |     </entry>
1033 |     <entry file="file://$PROJECT_DIR$/es_rnn/model.py">
1034 |       <provider selected="true" editor-type-id="text-editor">
1035 |         <state relative-caret-position="2339">
1036 |           <caret line="154" column="59" selection-start-line="154" selection-start-column="59" selection-end-line="154" selection-end-column="59" />
1037 |           <folding>
1038 |             <element signature="e#0#12#0" expanded="true" />
1039 |           </folding>
1040 |         </state>
1041 |       </provider>
1042 |     </entry>
1043 |     <entry file="file://$PROJECT_DIR$/slawek_github/c++/slstm.cpp">
1044 |       <provider selected="true" editor-type-id="text-editor">
1045 |         <state relative-caret-position="18">
1046 |           <caret line="3" column="83" selection-start-line="3" selection-start-column="83" selection-end-line="3" selection-end-column="83" />
1047 |         </state>
1048 |       </provider>
1049 |     </entry>
1050 |     <entry file="file://$PROJECT_DIR$/slawek_github/c++/ES_RNN.cc">
1051 |       <provider selected="true" editor-type-id="text-editor">
1052 |         <state relative-caret-position="162">
1053 |           <caret line="124" selection-start-line="124" selection-end-line="124" />
1054 |         </state>
1055 |       </provider>
1056 |     </entry>
1057 |     <entry file="file://$PROJECT_DIR$/es_rnn/data_loading.py">
1058 |       <provider selected="true" editor-type-id="text-editor">
1059 |         <state relative-caret-position="234">
1060 |           <caret line="19" column="16" selection-start-line="19" selection-start-column="16" selection-end-line="19" selection-end-column="16" />
1061 |           <folding>
1062 |             <element signature="e#0#18#0" expanded="true" />
1063 |           </folding>
1064 |         </state>
1065 |       </provider>
1066 |     </entry>
1067 |     <entry file="file://$PROJECT_DIR$/es_rnn/main.py">
1068 |       <provider selected="true" editor-type-id="text-editor">
1069 |         <state relative-caret-position="360">
1070 |           <caret line="20" column="78" selection-start-line="20" selection-start-column="78" selection-end-line="20" selection-end-column="78" />
1071 |           <folding>
1072 |             <element signature="e#0#19#0" expanded="true" />
1073 |           </folding>
1074 |         </state>
1075 |       </provider>
1076 |     </entry>
1077 |   </component>
1078 | </project>


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2018 Kaung M. Khin, Aldo Marini Macouzet and Andrew Redd
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Fast ES-RNN: A GPU Implementation of the ES-RNN Algorithm
 2 | 
 3 | A GPU-enabled version of the [hybrid ES-RNN model](https://eng.uber.com/m4-forecasting-competition/) by Slawek et al that won the M4 time-series forecasting competition by a large margin. The details of our implementation and the results are discussed in detail on this [paper](https://arxiv.org/abs/1907.03329)
 4 | 
 5 | ## Getting Started
 6 | 
 7 | ### Prerequisites
 8 | 
 9 | ```
10 | Python (3.5+)
11 | Tensorflow (1.12+ to 1.14)
12 | PyTorch (0.4.1)
13 | Zalando Research's Dilated RNN
14 | ```
15 | 
16 | ### Dataset
17 | 
18 | Please download the M4 competition dataset directly from [here](https://github.com/M4Competition/M4-methods/tree/master/Dataset) and put the files in the data directory.
19 | 
20 | ### Running the algorithm
21 | 
22 | Either use an IDE such as PyCharm or make sure to add the es\_rnn folder to your PYTHON PATH before running the [main.py](es_rnn/main.py) in the es\_rnn folder. You can change the configurations of the algorithm in the [config.py](es_rnn/config.py) file.
23 | 
24 | ## Built With
25 | 
26 | * [Python](https://www.python.org) - The *data science* language ;)
27 | * [PyTorch](https://www.pytorch.org/) - The dynamic framework for computation
28 | 
29 | 
30 | ## Authors
31 | 
32 | * **Andrew Redd** - [aredd-cmu](https://github.com/aredd-cmu)
33 | * **Kaung Khin** - [damitkwr](https://github.com/damitkwr)
34 | * **Aldo Marini** - [catapulta](https://github.com/catapulta)
35 | 
36 | ## License
37 | 
38 | This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details
39 | 
40 | ## Acknowledgments
41 | 
42 | * Thank you to the original author of the algorithm Smyl Slawek [slaweks17](https://github.com/slaweks17) for advice and for creating this amazing algorithm
43 | * Zalando Research [zalandoresearch](https://www.github.com/zalandoresearch) for their implementation of Dilated RNN
44 | 
45 | ## Citation
46 | 
47 | If you choose to use our implementation in your work please cite us as:
48 | 
49 | ```
50 | @article{ReddKhinMarini,
51 |        author = {{Redd}, Andrew and {Khin}, Kaung and {Marini}, Aldo},
52 |         title = "{Fast ES-RNN: A GPU Implementation of the ES-RNN Algorithm}",
53 |       journal = {arXiv e-prints},
54 |          year = "2019",
55 |         month = "Jul",
56 |           eid = {arXiv:1907.03329},
57 |         pages = {arXiv:1907.03329},
58 | archivePrefix = {arXiv},
59 |        eprint = {1907.03329},
60 |  primaryClass = {cs.LG}
61 | }
62 | ```
63 | 
64 | 
65 | #
66 | 


--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/damitkwr/ESRNN-GPU/918d192bf1516025978080aaea8e1ee68098c52a/__init__.py


--------------------------------------------------------------------------------
/es_rnn/DRNN.py:
--------------------------------------------------------------------------------
  1 | # lovingly borrowed from https://github.com/zalandoresearch/pytorch-dilated-rnn
  2 | 
  3 | import torch
  4 | import torch.nn as nn
  5 | import torch.autograd as autograd
  6 | 
  7 | use_cuda = torch.cuda.is_available()
  8 | 
  9 | 
 10 | class DRNN(nn.Module):
 11 | 
 12 |     def __init__(self, n_input, n_hidden, n_layers, dilations, dropout=0, cell_type='GRU', batch_first=False):
 13 | 
 14 |         super(DRNN, self).__init__()
 15 | 
 16 |         self.dilations = dilations
 17 |         self.cell_type = cell_type
 18 |         self.batch_first = batch_first
 19 | 
 20 |         layers = []
 21 |         if self.cell_type == "GRU":
 22 |             cell = nn.GRU
 23 |         elif self.cell_type == "RNN":
 24 |             cell = nn.RNN
 25 |         elif self.cell_type == "LSTM":
 26 |             cell = nn.LSTM
 27 |         else:
 28 |             raise NotImplementedError
 29 | 
 30 |         for i in range(n_layers):
 31 |             if i == 0:
 32 |                 c = cell(n_input, n_hidden, dropout=dropout)
 33 |             else:
 34 |                 c = cell(n_hidden, n_hidden, dropout=dropout)
 35 |             layers.append(c)
 36 |         self.cells = nn.Sequential(*layers)
 37 | 
 38 |     def forward(self, inputs, hidden=None):
 39 |         if self.batch_first:
 40 |             inputs = inputs.transpose(0, 1)
 41 |         outputs = []
 42 |         for i, (cell, dilation) in enumerate(zip(self.cells, self.dilations)):
 43 |             if hidden is None:
 44 |                 inputs, _ = self.drnn_layer(cell, inputs, dilation)
 45 |             else:
 46 |                 inputs, hidden[i] = self.drnn_layer(cell, inputs, dilation, hidden[i])
 47 | 
 48 |             outputs.append(inputs[-dilation:])
 49 | 
 50 |         if self.batch_first:
 51 |             inputs = inputs.transpose(0, 1)
 52 |         return inputs, outputs
 53 | 
 54 |     def drnn_layer(self, cell, inputs, rate, hidden=None):
 55 | 
 56 |         n_steps = len(inputs)
 57 |         batch_size = inputs[0].size(0)
 58 |         hidden_size = cell.hidden_size
 59 | 
 60 |         inputs, dilated_steps = self._pad_inputs(inputs, n_steps, rate)
 61 |         dilated_inputs = self._prepare_inputs(inputs, rate)
 62 | 
 63 |         if hidden is None:
 64 |             dilated_outputs, hidden = self._apply_cell(dilated_inputs, cell, batch_size, rate, hidden_size)
 65 |         else:
 66 |             hidden = self._prepare_inputs(hidden, rate)
 67 |             dilated_outputs, hidden = self._apply_cell(dilated_inputs, cell, batch_size, rate, hidden_size,
 68 |                                                        hidden=hidden)
 69 | 
 70 |         splitted_outputs = self._split_outputs(dilated_outputs, rate)
 71 |         outputs = self._unpad_outputs(splitted_outputs, n_steps)
 72 | 
 73 |         return outputs, hidden
 74 | 
 75 |     def _apply_cell(self, dilated_inputs, cell, batch_size, rate, hidden_size, hidden=None):
 76 |         if hidden is None:
 77 |             if self.cell_type == 'LSTM':
 78 |                 c, m = self.init_hidden(batch_size * rate, hidden_size)
 79 |                 hidden = (c.unsqueeze(0), m.unsqueeze(0))
 80 |             else:
 81 |                 hidden = self.init_hidden(batch_size * rate, hidden_size).unsqueeze(0)
 82 | 
 83 |         dilated_outputs, hidden = cell(dilated_inputs, hidden)
 84 | 
 85 |         return dilated_outputs, hidden
 86 | 
 87 |     def _unpad_outputs(self, splitted_outputs, n_steps):
 88 |         return splitted_outputs[:n_steps]
 89 | 
 90 |     def _split_outputs(self, dilated_outputs, rate):
 91 |         batchsize = dilated_outputs.size(1) // rate
 92 | 
 93 |         blocks = [dilated_outputs[:, i * batchsize: (i + 1) * batchsize, :] for i in range(rate)]
 94 | 
 95 |         interleaved = torch.stack((blocks)).transpose(1, 0).contiguous()
 96 |         interleaved = interleaved.view(dilated_outputs.size(0) * rate,
 97 |                                        batchsize,
 98 |                                        dilated_outputs.size(2))
 99 |         return interleaved
100 | 
101 |     def _pad_inputs(self, inputs, n_steps, rate):
102 |         iseven = (n_steps % rate) == 0
103 | 
104 |         if not iseven:
105 |             dilated_steps = n_steps // rate + 1
106 | 
107 |             zeros_ = torch.zeros(dilated_steps * rate - inputs.size(0),
108 |                                  inputs.size(1),
109 |                                  inputs.size(2))
110 |             if use_cuda:
111 |                 zeros_ = zeros_.cuda()
112 | 
113 |             inputs = torch.cat((inputs, autograd.Variable(zeros_)))
114 |         else:
115 |             dilated_steps = n_steps // rate
116 | 
117 |         return inputs, dilated_steps
118 | 
119 |     def _prepare_inputs(self, inputs, rate):
120 |         dilated_inputs = torch.cat([inputs[j::rate, :, :] for j in range(rate)], 1)
121 |         return dilated_inputs
122 | 
123 |     def init_hidden(self, batch_size, hidden_dim):
124 |         hidden = autograd.Variable(torch.zeros(batch_size, hidden_dim))
125 |         if use_cuda:
126 |             hidden = hidden.cuda()
127 |         if self.cell_type == "LSTM":
128 |             memory = autograd.Variable(torch.zeros(batch_size, hidden_dim))
129 |             if use_cuda:
130 |                 memory = memory.cuda()
131 |             return hidden, memory
132 |         else:
133 |             return hidden
134 | 
135 | 
136 | if __name__ == '__main__':
137 |     n_inp = 10
138 |     n_hidden = 16
139 |     n_layers = 3
140 | 
141 |     model = DRNN(n_inp, n_hidden, n_layers, cell_type='LSTM')
142 | 
143 |     test_x1 = torch.autograd.Variable(torch.randn(26, 2, n_inp))
144 |     test_x2 = torch.autograd.Variable(torch.randn(26, 2, n_inp))
145 | 
146 |     out, hidden = model(test_x1)
147 | 


--------------------------------------------------------------------------------
/es_rnn/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/damitkwr/ESRNN-GPU/918d192bf1516025978080aaea8e1ee68098c52a/es_rnn/__init__.py


--------------------------------------------------------------------------------
/es_rnn/config.py:
--------------------------------------------------------------------------------
 1 | from math import sqrt
 2 | 
 3 | import torch
 4 | 
 5 | 
 6 | def get_config(interval):
 7 |     config = {
 8 |         'prod': True,
 9 |         'device': ("cuda" if torch.cuda.is_available() else "cpu"),
10 |         'percentile': 50,
11 |         'training_percentile': 45,
12 |         'add_nl_layer': True,
13 |         'rnn_cell_type': 'LSTM',
14 |         'learning_rate': 1e-3,
15 |         'learning_rates': ((10, 1e-4)),
16 |         'num_of_train_epochs': 15,
17 |         'num_of_categories': 6,  # in data provided
18 |         'batch_size': 1024,
19 |         'gradient_clipping': 20,
20 |         'c_state_penalty': 0,
21 |         'min_learning_rate': 0.0001,
22 |         'lr_ratio': sqrt(10),
23 |         'lr_tolerance_multip': 1.005,
24 |         'min_epochs_before_changing_lrate': 2,
25 |         'print_train_batch_every': 5,
26 |         'print_output_stats': 3,
27 |         'lr_anneal_rate': 0.5,
28 |         'lr_anneal_step': 5
29 |     }
30 | 
31 |     if interval == 'Quarterly':
32 |         config.update({
33 |             'chop_val': 72,
34 |             'variable': "Quarterly",
35 |             'dilations': ((1, 2), (4, 8)),
36 |             'state_hsize': 40,
37 |             'seasonality': 4,
38 |             'input_size': 4,
39 |             'output_size': 8,
40 |             'level_variability_penalty': 80
41 |         })
42 |     elif interval == 'Monthly':
43 |         config.update({
44 |             #     RUNTIME PARAMETERS
45 |             'chop_val': 72,
46 |             'variable': "Monthly",
47 |             'dilations': ((1, 3), (6, 12)),
48 |             'state_hsize': 50,
49 |             'seasonality': 12,
50 |             'input_size': 12,
51 |             'output_size': 18,
52 |             'level_variability_penalty': 50
53 |         })
54 |     elif interval == 'Daily':
55 |         config.update({
56 |             #     RUNTIME PARAMETERS
57 |             'chop_val': 200,
58 |             'variable': "Daily",
59 |             'dilations': ((1, 7), (14, 28)),
60 |             'state_hsize': 50,
61 |             'seasonality': 7,
62 |             'input_size': 7,
63 |             'output_size': 14,
64 |             'level_variability_penalty': 50
65 |         })
66 |     elif interval == 'Yearly':
67 | 
68 |         config.update({
69 |             #     RUNTIME PARAMETERS
70 |             'chop_val': 25,
71 |             'variable': "Yearly",
72 |             'dilations': ((1, 2), (2, 6)),
73 |             'state_hsize': 30,
74 |             'seasonality': 1,
75 |             'input_size': 4,
76 |             'output_size': 6,
77 |             'level_variability_penalty': 0
78 |         })
79 |     else:
80 |         print("I don't have that config. :(")
81 | 
82 |     config['input_size_i'] = config['input_size']
83 |     config['output_size_i'] = config['output_size']
84 |     config['tau'] = config['percentile'] / 100
85 |     config['training_tau'] = config['training_percentile'] / 100
86 | 
87 |     if not config['prod']:
88 |         config['batch_size'] = 10
89 |         config['num_of_train_epochs'] = 15
90 | 
91 |     return config
92 | 


--------------------------------------------------------------------------------
/es_rnn/data_loading.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import torch
 3 | from torch.utils.data import Dataset
 4 | import pandas as pd
 5 | 
 6 | 
 7 | def read_file(file_location):
 8 |     series = []
 9 |     ids = []
10 |     with open(file_location, 'r') as file:
11 |         data = file.read().split("\n")
12 | 
13 |     for i in range(1, len(data) - 1):
14 |     # for i in range(1, len(data)):
15 |         row = data[i].replace('"', '').split(',')
16 |         series.append(np.array([float(j) for j in row[1:] if j != ""]))
17 |         ids.append(row[0])
18 | 
19 |     series = np.array(series)
20 |     return series
21 | 
22 | 
23 | def create_val_set(train, output_size):
24 |     val = []
25 |     for i in range(len(train)):
26 |         val.append(train[i][-output_size:])
27 |         train[i] = train[i][:-output_size]
28 |     return np.array(val)
29 | 
30 | 
31 | def chop_series(train, chop_val):
32 |     # CREATE MASK FOR VALUES TO BE CHOPPED
33 |     train_len_mask = [True if len(i) >= chop_val else False for i in train]
34 |     # FILTER AND CHOP TRAIN
35 |     train = [train[i][-chop_val:] for i in range(len(train)) if train_len_mask[i]]
36 |     return train, train_len_mask
37 | 
38 | 
39 | def create_datasets(train_file_location, test_file_location, output_size):
40 |     train = read_file(train_file_location)
41 |     test = read_file(test_file_location)
42 |     val = create_val_set(train, output_size)
43 |     return train, val, test
44 | 
45 | 
46 | class SeriesDataset(Dataset):
47 | 
48 |     def __init__(self, dataTrain, dataVal, dataTest, info, variable, chop_value, device):
49 |         dataTrain, mask = chop_series(dataTrain, chop_value)
50 | 
51 |         self.dataInfoCatOHE = pd.get_dummies(info[info['SP'] == variable]['category'])
52 |         self.dataInfoCatHeaders = np.array([i for i in self.dataInfoCatOHE.columns.values])
53 |         self.dataInfoCat = torch.from_numpy(self.dataInfoCatOHE[mask].values).float()
54 |         self.dataTrain = [torch.tensor(dataTrain[i]) for i in range(len(dataTrain))]  # ALREADY MASKED IN CHOP FUNCTION
55 |         self.dataVal = [torch.tensor(dataVal[i]) for i in range(len(dataVal)) if mask[i]]
56 |         self.dataTest = [torch.tensor(dataTest[i]) for i in range(len(dataTest)) if mask[i]]
57 |         self.device = device
58 | 
59 |     def __len__(self):
60 |         return len(self.dataTrain)
61 | 
62 |     def __getitem__(self, idx):
63 |         return self.dataTrain[idx].to(self.device), \
64 |                self.dataVal[idx].to(self.device), \
65 |                self.dataTest[idx].to(self.device), \
66 |                self.dataInfoCat[idx].to(self.device), \
67 |                idx
68 | 
69 | 
70 | def collate_lines(seq_list):
71 |     train_, val_, test_, info_cat_, idx_ = zip(*seq_list)
72 |     train_lens = [len(seq) for seq in train_]
73 |     seq_order = sorted(range(len(train_lens)), key=train_lens.__getitem__, reverse=True)
74 |     train = [train_[i] for i in seq_order]
75 |     val = [val_[i] for i in seq_order]
76 |     test = [test_[i] for i in seq_order]
77 |     info_cat = [info_cat_[i] for i in seq_order]
78 |     idx = [idx_[i] for i in seq_order]
79 |     return train, val, test, info_cat, idx
80 | 
81 | 


--------------------------------------------------------------------------------
/es_rnn/loss_modules.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import torch.nn as nn
  3 | import numpy as np
  4 | 
  5 | # Expression pinBallLoss(const Expression& out_ex, const Expression& actuals_ex) {//used by Dynet, learning loss function
  6 | #   vector<Expression> losses;
  7 | #   for (unsigned int indx = 0; indx<OUTPUT_SIZE; indx++) {
  8 | #     auto forec = pick(out_ex, indx);
  9 | #     auto actual = pick(actuals_ex, indx);
 10 | #     if (as_scalar(actual.value()) > as_scalar(forec.value()))
 11 | #       losses.push_back((actual - forec)*TRAINING_TAU);
 12 | #     else
 13 | #       losses.push_back((actual - forec)*(TRAINING_TAU - 1));
 14 | #   }
 15 | #   return sum(losses) / OUTPUT_SIZE * 2;
 16 | # }
 17 | 
 18 | #     as defined in the blog post --- https://eng.uber.com/m4-forecasting-competition/
 19 | 
 20 | 
 21 | class PinballLoss(nn.Module):
 22 | 
 23 |     def __init__(self, training_tau, output_size, device):
 24 |         super(PinballLoss, self).__init__()
 25 |         self.training_tau = training_tau
 26 |         self.output_size = output_size
 27 |         self.device = device
 28 | 
 29 |     def forward(self, predictions, actuals):
 30 |         cond = torch.zeros_like(predictions).to(self.device)
 31 |         loss = torch.sub(actuals, predictions).to(self.device)
 32 | 
 33 |         less_than = torch.mul(loss, torch.mul(torch.gt(loss, cond).type(torch.FloatTensor).to(self.device),
 34 |                                               self.training_tau))
 35 | 
 36 |         greater_than = torch.mul(loss, torch.mul(torch.lt(loss, cond).type(torch.FloatTensor).to(self.device),
 37 |                                                  (self.training_tau - 1)))
 38 | 
 39 |         final_loss = torch.add(less_than, greater_than)
 40 |         # losses = []
 41 |         # for i in range(self.output_size):
 42 |         #     prediction = predictions[i]
 43 |         #     actual = actuals[i]
 44 |         #     if actual > prediction:
 45 |         #         losses.append((actual - prediction) * self.training_tau)
 46 |         #     else:
 47 |         #         losses.append((actual - prediction) * (self.training_tau - 1))
 48 |         # loss = torch.Tensor(losses)
 49 |         return torch.sum(final_loss) / self.output_size * 2
 50 | 
 51 | 
 52 | # test1 = torch.rand(100)
 53 | # test2 = torch.rand(100)
 54 | # pb = PinballLoss(0.5, 100)
 55 | # pb(test1, test2)
 56 | 
 57 | 
 58 | ### sMAPE
 59 | 
 60 | # float sMAPE(vector<float>& out_vect, vector<float>& actuals_vect) {
 61 | #   float sumf = 0;
 62 | #   for (unsigned int indx = 0; indx<OUTPUT_SIZE; indx++) {
 63 | #     auto forec = out_vect[indx];
 64 | #     auto actual = actuals_vect[indx];
 65 | #     sumf+=abs(forec-actual)/(abs(forec)+abs(actual));
 66 | #   }
 67 | #   return sumf / OUTPUT_SIZE * 200;
 68 | # }
 69 | 
 70 | 
 71 | def non_sMAPE(predictions, actuals, output_size):
 72 |     sumf = 0
 73 |     for i in range(output_size):
 74 |         prediction = predictions[i]
 75 |         actual = actuals[i]
 76 |         sumf += abs(prediction - actual) / (abs(prediction) + abs(actual))
 77 |     return sumf / output_size * 200
 78 | 
 79 | 
 80 | def sMAPE(predictions, actuals, N):
 81 |     predictions = predictions.float()
 82 |     actuals = actuals.float()
 83 |     sumf = torch.sum(torch.abs(predictions - actuals) / (torch.abs(predictions) + torch.abs(actuals)))
 84 |     return ((2 * sumf) / N) * 100
 85 | 
 86 | 
 87 | def np_sMAPE(predictions, actuals, N):
 88 |     predictions = torch.from_numpy(np.array(predictions))
 89 |     actuals = torch.from_numpy(np.array(actuals))
 90 |     return float(sMAPE(predictions, actuals, N))
 91 | 
 92 | 
 93 | ### wQuantLoss
 94 | 
 95 | # float wQuantLoss(vector<float>& out_vect, vector<float>& actuals_vect) {
 96 | #   float sumf = 0; float suma=0;
 97 | #   for (unsigned int indx = 0; indx<OUTPUT_SIZE; indx++) {
 98 | #     auto forec = out_vect[indx];
 99 | #     auto actual = actuals_vect[indx];
100 | #     suma+= abs(actual);
101 | #     if (actual > forec)
102 | #       sumf = sumf + (actual - forec)*TAU;
103 | #     else
104 | #       sumf = sumf + (actual - forec)*(TAU - 1);
105 | #   }
106 | #   return sumf / suma * 200;
107 | # }
108 | 
109 | def wQuantLoss(predictions, actuals, output_size, training_tau):
110 |     sumf = 0
111 |     suma = 0
112 |     for i in range(output_size):
113 |         prediction = predictions[i]
114 |         actual = actuals[i]
115 | 
116 |         suma += abs(actual)
117 |         if (actual > prediction):
118 |             sumf = sumf + (actual - prediction) * training_tau
119 |         else:
120 |             sumf = sumf + (actual - prediction) * (training_tau - 1)
121 | 
122 |     return sumf / suma * 200
123 | 
124 | 
125 | # test1 = torch.rand(100)
126 | # test2 = torch.rand(100)
127 | # wQuantLoss(test1, test2, 100, 0.5)
128 | 
129 | 
130 | ### ErrorFunc
131 | 
132 | # float errorFunc(vector<float>& out_vect, vector<float>& actuals_vect) {
133 | #   if (PERCENTILE==50)
134 | #     return sMAPE(out_vect, actuals_vect);
135 | #   else
136 | #     return wQuantLoss(out_vect, actuals_vect);
137 | # }
138 | 
139 | def errorFunc(predictions, actuals, output_size, percentile):
140 |     if (percentile == 50):
141 |         return sMAPE(predictions, actuals, output_size)
142 |     else:
143 |         return wQuantLoss(predictions, actuals, output_size, percentile / 100)
144 | 
145 | 
146 | # test1 = torch.rand(100)
147 | # test2 = torch.rand(100)
148 | # print(errorFunc(test1, test2, 100, 48))
149 | # print(wQuantLoss(test1, test2, 100, 0.48))
150 | # print(errorFunc(test1, test2, 100, 50))
151 | # print(sMAPE(test1, test2, 100))
152 | 
153 | def main():
154 |     # Test vectorized calculation
155 |     test1 = torch.rand(100)
156 |     test2 = torch.rand(100)
157 |     cpu_loss = non_sMAPE(test1, test2, 100)
158 |     vec_loss = sMAPE(test1, test2, 100)
159 | 
160 | if __name__ == '__main__':
161 |     main()
162 | 


--------------------------------------------------------------------------------
/es_rnn/main.py:
--------------------------------------------------------------------------------
 1 | import pandas as pd
 2 | from torch.utils.data import DataLoader
 3 | from es_rnn.data_loading import create_datasets, SeriesDataset
 4 | from es_rnn.config import get_config
 5 | from es_rnn.trainer import ESRNNTrainer
 6 | from es_rnn.model import ESRNN
 7 | import time
 8 | 
 9 | print('loading config')
10 | config = get_config('Monthly')
11 | 
12 | print('loading data')
13 | info = pd.read_csv('../data/info.csv')
14 | 
15 | train_path = '../data/Train/%s-train.csv' % (config['variable'])
16 | test_path = '../data/Test/%s-test.csv' % (config['variable'])
17 | 
18 | train, val, test = create_datasets(train_path, test_path, config['output_size'])
19 | 
20 | dataset = SeriesDataset(train, val, test, info, config['variable'], config['chop_val'], config['device'])
21 | dataloader = DataLoader(dataset, batch_size=config['batch_size'], shuffle=True)
22 | 
23 | run_id = str(int(time.time()))
24 | model = ESRNN(num_series=len(dataset), config=config)
25 | tr = ESRNNTrainer(model, dataloader, run_id, config, ohe_headers=dataset.dataInfoCatHeaders)
26 | tr.train_epochs()
27 | 


--------------------------------------------------------------------------------
/es_rnn/model.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import torch.nn as nn
  3 | from es_rnn.DRNN import DRNN
  4 | 
  5 | 
  6 | class ESRNN(nn.Module):
  7 |     def __init__(self, num_series, config):
  8 |         super(ESRNN, self).__init__()
  9 |         self.config = config
 10 |         self.num_series = num_series
 11 |         self.add_nl_layer = self.config['add_nl_layer']
 12 | 
 13 |         init_lev_sms = []
 14 |         init_seas_sms = []
 15 |         init_seasonalities = []
 16 | 
 17 |         for i in range(num_series):
 18 |             init_lev_sms.append(nn.Parameter(torch.Tensor([0.5]), requires_grad=True))
 19 |             init_seas_sms.append(nn.Parameter(torch.Tensor([0.5]), requires_grad=True))
 20 |             init_seasonalities.append(nn.Parameter((torch.ones(config['seasonality']) * 0.5), requires_grad=True))
 21 | 
 22 |         self.init_lev_sms = nn.ParameterList(init_lev_sms)
 23 |         self.init_seas_sms = nn.ParameterList(init_seas_sms)
 24 |         self.init_seasonalities = nn.ParameterList(init_seasonalities)
 25 | 
 26 |         self.nl_layer = nn.Linear(config['state_hsize'],
 27 |                                   config['state_hsize'])
 28 |         self.act = nn.Tanh()
 29 |         self.scoring = nn.Linear(config['state_hsize'], config['output_size'])
 30 | 
 31 |         self.logistic = nn.Sigmoid()
 32 | 
 33 |         self.resid_drnn = ResidualDRNN(self.config)
 34 | 
 35 |     def forward(self, train, val, test, info_cat, idxs, testing=False):
 36 |         # GET THE PER SERIES PARAMETERS
 37 |         lev_sms = self.logistic(torch.stack([self.init_lev_sms[idx] for idx in idxs]).squeeze(1))
 38 |         seas_sms = self.logistic(torch.stack([self.init_seas_sms[idx] for idx in idxs]).squeeze(1))
 39 |         init_seasonalities = torch.stack([self.init_seasonalities[idx] for idx in idxs])
 40 | 
 41 |         seasonalities = []
 42 |         # PRIME SEASONALITY
 43 |         for i in range(self.config['seasonality']):
 44 |             seasonalities.append(torch.exp(init_seasonalities[:, i]))
 45 |         seasonalities.append(torch.exp(init_seasonalities[:, 0]))
 46 | 
 47 |         if testing:
 48 |             train = torch.cat((train, val), dim=1)
 49 | 
 50 |         train = train.float()
 51 | 
 52 |         levs = []
 53 |         log_diff_of_levels = []
 54 | 
 55 |         levs.append(train[:, 0] / seasonalities[0])
 56 |         for i in range(1, train.shape[1]):
 57 |             # CALCULATE LEVEL FOR CURRENT TIMESTEP TO NORMALIZE RNN
 58 |             new_lev = lev_sms * (train[:, i] / seasonalities[i]) + (1 - lev_sms) * levs[i - 1]
 59 |             levs.append(new_lev)
 60 | 
 61 |             # STORE DIFFERENCE TO PENALIZE LATER
 62 |             log_diff_of_levels.append(torch.log(new_lev / levs[i - 1]))
 63 | 
 64 |             # CALCULATE SEASONALITY TO DESEASONALIZE THE DATA FOR RNN
 65 |             seasonalities.append(seas_sms * (train[:, i] / new_lev) + (1 - seas_sms) * seasonalities[i])
 66 | 
 67 |         seasonalities_stacked = torch.stack(seasonalities).transpose(1, 0)
 68 |         levs_stacked = torch.stack(levs).transpose(1, 0)
 69 | 
 70 |         loss_mean_sq_log_diff_level = 0
 71 |         if self.config['level_variability_penalty'] > 0:
 72 |             sq_log_diff = torch.stack(
 73 |                 [(log_diff_of_levels[i] - log_diff_of_levels[i - 1]) ** 2 for i in range(1, len(log_diff_of_levels))])
 74 |             loss_mean_sq_log_diff_level = torch.mean(sq_log_diff)
 75 | 
 76 |         if self.config['output_size'] > self.config['seasonality']:
 77 |             start_seasonality_ext = seasonalities_stacked.shape[1] - self.config['seasonality']
 78 |             end_seasonality_ext = start_seasonality_ext + self.config['output_size'] - self.config['seasonality']
 79 |             seasonalities_stacked = torch.cat((seasonalities_stacked, seasonalities_stacked[:, start_seasonality_ext:end_seasonality_ext]),
 80 |                                               dim=1)
 81 | 
 82 |         window_input_list = []
 83 |         window_output_list = []
 84 |         for i in range(self.config['input_size'] - 1, train.shape[1]):
 85 |             input_window_start = i + 1 - self.config['input_size']
 86 |             input_window_end = i + 1
 87 | 
 88 |             train_deseas_window_input = train[:, input_window_start:input_window_end] / seasonalities_stacked[:,
 89 |                                                                                         input_window_start:input_window_end]
 90 |             train_deseas_norm_window_input = (train_deseas_window_input / levs_stacked[:, i].unsqueeze(1))
 91 |             train_deseas_norm_cat_window_input = torch.cat((train_deseas_norm_window_input, info_cat), dim=1)
 92 |             window_input_list.append(train_deseas_norm_cat_window_input)
 93 | 
 94 |             output_window_start = i + 1
 95 |             output_window_end = i + 1 + self.config['output_size']
 96 | 
 97 |             if i < train.shape[1] - self.config['output_size']:
 98 |                 train_deseas_window_output = train[:, output_window_start:output_window_end] / \
 99 |                                              seasonalities_stacked[:, output_window_start:output_window_end]
100 |                 train_deseas_norm_window_output = (train_deseas_window_output / levs_stacked[:, i].unsqueeze(1))
101 |                 window_output_list.append(train_deseas_norm_window_output)
102 | 
103 |         window_input = torch.cat([i.unsqueeze(0) for i in window_input_list], dim=0)
104 |         window_output = torch.cat([i.unsqueeze(0) for i in window_output_list], dim=0)
105 | 
106 |         self.train()
107 |         network_pred = self.series_forward(window_input[:-self.config['output_size']])
108 |         network_act = window_output
109 | 
110 |         self.eval()
111 |         network_output_non_train = self.series_forward(window_input)
112 | 
113 |         # USE THE LAST VALUE OF THE NETWORK OUTPUT TO COMPUTE THE HOLDOUT PREDICTIONS
114 |         hold_out_output_reseas = network_output_non_train[-1] * seasonalities_stacked[:, -self.config['output_size']:]
115 |         hold_out_output_renorm = hold_out_output_reseas * levs_stacked[:, -1].unsqueeze(1)
116 | 
117 |         hold_out_pred = hold_out_output_renorm * torch.gt(hold_out_output_renorm, 0).float()
118 |         hold_out_act = test if testing else val
119 | 
120 |         hold_out_act_deseas = hold_out_act.float() / seasonalities_stacked[:, -self.config['output_size']:]
121 |         hold_out_act_deseas_norm = hold_out_act_deseas / levs_stacked[:, -1].unsqueeze(1)
122 | 
123 |         self.train()
124 |         # RETURN JUST THE TRAINING INPUT RATHER THAN THE ENTIRE SET BECAUSE THE HOLDOUT IS BEING GENERATED WITH THE REST
125 |         return network_pred, \
126 |                network_act, \
127 |                (hold_out_pred, network_output_non_train), \
128 |                (hold_out_act, hold_out_act_deseas_norm), \
129 |                loss_mean_sq_log_diff_level
130 | 
131 |     def series_forward(self, data):
132 |         data = self.resid_drnn(data)
133 |         if self.add_nl_layer:
134 |             data = self.nl_layer(data)
135 |             data = self.act(data)
136 |         data = self.scoring(data)
137 |         return data
138 | 
139 | 
140 | class ResidualDRNN(nn.Module):
141 |     def __init__(self, config):
142 |         super(ResidualDRNN, self).__init__()
143 |         self.config = config
144 | 
145 |         layers = []
146 |         for grp_num in range(len(self.config['dilations'])):
147 | 
148 |             if grp_num == 0:
149 |                 input_size = self.config['input_size'] + self.config['num_of_categories']
150 |             else:
151 |                 input_size = self.config['state_hsize']
152 | 
153 |             l = DRNN(input_size,
154 |                      self.config['state_hsize'],
155 |                      n_layers=len(self.config['dilations'][grp_num]),
156 |                      dilations=self.config['dilations'][grp_num],
157 |                      cell_type=self.config['rnn_cell_type'])
158 | 
159 |             layers.append(l)
160 | 
161 |         self.rnn_stack = nn.Sequential(*layers)
162 | 
163 |     def forward(self, input_data):
164 |         for layer_num in range(len(self.rnn_stack)):
165 |             residual = input_data
166 |             out, _ = self.rnn_stack[layer_num](input_data)
167 |             if layer_num > 0:
168 |                 out += residual
169 |             input_data = out
170 |         return out
171 | 


--------------------------------------------------------------------------------
/es_rnn/trainer.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import time
  3 | import numpy as np
  4 | import copy
  5 | import torch
  6 | import torch.nn as nn
  7 | from es_rnn.loss_modules import PinballLoss, sMAPE, np_sMAPE
  8 | from utils.logger import Logger
  9 | import pandas as pd
 10 | 
 11 | 
 12 | class ESRNNTrainer(nn.Module):
 13 |     def __init__(self, model, dataloader, run_id, config, ohe_headers):
 14 |         super(ESRNNTrainer, self).__init__()
 15 |         self.model = model.to(config['device'])
 16 |         self.config = config
 17 |         self.dl = dataloader
 18 |         self.ohe_headers = ohe_headers
 19 |         self.optimizer = torch.optim.Adam(self.model.parameters(), lr=config['learning_rate'])
 20 |         # self.optimizer = torch.optim.ASGD(self.model.parameters(), lr=config['learning_rate'])
 21 |         self.scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer,
 22 |                                                          step_size=config['lr_anneal_step'],
 23 |                                                          gamma=config['lr_anneal_rate'])
 24 |         self.criterion = PinballLoss(self.config['training_tau'],
 25 |                                      self.config['output_size'] * self.config['batch_size'], self.config['device'])
 26 |         self.epochs = 0
 27 |         self.max_epochs = config['num_of_train_epochs']
 28 |         self.run_id = str(run_id)
 29 |         self.prod_str = 'prod' if config['prod'] else 'dev'
 30 |         self.log = Logger("../logs/train%s%s%s" % (self.config['variable'], self.prod_str, self.run_id))
 31 |         self.csv_save_path = None
 32 | 
 33 |     def train_epochs(self):
 34 |         max_loss = 1e8
 35 |         start_time = time.time()
 36 |         for e in range(self.max_epochs):
 37 |             self.scheduler.step()
 38 |             epoch_loss = self.train()
 39 |             if epoch_loss < max_loss:
 40 |                 self.save()
 41 |             epoch_val_loss = self.val()
 42 |             if e == 0:
 43 |                 file_path = os.path.join(self.csv_save_path, 'validation_losses.csv')
 44 |                 with open(file_path, 'w') as f:
 45 |                     f.write('epoch,training_loss,validation_loss\n')
 46 |             with open(file_path, 'a') as f:
 47 |                 f.write(','.join([str(e), str(epoch_loss), str(epoch_val_loss)]) + '\n')
 48 |         print('Total Training Mins: %5.2f' % ((time.time()-start_time)/60))
 49 | 
 50 |     def train(self):
 51 |         self.model.train()
 52 |         epoch_loss = 0
 53 |         for batch_num, (train, val, test, info_cat, idx) in enumerate(self.dl):
 54 |             start = time.time()
 55 |             print("Train_batch: %d" % (batch_num + 1))
 56 |             loss = self.train_batch(train, val, test, info_cat, idx)
 57 |             epoch_loss += loss
 58 |             end = time.time()
 59 |             self.log.log_scalar('Iteration time', end - start, batch_num + 1 * (self.epochs + 1))
 60 |         epoch_loss = epoch_loss / (batch_num + 1)
 61 |         self.epochs += 1
 62 | 
 63 |         # LOG EPOCH LEVEL INFORMATION
 64 |         print('[TRAIN]  Epoch [%d/%d]   Loss: %.4f' % (
 65 |             self.epochs, self.max_epochs, epoch_loss))
 66 |         info = {'loss': epoch_loss}
 67 | 
 68 |         self.log_values(info)
 69 |         self.log_hists()
 70 | 
 71 |         return epoch_loss
 72 | 
 73 |     def train_batch(self, train, val, test, info_cat, idx):
 74 |         self.optimizer.zero_grad()
 75 |         network_pred, network_act, _, _, loss_mean_sq_log_diff_level = self.model(train, val,
 76 |                                                                                   test, info_cat,
 77 |                                                                                   idx)
 78 | 
 79 |         loss = self.criterion(network_pred, network_act)
 80 |         loss.backward()
 81 |         nn.utils.clip_grad_value_(self.model.parameters(), self.config['gradient_clipping'])
 82 |         self.optimizer.step()
 83 |         return float(loss)
 84 | 
 85 |     def val(self):
 86 |         self.model.eval()
 87 |         with torch.no_grad():
 88 |             acts = []
 89 |             preds = []
 90 |             info_cats = []
 91 | 
 92 |             hold_out_loss = 0
 93 |             for batch_num, (train, val, test, info_cat, idx) in enumerate(self.dl):
 94 |                 _, _, (hold_out_pred, network_output_non_train), \
 95 |                 (hold_out_act, hold_out_act_deseas_norm), _ = self.model(train, val, test, info_cat, idx)
 96 |                 hold_out_loss += self.criterion(network_output_non_train.unsqueeze(0).float(),
 97 |                                                 hold_out_act_deseas_norm.unsqueeze(0).float())
 98 |                 acts.extend(hold_out_act.view(-1).cpu().detach().numpy())
 99 |                 preds.extend(hold_out_pred.view(-1).cpu().detach().numpy())
100 |                 info_cats.append(info_cat.cpu().detach().numpy())
101 |             hold_out_loss = hold_out_loss / (batch_num + 1)
102 | 
103 |             info_cat_overall = np.concatenate(info_cats, axis=0)
104 |             _hold_out_df = pd.DataFrame({'acts': acts, 'preds': preds})
105 |             cats = [val for val in self.ohe_headers[info_cat_overall.argmax(axis=1)] for _ in
106 |                     range(self.config['output_size'])]
107 |             _hold_out_df['category'] = cats
108 | 
109 |             overall_hold_out_df = copy.copy(_hold_out_df)
110 |             overall_hold_out_df['category'] = ['Overall' for _ in cats]
111 | 
112 |             overall_hold_out_df = pd.concat((_hold_out_df, overall_hold_out_df))
113 |             grouped_results = overall_hold_out_df.groupby(['category']).apply(
114 |                 lambda x: np_sMAPE(x.preds, x.acts, x.shape[0]))
115 | 
116 |             results = grouped_results.to_dict()
117 |             results['hold_out_loss'] = float(hold_out_loss.detach().cpu())
118 | 
119 |             self.log_values(results)
120 | 
121 |             file_path = os.path.join('..', 'grouped_results', self.run_id, self.prod_str)
122 |             os.makedirs(file_path, exist_ok=True)
123 | 
124 |             print(results)
125 |             grouped_path = os.path.join(file_path, 'grouped_results-{}.csv'.format(self.epochs))
126 |             grouped_results.to_csv(grouped_path)
127 |             self.csv_save_path = file_path
128 | 
129 |         return hold_out_loss.detach().cpu().item()
130 | 
131 |     def save(self, save_dir='..'):
132 |         print('Loss decreased, saving model!')
133 |         file_path = os.path.join(save_dir, 'models', self.run_id, self.prod_str)
134 |         model_path = os.path.join(file_path, 'model-{}.pyt'.format(self.epochs))
135 |         os.makedirs(file_path, exist_ok=True)
136 |         torch.save({'state_dict': self.model.state_dict()}, model_path)
137 | 
138 |     def log_values(self, info):
139 | 
140 |         # SCALAR
141 |         for tag, value in info.items():
142 |             self.log.log_scalar(tag, value, self.epochs + 1)
143 | 
144 |     def log_hists(self):
145 |         # HISTS
146 |         batch_params = dict()
147 |         for tag, value in self.model.named_parameters():
148 |             if value.grad is not None:
149 |                 if "init" in tag:
150 |                     name, _ = tag.split(".")
151 |                     if name not in batch_params.keys() or "%s/grad" % name not in batch_params.keys():
152 |                         batch_params[name] = []
153 |                         batch_params["%s/grad" % name] = []
154 |                     batch_params[name].append(value.data.cpu().numpy())
155 |                     batch_params["%s/grad" % name].append(value.grad.cpu().numpy())
156 |                 else:
157 |                     tag = tag.replace('.', '/')
158 |                     self.log.log_histogram(tag, value.data.cpu().numpy(), self.epochs + 1)
159 |                     self.log.log_histogram(tag + '/grad', value.grad.data.cpu().numpy(), self.epochs + 1)
160 |             else:
161 |                 print('Not printing %s because it\'s not updating' % tag)
162 | 
163 |         for tag, v in batch_params.items():
164 |             vals = np.concatenate(np.array(v))
165 |             self.log.log_histogram(tag, vals, self.epochs + 1)
166 | 


--------------------------------------------------------------------------------
/m4_baseline.R:
--------------------------------------------------------------------------------
 1 | # install.packages("devtools")
 2 | # devtools::install_github("carlanetto/M4comp2018")
 3 | # devtools::install_github("robjhyndman/M4metalearning")
 4 | # devtools::install_github("pmontman/tsfeatures")
 5 | # devtools::install_github("pmontman/customxgboost")
 6 | library(forecast)
 7 | 
 8 | ## basic example code
 9 | library(M4comp2018)
10 | data(M4)
11 | names(M4[[1]])
12 | #> [1] "st"     "x"      "n"      "type"   "h"      "period" "xx"
13 | #extract yearly series
14 | yearly_M4 <- Filter(function(l) l$period == "Yearly", M4)
15 | #plot one of the series, in red the future data
16 | #in black, the hitorical data
17 | library(ggplot2)
18 | library(forecast)
19 | plot(ts(c(M4[[40773]]$x, M4[[40773]]$xx),
20 |         start=start(M4[[40773]]$x), frequency = frequency(M4[[40773]]$x)),
21 |      col="red", type="l", ylab="")
22 | lines(M4[[40773]]$x, col="black")
23 | 
24 | #read the help file for documentation
25 | # ?M4comp2018
26 | 
27 | # from https://github.com/robjhyndman/M4metalearning/blob/master/docs/metalearning_example.md
28 | library(M4metalearning)
29 | library(M4comp2018)
30 | set.seed(31-05-2018)
31 | # we start by creating the training and test subsets
32 | indices <- sample(length(M4))
33 | # sample only 15 series for estimation
34 | M4_train <- M4[ indices[1:15]]
35 | M4_test <- M4[indices[16:25]]
36 | # we create the temporal holdout version of the training and test sets
37 | M4_train <- temp_holdout(M4_train)
38 | M4_test <- temp_holdout(M4_test)
39 | 
40 | # this will take time
41 | M4_train <- calc_forecasts(M4_train, c("auto_arima_forec", 'ets_forec', 'tbats_forec'), n.cores=4)
42 | # once we have the forecasts, we can calculate the errors
43 | M4_train2 <- calc_errors(M4_train)
44 | 


--------------------------------------------------------------------------------
/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/damitkwr/ESRNN-GPU/918d192bf1516025978080aaea8e1ee68098c52a/utils/__init__.py


--------------------------------------------------------------------------------
/utils/helper_funcs.py:
--------------------------------------------------------------------------------
 1 | import torch
 2 | 
 3 | 
 4 | def colwise_batch_mask(target_shape_tuple, target_lens):
 5 |     # takes in (seq_len, B) shape and returns mask of same shape with ones up to the target lens
 6 |     mask = torch.zeros(target_shape_tuple)
 7 |     for i in range(target_shape_tuple[1]):
 8 |         mask[:target_lens[i], i] = 1
 9 |     return mask
10 | 
11 | 
12 | def rowwise_batch_mask(target_shape_tuple, target_lens):
13 |     # takes in (B, seq_len) shape and returns mask of same shape with ones up to the target lens
14 |     mask = torch.zeros(target_shape_tuple)
15 |     for i in range(target_shape_tuple[0]):
16 |         mask[i, :target_lens[i]] = 1
17 |     return mask
18 | 
19 | 
20 | def unpad_sequence(padded_sequence, lens):
21 |     seqs = []
22 |     for i in range(padded_sequence.size(1)):
23 |         seqs.append(padded_sequence[:lens[i], i])
24 |     return seqs
25 | 
26 | 


--------------------------------------------------------------------------------
/utils/logger.py:
--------------------------------------------------------------------------------
 1 | import tensorflow as tf
 2 | import numpy as np
 3 | 
 4 | 
 5 | class Logger(object):
 6 |     """Logging in tensorboard without tensorflow ops."""
 7 | 
 8 |     def __init__(self, log_dir):
 9 |         self.writer = tf.summary.FileWriter(log_dir)
10 | 
11 |     def log_scalar(self, tag, value, step):
12 |         """Log a scalar variable.
13 |         Parameter
14 |         ----------
15 |         tag : Name of the scalar
16 |         value : value itself
17 |         step :  training iteration
18 |         """
19 |         # Notice we're using the Summary "class" instead of the "tf.summary" public API.
20 |         summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])
21 |         self.writer.add_summary(summary, step)
22 | 
23 |     def log_histogram(self, tag, values, step, bins=1000):
24 |         """Logs the histogram of a list/vector of values."""
25 |         # Convert to a numpy array
26 |         values = np.array(values)
27 | 
28 |         # Create histogram using numpy
29 |         counts, bin_edges = np.histogram(values, bins=bins)
30 | 
31 |         # Fill fields of histogram proto
32 |         hist = tf.HistogramProto()
33 |         hist.min = float(np.min(values))
34 |         hist.max = float(np.max(values))
35 |         hist.num = int(np.prod(values.shape))
36 |         hist.sum = float(np.sum(values))
37 |         hist.sum_squares = float(np.sum(values ** 2))
38 | 
39 |         # Requires equal number as bins, where the first goes from -DBL_MAX to bin_edges[1]
40 |         # See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto#L30
41 |         # Thus, we drop the start of the first bin
42 |         bin_edges = bin_edges[1:]
43 | 
44 |         # Add bin edges and counts
45 |         for edge in bin_edges:
46 |             hist.bucket_limit.append(edge)
47 |         for c in counts:
48 |             hist.bucket.append(c)
49 | 
50 |         # Create and write Summary
51 |         summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])
52 |         self.writer.add_summary(summary, step)
53 |         self.writer.flush()


--------------------------------------------------------------------------------