├── .idea
    ├── SRU-tensorflow.iml
    ├── misc.xml
    ├── modules.xml
    ├── vcs.xml
    └── workspace.xml
├── README.md
└── SRU_tensorflow.py


/.idea/SRU-tensorflow.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="jdk" jdkName="Python 2.7.12 (/usr/bin/python2.7)" jdkType="Python SDK" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="TestRunnerService">
 9 |     <option name="PROJECT_TEST_RUNNER" value="Unittests" />
10 |   </component>
11 | </module>


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.12 (/usr/bin/python2.7)" project-jdk-type="Python SDK" />
4 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/SRU-tensorflow.iml" filepath="$PROJECT_DIR$/.idea/SRU-tensorflow.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project version="4">
  3 |   <component name="ChangeListManager">
  4 |     <list default="true" id="57e23641-e33f-4566-8d3f-02d58a7ceb81" name="Default" comment="">
  5 |       <change type="MODIFICATION" beforePath="$PROJECT_DIR$/SRU_tensorflow.py" afterPath="$PROJECT_DIR$/SRU_tensorflow.py" />
  6 |     </list>
  7 |     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
  8 |     <option name="TRACKING_ENABLED" value="true" />
  9 |     <option name="SHOW_DIALOG" value="false" />
 10 |     <option name="HIGHLIGHT_CONFLICTS" value="true" />
 11 |     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
 12 |     <option name="LAST_RESOLUTION" value="IGNORE" />
 13 |   </component>
 14 |   <component name="DockManager">
 15 |     <window id="2">
 16 |       <content type="file-editors">
 17 |         <state>
 18 |           <leaf>
 19 |             <file leaf-file-name="rnn_cell_impl.py" pinned="false" current-in-tab="true">
 20 |               <entry file="file:///usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell_impl.py">
 21 |                 <provider selected="true" editor-type-id="text-editor">
 22 |                   <state relative-caret-position="68">
 23 |                     <caret line="22" column="4" lean-forward="false" selection-start-line="22" selection-start-column="0" selection-end-line="22" selection-end-column="4" />
 24 |                     <folding />
 25 |                   </state>
 26 |                 </provider>
 27 |               </entry>
 28 |             </file>
 29 |           </leaf>
 30 |         </state>
 31 |       </content>
 32 |     </window>
 33 |   </component>
 34 |   <component name="FileEditorManager">
 35 |     <leaf>
 36 |       <file leaf-file-name="SRU_tensorflow.py" pinned="false" current-in-tab="true">
 37 |         <entry file="file://$PROJECT_DIR$/SRU_tensorflow.py">
 38 |           <provider selected="true" editor-type-id="text-editor">
 39 |             <state relative-caret-position="204">
 40 |               <caret line="10" column="21" lean-forward="true" selection-start-line="10" selection-start-column="21" selection-end-line="10" selection-end-column="21" />
 41 |               <folding>
 42 |                 <element signature="e#0#23#0" expanded="true" />
 43 |               </folding>
 44 |             </state>
 45 |           </provider>
 46 |         </entry>
 47 |       </file>
 48 |     </leaf>
 49 |   </component>
 50 |   <component name="FindInProjectRecents">
 51 |     <findStrings>
 52 |       <find>BasicRNN</find>
 53 |     </findStrings>
 54 |   </component>
 55 |   <component name="Git.Settings">
 56 |     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
 57 |   </component>
 58 |   <component name="IdeDocumentHistory">
 59 |     <option name="CHANGED_PATHS">
 60 |       <list>
 61 |         <option value="$PROJECT_DIR$/SRU_tensorflow.py" />
 62 |       </list>
 63 |     </option>
 64 |   </component>
 65 |   <component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
 66 |   <component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
 67 |   <component name="JsGulpfileManager">
 68 |     <detection-done>true</detection-done>
 69 |     <sorting>DEFINITION_ORDER</sorting>
 70 |   </component>
 71 |   <component name="ProjectFrameBounds">
 72 |     <option name="x" value="1917" />
 73 |     <option name="y" value="344" />
 74 |     <option name="width" value="710" />
 75 |     <option name="height" value="907" />
 76 |   </component>
 77 |   <component name="ProjectLevelVcsManager">
 78 |     <ConfirmationsSetting value="2" id="Add" />
 79 |   </component>
 80 |   <component name="ProjectView">
 81 |     <navigator currentView="ProjectPane" proportions="" version="1">
 82 |       <flattenPackages />
 83 |       <showMembers />
 84 |       <showModules />
 85 |       <showLibraryContents />
 86 |       <hideEmptyPackages />
 87 |       <abbreviatePackageNames />
 88 |       <autoscrollToSource />
 89 |       <autoscrollFromSource />
 90 |       <sortByType />
 91 |       <manualOrder />
 92 |       <foldersAlwaysOnTop value="true" />
 93 |     </navigator>
 94 |     <panes>
 95 |       <pane id="ProjectPane">
 96 |         <subPane>
 97 |           <expand>
 98 |             <path>
 99 |               <item name="SRU-tensorflow" type="b2602c69:ProjectViewProjectNode" />
100 |               <item name="SRU-tensorflow" type="462c0819:PsiDirectoryNode" />
101 |             </path>
102 |           </expand>
103 |           <select />
104 |         </subPane>
105 |       </pane>
106 |       <pane id="Scope" />
107 |       <pane id="Scratches" />
108 |     </panes>
109 |   </component>
110 |   <component name="PropertiesComponent">
111 |     <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
112 |     <property name="WebServerToolWindowFactoryState" value="false" />
113 |   </component>
114 |   <component name="RunDashboard">
115 |     <option name="ruleStates">
116 |       <list>
117 |         <RuleState>
118 |           <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
119 |         </RuleState>
120 |         <RuleState>
121 |           <option name="name" value="StatusDashboardGroupingRule" />
122 |         </RuleState>
123 |       </list>
124 |     </option>
125 |   </component>
126 |   <component name="ShelveChangesManager" show_recycled="false">
127 |     <option name="remove_strategy" value="false" />
128 |   </component>
129 |   <component name="TaskManager">
130 |     <task active="true" id="Default" summary="Default task">
131 |       <changelist id="57e23641-e33f-4566-8d3f-02d58a7ceb81" name="Default" comment="" />
132 |       <created>1505547314989</created>
133 |       <option name="number" value="Default" />
134 |       <option name="presentableId" value="Default" />
135 |       <updated>1505547314989</updated>
136 |     </task>
137 |     <servers />
138 |   </component>
139 |   <component name="ToolWindowManager">
140 |     <frame x="1917" y="344" width="710" height="907" extended-state="0" />
141 |     <editor active="true" />
142 |     <layout>
143 |       <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.24929577" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
144 |       <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
145 |       <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
146 |       <window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
147 |       <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
148 |       <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
149 |       <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
150 |       <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
151 |       <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
152 |       <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
153 |       <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
154 |       <window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
155 |       <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
156 |       <window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
157 |       <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
158 |       <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
159 |       <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
160 |       <window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
161 |       <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
162 |     </layout>
163 |   </component>
164 |   <component name="TypeScriptGeneratedFilesManager">
165 |     <option name="version" value="1" />
166 |   </component>
167 |   <component name="VcsContentAnnotationSettings">
168 |     <option name="myLimit" value="2678400000" />
169 |   </component>
170 |   <component name="XDebuggerManager">
171 |     <breakpoint-manager />
172 |     <watches-manager />
173 |   </component>
174 |   <component name="editorHistoryManager">
175 |     <entry file="file:///usr/local/lib/python3.5/dist-packages/tensorflow/python/layers/base.py">
176 |       <provider selected="true" editor-type-id="text-editor">
177 |         <state relative-caret-position="-769">
178 |           <caret line="77" column="6" lean-forward="false" selection-start-line="77" selection-start-column="6" selection-end-line="77" selection-end-column="6" />
179 |           <folding />
180 |         </state>
181 |       </provider>
182 |     </entry>
183 |     <entry file="file:///usr/lib/python3.5/collections/__init__.py">
184 |       <provider selected="true" editor-type-id="text-editor">
185 |         <state relative-caret-position="0">
186 |           <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
187 |           <folding />
188 |         </state>
189 |       </provider>
190 |     </entry>
191 |     <entry file="file:///usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/rnn_cell_impl.py">
192 |       <provider selected="true" editor-type-id="text-editor">
193 |         <state relative-caret-position="-493">
194 |           <caret line="304" column="27" lean-forward="false" selection-start-line="304" selection-start-column="27" selection-end-line="304" selection-end-column="27" />
195 |           <folding>
196 |             <element signature="e#1070#1108#0" expanded="true" />
197 |           </folding>
198 |         </state>
199 |       </provider>
200 |     </entry>
201 |     <entry file="file://$USER_HOME$/.PyCharm2017.2/system/python_stubs/-1247972723/__builtin__.py">
202 |       <provider selected="true" editor-type-id="text-editor">
203 |         <state relative-caret-position="259">
204 |           <caret line="4337" column="8" lean-forward="false" selection-start-line="4337" selection-start-column="8" selection-end-line="4337" selection-end-column="8" />
205 |           <folding />
206 |         </state>
207 |       </provider>
208 |     </entry>
209 |     <entry file="file:///usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell_impl.py">
210 |       <provider selected="true" editor-type-id="text-editor">
211 |         <state relative-caret-position="68">
212 |           <caret line="22" column="4" lean-forward="false" selection-start-line="22" selection-start-column="0" selection-end-line="22" selection-end-column="4" />
213 |           <folding />
214 |         </state>
215 |       </provider>
216 |     </entry>
217 |     <entry file="file://$PROJECT_DIR$/SRU_tensorflow.py">
218 |       <provider selected="true" editor-type-id="text-editor">
219 |         <state relative-caret-position="204">
220 |           <caret line="10" column="21" lean-forward="true" selection-start-line="10" selection-start-column="21" selection-end-line="10" selection-end-column="21" />
221 |           <folding>
222 |             <element signature="e#0#23#0" expanded="true" />
223 |           </folding>
224 |         </state>
225 |       </provider>
226 |     </entry>
227 |   </component>
228 | </project>


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # SRU-tensorflow
2 | Training RNNs as fast as CNNs. An unofficial tensorflow implementation.
3 | 


--------------------------------------------------------------------------------
/SRU_tensorflow.py:
--------------------------------------------------------------------------------
 1 | import tensorflow as tf
 2 | import collections
 3 | 
 4 | from tensorflow.python.ops import variable_scope as vs
 5 | from tensorflow.contrib.rnn import RNNCell
 6 | 
 7 | class SRUCell(RNNCell):
 8 |     def __init__(self, num_units, activation=tf.nn.tanh, state_is_tuple=False, reuse=None):
 9 |         super(SRUCell, self).__init__(_reuse=reuse)
10 |         self.hidden_dim = num_units
11 |         self.state_is_tuple = state_is_tuple
12 |         self.g = activation
13 |         init_matrix = tf.orthogonal_initializer()
14 | 
15 |         self.Wr = tf.Variable(init_matrix([self.hidden_dim, self.hidden_dim]))
16 |         self.br = tf.Variable(self.init_matrix([self.hidden_dim]))
17 | 
18 |         self.Wf = tf.Variable(init_matrix([self.hidden_dim, self.hidden_dim]))
19 |         self.bf = tf.Variable(self.init_matrix([self.hidden_dim]))
20 | 
21 |         self.U = tf.Variable(init_matrix([self.hidden_dim, self.hidden_dim]))
22 | 
23 | 
24 |     @property
25 |     def state_size(self):
26 |         return (LSTMStateTuple(self.hidden_dim, self.hidden_dim)
27 |                 if self.state_is_tuple else self.hidden_dim)
28 | 
29 |     @property
30 |     def output_size(self):
31 |         return self.hidden_dim
32 | 
33 |     def __call__(self, inputs, state, scope=None):
34 |         with vs.variable_scope(scope or type(self).__name__):
35 |             if self.state_is_tuple:
36 |                 (c_prev, h_prev) = state
37 |             else:
38 |                 c_prev = state
39 |             # Forget Gate
40 |             f = tf.sigmoid(
41 |                 tf.matmul(inputs, self.Wf) + self.bf
42 |             )
43 | 
44 |             # Reset Gate
45 |             r = tf.sigmoid(
46 |                 tf.matmul(inputs, self.Wr) + self.br
47 |             )
48 | 
49 |             # Final Memory cell
50 |             c = f * c_prev + (1.0 - f) * tf.matmul(inputs, self.U)
51 | 
52 |             # Current Hidden state
53 |             current_hidden_state = r * self.g(c) + (1.0 - r) * inputs
54 |             if self.state_is_tuple:
55 |                 return current_hidden_state, LSTMStateTuple(c, current_hidden_state)
56 |             else:
57 |                 return current_hidden_state, c
58 | 
59 |     def init_matrix(self, shape):
60 |         return tf.random_normal(shape, stddev=0.1)
61 | 
62 | 
63 | _LSTMStateTuple = collections.namedtuple("LSTMStateTuple", ("c", "h"))
64 | 
65 | 
66 | class LSTMStateTuple(_LSTMStateTuple):
67 |   """Tuple used by LSTM Cells for `state_size`, `zero_state`, and output state.
68 | 
69 |   Stores two elements: `(c, h)`, in that order.
70 | 
71 |   Only used when `state_is_tuple=True`.
72 |   """
73 |   __slots__ = ()
74 | 
75 |   @property
76 |   def dtype(self):
77 |     (c, h) = self
78 |     if c.dtype != h.dtype:
79 |       raise TypeError("Inconsistent internal state: %s vs %s" %
80 |                       (str(c.dtype), str(h.dtype)))
81 |     return c.dtype
82 | 


--------------------------------------------------------------------------------