├── .gitignore ├── README.md ├── setup.py ├── test ├── test_single_stack_rnn.py ├── test_single_stack_rnnlm.py └── test_two_stack_rnn.py ├── tox.ini └── turnn ├── base ├── alphabet.py ├── string.py ├── symbol.py └── utils.py └── turing ├── pda.py ├── ppda.py ├── single_stack_rnn.py ├── single_stack_rnnlm.py ├── two_stack_rnn.py └── two_stack_rnnlm.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .ipynb_checkpoints 3 | notebooks/.ipynb_checkpoints 4 | turnn.egg-info 5 | .idea 6 | .DS_Store 7 | .vscode 8 | /data 9 | .history 10 | 11 | 12 | testing.ipynb -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Code accompanying the EMNLP 2023 publication ["On the Representational Capacity of Recurrent Neural Language Models"](https://arxiv.org/abs/2310.12942). 2 | 3 | This repository contains a library for proving the Turing completeness of recurrent neural network language models. 4 | Specifically, it includes implementations of weighted single-stack and two-stack pushdown automata encoded in recurrent neural nets (RNNs). 5 | Since two-stack pushdown automata are equivalent to Turing machines, this shows by construction that RNNs are Turing complete, and can even simulate certain kinds of probabilistic Turing machines. 6 | 7 | Implementation programmed by Ryan Cotterell and Anej Svete. 8 | The original inspiration for the "proof by code" is [Siegelmann and Sontag (1995)](https://binds.cs.umass.edu/papers/1995_Siegelmann_JComSysSci.pdf). 9 | 10 | ## Getting started with the code 11 | 12 | Clone the repository: 13 | 14 | ```bash 15 | $ git clone https://github.com/rycolab/rnn-turing-completeness.git 16 | $ cd rnn-turing-completeness 17 | $ pip install -e . 18 | ``` 19 | 20 | At this point it may be beneficial to create a new [Python virtual environment](https://docs.python.org/3.8/tutorial/venv.html). 21 | There are multiple solutions for this step, including [Miniconda](https://docs.conda.io/en/latest/miniconda.html). 22 | We aim at Python 3.10 version and above. 23 | 24 | Then you install the package _in editable mode_: 25 | ```bash 26 | $ pip install -e . 27 | ``` 28 | 29 | We use [black](https://github.com/psf/black) and [flake8](https://flake8.pycqa.org/en/latest/) to lint the code, [pytype](https://github.com/google/pytype) to check whether the types agree, and [pytest](https://docs.pytest.org) to unit test the code. 30 | 31 | To **unit-test** the code, run: 32 | ``` 33 | pytest . 34 | ``` 35 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | install_requires = [ 4 | "numpy", 5 | "sympy", 6 | # needed for the tests 7 | "pytest", 8 | ] 9 | 10 | 11 | setup( 12 | name="turnn", 13 | install_requires=install_requires, 14 | version="0.1", 15 | scripts=[], 16 | packages=["turnn"], 17 | ) 18 | -------------------------------------------------------------------------------- /test/test_single_stack_rnn.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from pytest import mark 4 | 5 | from turnn.base.alphabet import to_alphabet 6 | from turnn.base.symbol import EOS, BOT 7 | from turnn.base.string import String 8 | from turnn.turing.single_stack_rnn import ( 9 | SingleStackRNN, 10 | Index, 11 | cantor_decode, 12 | ) 13 | from turnn.turing.pda import SingleStackPDA 14 | 15 | 16 | Σ = to_alphabet({"a", "b"}) 17 | 18 | 19 | @mark.parametrize("n_automata", [32]) 20 | @mark.parametrize("n_steps", [16, 32, 64]) 21 | def test_conversion(n_automata: int, n_steps: int): 22 | for seed in range(n_automata): 23 | y = String([random.choice(list(Σ)) for _ in range(n_steps)]) 24 | 25 | pda = SingleStackPDA(seed=seed) 26 | rnn = SingleStackRNN(pda) 27 | 28 | for sym in y: 29 | pda.step(sym) 30 | rnn(sym) 31 | 32 | _, accept = rnn(EOS) 33 | 34 | assert pda.stack == cantor_decode(rnn.h[Index.STACK]) 35 | assert pda.stack == [BOT] and accept or pda.stack != [BOT] and not accept 36 | -------------------------------------------------------------------------------- /test/test_single_stack_rnnlm.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from turnn.base.alphabet import to_alphabet 4 | from turnn.base.string import String 5 | from turnn.turing.ppda import SingleStackPDA 6 | from turnn.turing.single_stack_rnnlm import SingleStackRNN 7 | 8 | Σ = to_alphabet({"a", "b"}) 9 | 10 | 11 | def test_conversion(): 12 | for seed in range(20): 13 | y = String([random.choice(list(Σ)) for _ in range(16)]) 14 | 15 | P = SingleStackPDA(seed=seed) 16 | R = SingleStackRNN(P) 17 | 18 | P_accept, P_logp = P(y) 19 | R_accept, R_logp = R(y) 20 | 21 | assert abs(P_logp - R_logp) < 1e-6 22 | 23 | assert P_accept and R_accept or not P_accept and not R_accept 24 | -------------------------------------------------------------------------------- /test/test_two_stack_rnn.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from pytest import mark 4 | 5 | from turnn.base.symbol import EOS, BOT 6 | from turnn.base.alphabet import to_alphabet 7 | from turnn.base.string import String 8 | from turnn.turing.two_stack_rnn import TwoStackRNN, Index, cantor_decode 9 | from turnn.turing.pda import TwoStackPDA 10 | 11 | 12 | Σ = to_alphabet({"a", "b"}) 13 | 14 | 15 | @mark.parametrize("n_automata", [32]) 16 | @mark.parametrize("n_steps", [16, 32, 64]) 17 | def test_siegelmann(n_automata: int, n_steps: int): 18 | for seed in range(n_automata): 19 | y = String([random.choice(list(Σ)) for _ in range(n_steps)]) 20 | 21 | pda = TwoStackPDA(seed=seed) 22 | rnn = TwoStackRNN(pda) 23 | 24 | for sym in y: 25 | pda.step(sym) 26 | rnn(sym) 27 | 28 | _, accept = rnn(EOS) 29 | 30 | assert pda.stacks[0] == cantor_decode(rnn.h[Index.STACK1]) 31 | assert pda.stacks[1] == cantor_decode(rnn.h[Index.STACK2]) 32 | 33 | assert ( 34 | pda.stacks == [[BOT], [BOT]] 35 | and accept 36 | or pda.stacks != [[BOT], [BOT]] 37 | and not accept 38 | ) 39 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | ignore = E712, W503, E203 4 | exclude = 5 | .git, 6 | .history, 7 | __pycache__, 8 | docs/source/conf.py, 9 | old, 10 | build, 11 | dist 12 | max-complexity = 8 13 | 14 | -------------------------------------------------------------------------------- /turnn/base/alphabet.py: -------------------------------------------------------------------------------- 1 | from typing import Union, List, Set 2 | 3 | from turnn.base.symbol import Sym, ε, to_sym 4 | 5 | 6 | class Alphabet: 7 | def __init__(self, symbols: Union[List[Sym], Set[Sym]]): 8 | assert len(symbols) > 0, "Alphabet must be non-empty." 9 | assert ε not in symbols, "ε must not be in the alphabet." 10 | self.symbols = symbols if isinstance(symbols, set) else set(symbols) 11 | 12 | def __str__(self): 13 | return "".join(str(sym) for sym in self.symbols) 14 | 15 | def __repr__(self): 16 | return "".join(str(sym) for sym in self.symbols) 17 | 18 | def __hash__(self): 19 | return hash(str(self)) 20 | 21 | def __eq__(self, other): 22 | return isinstance(other, Alphabet) and self.symbols == other.symbols 23 | 24 | def __len__(self): 25 | return len(self.symbols) 26 | 27 | def __contains__(self, item): 28 | return item in self.symbols 29 | 30 | def __iter__(self): 31 | return iter(self.symbols) 32 | 33 | 34 | def to_alphabet( 35 | symbols: Union[List[Union[Sym, str]], Set[Union[Sym, str]]] 36 | ) -> Alphabet: 37 | """Creates an Alphabet from a list of symbols. 38 | 39 | Args: 40 | symbols (Union[List[Union[Sym, str]], Set[Union[Sym, str]]]): _description_ 41 | 42 | Returns: 43 | Alphabet: _description_ 44 | """ 45 | 46 | if isinstance(symbols, Alphabet): 47 | return symbols 48 | else: 49 | return Alphabet( 50 | {to_sym(sym) if isinstance(sym, str) else sym for sym in symbols} 51 | ) 52 | -------------------------------------------------------------------------------- /turnn/base/string.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union 2 | 3 | from turnn.base.symbol import Sym, to_sym 4 | 5 | 6 | class String: 7 | def __init__(self, y: Union[str, List[Sym]]): 8 | self.y = y if isinstance(y, list) else [to_sym(sym) for sym in y] 9 | 10 | def __str__(self): 11 | return "".join(str(sym) for sym in self.y) 12 | 13 | def __repr__(self): 14 | return "".join(str(sym) for sym in self.y) 15 | 16 | def __hash__(self): 17 | return hash(str(self)) 18 | 19 | def __eq__(self, other): 20 | return isinstance(other, String) and self.y == other.y 21 | 22 | def __len__(self): 23 | return len(self.y) 24 | 25 | def __getitem__(self, key): 26 | return self.y[key] 27 | 28 | def __iter__(self): 29 | return iter(self.y) 30 | -------------------------------------------------------------------------------- /turnn/base/symbol.py: -------------------------------------------------------------------------------- 1 | class Sym: 2 | def __init__(self, sym): 3 | self.sym = sym 4 | 5 | def __str__(self): 6 | return str(self.sym) 7 | 8 | def __repr__(self): 9 | return str(self.sym) 10 | 11 | def __hash__(self): 12 | return hash(self.sym) 13 | 14 | def __eq__(self, other): 15 | return isinstance(other, Sym) and self.sym == other.sym 16 | 17 | def __invert__(self): 18 | return self 19 | 20 | 21 | ε = Sym("ε") 22 | 23 | # String sybols 24 | EOS = Sym("EOS") 25 | 26 | # Stack symbols 27 | BOT = Sym("⊥") 28 | 29 | 30 | def to_sym(s: str) -> Sym: 31 | """Converts a single character string to a symbol (Sym). 32 | 33 | Args: 34 | s (str): The input string 35 | 36 | Returns: 37 | Sym: Sym-ed version of the input string. 38 | """ 39 | if isinstance(s, Sym): 40 | return s 41 | else: 42 | return Sym(s) 43 | -------------------------------------------------------------------------------- /turnn/base/utils.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from turnn.base.symbol import BOT, Sym 4 | 5 | 6 | def cantor_decode(x) -> List[Sym]: 7 | """ 8 | This simply decodes the value of the cell in the hidden state representing a stack 9 | into a sequence of symbols. 10 | 11 | Args: 12 | x: The value to be decoded into a sequence of symbols. 13 | 14 | Returns: 15 | List[Sym]: The decoded sequence of symbols. 16 | """ 17 | stack = [BOT] 18 | if x.p == 0: 19 | return stack 20 | return stack + list( 21 | map(lambda x: Sym("0") if x == "1" else Sym("1"), reversed(list(str(x.p)))) 22 | ) 23 | -------------------------------------------------------------------------------- /turnn/turing/pda.py: -------------------------------------------------------------------------------- 1 | import random 2 | from enum import IntEnum, unique 3 | from itertools import product 4 | from typing import List, Tuple 5 | 6 | from turnn.base.string import String 7 | from turnn.base.symbol import BOT, Sym 8 | 9 | 10 | @unique 11 | class Action(IntEnum): 12 | PUSH = 0 13 | POP = 1 14 | NOOP = 2 15 | 16 | 17 | class SingleStackPDA: 18 | def __init__( 19 | self, 20 | Σ={Sym("a"), Sym("b")}, 21 | Γ={BOT, Sym("0"), Sym("1")}, 22 | seed: int = 42, 23 | randomize: bool = True, 24 | ): 25 | self.seed = seed 26 | 27 | self.Σ = Σ 28 | self.Γ = Γ 29 | 30 | self.stack = [BOT] 31 | self.δ = {} 32 | if randomize: 33 | self._random_δ() 34 | 35 | def step(self, sym: Sym): 36 | assert sym in self.Σ 37 | 38 | γ_top = self.stack[-1] 39 | action, γ_new = self.δ[sym][γ_top] 40 | 41 | if action == Action.PUSH: 42 | self.stack.append(γ_new) 43 | elif action == Action.POP: 44 | self.stack.pop() 45 | elif action == Action.NOOP: 46 | pass 47 | else: 48 | raise Exception 49 | 50 | def accept(self, y: String): 51 | for sym in y: 52 | self.step(sym) 53 | 54 | return self.stack == [BOT] 55 | 56 | def _random_δ(self): 57 | random.seed(self.seed) 58 | 59 | pushes = {(Action.PUSH, γ) for γ in self.Γ if γ != BOT} 60 | 61 | for sym in self.Σ: 62 | if sym not in self.δ: 63 | self.δ[sym] = {} 64 | for γ in self.Γ: 65 | if γ == BOT: 66 | flip = random.randint(0, 1) 67 | if flip == 0: 68 | self.δ[sym][γ] = (Action.NOOP, γ) 69 | else: 70 | self.δ[sym][γ] = random.choice(list(pushes)) 71 | else: 72 | flip = random.randint(0, 2) 73 | if flip == 0: 74 | self.δ[sym][γ] = (Action.NOOP, γ) 75 | elif flip == 1: 76 | self.δ[sym][γ] = (Action.POP, γ) 77 | else: 78 | self.δ[sym][γ] = random.choice(list(pushes)) 79 | 80 | 81 | class TwoStackPDA: 82 | def __init__( 83 | self, 84 | Σ={Sym("a"), Sym("b")}, 85 | Γ_1={BOT, Sym("0"), Sym("1")}, 86 | Γ_2={BOT, Sym("0"), Sym("1")}, 87 | seed: int = 42, 88 | ): 89 | self.seed = seed 90 | 91 | self.Σ = Σ 92 | self.Γ_1 = Γ_1 93 | self.Γ_2 = Γ_2 94 | 95 | self.stacks = [[BOT], [BOT]] 96 | self.δ = {} 97 | self._random_δ() 98 | 99 | def _execute_action(self, stack, action, γ_new): 100 | if action == Action.PUSH: 101 | stack.append(γ_new) 102 | elif action == Action.POP: 103 | stack.pop() 104 | elif action == Action.NOOP: 105 | pass 106 | else: 107 | raise Exception 108 | 109 | def step(self, y): 110 | assert y in self.Σ 111 | 112 | γ_top_1, γ_top_2 = self.stacks[0][-1], self.stacks[1][-1] 113 | (action_1, γ_new_1), (action_2, γ_new_2) = self.δ[y][(γ_top_1, γ_top_2)] 114 | 115 | self._execute_action(self.stacks[0], action_1, γ_new_1) 116 | self._execute_action(self.stacks[1], action_2, γ_new_2) 117 | 118 | def accept(self, y): 119 | for sym in y: 120 | self.step(sym) 121 | 122 | return self.stacks[0] == [BOT] and self.stacks[1] == [BOT] 123 | 124 | def _get_action(self, γ_top: Sym, pushes: List[Tuple[Action, Sym]]): 125 | if γ_top == BOT: 126 | flip = random.randint(0, 1) 127 | if flip == 0: 128 | action, γ_new = (Action.NOOP, γ_top) 129 | else: 130 | action, γ_new = random.choice(pushes) 131 | else: 132 | flip = random.randint(0, 2) 133 | if flip == 0: 134 | action, γ_new = (Action.NOOP, γ_top) 135 | elif flip == 1: 136 | action, γ_new = (Action.POP, γ_top) 137 | else: 138 | action, γ_new = random.choice(pushes) 139 | 140 | return action, γ_new 141 | 142 | def _random_δ(self): 143 | random.seed(self.seed) 144 | 145 | pushes = [ 146 | [(Action.PUSH, γ) for γ in Γ if γ != BOT] for Γ in [self.Γ_1, self.Γ_2] 147 | ] 148 | 149 | for a in self.Σ: 150 | if a not in self.δ: 151 | self.δ[a] = {} 152 | 153 | for γ_1, γ_2 in product(self.Γ_1, self.Γ_2): 154 | action_1, γ_new_1 = self._get_action(γ_1, pushes[0]) 155 | action_2, γ_new_2 = self._get_action(γ_2, pushes[1]) 156 | 157 | self.δ[a][(γ_1, γ_2)] = (action_1, γ_new_1), (action_2, γ_new_2) 158 | -------------------------------------------------------------------------------- /turnn/turing/ppda.py: -------------------------------------------------------------------------------- 1 | from enum import IntEnum, unique 2 | from itertools import product 3 | from math import log 4 | from typing import List, Tuple, Union 5 | 6 | import numpy as np 7 | 8 | from turnn.base.string import String 9 | from turnn.base.symbol import BOT, Sym 10 | 11 | 12 | @unique 13 | class Action(IntEnum): 14 | PUSH = 0 15 | POP = 1 16 | NOOP = 2 17 | 18 | 19 | class SingleStackPDA: 20 | def __init__( 21 | self, 22 | Σ={Sym("a"), Sym("b")}, 23 | Γ={BOT, Sym("0"), Sym("1")}, 24 | seed: int = 42, 25 | randomize: bool = True, 26 | ): 27 | self.seed = seed 28 | 29 | self.Σ = Σ 30 | self.Γ = Γ 31 | 32 | # δ: Σ × Γ → (({PUSH, POP, NOOP} × Γ) × R) 33 | self.δ = {sym: {γ: {} for γ in self.Γ} for sym in self.Σ} 34 | if randomize: 35 | self._random_δ() 36 | 37 | def step(self, stack: List[Sym], a: Sym) -> Tuple[List[Sym], float]: 38 | assert a in self.Σ 39 | 40 | γ = stack[-1] 41 | action, γʼ = self.δ[a][γ][0] 42 | 43 | # Modify the stack according to the action. 44 | if action == Action.PUSH: 45 | stack.append(γʼ) 46 | elif action == Action.POP: 47 | stack.pop() 48 | elif action == Action.NOOP: 49 | pass 50 | else: 51 | raise Exception 52 | 53 | # Return the new stack and the probability of the action. 54 | return stack, self.δ[a][γ][1] 55 | 56 | @property 57 | def probabilistic(self): 58 | """Checks if the PDA is probabilistic.""" 59 | d = {γ: 0 for γ in self.Γ} 60 | for a, γ in product(self.Σ, self.Γ): 61 | d[γ] += self.δ[a][γ][1] 62 | return all(abs(d[γ] - 1) < 1e-6 for γ in self.Γ) 63 | 64 | def accept(self, y: Union[str, String]) -> Tuple[bool, float]: 65 | """Computes the acceptance probability of a string. Returns a tuple of 66 | the acceptance status and the log probability. 67 | 68 | Args: 69 | y (Union[str, String]): The string to be accepted. 70 | 71 | Returns: 72 | Tuple[bool, float]: The acceptance status and the log probability. 73 | """ 74 | if isinstance(y, str): 75 | y = String(y) 76 | 77 | stack = [BOT] 78 | logp = 0 79 | 80 | # Simulate a run of the PDA. (Assumes that the PDA is deterministic.) 81 | for a in y: 82 | stack, p = self.step(stack, a) 83 | logp += log(p) 84 | 85 | return stack == [BOT], logp 86 | 87 | def __call__(self, y: Union[str, String]) -> Tuple[bool, float]: 88 | """Computes the acceptance probability of a string. Returns a tuple of 89 | the acceptance status and the log probability. 90 | It simply calls the accept method. 91 | 92 | Args: 93 | y (Union[str, String]): The string to be accepted. 94 | 95 | Returns: 96 | Tuple[bool, float]: The acceptance status and the log probability. 97 | """ 98 | return self.accept(y) 99 | 100 | def _random_δ(self): 101 | """Initializes a random transition function and with it a random PPDA.""" 102 | rng = np.random.default_rng(self.seed) 103 | 104 | pushes = list((Action.PUSH, γ) for γ in (self.Γ - {BOT})) 105 | 106 | for γ in self.Γ: 107 | # The possible actions have to form a probability distribution for every γ. 108 | α = rng.dirichlet(np.ones(len(self.Σ))) 109 | for ii, a in enumerate(self.Σ): 110 | if γ == BOT: 111 | flip = rng.integers(0, 1, endpoint=True) 112 | if flip == 0: 113 | self.δ[a][γ] = ((Action.NOOP, γ), α[ii]) 114 | else: 115 | self.δ[a][γ] = (tuple(rng.choice(pushes)), α[ii]) 116 | else: 117 | flip = rng.integers(0, 2, endpoint=True) 118 | if flip == 0: 119 | self.δ[a][γ] = ((Action.NOOP, γ), α[ii]) 120 | elif flip == 1: 121 | self.δ[a][γ] = ((Action.POP, γ), α[ii]) 122 | else: 123 | self.δ[a][γ] = (tuple(rng.choice(pushes)), α[ii]) 124 | 125 | 126 | class TwoStackPDA: 127 | def __init__( 128 | self, 129 | Σ={Sym("a"), Sym("b")}, 130 | Γ_1={BOT, Sym("0"), Sym("1")}, 131 | Γ_2={BOT, Sym("0"), Sym("1")}, 132 | seed: int = 42, 133 | randomize: bool = True, 134 | ): 135 | self.seed = seed 136 | 137 | self.Σ = Σ 138 | self.Γ_1 = Γ_1 139 | self.Γ_2 = Γ_2 140 | 141 | # δ: Σ × (Γ_1 × Γ_2) → (({PUSH, POP, NOOP} × Γ_1 × Γ_2) × R) 142 | self.δ = { 143 | sym: {(γ_1, γ_2): {} for (γ_1, γ_2) in product(self.Γ_1, self.Γ_2)} 144 | for sym in self.Σ 145 | } 146 | if randomize: 147 | self._random_δ() 148 | 149 | def _execute_action(self, stack: List[Sym], action: Action, γ_new: Sym): 150 | """Commits an action to a the current stack configuration.""" 151 | if action == Action.PUSH: 152 | stack.append(γ_new) 153 | elif action == Action.POP: 154 | stack.pop() 155 | elif action == Action.NOOP: 156 | pass 157 | else: 158 | raise Exception 159 | 160 | def step( 161 | self, stacks: Tuple[List[Sym], List[Sym]], a: Sym 162 | ) -> Tuple[Tuple[List[Sym], List[Sym]], float]: 163 | """Executes a step of the PDA. Returns a tuple of the new stacks and the 164 | probability of the action. 165 | 166 | Args: 167 | stacks (Tuple[List[Sym], List[Sym]]): The current stacks. 168 | a (Sym): The current symbol. 169 | 170 | Returns: 171 | Tuple[Tuple[List[Sym], List[Sym]], float]: The new stacks and the 172 | probability of the action. 173 | """ 174 | assert a in self.Σ 175 | 176 | γ_1, γ_2 = stacks[0][-1], stacks[1][-1] 177 | (action_1, γ_1ʼ), (action_2, γ_2ʼ) = self.δ[a][(γ_1, γ_2)][:2] 178 | 179 | self._execute_action(stacks[0], action_1, γ_1ʼ) 180 | self._execute_action(stacks[1], action_2, γ_2ʼ) 181 | 182 | return stacks, self.δ[a][(γ_1, γ_2)][2] 183 | 184 | def accept(self, y: Union[str, String]) -> Tuple[bool, float]: 185 | """Computes the acceptance probability of a string. Returns a tuple of 186 | the acceptance status and the log probability. 187 | 188 | Args: 189 | y (Union[str, String]): The string to be accepted. 190 | 191 | Returns: 192 | Tuple[bool, float]: The acceptance status and the log probability. 193 | """ 194 | if isinstance(y, str): 195 | y = String(y) 196 | 197 | stacks = [BOT], [BOT] 198 | logp = 0 199 | 200 | for a in y: 201 | stacks, p = self.step(stacks, a) 202 | logp += log(p) 203 | 204 | return stacks[0] == [BOT] and stacks[1] == [BOT], logp 205 | 206 | def __call__(self, y: Union[str, String]) -> Tuple[bool, float]: 207 | """Computes the acceptance probability of a string. Returns a tuple of 208 | the acceptance status and the log probability. 209 | It simply calls the accept method. 210 | 211 | Args: 212 | y (Union[str, String]): The string to be accepted. 213 | 214 | Returns: 215 | Tuple[bool, float]: The acceptance status and the log probability. 216 | """ 217 | return self.accept(y) 218 | 219 | def _get_action( 220 | self, γ_top: Sym, pushes: List[Tuple[Action, Sym]], rng: np.random.Generator 221 | ): 222 | """Returns a random action and a random symbol for a given stack top.""" 223 | if γ_top == BOT: 224 | flip = rng.integers(0, 1, endpoint=True) 225 | if flip == 0: 226 | action, γ_new = (Action.NOOP, γ_top) 227 | else: 228 | action, γ_new = tuple(rng.choice(pushes)) 229 | else: 230 | flip = rng.integers(0, 2, endpoint=True) 231 | if flip == 0: 232 | action, γ_new = (Action.NOOP, γ_top) 233 | elif flip == 1: 234 | action, γ_new = (Action.POP, γ_top) 235 | else: 236 | action, γ_new = tuple(rng.choice(pushes)) 237 | 238 | return action, γ_new 239 | 240 | def _random_δ(self): 241 | """Initializes a random transition function and with it a random PPDA.""" 242 | rng = np.random.default_rng(self.seed) 243 | 244 | pushes = [ 245 | [(Action.PUSH, γ) for γ in Γ] for Γ in [self.Γ_1 - {BOT}, self.Γ_2 - {BOT}] 246 | ] 247 | 248 | for γ_1, γ_2 in product(self.Γ_1, self.Γ_2): 249 | α = rng.dirichlet(np.ones(len(self.Σ))) 250 | for ii, a in enumerate(self.Σ): 251 | action_1, γ_1ʼ = self._get_action(γ_1, pushes[0], rng) 252 | action_2, γ_2ʼ = self._get_action(γ_2, pushes[1], rng) 253 | 254 | self.δ[a][(γ_1, γ_2)] = (action_1, γ_1ʼ), (action_2, γ_2ʼ), α[ii] 255 | -------------------------------------------------------------------------------- /turnn/turing/single_stack_rnn.py: -------------------------------------------------------------------------------- 1 | """ This is the *non-LM* implementation of an RNN simulating an 2 | *unweighted* PDA with a single stack. 3 | See `turnn/turing/single_stack_rnnlm.py` for the *LM* implementation.""" 4 | 5 | from enum import IntEnum, unique 6 | 7 | from sympy import Abs, Matrix, Piecewise, Rational, Symbol, eye, sympify, zeros 8 | 9 | from turnn.base.string import String 10 | from turnn.base.symbol import BOT, EOS, Sym 11 | from turnn.base.utils import cantor_decode 12 | from turnn.turing.pda import Action, SingleStackPDA 13 | 14 | x = Symbol("x") 15 | σ = Piecewise((Rational(0), x <= 0), (Rational(1), x >= 1), (Abs(x) <= sympify(1))) 16 | 17 | 18 | @unique 19 | class Index(IntEnum): 20 | # commands 21 | STACK = 0 22 | BUFFER1 = 1 23 | BUFFER2 = 2 24 | 25 | PHASE1 = 3 26 | PHASE2 = 4 27 | PHASE3 = 5 28 | PHASE4 = 6 29 | 30 | STACK_EMPTY = 7 31 | STACK_ZERO = 8 32 | STACK_ONE = 9 33 | 34 | # `CONF_γa` corresponds to the PDA configuration 35 | # in which the top of the stack is `γ` and the next symbol is `a`` 36 | CONF_BOT_a = 10 37 | CONF_BOT_b = 11 38 | CONF_0_a = 12 39 | CONF_0_b = 13 40 | CONF_1_a = 14 41 | CONF_1_b = 15 42 | CONF_BOT_EOS = 16 43 | CONF_0_EOS = 17 44 | CONF_1_EOS = 18 45 | 46 | PUSH_0 = 19 47 | PUSH_1 = 20 48 | POP_0 = 21 49 | POP_1 = 22 50 | NOOP = 23 51 | 52 | # Signals the acceptance of the input string 53 | ACCEPT = 24 54 | 55 | 56 | def enc_peek(x): # noqa: C901 57 | if x == Index.CONF_BOT_a: 58 | print("(⊥, a)") 59 | elif x == Index.CONF_BOT_b: 60 | print("(⊥, b)") 61 | elif x == Index.CONF_0_a: 62 | print("(0, a)") 63 | elif x == Index.CONF_0_b: 64 | print("(0, b)") 65 | elif x == Index.CONF_1_a: 66 | print("(1, a)") 67 | elif x == Index.CONF_1_b: 68 | print("(1, b)") 69 | elif x == Index.CONF_BOT_EOS: 70 | print("(⊥, EOS)") 71 | elif x == Index.CONF_0_EOS: 72 | print("(0, EOS)") 73 | elif x == Index.CONF_1_EOS: 74 | print("(1, EOS)") 75 | 76 | raise Exception 77 | 78 | 79 | def conf2idx(γ: Sym, sym: Sym): # noqa: C901 80 | if (sym, γ) == (Sym("a"), BOT): 81 | return Index.CONF_BOT_a 82 | elif (sym, γ) == (Sym("b"), BOT): 83 | return Index.CONF_BOT_b 84 | elif (sym, γ) == (Sym("a"), Sym("0")): 85 | return Index.CONF_0_a 86 | elif (sym, γ) == (Sym("b"), Sym("0")): 87 | return Index.CONF_0_b 88 | elif (sym, γ) == (Sym("a"), Sym("1")): 89 | return Index.CONF_1_a 90 | elif (sym, γ) == (Sym("b"), Sym("1")): 91 | return Index.CONF_1_b 92 | elif (sym, γ) == (EOS, BOT): 93 | return Index.CONF_BOT_EOS 94 | elif (sym, γ) == (EOS, Sym("0")): 95 | return Index.CONF_0_EOS 96 | elif (sym, γ) == (EOS, Sym("1")): 97 | return Index.CONF_1_EOS 98 | raise Exception 99 | 100 | 101 | class SingleStackRNN: 102 | def __init__(self, pda: SingleStackPDA): 103 | self.pda = pda 104 | self.Σ = pda.Σ.union({EOS}) # In contrast to the globally normalized PDA, 105 | # the RNN works with an EOS symbol 106 | 107 | self.sym2idx = {sym: idx for idx, sym in enumerate(self.Σ)} 108 | 109 | # The dimension of the hidden state 110 | self.D = len(Index) 111 | 112 | # Recurrence matrix U 113 | self.U = zeros(self.D, self.D, dtype=Rational(0)) 114 | # Input matrix V 115 | self.V = zeros(self.D, len(self.Σ), dtype=Rational(0)) 116 | # Bias vector b 117 | self.b = zeros(self.D, 1, dtype=Rational(0)) 118 | 119 | self.make_U() 120 | self.make_b() 121 | self.make_V() 122 | 123 | # One-hot encoding of the input symbols 124 | self.one_hot = eye(len(self.Σ), dtype=Rational(0)) 125 | 126 | # Embedding matrix of the input symbols 127 | self.E = zeros(self.D, len(self.Σ), dtype=Rational(0)) 128 | 129 | # We start with an zero hidden state in phase 1 130 | self.h = zeros(self.D, 1, dtype=Rational(0)) 131 | self.reset() 132 | 133 | def reset(self): 134 | """Sets the initial hidden state of the RNN to the zero vector 135 | indicating the first phase.""" 136 | self.h = zeros(self.D, 1, dtype=Rational(0)) 137 | self.h[Index.PHASE1] = Rational(1) 138 | 139 | def accept(self, y: String): 140 | """Checks whether the RNN accepts the input string `y`. 141 | 142 | Args: 143 | y (String): The input string. 144 | 145 | Returns: 146 | bool: True if the RNN accepts the input string, False otherwise. 147 | """ 148 | self.reset() 149 | for sym in y: 150 | self(sym) 151 | return self.h[Index.ACCEPT] == Rational(1) 152 | 153 | def disp(self, h: Matrix): 154 | """Prints the content of the hidden state of the RNN, separated by 155 | the different components. 156 | 157 | Args: 158 | h (Matrix): The hidden state of the RNN. 159 | """ 160 | print(f"Stack:\t{h[Index.STACK]}") 161 | print(f"Buffer 1:\t{h[Index.BUFFER1]}") 162 | print(f"Buffer 2:\t{h[Index.BUFFER2]}") 163 | 164 | print(f"Phase 1:\t{h[Index.PHASE1]}") 165 | print(f"Phase 2:\t{h[Index.PHASE2]}") 166 | print(f"Phase 3:\t{h[Index.PHASE3]}") 167 | print(f"Phase 4:\t{h[Index.PHASE4]}") 168 | 169 | print(f"1 Peek EMPTY:\t{h[Index.STACK_EMPTY]}") 170 | print(f"1 Peek 0:\t{h[Index.STACK_ZERO]}") 171 | print(f"1 Peek 1:\t{h[Index.STACK_ONE]}") 172 | 173 | print(f"2 (⊥, a):\t{h[Index.CONF_BOT_a]}") 174 | print(f"2 (⊥, b):\t{h[Index.CONF_BOT_b]}") 175 | print(f"2 (0, a):\t{h[Index.CONF_0_a]}") 176 | print(f"2 (0, b):\t{h[Index.CONF_0_b]}") 177 | print(f"2 (1, a):\t{h[Index.CONF_1_a]}") 178 | print(f"2 (1, b):\t{h[Index.CONF_1_b]}") 179 | print(f"2 (BOT, EOS):\t{h[Index.CONF_BOT_EOS]}") 180 | print(f"2 (0, EOS):\t{h[Index.CONF_0_EOS]}") 181 | print(f"2 (1, EOS):\t{h[Index.CONF_1_EOS]}") 182 | 183 | print(f"3 Push 0:\t{h[Index.PUSH_0]}") 184 | print(f"3 Push 1:\t{h[Index.PUSH_1]}") 185 | print(f"3 Pop 0:\t{h[Index.POP_0]}") 186 | print(f"3 Pop 1:\t{h[Index.POP_1]}") 187 | print(f"3 NOOP:\t{h[Index.NOOP]}") 188 | 189 | print(f"4 ACCEPT:\t{h[Index.ACCEPT]}") 190 | 191 | print() 192 | 193 | def make_stack_transitions(self): 194 | # This submatrix encodes the copying of the stack contents between 195 | # the three different stacks simulated by the RNN in the hidden state 196 | # while it moves through the 4 phases of the hidden state update. 197 | # Each of the stacks is simply a dimension in the hidden state. 198 | # Each of the three stacks is copied to the next one, 199 | # and the last one is zeroed out. 200 | self.U[Index.STACK, Index.STACK] = Rational(0) # delete 201 | self.U[Index.BUFFER1, Index.STACK] = Rational(1) # copy over to the next one 202 | self.U[Index.BUFFER1, Index.BUFFER1] = Rational(0) 203 | self.U[Index.BUFFER2, Index.BUFFER1] = Rational(1) 204 | self.U[Index.BUFFER2, Index.BUFFER2] = Rational(0) 205 | 206 | def make_phase_transitions(self): 207 | # These two blocks encode the state---it's a one-hot encoding 208 | self.U[Index.PHASE1, Index.PHASE1] = Rational(0) 209 | self.U[Index.PHASE2, Index.PHASE2] = Rational(0) 210 | self.U[Index.PHASE3, Index.PHASE3] = Rational(0) 211 | self.U[Index.PHASE4, Index.PHASE4] = Rational(0) 212 | self.U[Index.PHASE2, Index.PHASE1] = Rational(1) 213 | self.U[Index.PHASE3, Index.PHASE2] = Rational(1) 214 | self.U[Index.PHASE4, Index.PHASE3] = Rational(1) 215 | self.U[Index.PHASE1, Index.PHASE4] = Rational(1) 216 | 217 | def make_stack_detector(self): 218 | # PHASE 1: 219 | # This detects an empty stack 220 | self.U[Index.STACK_EMPTY, Index.STACK] = Rational(-10) 221 | # This detects whether the top of the stack is a 0 or a 1 222 | self.U[Index.STACK_ZERO, Index.STACK] = Rational(-10) 223 | self.U[Index.STACK_ONE, Index.STACK] = Rational(10) 224 | 225 | def make_configuration_detector(self): 226 | # PHASE 2: This submatrix corresponds to the actions available given any 227 | # configuration of the stack, captured at EMPTY/PEEK locations. 228 | # This corresponds to the emmisions 229 | # These then get "intersected" with the symbol embeddings to select a or b 230 | self.U[Index.CONF_BOT_a, Index.STACK_EMPTY] = Rational(1) 231 | self.U[Index.CONF_BOT_b, Index.STACK_EMPTY] = Rational(1) 232 | self.U[Index.CONF_BOT_EOS, Index.STACK_EMPTY] = Rational(1) 233 | # E.g., given that we have peeked 0, we can either read in an ``a'' or a ``b'' 234 | self.U[Index.CONF_0_a, Index.STACK_ZERO] = Rational(1) 235 | self.U[Index.CONF_0_b, Index.STACK_ZERO] = Rational(1) 236 | self.U[Index.CONF_0_EOS, Index.STACK_ZERO] = Rational(1) 237 | self.U[Index.CONF_1_a, Index.STACK_ONE] = Rational(1) 238 | self.U[Index.CONF_1_b, Index.STACK_ONE] = Rational(1) 239 | self.U[Index.CONF_1_EOS, Index.STACK_ONE] = Rational(1) 240 | 241 | # This bit ensures that we can't be empty and non-empty 242 | # It erases the effects of always setting the PEEK0 bit to 1 243 | self.U[Index.CONF_0_a, Index.STACK_EMPTY] = Rational(-1) 244 | self.U[Index.CONF_0_b, Index.STACK_EMPTY] = Rational(-1) 245 | self.U[Index.CONF_0_EOS, Index.STACK_EMPTY] = Rational(-1) 246 | # self.U[Index.CONF_1_a, Index.STACK_EMPTY] = Rational(-10) 247 | # self.U[Index.CONF_1_b, Index.STACK_EMPTY] = Rational(-10) 248 | # self.U[Index.CONF_1_EOS, Index.STACK_EMPTY] = Rational(-10) 249 | 250 | def initialize_transition_function(self): 251 | # transition function 252 | for i in range(Index.PUSH_0, Index.NOOP + 1): 253 | for j in range(Index.CONF_BOT_a, Index.CONF_1_EOS + 1): 254 | # As soon as the configuration has been determined, reset 255 | # the information about what action should be taken, 256 | # so that only the correct action is considered (by overwriting 257 | # the -10's with 0's below). 258 | self.U[i, j] = Rational(-10) 259 | 260 | def make_action_detector(self): # noqa: C901 261 | self.initialize_transition_function() 262 | 263 | # This is the transition matrix 264 | for sym in self.Σ: 265 | if sym == EOS: 266 | # If we read the EOS symbol, we don't do anything. 267 | # We set those manually, since EOS-transitions are not in the PDA. 268 | self.U[Index.NOOP, conf2idx(BOT, EOS)] = Rational(0) 269 | self.U[Index.NOOP, conf2idx(Sym("0"), EOS)] = Rational(0) 270 | self.U[Index.NOOP, conf2idx(Sym("1"), EOS)] = Rational(0) 271 | continue 272 | 273 | for γ_top in self.pda.δ[sym]: 274 | action, γ_new = self.pda.δ[sym][γ_top] 275 | conf = conf2idx(γ_top, sym) 276 | 277 | if action == Action.PUSH and γ_new == Sym("0"): 278 | # If the PDA stack top is γ1 and sym is read, simulate PUSHing 0 279 | # According to the PDA: if we were in the configuration encoded by 280 | # conf, and we read sym, we would push 0 onto the stack. 281 | self.U[Index.PUSH_0, conf] = Rational(0) 282 | elif action == Action.PUSH and γ_new == Sym("1"): 283 | # If the PDA stack top is γ1 and sym is read, simulate PUSHing 1 284 | self.U[Index.PUSH_1, conf] = Rational(0) 285 | elif action == Action.POP and γ_new == Sym("0"): 286 | # If the PDA stack top is γ1 and sym is read, 287 | # simulate POPping γ1 (0) 288 | self.U[Index.POP_0, conf] = Rational(0) 289 | elif action == Action.POP and γ_new == Sym("1"): 290 | # If the PDA stack top is γ1 and sym is read, 291 | # simulate POPping γ1 (1) 292 | self.U[Index.POP_1, conf] = Rational(0) 293 | elif action == Action.NOOP: 294 | self.U[Index.NOOP, conf] = Rational(0) 295 | else: 296 | raise ValueError( 297 | "Unknown action: action: {}, γ_new: {}.".format(action, γ_new) 298 | ) 299 | 300 | def make_action_executor(self): 301 | # PHASE 3: we execute all possible actions 302 | # These two take whatever is on the stack so far (primarily it would be on 303 | # stack 1, but by phase 4, it has been moved to stack 3) and move it 304 | # down by dividing by 10. 305 | self.U[Index.PUSH_0, Index.BUFFER2] = Rational("1/10") 306 | self.U[Index.PUSH_1, Index.BUFFER2] = Rational("1/10") 307 | # These two take whatever is on the stack and move it up by multiplying by 10. 308 | self.U[Index.POP_0, Index.BUFFER2] = Rational(10) 309 | self.U[Index.POP_1, Index.BUFFER2] = Rational(10) 310 | 311 | # Copy the last buffer into the NOOP cell, but only if none of the other actions 312 | # are active (handled in `make_action_detector`). 313 | self.U[Index.NOOP, Index.BUFFER2] = Rational(1) 314 | 315 | def make_action_committer(self): 316 | # PHASE 4: The final phase---commit the action from the individual 317 | # action cells to the stack. 318 | # Only one of the four entries PUSH_0, PUSH_1, POP_0, POP_1 319 | # will be "active" (non-zero), the rest will be 0. 320 | 321 | # This submatrix encodes the copying of the actions from the ``actions'' part 322 | # of the hidden state to the part of the hidden state corresponding to stack 1. 323 | self.U[Index.STACK, Index.PUSH_0] = Rational(1) 324 | self.U[Index.STACK, Index.PUSH_1] = Rational(1) 325 | self.U[Index.STACK, Index.POP_0] = Rational(1) 326 | self.U[Index.STACK, Index.POP_1] = Rational(1) 327 | self.U[Index.STACK, Index.NOOP] = Rational(1) 328 | 329 | def make_acceptor(self): 330 | # PHASE 5: acceptor 331 | # This is the final state of the RNN. 332 | # It accepts if the stack is empty (encoded in the computation component) 333 | # and the input is empty, i.e., the previous input was EOS. 334 | 335 | # Checks the emptiness of the computation component 336 | self.U[Index.ACCEPT, Index.PUSH_0] = Rational(-10) 337 | self.U[Index.ACCEPT, Index.PUSH_1] = Rational(-10) 338 | self.U[Index.ACCEPT, Index.POP_0] = Rational(-10) 339 | self.U[Index.ACCEPT, Index.POP_1] = Rational(-10) 340 | self.U[Index.ACCEPT, Index.NOOP] = Rational(-10) 341 | 342 | # Checks that the input was EOS 343 | self.U[Index.ACCEPT, Index.CONF_BOT_a] = Rational(-10) 344 | self.U[Index.ACCEPT, Index.CONF_BOT_b] = Rational(-10) 345 | self.U[Index.ACCEPT, Index.CONF_0_a] = Rational(-10) 346 | self.U[Index.ACCEPT, Index.CONF_0_b] = Rational(-10) 347 | self.U[Index.ACCEPT, Index.CONF_1_a] = Rational(-10) 348 | self.U[Index.ACCEPT, Index.CONF_1_b] = Rational(-10) 349 | 350 | def make_U(self): 351 | """Constructs the recurrence matrix U of the RNN.""" 352 | 353 | # ORCHESTRATION 354 | self.make_stack_transitions() 355 | self.make_phase_transitions() 356 | 357 | # PHASE 1 358 | self.make_stack_detector() 359 | 360 | # PHASE 2 361 | self.make_configuration_detector() 362 | 363 | # PHASE 3 364 | self.make_action_detector() 365 | self.make_action_executor() 366 | 367 | # PHASE 4 368 | self.make_action_committer() 369 | self.make_acceptor() 370 | 371 | def make_V(self): 372 | """Constructs the emission matrix V of the RNN.""" 373 | # Enables setting the hidden state to 1 at the indices 374 | # corresponding to all possible combinations of the stack configurations 375 | # (top of the stack) and the incomping input symbol. 376 | # For input symbol a 377 | self.V[Index.CONF_BOT_a, self.sym2idx[Sym("a")]] = Rational(1) 378 | self.V[Index.CONF_0_a, self.sym2idx[Sym("a")]] = Rational(1) 379 | self.V[Index.CONF_1_a, self.sym2idx[Sym("a")]] = Rational(1) 380 | # For input symbol b 381 | self.V[Index.CONF_BOT_b, self.sym2idx[Sym("b")]] = Rational(1) 382 | self.V[Index.CONF_0_b, self.sym2idx[Sym("b")]] = Rational(1) 383 | self.V[Index.CONF_1_b, self.sym2idx[Sym("b")]] = Rational(1) 384 | # For input symbol EOS 385 | self.V[Index.CONF_BOT_EOS, self.sym2idx[EOS]] = Rational(1) 386 | self.V[Index.CONF_0_EOS, self.sym2idx[EOS]] = Rational(1) 387 | self.V[Index.CONF_1_EOS, self.sym2idx[EOS]] = Rational(1) 388 | 389 | def make_b(self): 390 | """Constructs the bias vector b of the RNN.""" 391 | self.b[Index.STACK_EMPTY, 0] = Rational(1) 392 | # If the top of the stack is a 0, the encoding will be 0.1... 393 | # This means that -10 * stack encoding (self.U[Index.STACK_ZERO, Index.STACK1]) 394 | # will be at least -2. 395 | # We therefore map it so a value >= 1. 396 | # Otherwise, the encoding will be 0.3... and this, together with the 397 | # multiplication and addition will not be >= 1. 398 | self.b[Index.STACK_ZERO, 0] = Rational(3) 399 | # If the top of the stack is a 1, the encoding will be 0.3... 400 | # This means that 10 * stack encoding (self.U[Index.STACK_ONE, Index.STACK1]) 401 | # will be at least 3. 402 | # We therefore map it so a value >= 1. 403 | # Otherwise, the encoding will be 0.1... and this, together with the 404 | # multiplication and subtraction will not be >= 1. 405 | self.b[Index.STACK_ONE, 0] = Rational(-2) 406 | 407 | # This provides the base of the quantities that get added to 408 | # the previous encoding of the stack after it is divided by 10. 409 | # Add the encoding of the new top of the stack 410 | self.b[Index.PUSH_0, 0] = Rational("1/10") 411 | self.b[Index.PUSH_1, 0] = Rational("3/10") 412 | # Remove the top encoding of the stack, after it has been multiplied by 10. 413 | self.b[Index.POP_0, 0] = Rational(-1) 414 | self.b[Index.POP_1, 0] = Rational(-3) 415 | 416 | self.b[Index.NOOP, 0] = Rational(0) 417 | 418 | # This zeroes out the configurations which are not active 419 | # even though some of the "triggering" cells in the hidden state are active. 420 | self.b[Index.CONF_BOT_a, 0] = Rational(-1) 421 | self.b[Index.CONF_BOT_b, 0] = Rational(-1) 422 | self.b[Index.CONF_BOT_EOS, 0] = Rational(-1) 423 | self.b[Index.CONF_0_a, 0] = Rational(-1) 424 | self.b[Index.CONF_0_b, 0] = Rational(-1) 425 | self.b[Index.CONF_0_EOS, 0] = Rational(-1) 426 | self.b[Index.CONF_1_a, 0] = Rational(-1) 427 | self.b[Index.CONF_1_b, 0] = Rational(-1) 428 | self.b[Index.CONF_1_EOS, 0] = Rational(-1) 429 | 430 | self.b[Index.ACCEPT, 0] = Rational(1) 431 | 432 | def __call__(self, a: Sym): 433 | """Performs a step of the RNN.""" 434 | assert a in self.Σ 435 | 436 | # We have four stages in which the hidden state is updated 437 | # 1) Peek 438 | # 2) Combine 439 | # 3) Transition 440 | # 4) Consolidate 441 | 442 | def apply(_h): 443 | tmp = self.U * _h + self.V * self.one_hot.col(self.sym2idx[a]) + self.b 444 | # print(self.disp(tmp)) 445 | h2 = zeros(self.D, 1, dtype=Rational(0)) 446 | 447 | for i in range(self.D): 448 | h2[i] = σ.subs(x, tmp[i]) 449 | return h2 450 | 451 | h = self.h 452 | print("\n\n\n\n\n\n >>>> START:") 453 | self.disp(h) 454 | print(cantor_decode(h[Index.STACK])) 455 | 456 | print("\n\n\n\n----------------------------\n>>>>> PHASE 1") 457 | h = apply(h) 458 | print(cantor_decode(h[Index.STACK])) 459 | self.disp(h) 460 | print("\n\n\n\n----------------------------\n>>>>> PHASE 2") 461 | h = apply(h) 462 | print(cantor_decode(h[Index.STACK])) 463 | self.disp(h) 464 | print("\n\n\n\n----------------------------\n>>>>> PHASE 3") 465 | h = apply(h) 466 | print(cantor_decode(h[Index.STACK])) 467 | self.disp(h) 468 | print("\n\n\n\n----------------------------\n>>>>> PHASE 4") 469 | h = apply(h) 470 | print(cantor_decode(h[Index.STACK])) 471 | self.disp(h) 472 | print(cantor_decode(h[Index.STACK])) 473 | self.disp(h) 474 | self.h = h 475 | 476 | return self.h, self.h[Index.ACCEPT] 477 | -------------------------------------------------------------------------------- /turnn/turing/single_stack_rnnlm.py: -------------------------------------------------------------------------------- 1 | """ Implements an RNN language model simulating a single-stack 2 | *probabilistic* PDA. 3 | See `turnn/turing/single_stack_rnn.py` for the unweighted version of the 4 | same construction.""" 5 | 6 | from enum import IntEnum, unique 7 | from typing import Tuple, Union 8 | 9 | from sympy import (Abs, Matrix, Piecewise, Rational, Symbol, eye, log, sympify, 10 | zeros) 11 | 12 | from turnn.base.string import String 13 | from turnn.base.symbol import BOT, EOS, Sym, ε 14 | # from turnn.base.utils import cantor_decode 15 | from turnn.turing.pda import Action, SingleStackPDA 16 | 17 | # The element-wise saturated sigmoid function 18 | x = Symbol("x") 19 | σ = Piecewise((Rational(0), x <= 0), (Rational(1), x >= 1), (Abs(x) <= sympify(1))) 20 | 21 | 22 | @unique 23 | class Index(IntEnum): 24 | """This class is used to index the hidden state of the RNN for two 25 | stacks by naming the individual dimensions of the hidden state with the role 26 | they play in simulating the PDA. 27 | """ 28 | 29 | # (1) Data component 30 | STACK = 0 31 | BUFFER1 = 1 32 | BUFFER2 = 2 33 | 34 | # (2) Phase component 35 | PHASE1 = 3 36 | PHASE2 = 4 37 | PHASE3 = 5 38 | PHASE4 = 6 39 | 40 | # (3) Stack configuration component 41 | # Since we assume a single-state machine, this is also 42 | # the full configuration of the machine. 43 | STACK_EMPTY = 7 44 | STACK_ZERO = 8 45 | STACK_ONE = 9 46 | 47 | # (4) Stack and input configuration component 48 | # `CONF_γ_a` corresponds to the configuration 49 | # where the top of the stack is `γ` and the next symbol is `a` 50 | CONF_BOT_ε = 10 51 | CONF_BOT_a = 11 52 | CONF_BOT_b = 12 53 | CONF_BOT_EOS = 13 54 | CONF_0_ε = 14 55 | CONF_0_a = 15 56 | CONF_0_b = 16 57 | CONF_0_EOS = 17 58 | CONF_1_ε = 18 59 | CONF_1_a = 19 60 | CONF_1_b = 20 61 | CONF_1_EOS = 21 62 | 63 | # (5) Computation component 64 | PUSH_0 = 22 65 | PUSH_1 = 23 66 | POP_0 = 24 67 | POP_1 = 25 68 | NOOP = 26 69 | 70 | # (6) Acceptance component 71 | ACCEPT = 27 72 | 73 | 74 | def enc_peek(x): # noqa: C901 75 | if x == Index.CONF_BOT_a: 76 | print("(⊥, a)") 77 | elif x == Index.CONF_BOT_b: 78 | print("(⊥, b)") 79 | elif x == Index.CONF_0_a: 80 | print("(0, a)") 81 | elif x == Index.CONF_0_b: 82 | print("(0, b)") 83 | elif x == Index.CONF_1_a: 84 | print("(1, a)") 85 | elif x == Index.CONF_1_b: 86 | print("(1, b)") 87 | elif x == Index.CONF_BOT_EOS: 88 | print("(⊥, EOS)") 89 | elif x == Index.CONF_0_EOS: 90 | print("(0, EOS)") 91 | elif x == Index.CONF_1_EOS: 92 | print("(1, EOS)") 93 | 94 | raise Exception 95 | 96 | 97 | def conf2idx(γ: Sym, sym: Sym): # noqa: C901 98 | if (sym, γ) == (ε, BOT): 99 | return Index.CONF_BOT_ε 100 | elif (sym, γ) == (Sym("a"), BOT): 101 | return Index.CONF_BOT_a 102 | elif (sym, γ) == (Sym("b"), BOT): 103 | return Index.CONF_BOT_b 104 | elif (sym, γ) == (EOS, BOT): 105 | return Index.CONF_BOT_EOS 106 | elif (sym, γ) == (ε, Sym("0")): 107 | return Index.CONF_0_ε 108 | elif (sym, γ) == (Sym("a"), Sym("0")): 109 | return Index.CONF_0_a 110 | elif (sym, γ) == (Sym("b"), Sym("0")): 111 | return Index.CONF_0_b 112 | elif (sym, γ) == (EOS, Sym("0")): 113 | return Index.CONF_0_EOS 114 | elif (sym, γ) == (ε, Sym("1")): 115 | return Index.CONF_1_ε 116 | elif (sym, γ) == (Sym("a"), Sym("1")): 117 | return Index.CONF_1_a 118 | elif (sym, γ) == (Sym("b"), Sym("1")): 119 | return Index.CONF_1_b 120 | elif (sym, γ) == (EOS, Sym("1")): 121 | return Index.CONF_1_EOS 122 | raise ValueError("Unknown configuration: ({}, {}).".format(γ, sym)) 123 | 124 | 125 | class SingleStackRNN: 126 | """ 127 | An implementation of an Elman RNN language model that simulates a probabilistic PDA. 128 | It computes the next state h_t+1 from the current state h_t by applying the 129 | Elman update rule four times: 130 | h' = σ(U * h + V * y + b) 131 | where the matrices U, V and the vector b are computed from the PDA and 132 | the input symbol y is one-hot encoded. 133 | After four applications of the update rule, the new hidden state represents the 134 | new configuration of the PDA. 135 | 136 | The hidden state is composed of six components: 137 | 1. The data component 138 | 2. The phase component 139 | 3. The stack configurations component 140 | 4. The stacks and input configuration component 141 | 5. The computation component 142 | 6. The acceptance component 143 | 144 | 1. The first component contains the cells containing the encodings of the stack 145 | along with buffer cells through which the stack is passed during the 146 | four-stage computation. 147 | 2. The second component simply denotes the current phase of the computation, and is 148 | only included for keeping track of the computation. 149 | 3. The third component contains the encoding of the stack at the current time step, 150 | i.e., it flags whether the stacks are empty or have a 0/1 on the top. 151 | 4. The fourth component contains the encoding of the current input symbol together 152 | with the stacks. 153 | 5. The fifth component contains cells to which the current stack configuration is 154 | copied and modified according to the action of the PDA. 155 | 6. The sixth component contains a cell that are set to 1 after reading in the EOS 156 | symbol if the PDA accepts the input string appearing before EOS. 157 | 158 | The hidden state is then used to index the output matrix of (log) probabilities 159 | such that the probability of the next symbol is computed as 160 | p(y_t+1 | y_t, h_t) = exp(E * h_t+1) / sum(exp(E * h_t+1)) 161 | where E is the emission matrix. 162 | The log probabilities are then summed up over the entire string to form the log 163 | probability of the string, which matches the one of the simulated PPDA. 164 | """ 165 | 166 | def __init__(self, pda: SingleStackPDA): 167 | self.pda = pda 168 | self.Σ = pda.Σ.union({EOS}) # In contrast to the globally normalized PDA, 169 | # the RNN works with an EOS symbol 170 | 171 | self.sym2idx = {sym: idx for idx, sym in enumerate(self.Σ)} 172 | 173 | # The dimension of the hidden state 174 | self.D = len(Index) 175 | 176 | # Recurrence matrix U 177 | self.U = zeros(self.D, self.D, dtype=Rational(0)) 178 | # Input matrix V 179 | self.V = zeros(self.D, len(self.Σ), dtype=Rational(0)) 180 | # Bias vector b 181 | self.b = zeros(self.D, 1, dtype=Rational(0)) 182 | 183 | self.make_U() 184 | self.make_b() 185 | self.make_V() 186 | 187 | # Emission matrix E 188 | self.E = zeros(len(self.Σ), self.D, dtype=Rational(0)) 189 | self.make_E() 190 | 191 | # One-hot encoding of the input symbols 192 | self.one_hot = eye(len(self.Σ), dtype=Rational(0)) 193 | 194 | def disp(self, h: Matrix): 195 | """Prints the content of the hidden state of the RNN, separated by 196 | the different components. 197 | 198 | Args: 199 | h (Matrix): The hidden state of the RNN. 200 | """ 201 | print(f"Stack:\t{h[Index.STACK]}") 202 | print(f"Buffer 1:\t{h[Index.BUFFER1]}") 203 | print(f"Buffer 2:\t{h[Index.BUFFER2]}") 204 | 205 | print(f"Phase 1:\t{h[Index.PHASE1]}") 206 | print(f"Phase 2:\t{h[Index.PHASE2]}") 207 | print(f"Phase 3:\t{h[Index.PHASE3]}") 208 | print(f"Phase 4:\t{h[Index.PHASE4]}") 209 | 210 | print(f"1 Peek EMPTY:\t{h[Index.STACK_EMPTY]}") 211 | print(f"1 Peek 0:\t{h[Index.STACK_ZERO]}") 212 | print(f"1 Peek 1:\t{h[Index.STACK_ONE]}") 213 | 214 | print(f"2 (⊥, a):\t{h[Index.CONF_BOT_a]}") 215 | print(f"2 (⊥, b):\t{h[Index.CONF_BOT_b]}") 216 | print(f"2 (0, a):\t{h[Index.CONF_0_a]}") 217 | print(f"2 (0, b):\t{h[Index.CONF_0_b]}") 218 | print(f"2 (1, a):\t{h[Index.CONF_1_a]}") 219 | print(f"2 (1, b):\t{h[Index.CONF_1_b]}") 220 | print(f"2 (BOT, EOS):\t{h[Index.CONF_BOT_EOS]}") 221 | print(f"2 (0, EOS):\t{h[Index.CONF_0_EOS]}") 222 | print(f"2 (1, EOS):\t{h[Index.CONF_1_EOS]}") 223 | 224 | print(f"3 Push 0:\t{h[Index.PUSH_0]}") 225 | print(f"3 Push 1:\t{h[Index.PUSH_1]}") 226 | print(f"3 Pop 0:\t{h[Index.POP_0]}") 227 | print(f"3 Pop 1:\t{h[Index.POP_1]}") 228 | print(f"3 NOOP:\t{h[Index.NOOP]}") 229 | 230 | print(f"4 ACCEPT:\t{h[Index.ACCEPT]}") 231 | 232 | print() 233 | 234 | def make_stack_transitions(self): 235 | # This submatrix encodes the copying of the stack contents between 236 | # the three different stacks simulated by the RNN in the hidden state 237 | # while it moves through the 4 phases of the hidden state update. 238 | # Each of the stacks is simply a dimension in the hidden state. 239 | # Each of the three stacks is copied to the next one, 240 | # and the last one is zeroed out. 241 | self.U[Index.STACK, Index.STACK] = Rational(0) # delete 242 | self.U[Index.BUFFER1, Index.STACK] = Rational(1) # copy over to the next one 243 | self.U[Index.BUFFER1, Index.BUFFER1] = Rational(0) 244 | self.U[Index.BUFFER2, Index.BUFFER1] = Rational(1) 245 | self.U[Index.BUFFER2, Index.BUFFER2] = Rational(0) 246 | 247 | def make_phase_transitions(self): 248 | # These two blocks encode the state---it's a one-hot encoding 249 | self.U[Index.PHASE1, Index.PHASE1] = Rational(0) 250 | self.U[Index.PHASE2, Index.PHASE2] = Rational(0) 251 | self.U[Index.PHASE3, Index.PHASE3] = Rational(0) 252 | self.U[Index.PHASE4, Index.PHASE4] = Rational(0) 253 | self.U[Index.PHASE2, Index.PHASE1] = Rational(1) 254 | self.U[Index.PHASE3, Index.PHASE2] = Rational(1) 255 | self.U[Index.PHASE4, Index.PHASE3] = Rational(1) 256 | self.U[Index.PHASE1, Index.PHASE4] = Rational(1) 257 | 258 | def make_stack_detector(self): 259 | # PHASE 1: 260 | # This detects an empty stack 261 | self.U[Index.STACK_EMPTY, Index.STACK] = Rational(-10) 262 | # This detects whether the top of the stack is a 0 or a 1 263 | self.U[Index.STACK_ZERO, Index.STACK] = Rational(-10) 264 | self.U[Index.STACK_ONE, Index.STACK] = Rational(10) 265 | 266 | def make_configuration_detector(self): 267 | # PHASE 2: This submatrix corresponds to the actions available given any 268 | # configuration of the stack, captured at EMPTY/PEEK locations. 269 | # This corresponds to the emissions 270 | # These then get "intersected" with the symbol embeddings to select a or b 271 | self.U[Index.CONF_BOT_a, Index.STACK_EMPTY] = Rational(1) 272 | self.U[Index.CONF_BOT_b, Index.STACK_EMPTY] = Rational(1) 273 | self.U[Index.CONF_BOT_EOS, Index.STACK_EMPTY] = Rational(1) 274 | # E.g., given that we have peeked 0, we can either read in an ``a'' or a ``b'' 275 | self.U[Index.CONF_0_a, Index.STACK_ZERO] = Rational(1) 276 | self.U[Index.CONF_0_b, Index.STACK_ZERO] = Rational(1) 277 | self.U[Index.CONF_0_EOS, Index.STACK_ZERO] = Rational(1) 278 | self.U[Index.CONF_1_a, Index.STACK_ONE] = Rational(1) 279 | self.U[Index.CONF_1_b, Index.STACK_ONE] = Rational(1) 280 | self.U[Index.CONF_1_EOS, Index.STACK_ONE] = Rational(1) 281 | 282 | # This bit ensures that we can't be empty and non-empty 283 | # It erases the effects of always setting the PEEK0 bit to 1 284 | self.U[Index.STACK_ZERO, Index.STACK_EMPTY] = Rational(-3) 285 | self.U[Index.CONF_0_a, Index.STACK_EMPTY] = Rational(-1) 286 | self.U[Index.CONF_0_b, Index.STACK_EMPTY] = Rational(-1) 287 | self.U[Index.CONF_0_EOS, Index.STACK_EMPTY] = Rational(-1) 288 | 289 | def initialize_transition_function(self): 290 | # transition function 291 | for i in range(Index.PUSH_0, Index.NOOP + 1): 292 | for j in range(Index.CONF_BOT_a, Index.CONF_1_EOS + 1): 293 | # As soon as the configuration has been determined, reset 294 | # the information about what action should be taken, 295 | # so that only the correct action is considered (by overwriting 296 | # the -10's with 0's below). 297 | self.U[i, j] = Rational(-10) 298 | 299 | def make_action_detector(self): # noqa: C901 300 | self.initialize_transition_function() 301 | 302 | # This is the transition matrix 303 | for sym in self.Σ: 304 | if sym == EOS: 305 | # If we read the EOS symbol, we don't do anything. 306 | # We set those manually, since EOS-transitions are not in the PDA. 307 | self.U[Index.NOOP, conf2idx(BOT, EOS)] = Rational(0) 308 | self.U[Index.NOOP, conf2idx(Sym("0"), EOS)] = Rational(0) 309 | self.U[Index.NOOP, conf2idx(Sym("1"), EOS)] = Rational(0) 310 | continue 311 | 312 | for γ in self.pda.δ[sym]: 313 | action, γʼ = self.pda.δ[sym][γ][0] 314 | conf = conf2idx(γ, sym) 315 | 316 | if action == Action.PUSH and γʼ == Sym("0"): 317 | # If the PDA stack top is γ1 and sym is read, simulate PUSHing 0 318 | # According to the PDA: if we were in the configuration encoded by 319 | # conf, and we read sym, we would push 0 onto the stack. 320 | self.U[Index.PUSH_0, conf] = Rational(0) 321 | elif action == Action.PUSH and γʼ == Sym("1"): 322 | # If the PDA stack top is γ1 and sym is read, simulate PUSHing 1 323 | self.U[Index.PUSH_1, conf] = Rational(0) 324 | elif action == Action.POP and γʼ == Sym("0"): 325 | # If the PDA stack top is γ1 and sym is read, 326 | # simulate POPping γ1 (0) 327 | self.U[Index.POP_0, conf] = Rational(0) 328 | elif action == Action.POP and γʼ == Sym("1"): 329 | # If the PDA stack top is γ1 and sym is read, 330 | # simulate POPping γ1 (1) 331 | self.U[Index.POP_1, conf] = Rational(0) 332 | elif action == Action.NOOP: 333 | self.U[Index.NOOP, conf] = Rational(0) 334 | else: 335 | raise ValueError( 336 | "Unknown action: action: {}, γʼ: {}.".format(action, γʼ) 337 | ) 338 | 339 | def make_action_executor(self): 340 | # PHASE 3: we execute all possible actions 341 | # These two take whatever is on the stack so far (primarily it would be on 342 | # stack 1, but by phase 4, it has been moved to stack 3) and move it 343 | # down by dividing by 10. 344 | self.U[Index.PUSH_0, Index.BUFFER2] = Rational("1/10") 345 | self.U[Index.PUSH_1, Index.BUFFER2] = Rational("1/10") 346 | # These two take whatever is on the stack and move it up by multiplying by 10. 347 | self.U[Index.POP_0, Index.BUFFER2] = Rational(10) 348 | self.U[Index.POP_1, Index.BUFFER2] = Rational(10) 349 | 350 | # Copy the last buffer into the NOOP cell, but only if none of the other actions 351 | # are active (handled in `make_action_detector`). 352 | self.U[Index.NOOP, Index.BUFFER2] = Rational(1) 353 | 354 | def make_action_committer(self): 355 | # PHASE 4: The final phase---commit the action from the individual 356 | # action cells to the stack. 357 | # Only one of the four entries PUSH_0, PUSH_1, POP_0, POP_1 358 | # will be "active" (non-zero), the rest will be 0. 359 | 360 | # This submatrix encodes the copying of the actions from the ``actions'' part 361 | # of the hidden state to the part of the hidden state corresponding to stack 1. 362 | self.U[Index.STACK, Index.PUSH_0] = Rational(1) 363 | self.U[Index.STACK, Index.PUSH_1] = Rational(1) 364 | self.U[Index.STACK, Index.POP_0] = Rational(1) 365 | self.U[Index.STACK, Index.POP_1] = Rational(1) 366 | self.U[Index.STACK, Index.NOOP] = Rational(1) 367 | 368 | def make_acceptor(self): 369 | # PHASE 5: acceptor 370 | # This is the final state of the RNN. 371 | # It accepts if the stack is empty (encoded in the computation component) 372 | # and the input is empty, i.e., the previous input was EOS. 373 | 374 | # Checks the emptiness of the computation component 375 | self.U[Index.ACCEPT, Index.PUSH_0] = Rational(-10) 376 | self.U[Index.ACCEPT, Index.PUSH_1] = Rational(-10) 377 | self.U[Index.ACCEPT, Index.POP_0] = Rational(-10) 378 | self.U[Index.ACCEPT, Index.POP_1] = Rational(-10) 379 | self.U[Index.ACCEPT, Index.NOOP] = Rational(-10) 380 | 381 | # Checks that the input was EOS 382 | self.U[Index.ACCEPT, Index.CONF_BOT_a] = Rational(-10) 383 | self.U[Index.ACCEPT, Index.CONF_BOT_b] = Rational(-10) 384 | self.U[Index.ACCEPT, Index.CONF_0_a] = Rational(-10) 385 | self.U[Index.ACCEPT, Index.CONF_0_b] = Rational(-10) 386 | self.U[Index.ACCEPT, Index.CONF_1_a] = Rational(-10) 387 | self.U[Index.ACCEPT, Index.CONF_1_b] = Rational(-10) 388 | 389 | def make_U(self): 390 | """Constructs the recurrence matrix U of the RNN.""" 391 | 392 | # ORCHESTRATION 393 | self.make_stack_transitions() 394 | self.make_phase_transitions() 395 | 396 | # PHASE 1 397 | self.make_stack_detector() 398 | 399 | # PHASE 2 400 | self.make_configuration_detector() 401 | 402 | # PHASE 3 403 | self.make_action_detector() 404 | self.make_action_executor() 405 | 406 | # PHASE 4 407 | self.make_action_committer() 408 | self.make_acceptor() 409 | 410 | def make_V(self): 411 | """Constructs the emission matrix V of the RNN.""" 412 | # Enables setting the hidden state to 1 at the indices 413 | # corresponding to all possible combinations of the stack configurations 414 | # (top of the stack) and the incomping input symbol. 415 | # For input symbol a 416 | self.V[Index.CONF_BOT_a, self.sym2idx[Sym("a")]] = Rational(1) 417 | self.V[Index.CONF_0_a, self.sym2idx[Sym("a")]] = Rational(1) 418 | self.V[Index.CONF_1_a, self.sym2idx[Sym("a")]] = Rational(1) 419 | # For input symbol b 420 | self.V[Index.CONF_BOT_b, self.sym2idx[Sym("b")]] = Rational(1) 421 | self.V[Index.CONF_0_b, self.sym2idx[Sym("b")]] = Rational(1) 422 | self.V[Index.CONF_1_b, self.sym2idx[Sym("b")]] = Rational(1) 423 | # For input symbol EOS 424 | self.V[Index.CONF_BOT_EOS, self.sym2idx[EOS]] = Rational(1) 425 | self.V[Index.CONF_0_EOS, self.sym2idx[EOS]] = Rational(1) 426 | self.V[Index.CONF_1_EOS, self.sym2idx[EOS]] = Rational(1) 427 | 428 | def make_b(self): 429 | """Constructs the bias vector b of the RNN.""" 430 | self.b[Index.STACK_EMPTY, 0] = Rational(1) 431 | # If the top of the stack is a 0, the encoding will be 0.1... 432 | # This means that -10 * stack encoding (self.U[Index.STACK_ZERO, Index.STACK1]) 433 | # will be at least -2. 434 | # We therefore map it so a value >= 1. 435 | # Otherwise, the encoding will be 0.3... and this, together with the 436 | # multiplication and addition will not be >= 1. 437 | self.b[Index.STACK_ZERO, 0] = Rational(3) 438 | # If the top of the stack is a 1, the encoding will be 0.3... 439 | # This means that 10 * stack encoding (self.U[Index.STACK_ONE, Index.STACK1]) 440 | # will be at least 3. 441 | # We therefore map it so a value >= 1. 442 | # Otherwise, the encoding will be 0.1... and this, together with the 443 | # multiplication and subtraction will not be >= 1. 444 | self.b[Index.STACK_ONE, 0] = Rational(-2) 445 | 446 | # This provides the base of the quantities that get added to 447 | # the previous encoding of the stack after it is divided by 10. 448 | # Add the encoding of the new top of the stack 449 | self.b[Index.PUSH_0, 0] = Rational("1/10") 450 | self.b[Index.PUSH_1, 0] = Rational("3/10") 451 | # Remove the top encoding of the stack, after it has been multiplied by 10. 452 | self.b[Index.POP_0, 0] = Rational(-1) 453 | self.b[Index.POP_1, 0] = Rational(-3) 454 | 455 | self.b[Index.NOOP, 0] = Rational(0) 456 | 457 | # This zeroes out the configurations which are not active 458 | # even though some of the "triggering" cells in the hidden state are active. 459 | self.b[Index.CONF_BOT_a, 0] = Rational(-1) 460 | self.b[Index.CONF_BOT_b, 0] = Rational(-1) 461 | self.b[Index.CONF_BOT_EOS, 0] = Rational(-1) 462 | self.b[Index.CONF_0_a, 0] = Rational(-1) 463 | self.b[Index.CONF_0_b, 0] = Rational(-1) 464 | self.b[Index.CONF_0_EOS, 0] = Rational(-1) 465 | self.b[Index.CONF_1_a, 0] = Rational(-1) 466 | self.b[Index.CONF_1_b, 0] = Rational(-1) 467 | self.b[Index.CONF_1_EOS, 0] = Rational(-1) 468 | 469 | self.b[Index.ACCEPT, 0] = Rational(1) 470 | 471 | def make_E(self): 472 | # Indexes the relevant (log) probabilities in output matrix based on the 473 | # one-hot representation of the stack configurations. 474 | for a in self.Σ - {EOS}: 475 | d = self.sym2idx[a] 476 | self.E[d, Index.STACK_EMPTY] = log(self.pda.δ[a][BOT][1]) 477 | self.E[d, Index.STACK_ZERO] = log(self.pda.δ[a][Sym("0")][1]) 478 | self.E[d, Index.STACK_ONE] = log(self.pda.δ[a][Sym("1")][1]) 479 | 480 | def initial_hidden_state(self) -> Matrix: 481 | """Sets the initial hidden state of the RNN to the zero vector 482 | indicating the first phase.""" 483 | # We start with an zero hidden state in phase 1 484 | h = zeros(self.D, 1, dtype=Rational(0)) 485 | h[Index.PHASE1] = Rational(1) 486 | return h 487 | 488 | def accept(self, y: Union[str, String]) -> Tuple[bool, float]: 489 | """Computes the acceptance probability of a string and whether it is accepted. 490 | 491 | Args: 492 | y (Union[str, String]): The string to be accepted. 493 | 494 | Returns: 495 | Tuple[bool, float]: Whether the string is accepted 496 | and the acceptance probability. 497 | """ 498 | if isinstance(y, str): 499 | y = String(y) 500 | y.y += [EOS] 501 | 502 | h, logp = self.initial_hidden_state(), Rational(0) 503 | for a in y: 504 | print(f"sym = {a}") 505 | h, _acc, _logp = self.step(h, a) 506 | print(f"logp = {_logp}") 507 | logp += _logp 508 | 509 | return h[Index.ACCEPT] == Rational(1), logp 510 | 511 | def __call__(self, y: Union[str, String]) -> Tuple[bool, float]: 512 | """Computes the acceptance probability of a string and whether it is accepted 513 | by simply calling the `accept` method. 514 | 515 | Args: 516 | y (Union[str, String]): The string to be accepted. 517 | 518 | Returns: 519 | Tuple[bool, float]: Whether the string is accepted 520 | and the acceptance probability. 521 | """ 522 | return self.accept(y) 523 | 524 | def step(self, h: Matrix, a: Sym) -> Tuple[Matrix, Rational, Rational]: 525 | """Performs a single whole step of the Siegelmann RNN composed of the four 526 | sub-steps/phases. 527 | 528 | Args: 529 | h (Matrix): The current hidden state of the RNN. 530 | a (Sym): The current input symbol. 531 | 532 | Returns: 533 | Tuple[Matrix, bool, float]: The new hidden state, whether the string is 534 | accepted and the acceptance probability. 535 | """ 536 | assert a in self.Σ 537 | 538 | # We have four stages in which the hidden state is updated 539 | # 1) Peek 540 | # 2) Combine 541 | # 3) Transition 542 | # 4) Consolidate 543 | 544 | def apply(_h: Matrix) -> Matrix: 545 | """Performs the update (sub-)step of the Siegelmann RNN on the current 546 | hidden state _h.""" 547 | z = self.U * _h + self.V * self.one_hot.col(self.sym2idx[a]) + self.b 548 | hʼ = zeros(self.D, 1, dtype=Rational(0)) 549 | 550 | for i in range(self.D): 551 | hʼ[i] = σ.subs(x, z[i]) 552 | return hʼ 553 | 554 | # print("\n\n\n\n\n\n >>>> START:") 555 | # self.disp(h) 556 | # print(cantor_decode(h[Index.STACK])) 557 | 558 | # print("\n\n\n\n----------------------------\n>>>>> PHASE 1") 559 | h = apply(h) 560 | # print(cantor_decode(h[Index.STACK])) 561 | # self.disp(h) 562 | # print("\n\n\n\n----------------------------\n>>>>> PHASE 2") 563 | h = apply(h) 564 | # print(cantor_decode(h[Index.STACK])) 565 | # self.disp(h) 566 | logits = self.E * h 567 | # print("\n\n\n\n----------------------------\n>>>>> PHASE 3") 568 | h = apply(h) 569 | # print(cantor_decode(h[Index.STACK])) 570 | # self.disp(h) 571 | # print("\n\n\n\n----------------------------\n>>>>> PHASE 4") 572 | h = apply(h) 573 | # print(cantor_decode(h[Index.STACK])) 574 | # self.disp(h) 575 | # print(cantor_decode(h[Index.STACK])) 576 | # self.disp(h) 577 | 578 | return h, h[Index.ACCEPT], logits[self.sym2idx[a]] 579 | -------------------------------------------------------------------------------- /turnn/turing/two_stack_rnn.py: -------------------------------------------------------------------------------- 1 | """This module implements a *non-LM* RNN simulating an *unweighted* two-stack PDA. 2 | The construction is original but based on the one in Siegelmann and Sontag (1995) 3 | [https://binds.cs.umass.edu/papers/1995_Siegelmann_JComSysSci.pdf] 4 | 5 | See `turnn/turing/two_stack_rnnlm.py` for the *language model* version of this RNN. 6 | 7 | """ 8 | 9 | from enum import IntEnum, unique 10 | from itertools import product 11 | 12 | from sympy import Abs, Matrix, Piecewise, Rational, Symbol, eye, sympify, zeros 13 | 14 | from turnn.base.symbol import BOT, EOS, Sym 15 | from turnn.base.utils import cantor_decode 16 | from turnn.turing.pda import Action, TwoStackPDA 17 | 18 | x = Symbol("x") 19 | # This is the saturated sigmoid function 20 | σ = Piecewise((Rational(0), x <= 0), (Rational(1), x >= 1), (Abs(x) <= sympify(1))) 21 | 22 | 23 | @unique 24 | class Index(IntEnum): 25 | """This class is used to index the hidden state of the RNN for two 26 | stacks by naming the individual dimensions of the hidden state with the role 27 | they play in simulating the two-stack PDA. 28 | """ 29 | 30 | # (1) Data component 31 | STACK1 = 0 32 | STACK2 = 1 33 | BUFFER11 = 2 34 | BUFFER12 = 3 35 | BUFFER21 = 4 36 | BUFFER22 = 5 37 | 38 | # (2) Phase component 39 | PHASE1 = 6 40 | PHASE2 = 7 41 | PHASE3 = 8 42 | PHASE4 = 9 43 | 44 | # (3) Stack configuration component 45 | STACK1_EMPTY = 10 46 | STACK1_ZERO = 11 47 | STACK1_ONE = 12 48 | STACK2_EMPTY = 13 49 | STACK2_ZERO = 14 50 | STACK2_ONE = 15 51 | 52 | # (4) Stack and input configuration component 53 | # `CONF_γ1_γ2_a` corresponds to the configuration 54 | # where the tops of the two stacks are `γ1` and `γ2` and the next symbol is `a` 55 | CONF_BOT_BOT_EOS = 16 56 | CONF_BOT_0_EOS = 17 57 | CONF_BOT_1_EOS = 18 58 | CONF_0_BOT_EOS = 19 59 | CONF_1_BOT_EOS = 20 60 | CONF_0_0_EOS = 21 61 | CONF_0_1_EOS = 22 62 | CONF_1_0_EOS = 23 63 | CONF_1_1_EOS = 24 64 | CONF_BOT_BOT_a = 25 65 | CONF_BOT_0_a = 26 66 | CONF_BOT_1_a = 27 67 | CONF_0_BOT_a = 28 68 | CONF_1_BOT_a = 29 69 | CONF_0_0_a = 30 70 | CONF_0_1_a = 31 71 | CONF_1_0_a = 32 72 | CONF_1_1_a = 33 73 | CONF_BOT_BOT_b = 34 74 | CONF_BOT_0_b = 35 75 | CONF_BOT_1_b = 36 76 | CONF_0_BOT_b = 37 77 | CONF_1_BOT_b = 38 78 | CONF_0_0_b = 39 79 | CONF_0_1_b = 40 80 | CONF_1_0_b = 41 81 | CONF_1_1_b = 42 82 | 83 | # (5) Computation component 84 | STACK1_PUSH_0 = 43 85 | STACK1_PUSH_1 = 44 86 | STACK1_POP_0 = 45 87 | STACK1_POP_1 = 46 88 | STACK1_NOOP = 47 89 | STACK2_PUSH_0 = 48 90 | STACK2_PUSH_1 = 49 91 | STACK2_POP_0 = 50 92 | STACK2_POP_1 = 51 93 | STACK2_NOOP = 52 94 | 95 | # (6) Acceptance component 96 | ACCEPT = 53 97 | 98 | 99 | # We use a two-letter alphabet {a, b} and the EOS symbol. 100 | sym2idx = { 101 | Sym("a"): 0, 102 | Sym("b"): 1, 103 | EOS: 2, 104 | } 105 | 106 | 107 | def conf_peek(x: Index): # noqa: C901 108 | """Converts the index of the configuration into a human-readable string. 109 | 110 | Args: 111 | x (Index): THe index of the configuration. 112 | """ 113 | if x == Index.CONF_BOT_BOT_EOS: 114 | print("(⊥, ⊥, EOS)") 115 | elif x == Index.CONF_BOT_0_EOS: 116 | print("(⊥, 0, EOS)") 117 | elif x == Index.CONF_BOT_1_EOS: 118 | print("(⊥, 1, EOS)") 119 | elif x == Index.CONF_0_BOT_EOS: 120 | print("(0, ⊥, EOS)") 121 | elif x == Index.CONF_1_BOT_EOS: 122 | print("(1, ⊥, EOS)") 123 | elif x == Index.CONF_BOT_BOT_a: 124 | print("(⊥, ⊥, a)") 125 | elif x == Index.CONF_BOT_BOT_b: 126 | print("(⊥, ⊥, b)") 127 | elif x == Index.CONF_BOT_0_a: 128 | print("(⊥, 0, a)") 129 | elif x == Index.CONF_BOT_0_b: 130 | print("(⊥, 0, b)") 131 | elif x == Index.CONF_BOT_1_a: 132 | print("(⊥, 1, a)") 133 | elif x == Index.CONF_BOT_1_b: 134 | print("(⊥, 1, b)") 135 | elif x == Index.CONF_0_BOT_a: 136 | print("(0, ⊥, a)") 137 | elif x == Index.CONF_0_BOT_b: 138 | print("(0, ⊥, b)") 139 | elif x == Index.CONF_1_BOT_a: 140 | print("(1, ⊥, a)") 141 | elif x == Index.CONF_1_BOT_b: 142 | print("(1, ⊥, b)") 143 | elif x == Index.CONF_0_0_a: 144 | print("(0, 0, a)") 145 | elif x == Index.CONF_0_0_b: 146 | print("(0, 0, b)") 147 | elif x == Index.CONF_0_1_a: 148 | print("(0, 1, a)") 149 | elif x == Index.CONF_0_1_b: 150 | print("(0, 1, b)") 151 | elif x == Index.CONF_1_0_a: 152 | print("(1, 0, a)") 153 | elif x == Index.CONF_1_0_b: 154 | print("(1, 0, b)") 155 | elif x == Index.CONF_1_1_a: 156 | print("(1, 1, a)") 157 | elif x == Index.CONF_1_1_b: 158 | print("(1, 1, b)") 159 | 160 | raise NotImplementedError 161 | 162 | 163 | def conf2idx(γ1: Sym, γ2: Sym, sym: Sym) -> Index: # noqa: C901 164 | """Converts a configuration (tops of the stacks and the input symbol) 165 | into an index in the matrices. 166 | 167 | Args: 168 | γ1 (Sym): The top of the first stack. 169 | γ2 (Sym): The top of the second stack. 170 | sym (Sym): The input symbol. 171 | 172 | Raises: 173 | NotImplementedError: If the configuration is not supported. 174 | 175 | Returns: 176 | Index: The index of the configuration. 177 | """ 178 | if (γ1, γ2, sym) == (BOT, BOT, EOS): 179 | return Index.CONF_BOT_BOT_EOS 180 | elif (γ1, γ2, sym) == (BOT, Sym("0"), EOS): 181 | return Index.CONF_BOT_0_EOS 182 | elif (γ1, γ2, sym) == (BOT, Sym("1"), EOS): 183 | return Index.CONF_BOT_1_EOS 184 | elif (γ1, γ2, sym) == (Sym("0"), BOT, EOS): 185 | return Index.CONF_0_BOT_EOS 186 | elif (γ1, γ2, sym) == (Sym("1"), BOT, EOS): 187 | return Index.CONF_1_BOT_EOS 188 | elif (γ1, γ2, sym) == (Sym("0"), Sym("0"), EOS): 189 | return Index.CONF_0_0_EOS 190 | elif (γ1, γ2, sym) == (Sym("0"), Sym("1"), EOS): 191 | return Index.CONF_0_1_EOS 192 | elif (γ1, γ2, sym) == (Sym("1"), Sym("0"), EOS): 193 | return Index.CONF_1_0_EOS 194 | elif (γ1, γ2, sym) == (Sym("1"), Sym("1"), EOS): 195 | return Index.CONF_1_1_EOS 196 | elif (γ1, γ2, sym) == (BOT, BOT, Sym("a")): 197 | return Index.CONF_BOT_BOT_a 198 | elif (γ1, γ2, sym) == (BOT, Sym("0"), Sym("a")): 199 | return Index.CONF_BOT_0_a 200 | elif (γ1, γ2, sym) == (BOT, Sym("1"), Sym("a")): 201 | return Index.CONF_BOT_1_a 202 | elif (γ1, γ2, sym) == (Sym("0"), BOT, Sym("a")): 203 | return Index.CONF_0_BOT_a 204 | elif (γ1, γ2, sym) == (Sym("1"), BOT, Sym("a")): 205 | return Index.CONF_1_BOT_a 206 | elif (γ1, γ2, sym) == (Sym("0"), Sym("0"), Sym("a")): 207 | return Index.CONF_0_0_a 208 | elif (γ1, γ2, sym) == (Sym("0"), Sym("1"), Sym("a")): 209 | return Index.CONF_0_1_a 210 | elif (γ1, γ2, sym) == (Sym("1"), Sym("0"), Sym("a")): 211 | return Index.CONF_1_0_a 212 | elif (γ1, γ2, sym) == (Sym("1"), Sym("1"), Sym("a")): 213 | return Index.CONF_1_1_a 214 | elif (γ1, γ2, sym) == (BOT, BOT, Sym("b")): 215 | return Index.CONF_BOT_BOT_b 216 | elif (γ1, γ2, sym) == (BOT, Sym("0"), Sym("b")): 217 | return Index.CONF_BOT_0_b 218 | elif (γ1, γ2, sym) == (BOT, Sym("1"), Sym("b")): 219 | return Index.CONF_BOT_1_b 220 | elif (γ1, γ2, sym) == (Sym("0"), BOT, Sym("b")): 221 | return Index.CONF_0_BOT_b 222 | elif (γ1, γ2, sym) == (Sym("1"), BOT, Sym("b")): 223 | return Index.CONF_1_BOT_b 224 | elif (γ1, γ2, sym) == (Sym("0"), Sym("0"), Sym("b")): 225 | return Index.CONF_0_0_b 226 | elif (γ1, γ2, sym) == (Sym("0"), Sym("1"), Sym("b")): 227 | return Index.CONF_0_1_b 228 | elif (γ1, γ2, sym) == (Sym("1"), Sym("0"), Sym("b")): 229 | return Index.CONF_1_0_b 230 | elif (γ1, γ2, sym) == (Sym("1"), Sym("1"), Sym("b")): 231 | return Index.CONF_1_1_b 232 | 233 | raise NotImplementedError 234 | 235 | 236 | class TwoStackRNN: 237 | """ 238 | An implementation of an Elman RNN that simulates a two-stack PDA. 239 | It computes the next state h_t+1 from the current state h_t by applying the 240 | Elman update rule four times: 241 | h' = σ(U * h + V * y + b) 242 | where the matrices U, V and the vector b are computed from the PDA and 243 | the input symbol y is one-hot encoded. 244 | After four applications of the update rule, the new hidden state represents the 245 | new configuration of the PDA. 246 | 247 | The hidden state is composed of six components: 248 | 1. The data component 249 | 2. The phase component 250 | 3. The stack configurations component 251 | 4. The stacks and input configuration component 252 | 5. The computation component 253 | 6. The acceptance component 254 | 255 | 1. The first component contains the cells containing the encodings of the two stacks 256 | along with two buffer cells for each stack through which the stack is passed 257 | during the four-stage computation. 258 | 2. The second component simply denotes the current phase of the computation, and is 259 | only included for keeping track of the computation. 260 | 3. The third component contains the encoding of the stack at the current time step, 261 | i.e., it flags whether the stacks are empty or have a 0/1 on the top. 262 | 4. The fourth component contains the encoding of the current input symbol together 263 | with the stacks. 264 | 5. The fifth component contains cells to which the current stack configuration is 265 | copied and modified according to the action of the PDA. 266 | 6. The sixth component contains a cell that are set to 1 after reading in the EOS 267 | symbol if the PDA accepts the input string appearing before EOS. 268 | """ 269 | 270 | def __init__(self, pda: TwoStackPDA): 271 | self.pda = pda 272 | self.Σ = pda.Σ.union({EOS}) # In contrast to the globally normalized PDA, 273 | # the RNN works with an EOS symbol 274 | 275 | # The dimension of the hidden state 276 | self.D = len(Index) 277 | 278 | # Recurrence matrix U 279 | self.U = zeros(self.D, self.D, dtype=Rational(0)) 280 | # Input matrix V 281 | self.V = zeros(self.D, len(self.Σ), dtype=Rational(0)) 282 | # Bias vector b 283 | self.b = zeros(self.D, 1, dtype=Rational(0)) 284 | 285 | self.make_U() 286 | self.make_b() 287 | self.make_V() 288 | 289 | # One-hot encoding of the input symbols 290 | self.one_hot = eye(len(self.Σ), dtype=Rational(0)) 291 | 292 | # We start with an zero hidden state in phase 1 293 | self.h = zeros(self.D, 1, dtype=Rational(0)) 294 | self.reset() 295 | 296 | def reset(self): 297 | """Sets the initial hidden state of the RNN to the zero vector 298 | indicating the first phase.""" 299 | self.h = zeros(self.D, 1, dtype=Rational(0)) 300 | self.h[Index.PHASE1] = Rational(1) 301 | 302 | def disp(self, h: Matrix): 303 | """Prints the content of the hidden state of the Siegelmann RNN, separated by 304 | the different components. 305 | 306 | Args: 307 | h (Matrix): The hidden state of the Siegelmann RNN. 308 | """ 309 | print(f"Stack 1:\t{h[Index.STACK1]}") 310 | print(f"Buffer 11:\t{h[Index.BUFFER11]}") 311 | print(f"Buffer 21:\t{h[Index.BUFFER21]}") 312 | 313 | print(f"Stack 2:\t{h[Index.STACK2]}") 314 | print(f"Buffer 12:\t{h[Index.BUFFER12]}") 315 | print(f"Buffer 22:\t{h[Index.BUFFER22]}") 316 | 317 | print(f"Phase 1:\t{h[Index.PHASE1]}") 318 | print(f"Phase 2:\t{h[Index.PHASE2]}") 319 | print(f"Phase 3:\t{h[Index.PHASE3]}") 320 | print(f"Phase 4:\t{h[Index.PHASE4]}") 321 | 322 | print(f"1: STACK 1 - EMPTY:\t{h[Index.STACK1_EMPTY]}") 323 | print(f"1: STACK 1 - 0:\t{h[Index.STACK1_ZERO]}") 324 | print(f"1: STACK 1 - 1:\t{h[Index.STACK1_ONE]}") 325 | print(f"1: STACK 2 - EMPTY:\t{h[Index.STACK2_EMPTY]}") 326 | print(f"1: STACK 2 - 0:\t{h[Index.STACK2_ZERO]}") 327 | print(f"1: STACK 2 - 1:\t{h[Index.STACK2_ONE]}") 328 | 329 | print(f"2: (⊥, ⊥, EOS):\t{h[Index.CONF_BOT_BOT_EOS]}") 330 | print(f"2: (⊥, 0, EOS):\t{h[Index.CONF_BOT_0_EOS]}") 331 | print(f"2: (⊥, 1, EOS):\t{h[Index.CONF_BOT_1_EOS]}") 332 | print(f"2: (0, ⊥, EOS):\t{h[Index.CONF_0_BOT_EOS]}") 333 | print(f"2: (1, ⊥, EOS):\t{h[Index.CONF_1_BOT_EOS]}") 334 | print(f"2: (0, 0, EOS):\t{h[Index.CONF_0_0_EOS]}") 335 | print(f"2: (0, 1, EOS):\t{h[Index.CONF_0_1_EOS]}") 336 | print(f"2: (1, 0, EOS):\t{h[Index.CONF_1_0_EOS]}") 337 | print(f"2: (1, 1, EOS):\t{h[Index.CONF_1_1_EOS]}") 338 | print(f"2: (⊥, ⊥, a):\t{h[Index.CONF_BOT_BOT_a]}") 339 | print(f"2: (⊥, 0, a):\t{h[Index.CONF_BOT_0_a]}") 340 | print(f"2: (⊥, 1, a):\t{h[Index.CONF_BOT_1_a]}") 341 | print(f"2: (0, ⊥, a):\t{h[Index.CONF_0_BOT_a]}") 342 | print(f"2: (1, ⊥, a):\t{h[Index.CONF_1_BOT_a]}") 343 | print(f"2: (0, 0, a):\t{h[Index.CONF_0_0_a]}") 344 | print(f"2: (0, 1, a):\t{h[Index.CONF_0_1_a]}") 345 | print(f"2: (1, 0, a):\t{h[Index.CONF_1_0_a]}") 346 | print(f"2: (1, 1, a):\t{h[Index.CONF_1_1_a]}") 347 | print(f"2: (⊥, ⊥, b):\t{h[Index.CONF_BOT_BOT_b]}") 348 | print(f"2: (⊥, 0, b):\t{h[Index.CONF_BOT_0_b]}") 349 | print(f"2: (⊥, 1, b):\t{h[Index.CONF_BOT_1_b]}") 350 | print(f"2: (0, ⊥, b):\t{h[Index.CONF_0_BOT_b]}") 351 | print(f"2: (1, ⊥, b):\t{h[Index.CONF_1_BOT_b]}") 352 | print(f"2: (0, 0, b):\t{h[Index.CONF_0_0_b]}") 353 | print(f"2: (0, 1, b):\t{h[Index.CONF_0_1_b]}") 354 | print(f"2: (1, 0, b):\t{h[Index.CONF_1_0_b]}") 355 | print(f"2: (1, 1, b):\t{h[Index.CONF_1_1_b]}") 356 | 357 | print(f"3: STACK 1 Push 0:\t{h[Index.STACK1_PUSH_0]}") 358 | print(f"3: STACK 1 Push 1:\t{h[Index.STACK1_PUSH_1]}") 359 | print(f"3: STACK 1 Pop 0:\t{h[Index.STACK1_POP_0]}") 360 | print(f"3: STACK 1 Pop 1:\t{h[Index.STACK1_POP_1]}") 361 | 362 | print(f"3: STACK 2 Push 0:\t{h[Index.STACK2_PUSH_0]}") 363 | print(f"3: STACK 2 Push 1:\t{h[Index.STACK2_PUSH_1]}") 364 | print(f"3: STACK 2 Pop 0:\t{h[Index.STACK2_POP_0]}") 365 | print(f"3: STACK 2 Pop 1:\t{h[Index.STACK2_POP_1]}") 366 | 367 | print() 368 | 369 | def make_buffer_transitions(self): 370 | """ 371 | Constructs the submatrix encodes the copying of the stack contents between 372 | the three different stacks simulated by the RNN in the hidden state 373 | while it moves through the 4 phases of the hidden state update. 374 | Each of the stacks is simply a dimension in the hidden state. 375 | Each of the three stacks is copied to the next one, 376 | and the last one is zeroed out. 377 | """ 378 | self.U[Index.STACK1, Index.STACK1] = Rational(0) # delete 379 | self.U[Index.BUFFER11, Index.STACK1] = Rational(1) # copy over to the next one 380 | self.U[Index.BUFFER11, Index.BUFFER11] = Rational(0) 381 | self.U[Index.BUFFER21, Index.BUFFER11] = Rational(1) 382 | self.U[Index.BUFFER21, Index.BUFFER21] = Rational(0) 383 | 384 | self.U[Index.STACK2, Index.STACK2] = Rational(0) # delete 385 | self.U[Index.BUFFER12, Index.STACK2] = Rational(1) # copy over to the next one 386 | self.U[Index.BUFFER12, Index.BUFFER12] = Rational(0) 387 | self.U[Index.BUFFER22, Index.BUFFER12] = Rational(1) 388 | self.U[Index.BUFFER22, Index.BUFFER22] = Rational(0) 389 | 390 | def make_phase_transitions(self): 391 | # These two blocks encode the state---it's a one-hot encoding 392 | self.U[Index.PHASE1, Index.PHASE1] = Rational(0) 393 | self.U[Index.PHASE2, Index.PHASE2] = Rational(0) 394 | self.U[Index.PHASE3, Index.PHASE3] = Rational(0) 395 | self.U[Index.PHASE4, Index.PHASE4] = Rational(0) 396 | self.U[Index.PHASE2, Index.PHASE1] = Rational(1) 397 | self.U[Index.PHASE3, Index.PHASE2] = Rational(1) 398 | self.U[Index.PHASE4, Index.PHASE3] = Rational(1) 399 | self.U[Index.PHASE1, Index.PHASE4] = Rational(1) 400 | 401 | def make_stack_detector(self): 402 | # PHASE 1: 403 | # This detects an empty stack 404 | self.U[Index.STACK1_EMPTY, Index.STACK1] = Rational(-10) 405 | self.U[Index.STACK2_EMPTY, Index.STACK2] = Rational(-10) 406 | 407 | # This detects whether the top of the stack is a 0 or a 1 408 | self.U[Index.STACK1_ZERO, Index.STACK1] = Rational(-10) 409 | self.U[Index.STACK1_ONE, Index.STACK1] = Rational(10) 410 | self.U[Index.STACK2_ZERO, Index.STACK2] = Rational(-10) 411 | self.U[Index.STACK2_ONE, Index.STACK2] = Rational(10) 412 | 413 | def make_configuration_detector(self): 414 | # PHASE 2: This submatrix corresponds to the actions available given any 415 | # configuration of the stack, captured at EMPTY/PEEK locations. 416 | # This corresponds to the emmisions 417 | # These then get "intersected" with the symbol embeddings to select a or b 418 | self.U[Index.CONF_BOT_BOT_a, Index.STACK1_EMPTY] = Rational(1) 419 | self.U[Index.CONF_BOT_0_a, Index.STACK1_EMPTY] = Rational(1) 420 | self.U[Index.CONF_BOT_1_a, Index.STACK1_EMPTY] = Rational(1) 421 | self.U[Index.CONF_BOT_BOT_b, Index.STACK1_EMPTY] = Rational(1) 422 | self.U[Index.CONF_BOT_0_b, Index.STACK1_EMPTY] = Rational(1) 423 | self.U[Index.CONF_BOT_1_b, Index.STACK1_EMPTY] = Rational(1) 424 | self.U[Index.CONF_BOT_BOT_EOS, Index.STACK1_EMPTY] = Rational(1) 425 | self.U[Index.CONF_BOT_0_EOS, Index.STACK1_EMPTY] = Rational(1) 426 | self.U[Index.CONF_BOT_1_EOS, Index.STACK1_EMPTY] = Rational(1) 427 | self.U[Index.CONF_0_BOT_a, Index.STACK1_ZERO] = Rational(1) 428 | self.U[Index.CONF_0_0_a, Index.STACK1_ZERO] = Rational(1) 429 | self.U[Index.CONF_0_1_a, Index.STACK1_ZERO] = Rational(1) 430 | self.U[Index.CONF_0_BOT_b, Index.STACK1_ZERO] = Rational(1) 431 | self.U[Index.CONF_0_0_b, Index.STACK1_ZERO] = Rational(1) 432 | self.U[Index.CONF_0_1_b, Index.STACK1_ZERO] = Rational(1) 433 | self.U[Index.CONF_0_BOT_EOS, Index.STACK1_ZERO] = Rational(1) 434 | self.U[Index.CONF_0_0_EOS, Index.STACK1_ZERO] = Rational(1) 435 | self.U[Index.CONF_0_1_EOS, Index.STACK1_ZERO] = Rational(1) 436 | self.U[Index.CONF_1_BOT_a, Index.STACK1_ONE] = Rational(1) 437 | self.U[Index.CONF_1_0_a, Index.STACK1_ONE] = Rational(1) 438 | self.U[Index.CONF_1_1_a, Index.STACK1_ONE] = Rational(1) 439 | self.U[Index.CONF_1_BOT_b, Index.STACK1_ONE] = Rational(1) 440 | self.U[Index.CONF_1_0_b, Index.STACK1_ONE] = Rational(1) 441 | self.U[Index.CONF_1_1_b, Index.STACK1_ONE] = Rational(1) 442 | self.U[Index.CONF_1_BOT_EOS, Index.STACK1_ONE] = Rational(1) 443 | self.U[Index.CONF_1_0_EOS, Index.STACK1_ONE] = Rational(1) 444 | self.U[Index.CONF_1_1_EOS, Index.STACK1_ONE] = Rational(1) 445 | 446 | self.U[Index.CONF_BOT_BOT_a, Index.STACK2_EMPTY] = Rational(1) 447 | self.U[Index.CONF_BOT_BOT_b, Index.STACK2_EMPTY] = Rational(1) 448 | self.U[Index.CONF_0_BOT_b, Index.STACK2_EMPTY] = Rational(1) 449 | self.U[Index.CONF_0_BOT_a, Index.STACK2_EMPTY] = Rational(1) 450 | self.U[Index.CONF_1_BOT_a, Index.STACK2_EMPTY] = Rational(1) 451 | self.U[Index.CONF_1_BOT_b, Index.STACK2_EMPTY] = Rational(1) 452 | self.U[Index.CONF_BOT_0_a, Index.STACK2_ZERO] = Rational(1) 453 | self.U[Index.CONF_BOT_0_b, Index.STACK2_ZERO] = Rational(1) 454 | self.U[Index.CONF_0_0_a, Index.STACK2_ZERO] = Rational(1) 455 | self.U[Index.CONF_0_0_b, Index.STACK2_ZERO] = Rational(1) 456 | self.U[Index.CONF_1_0_a, Index.STACK2_ZERO] = Rational(1) 457 | self.U[Index.CONF_1_0_b, Index.STACK2_ZERO] = Rational(1) 458 | self.U[Index.CONF_BOT_1_a, Index.STACK2_ONE] = Rational(1) 459 | self.U[Index.CONF_BOT_1_b, Index.STACK2_ONE] = Rational(1) 460 | self.U[Index.CONF_0_1_a, Index.STACK2_ONE] = Rational(1) 461 | self.U[Index.CONF_0_1_b, Index.STACK2_ONE] = Rational(1) 462 | self.U[Index.CONF_1_1_a, Index.STACK2_ONE] = Rational(1) 463 | self.U[Index.CONF_1_1_b, Index.STACK2_ONE] = Rational(1) 464 | self.U[Index.CONF_BOT_BOT_EOS, Index.STACK2_EMPTY] = Rational(1) 465 | self.U[Index.CONF_0_BOT_EOS, Index.STACK2_EMPTY] = Rational(1) 466 | self.U[Index.CONF_1_BOT_EOS, Index.STACK2_EMPTY] = Rational(1) 467 | self.U[Index.CONF_BOT_0_EOS, Index.STACK2_ZERO] = Rational(1) 468 | self.U[Index.CONF_0_0_EOS, Index.STACK2_ZERO] = Rational(1) 469 | self.U[Index.CONF_1_1_EOS, Index.STACK2_ZERO] = Rational(1) 470 | self.U[Index.CONF_BOT_1_EOS, Index.STACK2_ONE] = Rational(1) 471 | self.U[Index.CONF_0_1_EOS, Index.STACK2_ONE] = Rational(1) 472 | self.U[Index.CONF_1_1_EOS, Index.STACK2_ONE] = Rational(1) 473 | 474 | # This bit ensures that we can't be empty and non-empty 475 | # It erases the effects of always setting the PEEK0 bit to 1 476 | self.U[Index.CONF_0_BOT_a, Index.STACK1_EMPTY] = Rational(-10) 477 | self.U[Index.CONF_0_0_a, Index.STACK1_EMPTY] = Rational(-10) 478 | self.U[Index.CONF_0_1_a, Index.STACK1_EMPTY] = Rational(-10) 479 | self.U[Index.CONF_0_BOT_b, Index.STACK1_EMPTY] = Rational(-10) 480 | self.U[Index.CONF_0_0_b, Index.STACK1_EMPTY] = Rational(-10) 481 | self.U[Index.CONF_0_1_b, Index.STACK1_EMPTY] = Rational(-10) 482 | self.U[Index.CONF_1_BOT_a, Index.STACK1_EMPTY] = Rational(-10) 483 | self.U[Index.CONF_1_0_a, Index.STACK1_EMPTY] = Rational(-10) 484 | self.U[Index.CONF_1_1_a, Index.STACK1_EMPTY] = Rational(-10) 485 | self.U[Index.CONF_1_BOT_b, Index.STACK1_EMPTY] = Rational(-10) 486 | self.U[Index.CONF_1_0_b, Index.STACK1_EMPTY] = Rational(-10) 487 | self.U[Index.CONF_1_1_b, Index.STACK1_EMPTY] = Rational(-10) 488 | 489 | self.U[Index.CONF_BOT_0_a, Index.STACK2_EMPTY] = Rational(-10) 490 | self.U[Index.CONF_0_0_a, Index.STACK2_EMPTY] = Rational(-10) 491 | self.U[Index.CONF_1_0_a, Index.STACK2_EMPTY] = Rational(-10) 492 | self.U[Index.CONF_BOT_0_b, Index.STACK2_EMPTY] = Rational(-10) 493 | self.U[Index.CONF_0_0_b, Index.STACK2_EMPTY] = Rational(-10) 494 | self.U[Index.CONF_1_0_b, Index.STACK2_EMPTY] = Rational(-10) 495 | self.U[Index.CONF_BOT_1_a, Index.STACK2_EMPTY] = Rational(-10) 496 | self.U[Index.CONF_0_1_a, Index.STACK2_EMPTY] = Rational(-10) 497 | self.U[Index.CONF_1_1_a, Index.STACK2_EMPTY] = Rational(-10) 498 | self.U[Index.CONF_BOT_1_b, Index.STACK2_EMPTY] = Rational(-10) 499 | self.U[Index.CONF_0_1_b, Index.STACK2_EMPTY] = Rational(-10) 500 | self.U[Index.CONF_1_1_b, Index.STACK2_EMPTY] = Rational(-10) 501 | 502 | def initialize_transition_function(self): 503 | # transition function 504 | for i in range(Index.STACK1_PUSH_0, Index.STACK1_NOOP + 1): 505 | for j in range(Index.CONF_BOT_BOT_EOS, Index.CONF_1_1_b + 1): 506 | # As soon as the configuration has been determined, reset 507 | # the information about what action should be taken, 508 | # so that only the correct action is considered (by overwriting 509 | # the -10's with 0's below). 510 | self.U[i, j] = Rational(-10) 511 | for i in range(Index.STACK2_PUSH_0, Index.STACK2_NOOP + 1): 512 | for j in range(Index.CONF_BOT_BOT_EOS, Index.CONF_1_1_b + 1): 513 | self.U[i, j] = Rational(-10) 514 | 515 | def make_action_detector(self): # noqa: C901 516 | self.initialize_transition_function() 517 | 518 | # This is the transition matrix 519 | for sym in self.Σ: 520 | if sym == EOS: 521 | # If we read the EOS symbol, we don't do anything. 522 | # We set those manually, since EOS-transitions are not in the PDA. 523 | for action in [Index.STACK1_NOOP, Index.STACK2_NOOP]: 524 | for γ_top_1, γ_top_2 in product(self.pda.Γ_1, self.pda.Γ_2): 525 | self.U[action, conf2idx(γ_top_1, γ_top_2, EOS)] = Rational(0) 526 | continue 527 | 528 | for γ_top_1, γ_top_2 in self.pda.δ[sym]: 529 | (action_1, γ_new_1), (action_2, γ_new_2) = self.pda.δ[sym][ 530 | (γ_top_1, γ_top_2) 531 | ] 532 | 533 | conf = conf2idx(γ_top_1, γ_top_2, sym) 534 | 535 | if action_1 == Action.PUSH and γ_new_1 == Sym("0"): 536 | self.U[Index.STACK1_PUSH_0, conf] = Rational(0) 537 | elif action_1 == Action.PUSH and γ_new_1 == Sym("1"): 538 | self.U[Index.STACK1_PUSH_1, conf] = Rational(0) 539 | elif action_1 == Action.POP and γ_new_1 == Sym("0"): 540 | self.U[Index.STACK1_POP_0, conf] = Rational(0) 541 | elif action_1 == Action.POP and γ_new_1 == Sym("1"): 542 | self.U[Index.STACK1_POP_1, conf] = Rational(0) 543 | elif action_1 == Action.NOOP: 544 | self.U[Index.STACK1_NOOP, conf] = Rational(0) 545 | else: 546 | raise ValueError( 547 | "Unknown action: action: {}, γ_new: {}.".format( 548 | action_1, γ_new_1 549 | ) 550 | ) 551 | 552 | if action_2 == Action.PUSH and γ_new_2 == Sym("0"): 553 | self.U[Index.STACK2_PUSH_0, conf] = Rational(0) 554 | elif action_2 == Action.PUSH and γ_new_2 == Sym("1"): 555 | self.U[Index.STACK2_PUSH_1, conf] = Rational(0) 556 | elif action_2 == Action.POP and γ_new_2 == Sym("0"): 557 | self.U[Index.STACK2_POP_0, conf] = Rational(0) 558 | elif action_2 == Action.POP and γ_new_2 == Sym("1"): 559 | self.U[Index.STACK2_POP_1, conf] = Rational(0) 560 | elif action_2 == Action.NOOP: 561 | self.U[Index.STACK2_NOOP, conf] = Rational(0) 562 | else: 563 | raise ValueError( 564 | "Unknown action: action: {}, γ_new: {}.".format( 565 | action_2, γ_new_2 566 | ) 567 | ) 568 | 569 | def make_action_executor(self): 570 | # PHASE 3: we execute all possible actions 571 | # These two take whatever is on the stack so far (primarily it would be on 572 | # stack 1, but by phase 4, it has been moved to stack 3) and move it 573 | # down by dividing by 10. 574 | self.U[Index.STACK1_PUSH_0, Index.BUFFER21] = Rational("1/10") 575 | self.U[Index.STACK1_PUSH_1, Index.BUFFER21] = Rational("1/10") 576 | 577 | self.U[Index.STACK2_PUSH_0, Index.BUFFER22] = Rational("1/10") 578 | self.U[Index.STACK2_PUSH_1, Index.BUFFER22] = Rational("1/10") 579 | 580 | # These two take whatever is on the stack and move it up by multiplying by 10. 581 | self.U[Index.STACK1_POP_0, Index.BUFFER21] = Rational(10) 582 | self.U[Index.STACK1_POP_1, Index.BUFFER21] = Rational(10) 583 | 584 | self.U[Index.STACK2_POP_0, Index.BUFFER22] = Rational(10) 585 | self.U[Index.STACK2_POP_1, Index.BUFFER22] = Rational(10) 586 | 587 | # Copy the last buffer into the NOOP cells, 588 | # but only if none of the other actions 589 | # are active (handled in `make_action_detector`). 590 | self.U[Index.STACK1_NOOP, Index.BUFFER21] = Rational(1) 591 | self.U[Index.STACK2_NOOP, Index.BUFFER22] = Rational(1) 592 | 593 | def make_action_committer(self): 594 | # PHASE 4: The final phase---commit the action from the individual 595 | # action cells to the stack. 596 | # Only one of the four entries PUSH_0, PUSH_1, POP_0, POP_1 597 | # will be "active" (non-zero), the rest will be 0. 598 | 599 | # This submatrix encodes the copying of the actions from the ``actions'' part 600 | # of the hidden state to the part of the hidden state corresponding to stack 1. 601 | 602 | self.U[Index.STACK1, Index.STACK1_PUSH_0] = Rational(1) 603 | self.U[Index.STACK1, Index.STACK1_PUSH_1] = Rational(1) 604 | self.U[Index.STACK2, Index.STACK2_PUSH_0] = Rational(1) 605 | self.U[Index.STACK2, Index.STACK2_PUSH_1] = Rational(1) 606 | 607 | self.U[Index.STACK1, Index.STACK1_POP_0] = Rational(1) 608 | self.U[Index.STACK1, Index.STACK1_POP_1] = Rational(1) 609 | self.U[Index.STACK2, Index.STACK2_POP_0] = Rational(1) 610 | self.U[Index.STACK2, Index.STACK2_POP_1] = Rational(1) 611 | 612 | self.U[Index.STACK1, Index.STACK1_NOOP] = Rational(1) 613 | self.U[Index.STACK2, Index.STACK2_NOOP] = Rational(1) 614 | 615 | def make_acceptor(self): 616 | # PHASE 5: acceptor 617 | # This is the final state of the RNN. 618 | # It accepts if the stack is empty (encoded in the computation component) 619 | # and the input is empty, i.e., the previous input was EOS. 620 | 621 | # Checks the emptiness of the computation component 622 | self.U[Index.ACCEPT, Index.STACK1_PUSH_0] = Rational(-10) 623 | self.U[Index.ACCEPT, Index.STACK1_PUSH_1] = Rational(-10) 624 | self.U[Index.ACCEPT, Index.STACK1_POP_0] = Rational(-10) 625 | self.U[Index.ACCEPT, Index.STACK1_POP_1] = Rational(-10) 626 | self.U[Index.ACCEPT, Index.STACK1_NOOP] = Rational(-10) 627 | self.U[Index.ACCEPT, Index.STACK2_PUSH_0] = Rational(-10) 628 | self.U[Index.ACCEPT, Index.STACK2_PUSH_1] = Rational(-10) 629 | self.U[Index.ACCEPT, Index.STACK2_POP_0] = Rational(-10) 630 | self.U[Index.ACCEPT, Index.STACK2_POP_1] = Rational(-10) 631 | self.U[Index.ACCEPT, Index.STACK2_NOOP] = Rational(-10) 632 | 633 | # Checks that the input was EOS 634 | self.U[Index.ACCEPT, Index.CONF_BOT_BOT_a] = Rational(-10) 635 | self.U[Index.ACCEPT, Index.CONF_BOT_0_a] = Rational(-10) 636 | self.U[Index.ACCEPT, Index.CONF_BOT_1_a] = Rational(-10) 637 | self.U[Index.ACCEPT, Index.CONF_BOT_BOT_b] = Rational(-10) 638 | self.U[Index.ACCEPT, Index.CONF_BOT_0_b] = Rational(-10) 639 | self.U[Index.ACCEPT, Index.CONF_BOT_1_b] = Rational(-10) 640 | self.U[Index.ACCEPT, Index.CONF_0_BOT_a] = Rational(-10) 641 | self.U[Index.ACCEPT, Index.CONF_0_0_a] = Rational(-10) 642 | self.U[Index.ACCEPT, Index.CONF_0_1_a] = Rational(-10) 643 | self.U[Index.ACCEPT, Index.CONF_0_BOT_b] = Rational(-10) 644 | self.U[Index.ACCEPT, Index.CONF_0_0_b] = Rational(-10) 645 | self.U[Index.ACCEPT, Index.CONF_0_1_b] = Rational(-10) 646 | self.U[Index.ACCEPT, Index.CONF_1_BOT_a] = Rational(-10) 647 | self.U[Index.ACCEPT, Index.CONF_1_0_a] = Rational(-10) 648 | self.U[Index.ACCEPT, Index.CONF_1_1_a] = Rational(-10) 649 | self.U[Index.ACCEPT, Index.CONF_1_BOT_b] = Rational(-10) 650 | self.U[Index.ACCEPT, Index.CONF_1_0_b] = Rational(-10) 651 | self.U[Index.ACCEPT, Index.CONF_1_1_b] = Rational(-10) 652 | 653 | def make_U(self): 654 | """Constructs the recurrence matrix U of the Siegelmann RNN.""" 655 | 656 | # ORCHESTRATION 657 | self.make_buffer_transitions() 658 | self.make_phase_transitions() 659 | 660 | # PHASE 1 661 | self.make_stack_detector() 662 | 663 | # PHASE 2 664 | self.make_configuration_detector() 665 | 666 | # PHASE 3 667 | self.make_action_detector() 668 | self.make_action_executor() 669 | 670 | # PHASE 4 671 | self.make_action_committer() 672 | self.make_acceptor() 673 | 674 | def make_V(self): 675 | """Constructs the emission matrix V of the Siegelmann RNN.""" 676 | # Enables setting the hidden state to 1 at the indices 677 | # corresponding to all possible combinations of the stack configurations 678 | # (top of the stack) and the incomping input symbol. 679 | # For input symbol a 680 | self.V[Index.CONF_BOT_BOT_a, sym2idx[Sym("a")]] = Rational(1) 681 | self.V[Index.CONF_BOT_0_a, sym2idx[Sym("a")]] = Rational(1) 682 | self.V[Index.CONF_BOT_1_a, sym2idx[Sym("a")]] = Rational(1) 683 | self.V[Index.CONF_0_BOT_a, sym2idx[Sym("a")]] = Rational(1) 684 | self.V[Index.CONF_0_0_a, sym2idx[Sym("a")]] = Rational(1) 685 | self.V[Index.CONF_0_1_a, sym2idx[Sym("a")]] = Rational(1) 686 | self.V[Index.CONF_1_BOT_a, sym2idx[Sym("a")]] = Rational(1) 687 | self.V[Index.CONF_1_0_a, sym2idx[Sym("a")]] = Rational(1) 688 | self.V[Index.CONF_1_1_a, sym2idx[Sym("a")]] = Rational(1) 689 | # For input symbol b 690 | self.V[Index.CONF_BOT_BOT_b, sym2idx[Sym("b")]] = Rational(1) 691 | self.V[Index.CONF_BOT_0_b, sym2idx[Sym("b")]] = Rational(1) 692 | self.V[Index.CONF_BOT_1_b, sym2idx[Sym("b")]] = Rational(1) 693 | self.V[Index.CONF_0_BOT_b, sym2idx[Sym("b")]] = Rational(1) 694 | self.V[Index.CONF_0_0_b, sym2idx[Sym("b")]] = Rational(1) 695 | self.V[Index.CONF_0_1_b, sym2idx[Sym("b")]] = Rational(1) 696 | self.V[Index.CONF_1_BOT_b, sym2idx[Sym("b")]] = Rational(1) 697 | self.V[Index.CONF_1_0_b, sym2idx[Sym("b")]] = Rational(1) 698 | self.V[Index.CONF_1_1_b, sym2idx[Sym("b")]] = Rational(1) 699 | # For input symbol EOS 700 | self.V[Index.CONF_BOT_BOT_EOS, sym2idx[EOS]] = Rational(1) 701 | self.V[Index.CONF_BOT_0_EOS, sym2idx[EOS]] = Rational(1) 702 | self.V[Index.CONF_BOT_1_EOS, sym2idx[EOS]] = Rational(1) 703 | self.V[Index.CONF_0_BOT_EOS, sym2idx[EOS]] = Rational(1) 704 | self.V[Index.CONF_0_0_EOS, sym2idx[EOS]] = Rational(1) 705 | self.V[Index.CONF_0_1_EOS, sym2idx[EOS]] = Rational(1) 706 | self.V[Index.CONF_1_BOT_EOS, sym2idx[EOS]] = Rational(1) 707 | self.V[Index.CONF_1_0_EOS, sym2idx[EOS]] = Rational(1) 708 | self.V[Index.CONF_1_1_EOS, sym2idx[EOS]] = Rational(1) 709 | 710 | def make_b(self): 711 | """Constructs the bias vector b of the Siegelmann RNN.""" 712 | 713 | self.b[Index.STACK1_EMPTY, 0] = Rational(1) 714 | self.b[Index.STACK2_EMPTY, 0] = Rational(1) 715 | self.b[Index.STACK1_ZERO, 0] = Rational(3) 716 | self.b[Index.STACK2_ZERO, 0] = Rational(3) 717 | self.b[Index.STACK1_ONE, 0] = Rational(-2) 718 | self.b[Index.STACK2_ONE, 0] = Rational(-2) 719 | 720 | # This provides the base of the quantities that get added to 721 | # the previous encoding of the stack after it is divided by 10. 722 | # Add the encoding of the new top of the stack 723 | self.b[Index.STACK1_PUSH_0, 0] = Rational("1/10") 724 | self.b[Index.STACK1_PUSH_1, 0] = Rational("3/10") 725 | 726 | self.b[Index.STACK2_PUSH_0, 0] = Rational("1/10") 727 | self.b[Index.STACK2_PUSH_1, 0] = Rational("3/10") 728 | 729 | # Remove the top encoding of the stack, after it has been multiplied by 10. 730 | self.b[Index.STACK1_POP_0, 0] = Rational(-1) 731 | self.b[Index.STACK1_POP_1, 0] = Rational(-3) 732 | 733 | self.b[Index.STACK2_POP_0, 0] = Rational(-1) 734 | self.b[Index.STACK2_POP_1, 0] = Rational(-3) 735 | 736 | self.b[Index.STACK1_NOOP, 0] = Rational(0) 737 | self.b[Index.STACK2_NOOP, 0] = Rational(0) 738 | 739 | # This zeroes out the configurations which are not active 740 | # even though some of the "triggering" cells in the hidden state are active. 741 | self.b[Index.CONF_BOT_BOT_a, 0] = Rational(-2) 742 | self.b[Index.CONF_BOT_0_a, 0] = Rational(-2) 743 | self.b[Index.CONF_BOT_1_a, 0] = Rational(-2) 744 | self.b[Index.CONF_BOT_BOT_b, 0] = Rational(-2) 745 | self.b[Index.CONF_BOT_0_b, 0] = Rational(-2) 746 | self.b[Index.CONF_BOT_1_b, 0] = Rational(-2) 747 | self.b[Index.CONF_BOT_BOT_EOS, 0] = Rational(-2) 748 | self.b[Index.CONF_BOT_0_EOS, 0] = Rational(-2) 749 | self.b[Index.CONF_BOT_1_EOS, 0] = Rational(-2) 750 | self.b[Index.CONF_0_BOT_a, 0] = Rational(-2) 751 | self.b[Index.CONF_0_0_a, 0] = Rational(-2) 752 | self.b[Index.CONF_0_1_a, 0] = Rational(-2) 753 | self.b[Index.CONF_0_BOT_b, 0] = Rational(-2) 754 | self.b[Index.CONF_0_0_b, 0] = Rational(-2) 755 | self.b[Index.CONF_0_1_b, 0] = Rational(-2) 756 | self.b[Index.CONF_0_BOT_EOS, 0] = Rational(-2) 757 | self.b[Index.CONF_0_0_EOS, 0] = Rational(-2) 758 | self.b[Index.CONF_0_1_EOS, 0] = Rational(-2) 759 | self.b[Index.CONF_1_BOT_a, 0] = Rational(-2) 760 | self.b[Index.CONF_1_0_a, 0] = Rational(-2) 761 | self.b[Index.CONF_1_1_a, 0] = Rational(-2) 762 | self.b[Index.CONF_1_BOT_b, 0] = Rational(-2) 763 | self.b[Index.CONF_1_0_b, 0] = Rational(-2) 764 | self.b[Index.CONF_1_1_b, 0] = Rational(-2) 765 | self.b[Index.CONF_1_BOT_EOS, 0] = Rational(-2) 766 | self.b[Index.CONF_1_0_EOS, 0] = Rational(-2) 767 | self.b[Index.CONF_1_1_EOS, 0] = Rational(-2) 768 | 769 | self.b[Index.ACCEPT, 0] = Rational(1) 770 | 771 | def __call__(self, a: Sym): 772 | """Performs a step of the Siegelmann RNN.""" 773 | assert a in self.Σ 774 | 775 | # We have four stages in which the hidden state is updated 776 | # 1) Peek 777 | # 2) Combine 778 | # 3) Transition 779 | # 4) Consolidate 780 | 781 | def apply(_h): 782 | tmp = self.U * _h + self.V * self.one_hot.col(sym2idx[a]) + self.b 783 | 784 | self.disp(tmp) 785 | 786 | h2 = zeros(self.D, 1, dtype=Rational(0)) 787 | 788 | for i in range(self.D): 789 | h2[i] = σ.subs(x, tmp[i]) 790 | return h2 791 | 792 | h = self.h 793 | print("\n\n\n\n >>>> START:") 794 | self.disp(h) 795 | print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 796 | 797 | print("\n\n----------------------------\n>>>>> PHASE 1") 798 | h = apply(h) 799 | print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 800 | self.disp(h) 801 | print("\n\n----------------------------\n>>>>> PHASE 2") 802 | h = apply(h) 803 | print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 804 | self.disp(h) 805 | print("\n\n----------------------------\n>>>>> PHASE 3") 806 | h = apply(h) 807 | print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 808 | self.disp(h) 809 | print("\n\n----------------------------\n>>>>> PHASE 4") 810 | h = apply(h) 811 | print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 812 | self.disp(h) 813 | print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 814 | self.disp(h) 815 | self.h = h 816 | print("\n\n\n\n\n") 817 | 818 | return self.h, self.h[Index.ACCEPT] 819 | -------------------------------------------------------------------------------- /turnn/turing/two_stack_rnnlm.py: -------------------------------------------------------------------------------- 1 | """This module implements an RNN *language model* simulating a 2 | *probabilistic* two-stack PDA. 3 | The construction of the controller (unweighted part) is original but based on the one in 4 | Siegelmann and Sontag (1995) 5 | [https://binds.cs.umass.edu/papers/1995_Siegelmann_JComSysSci.pdf] 6 | while the weighted part is original. 7 | 8 | See `turnn/turing/two_stack_rnn.py` for the unweighted version of the 9 | same construction. 10 | """ 11 | 12 | from enum import IntEnum, unique 13 | from itertools import product 14 | from typing import Tuple, Union 15 | 16 | from sympy import Abs, Matrix, Piecewise, Rational, Symbol, eye, log, sympify, zeros 17 | 18 | from turnn.base.string import String 19 | from turnn.base.symbol import BOT, EOS, Sym 20 | 21 | # from turnn.base.utils import cantor_decode 22 | from turnn.turing.pda import Action, TwoStackPDA 23 | 24 | # The element-wise saturated sigmoid function 25 | x = Symbol("x") 26 | σ = Piecewise((Rational(0), x <= 0), (Rational(1), x >= 1), (Abs(x) <= sympify(1))) 27 | 28 | 29 | @unique 30 | class Index(IntEnum): 31 | """This class is used to index the hidden state of the RNN for two 32 | stacks by naming the individual dimensions of the hidden state with the role 33 | they play in simulating the two-stack PDA. 34 | """ 35 | 36 | # (1) Data component 37 | STACK1 = 0 38 | STACK2 = 1 39 | BUFFER11 = 2 40 | BUFFER12 = 3 41 | BUFFER21 = 4 42 | BUFFER22 = 5 43 | 44 | # (2) Phase component 45 | PHASE1 = 6 46 | PHASE2 = 7 47 | PHASE3 = 8 48 | PHASE4 = 9 49 | 50 | # (3) Stack configuration component 51 | STACK1_EMPTY = 10 52 | STACK1_ZERO = 11 53 | STACK1_ONE = 12 54 | STACK2_EMPTY = 13 55 | STACK2_ZERO = 14 56 | STACK2_ONE = 15 57 | 58 | # (4) Stack and input configuration component 59 | # `CONF_γ1_γ2_a` corresponds to the configuration 60 | # where the tops of the two stacks are `γ1` and `γ2` and the next symbol is `a` 61 | CONF_BOT_BOT_EOS = 16 62 | CONF_BOT_0_EOS = 17 63 | CONF_BOT_1_EOS = 18 64 | CONF_0_BOT_EOS = 19 65 | CONF_1_BOT_EOS = 20 66 | CONF_0_0_EOS = 21 67 | CONF_0_1_EOS = 22 68 | CONF_1_0_EOS = 23 69 | CONF_1_1_EOS = 24 70 | CONF_BOT_BOT_a = 25 71 | CONF_BOT_0_a = 26 72 | CONF_BOT_1_a = 27 73 | CONF_0_BOT_a = 28 74 | CONF_1_BOT_a = 29 75 | CONF_0_0_a = 30 76 | CONF_0_1_a = 31 77 | CONF_1_0_a = 32 78 | CONF_1_1_a = 33 79 | CONF_BOT_BOT_b = 34 80 | CONF_BOT_0_b = 35 81 | CONF_BOT_1_b = 36 82 | CONF_0_BOT_b = 37 83 | CONF_1_BOT_b = 38 84 | CONF_0_0_b = 39 85 | CONF_0_1_b = 40 86 | CONF_1_0_b = 41 87 | CONF_1_1_b = 42 88 | 89 | # (5) Computation component 90 | STACK1_PUSH_0 = 43 91 | STACK1_PUSH_1 = 44 92 | STACK1_POP_0 = 45 93 | STACK1_POP_1 = 46 94 | STACK1_NOOP = 47 95 | STACK2_PUSH_0 = 48 96 | STACK2_PUSH_1 = 49 97 | STACK2_POP_0 = 50 98 | STACK2_POP_1 = 51 99 | STACK2_NOOP = 52 100 | 101 | # (6) Acceptance component 102 | ACCEPT = 53 103 | 104 | 105 | class EmissionIndex(IntEnum): 106 | CONF_BOTBOT = 0 107 | CONF_BOT0 = 1 108 | CONF_BOT1 = 2 109 | CONF_0BOT = 3 110 | CONF_1BOT = 4 111 | CONF_00 = 5 112 | CONF_01 = 6 113 | CONF_10 = 7 114 | CONF_11 = 8 115 | 116 | 117 | # We use a two-letter alphabet {a, b} and the EOS symbol. 118 | sym2idx = { 119 | Sym("a"): 0, 120 | Sym("b"): 1, 121 | EOS: 2, 122 | } 123 | 124 | 125 | def conf_peek(x: Index): # noqa: C901 126 | """Converts the index of the configuration into a human-readable string. 127 | 128 | Args: 129 | x (Index): THe index of the configuration. 130 | """ 131 | if x == Index.CONF_BOT_BOT_EOS: 132 | print("(⊥, ⊥, EOS)") 133 | elif x == Index.CONF_BOT_0_EOS: 134 | print("(⊥, 0, EOS)") 135 | elif x == Index.CONF_BOT_1_EOS: 136 | print("(⊥, 1, EOS)") 137 | elif x == Index.CONF_0_BOT_EOS: 138 | print("(0, ⊥, EOS)") 139 | elif x == Index.CONF_1_BOT_EOS: 140 | print("(1, ⊥, EOS)") 141 | elif x == Index.CONF_BOT_BOT_a: 142 | print("(⊥, ⊥, a)") 143 | elif x == Index.CONF_BOT_BOT_b: 144 | print("(⊥, ⊥, b)") 145 | elif x == Index.CONF_BOT_0_a: 146 | print("(⊥, 0, a)") 147 | elif x == Index.CONF_BOT_0_b: 148 | print("(⊥, 0, b)") 149 | elif x == Index.CONF_BOT_1_a: 150 | print("(⊥, 1, a)") 151 | elif x == Index.CONF_BOT_1_b: 152 | print("(⊥, 1, b)") 153 | elif x == Index.CONF_0_BOT_a: 154 | print("(0, ⊥, a)") 155 | elif x == Index.CONF_0_BOT_b: 156 | print("(0, ⊥, b)") 157 | elif x == Index.CONF_1_BOT_a: 158 | print("(1, ⊥, a)") 159 | elif x == Index.CONF_1_BOT_b: 160 | print("(1, ⊥, b)") 161 | elif x == Index.CONF_0_0_a: 162 | print("(0, 0, a)") 163 | elif x == Index.CONF_0_0_b: 164 | print("(0, 0, b)") 165 | elif x == Index.CONF_0_1_a: 166 | print("(0, 1, a)") 167 | elif x == Index.CONF_0_1_b: 168 | print("(0, 1, b)") 169 | elif x == Index.CONF_1_0_a: 170 | print("(1, 0, a)") 171 | elif x == Index.CONF_1_0_b: 172 | print("(1, 0, b)") 173 | elif x == Index.CONF_1_1_a: 174 | print("(1, 1, a)") 175 | elif x == Index.CONF_1_1_b: 176 | print("(1, 1, b)") 177 | 178 | raise NotImplementedError 179 | 180 | 181 | def conf2idx(γ1: Sym, γ2: Sym, sym: Sym) -> Index: # noqa: C901 182 | """Converts a configuration (tops of the stacks and the input symbol) 183 | into an index in the matrices. 184 | 185 | Args: 186 | γ1 (Sym): The top of the first stack. 187 | γ2 (Sym): The top of the second stack. 188 | sym (Sym): The input symbol. 189 | 190 | Raises: 191 | NotImplementedError: If the configuration is not supported. 192 | 193 | Returns: 194 | Index: The index of the configuration. 195 | """ 196 | if (γ1, γ2, sym) == (BOT, BOT, EOS): 197 | return Index.CONF_BOT_BOT_EOS 198 | elif (γ1, γ2, sym) == (BOT, Sym("0"), EOS): 199 | return Index.CONF_BOT_0_EOS 200 | elif (γ1, γ2, sym) == (BOT, Sym("1"), EOS): 201 | return Index.CONF_BOT_1_EOS 202 | elif (γ1, γ2, sym) == (Sym("0"), BOT, EOS): 203 | return Index.CONF_0_BOT_EOS 204 | elif (γ1, γ2, sym) == (Sym("1"), BOT, EOS): 205 | return Index.CONF_1_BOT_EOS 206 | elif (γ1, γ2, sym) == (Sym("0"), Sym("0"), EOS): 207 | return Index.CONF_0_0_EOS 208 | elif (γ1, γ2, sym) == (Sym("0"), Sym("1"), EOS): 209 | return Index.CONF_0_1_EOS 210 | elif (γ1, γ2, sym) == (Sym("1"), Sym("0"), EOS): 211 | return Index.CONF_1_0_EOS 212 | elif (γ1, γ2, sym) == (Sym("1"), Sym("1"), EOS): 213 | return Index.CONF_1_1_EOS 214 | elif (γ1, γ2, sym) == (BOT, BOT, Sym("a")): 215 | return Index.CONF_BOT_BOT_a 216 | elif (γ1, γ2, sym) == (BOT, Sym("0"), Sym("a")): 217 | return Index.CONF_BOT_0_a 218 | elif (γ1, γ2, sym) == (BOT, Sym("1"), Sym("a")): 219 | return Index.CONF_BOT_1_a 220 | elif (γ1, γ2, sym) == (Sym("0"), BOT, Sym("a")): 221 | return Index.CONF_0_BOT_a 222 | elif (γ1, γ2, sym) == (Sym("1"), BOT, Sym("a")): 223 | return Index.CONF_1_BOT_a 224 | elif (γ1, γ2, sym) == (Sym("0"), Sym("0"), Sym("a")): 225 | return Index.CONF_0_0_a 226 | elif (γ1, γ2, sym) == (Sym("0"), Sym("1"), Sym("a")): 227 | return Index.CONF_0_1_a 228 | elif (γ1, γ2, sym) == (Sym("1"), Sym("0"), Sym("a")): 229 | return Index.CONF_1_0_a 230 | elif (γ1, γ2, sym) == (Sym("1"), Sym("1"), Sym("a")): 231 | return Index.CONF_1_1_a 232 | elif (γ1, γ2, sym) == (BOT, BOT, Sym("b")): 233 | return Index.CONF_BOT_BOT_b 234 | elif (γ1, γ2, sym) == (BOT, Sym("0"), Sym("b")): 235 | return Index.CONF_BOT_0_b 236 | elif (γ1, γ2, sym) == (BOT, Sym("1"), Sym("b")): 237 | return Index.CONF_BOT_1_b 238 | elif (γ1, γ2, sym) == (Sym("0"), BOT, Sym("b")): 239 | return Index.CONF_0_BOT_b 240 | elif (γ1, γ2, sym) == (Sym("1"), BOT, Sym("b")): 241 | return Index.CONF_1_BOT_b 242 | elif (γ1, γ2, sym) == (Sym("0"), Sym("0"), Sym("b")): 243 | return Index.CONF_0_0_b 244 | elif (γ1, γ2, sym) == (Sym("0"), Sym("1"), Sym("b")): 245 | return Index.CONF_0_1_b 246 | elif (γ1, γ2, sym) == (Sym("1"), Sym("0"), Sym("b")): 247 | return Index.CONF_1_0_b 248 | elif (γ1, γ2, sym) == (Sym("1"), Sym("1"), Sym("b")): 249 | return Index.CONF_1_1_b 250 | 251 | raise NotImplementedError 252 | 253 | 254 | class TwoStackRNN: 255 | """ 256 | An implementation of an Elman RNN language model that simulates a probabilistic 257 | two-stack PDA. 258 | It computes the next state h_t+1 from the current state h_t by applying the 259 | Elman update rule four times: 260 | h' = σ(U * h + V * y + b) 261 | where the matrices U, V and the vector b are computed from the PDA and 262 | the input symbol y is one-hot encoded. 263 | After four applications of the update rule, the new hidden state represents the 264 | new configuration of the PDA. 265 | 266 | The hidden state is composed of six components: 267 | 1. The data component 268 | 2. The phase component 269 | 3. The stack configurations component 270 | 4. The stacks and input configuration component 271 | 5. The computation component 272 | 6. The acceptance component 273 | 274 | 1. The first component contains the cells containing the encodings of the two stacks 275 | along with two buffer cells for each stack through which the stack is passed 276 | during the four-stage computation. 277 | 2. The second component simply denotes the current phase of the computation, and is 278 | only included for keeping track of the computation. 279 | 3. The third component contains the encoding of the stack at the current time step, 280 | i.e., it flags whether the stacks are empty or have a 0/1 on the top. 281 | 4. The fourth component contains the encoding of the current input symbol together 282 | with the stacks. 283 | 5. The fifth component contains cells to which the current stack configuration is 284 | copied and modified according to the action of the PDA. 285 | 6. The sixth component contains a cell that are set to 1 after reading in the EOS 286 | symbol if the PDA accepts the input string appearing before EOS. 287 | 288 | The hidden state is then used to index the output matrix of (log) probabilities 289 | such that the probability of the next symbol is computed as 290 | p(y_t+1 | y_t, h_t) = exp(E * σ(W * h_t+1 + bw)) / sum(exp(E * σ(W * h_t+1 + bw))) 291 | where E is the emission matrix and W and bw are the parameters of the MLP projecting 292 | the hidden state to a one-hot encoding of the configuration of the two stacks. 293 | The log probabilities are then summed up over the entire string to form the log 294 | probability of the string, which matches the one of the simulated PPDA. 295 | """ 296 | 297 | def __init__(self, pda: TwoStackPDA): 298 | self.pda = pda 299 | self.Σ = pda.Σ.union({EOS}) # In contrast to the globally normalized PDA, 300 | # the RNN works with an EOS symbol 301 | 302 | self.sym2idx = {sym: idx for idx, sym in enumerate(self.Σ)} 303 | 304 | # The dimension of the hidden state 305 | self.D = len(Index) 306 | 307 | # Recurrence matrix U 308 | self.U = zeros(self.D, self.D, dtype=Rational(0)) 309 | # Input matrix V 310 | self.V = zeros(self.D, len(self.Σ), dtype=Rational(0)) 311 | # Bias vector b 312 | self.b = zeros(self.D, 1, dtype=Rational(0)) 313 | 314 | self.make_U() 315 | self.make_b() 316 | self.make_V() 317 | 318 | # Emission matrix E together with the MLP (defined through W and bw) that 319 | # projects the hidden state into a usable one-hot encoding 320 | self.W = zeros(9, self.D, dtype=Rational(0)) 321 | self.bw = zeros(9, 1, dtype=Rational(0)) 322 | self.E = zeros(len(self.Σ), 9, dtype=Rational(0)) 323 | self.make_emission_params() 324 | 325 | # One-hot encoding of the input symbols 326 | self.one_hot = eye(len(self.Σ), dtype=Rational(0)) 327 | 328 | def disp(self, h: Matrix): 329 | """Prints the content of the hidden state of the Siegelmann RNN, separated by 330 | the different components. 331 | 332 | Args: 333 | h (Matrix): The hidden state of the Siegelmann RNN. 334 | """ 335 | print(f"Stack 1:\t{h[Index.STACK1]}") 336 | print(f"Buffer 11:\t{h[Index.BUFFER11]}") 337 | print(f"Buffer 21:\t{h[Index.BUFFER21]}") 338 | 339 | print(f"Stack 2:\t{h[Index.STACK2]}") 340 | print(f"Buffer 12:\t{h[Index.BUFFER12]}") 341 | print(f"Buffer 22:\t{h[Index.BUFFER22]}") 342 | 343 | print(f"Phase 1:\t{h[Index.PHASE1]}") 344 | print(f"Phase 2:\t{h[Index.PHASE2]}") 345 | print(f"Phase 3:\t{h[Index.PHASE3]}") 346 | print(f"Phase 4:\t{h[Index.PHASE4]}") 347 | 348 | print(f"1: STACK 1 - EMPTY:\t{h[Index.STACK1_EMPTY]}") 349 | print(f"1: STACK 1 - 0:\t{h[Index.STACK1_ZERO]}") 350 | print(f"1: STACK 1 - 1:\t{h[Index.STACK1_ONE]}") 351 | print(f"1: STACK 2 - EMPTY:\t{h[Index.STACK2_EMPTY]}") 352 | print(f"1: STACK 2 - 0:\t{h[Index.STACK2_ZERO]}") 353 | print(f"1: STACK 2 - 1:\t{h[Index.STACK2_ONE]}") 354 | 355 | print(f"2: (⊥, ⊥, EOS):\t{h[Index.CONF_BOT_BOT_EOS]}") 356 | print(f"2: (⊥, 0, EOS):\t{h[Index.CONF_BOT_0_EOS]}") 357 | print(f"2: (⊥, 1, EOS):\t{h[Index.CONF_BOT_1_EOS]}") 358 | print(f"2: (0, ⊥, EOS):\t{h[Index.CONF_0_BOT_EOS]}") 359 | print(f"2: (1, ⊥, EOS):\t{h[Index.CONF_1_BOT_EOS]}") 360 | print(f"2: (0, 0, EOS):\t{h[Index.CONF_0_0_EOS]}") 361 | print(f"2: (0, 1, EOS):\t{h[Index.CONF_0_1_EOS]}") 362 | print(f"2: (1, 0, EOS):\t{h[Index.CONF_1_0_EOS]}") 363 | print(f"2: (1, 1, EOS):\t{h[Index.CONF_1_1_EOS]}") 364 | print(f"2: (⊥, ⊥, a):\t{h[Index.CONF_BOT_BOT_a]}") 365 | print(f"2: (⊥, 0, a):\t{h[Index.CONF_BOT_0_a]}") 366 | print(f"2: (⊥, 1, a):\t{h[Index.CONF_BOT_1_a]}") 367 | print(f"2: (0, ⊥, a):\t{h[Index.CONF_0_BOT_a]}") 368 | print(f"2: (1, ⊥, a):\t{h[Index.CONF_1_BOT_a]}") 369 | print(f"2: (0, 0, a):\t{h[Index.CONF_0_0_a]}") 370 | print(f"2: (0, 1, a):\t{h[Index.CONF_0_1_a]}") 371 | print(f"2: (1, 0, a):\t{h[Index.CONF_1_0_a]}") 372 | print(f"2: (1, 1, a):\t{h[Index.CONF_1_1_a]}") 373 | print(f"2: (⊥, ⊥, b):\t{h[Index.CONF_BOT_BOT_b]}") 374 | print(f"2: (⊥, 0, b):\t{h[Index.CONF_BOT_0_b]}") 375 | print(f"2: (⊥, 1, b):\t{h[Index.CONF_BOT_1_b]}") 376 | print(f"2: (0, ⊥, b):\t{h[Index.CONF_0_BOT_b]}") 377 | print(f"2: (1, ⊥, b):\t{h[Index.CONF_1_BOT_b]}") 378 | print(f"2: (0, 0, b):\t{h[Index.CONF_0_0_b]}") 379 | print(f"2: (0, 1, b):\t{h[Index.CONF_0_1_b]}") 380 | print(f"2: (1, 0, b):\t{h[Index.CONF_1_0_b]}") 381 | print(f"2: (1, 1, b):\t{h[Index.CONF_1_1_b]}") 382 | 383 | print(f"3: STACK 1 Push 0:\t{h[Index.STACK1_PUSH_0]}") 384 | print(f"3: STACK 1 Push 1:\t{h[Index.STACK1_PUSH_1]}") 385 | print(f"3: STACK 1 Pop 0:\t{h[Index.STACK1_POP_0]}") 386 | print(f"3: STACK 1 Pop 1:\t{h[Index.STACK1_POP_1]}") 387 | 388 | print(f"3: STACK 2 Push 0:\t{h[Index.STACK2_PUSH_0]}") 389 | print(f"3: STACK 2 Push 1:\t{h[Index.STACK2_PUSH_1]}") 390 | print(f"3: STACK 2 Pop 0:\t{h[Index.STACK2_POP_0]}") 391 | print(f"3: STACK 2 Pop 1:\t{h[Index.STACK2_POP_1]}") 392 | 393 | print() 394 | 395 | def make_buffer_transitions(self): 396 | """ 397 | Constructs the submatrix encodes the copying of the stack contents between 398 | the three different stacks simulated by the RNN in the hidden state 399 | while it moves through the 4 phases of the hidden state update. 400 | Each of the stacks is simply a dimension in the hidden state. 401 | Each of the three stacks is copied to the next one, 402 | and the last one is zeroed out. 403 | """ 404 | self.U[Index.STACK1, Index.STACK1] = Rational(0) # delete 405 | self.U[Index.BUFFER11, Index.STACK1] = Rational(1) # copy over to the next one 406 | self.U[Index.BUFFER11, Index.BUFFER11] = Rational(0) 407 | self.U[Index.BUFFER21, Index.BUFFER11] = Rational(1) 408 | self.U[Index.BUFFER21, Index.BUFFER21] = Rational(0) 409 | 410 | self.U[Index.STACK2, Index.STACK2] = Rational(0) # delete 411 | self.U[Index.BUFFER12, Index.STACK2] = Rational(1) # copy over to the next one 412 | self.U[Index.BUFFER12, Index.BUFFER12] = Rational(0) 413 | self.U[Index.BUFFER22, Index.BUFFER12] = Rational(1) 414 | self.U[Index.BUFFER22, Index.BUFFER22] = Rational(0) 415 | 416 | def make_phase_transitions(self): 417 | # These two blocks encode the state---it's a one-hot encoding 418 | self.U[Index.PHASE1, Index.PHASE1] = Rational(0) 419 | self.U[Index.PHASE2, Index.PHASE2] = Rational(0) 420 | self.U[Index.PHASE3, Index.PHASE3] = Rational(0) 421 | self.U[Index.PHASE4, Index.PHASE4] = Rational(0) 422 | self.U[Index.PHASE2, Index.PHASE1] = Rational(1) 423 | self.U[Index.PHASE3, Index.PHASE2] = Rational(1) 424 | self.U[Index.PHASE4, Index.PHASE3] = Rational(1) 425 | self.U[Index.PHASE1, Index.PHASE4] = Rational(1) 426 | 427 | def make_stack_detector(self): 428 | # PHASE 1: 429 | # This detects an empty stack 430 | self.U[Index.STACK1_EMPTY, Index.STACK1] = Rational(-10) 431 | self.U[Index.STACK2_EMPTY, Index.STACK2] = Rational(-10) 432 | 433 | # This detects whether the top of the stack is a 0 or a 1 434 | self.U[Index.STACK1_ZERO, Index.STACK1] = Rational(-10) 435 | self.U[Index.STACK1_ONE, Index.STACK1] = Rational(10) 436 | self.U[Index.STACK2_ZERO, Index.STACK2] = Rational(-10) 437 | self.U[Index.STACK2_ONE, Index.STACK2] = Rational(10) 438 | 439 | def make_configuration_detector(self): 440 | # PHASE 2: This submatrix corresponds to the actions available given any 441 | # configuration of the stack, captured at EMPTY/PEEK locations. 442 | # This corresponds to the emissions 443 | # These then get "intersected" with the symbol embeddings to select a or b 444 | self.U[Index.CONF_BOT_BOT_a, Index.STACK1_EMPTY] = Rational(1) 445 | self.U[Index.CONF_BOT_0_a, Index.STACK1_EMPTY] = Rational(1) 446 | self.U[Index.CONF_BOT_1_a, Index.STACK1_EMPTY] = Rational(1) 447 | self.U[Index.CONF_BOT_BOT_b, Index.STACK1_EMPTY] = Rational(1) 448 | self.U[Index.CONF_BOT_0_b, Index.STACK1_EMPTY] = Rational(1) 449 | self.U[Index.CONF_BOT_1_b, Index.STACK1_EMPTY] = Rational(1) 450 | self.U[Index.CONF_BOT_BOT_EOS, Index.STACK1_EMPTY] = Rational(1) 451 | self.U[Index.CONF_BOT_0_EOS, Index.STACK1_EMPTY] = Rational(1) 452 | self.U[Index.CONF_BOT_1_EOS, Index.STACK1_EMPTY] = Rational(1) 453 | self.U[Index.CONF_0_BOT_a, Index.STACK1_ZERO] = Rational(1) 454 | self.U[Index.CONF_0_0_a, Index.STACK1_ZERO] = Rational(1) 455 | self.U[Index.CONF_0_1_a, Index.STACK1_ZERO] = Rational(1) 456 | self.U[Index.CONF_0_BOT_b, Index.STACK1_ZERO] = Rational(1) 457 | self.U[Index.CONF_0_0_b, Index.STACK1_ZERO] = Rational(1) 458 | self.U[Index.CONF_0_1_b, Index.STACK1_ZERO] = Rational(1) 459 | self.U[Index.CONF_0_BOT_EOS, Index.STACK1_ZERO] = Rational(1) 460 | self.U[Index.CONF_0_0_EOS, Index.STACK1_ZERO] = Rational(1) 461 | self.U[Index.CONF_0_1_EOS, Index.STACK1_ZERO] = Rational(1) 462 | self.U[Index.CONF_1_BOT_a, Index.STACK1_ONE] = Rational(1) 463 | self.U[Index.CONF_1_0_a, Index.STACK1_ONE] = Rational(1) 464 | self.U[Index.CONF_1_1_a, Index.STACK1_ONE] = Rational(1) 465 | self.U[Index.CONF_1_BOT_b, Index.STACK1_ONE] = Rational(1) 466 | self.U[Index.CONF_1_0_b, Index.STACK1_ONE] = Rational(1) 467 | self.U[Index.CONF_1_1_b, Index.STACK1_ONE] = Rational(1) 468 | self.U[Index.CONF_1_BOT_EOS, Index.STACK1_ONE] = Rational(1) 469 | self.U[Index.CONF_1_0_EOS, Index.STACK1_ONE] = Rational(1) 470 | self.U[Index.CONF_1_1_EOS, Index.STACK1_ONE] = Rational(1) 471 | 472 | self.U[Index.CONF_BOT_BOT_a, Index.STACK2_EMPTY] = Rational(1) 473 | self.U[Index.CONF_BOT_BOT_b, Index.STACK2_EMPTY] = Rational(1) 474 | self.U[Index.CONF_0_BOT_b, Index.STACK2_EMPTY] = Rational(1) 475 | self.U[Index.CONF_0_BOT_a, Index.STACK2_EMPTY] = Rational(1) 476 | self.U[Index.CONF_1_BOT_a, Index.STACK2_EMPTY] = Rational(1) 477 | self.U[Index.CONF_1_BOT_b, Index.STACK2_EMPTY] = Rational(1) 478 | self.U[Index.CONF_BOT_0_a, Index.STACK2_ZERO] = Rational(1) 479 | self.U[Index.CONF_BOT_0_b, Index.STACK2_ZERO] = Rational(1) 480 | self.U[Index.CONF_0_0_a, Index.STACK2_ZERO] = Rational(1) 481 | self.U[Index.CONF_0_0_b, Index.STACK2_ZERO] = Rational(1) 482 | self.U[Index.CONF_1_0_a, Index.STACK2_ZERO] = Rational(1) 483 | self.U[Index.CONF_1_0_b, Index.STACK2_ZERO] = Rational(1) 484 | self.U[Index.CONF_BOT_1_a, Index.STACK2_ONE] = Rational(1) 485 | self.U[Index.CONF_BOT_1_b, Index.STACK2_ONE] = Rational(1) 486 | self.U[Index.CONF_0_1_a, Index.STACK2_ONE] = Rational(1) 487 | self.U[Index.CONF_0_1_b, Index.STACK2_ONE] = Rational(1) 488 | self.U[Index.CONF_1_1_a, Index.STACK2_ONE] = Rational(1) 489 | self.U[Index.CONF_1_1_b, Index.STACK2_ONE] = Rational(1) 490 | self.U[Index.CONF_BOT_BOT_EOS, Index.STACK2_EMPTY] = Rational(1) 491 | self.U[Index.CONF_0_BOT_EOS, Index.STACK2_EMPTY] = Rational(1) 492 | self.U[Index.CONF_1_BOT_EOS, Index.STACK2_EMPTY] = Rational(1) 493 | self.U[Index.CONF_BOT_0_EOS, Index.STACK2_ZERO] = Rational(1) 494 | self.U[Index.CONF_0_0_EOS, Index.STACK2_ZERO] = Rational(1) 495 | self.U[Index.CONF_1_1_EOS, Index.STACK2_ZERO] = Rational(1) 496 | self.U[Index.CONF_BOT_1_EOS, Index.STACK2_ONE] = Rational(1) 497 | self.U[Index.CONF_0_1_EOS, Index.STACK2_ONE] = Rational(1) 498 | self.U[Index.CONF_1_1_EOS, Index.STACK2_ONE] = Rational(1) 499 | 500 | # This bit ensures that we can't be empty and non-empty 501 | # It erases the effects of always setting the PEEK0 bit to 1 502 | self.U[Index.CONF_0_BOT_a, Index.STACK1_EMPTY] = Rational(-10) 503 | self.U[Index.CONF_0_0_a, Index.STACK1_EMPTY] = Rational(-10) 504 | self.U[Index.CONF_0_1_a, Index.STACK1_EMPTY] = Rational(-10) 505 | self.U[Index.CONF_0_BOT_b, Index.STACK1_EMPTY] = Rational(-10) 506 | self.U[Index.CONF_0_0_b, Index.STACK1_EMPTY] = Rational(-10) 507 | self.U[Index.CONF_0_1_b, Index.STACK1_EMPTY] = Rational(-10) 508 | self.U[Index.CONF_1_BOT_a, Index.STACK1_EMPTY] = Rational(-10) 509 | self.U[Index.CONF_1_0_a, Index.STACK1_EMPTY] = Rational(-10) 510 | self.U[Index.CONF_1_1_a, Index.STACK1_EMPTY] = Rational(-10) 511 | self.U[Index.CONF_1_BOT_b, Index.STACK1_EMPTY] = Rational(-10) 512 | self.U[Index.CONF_1_0_b, Index.STACK1_EMPTY] = Rational(-10) 513 | self.U[Index.CONF_1_1_b, Index.STACK1_EMPTY] = Rational(-10) 514 | 515 | self.U[Index.CONF_BOT_0_a, Index.STACK2_EMPTY] = Rational(-10) 516 | self.U[Index.CONF_0_0_a, Index.STACK2_EMPTY] = Rational(-10) 517 | self.U[Index.CONF_1_0_a, Index.STACK2_EMPTY] = Rational(-10) 518 | self.U[Index.CONF_BOT_0_b, Index.STACK2_EMPTY] = Rational(-10) 519 | self.U[Index.CONF_0_0_b, Index.STACK2_EMPTY] = Rational(-10) 520 | self.U[Index.CONF_1_0_b, Index.STACK2_EMPTY] = Rational(-10) 521 | self.U[Index.CONF_BOT_1_a, Index.STACK2_EMPTY] = Rational(-10) 522 | self.U[Index.CONF_0_1_a, Index.STACK2_EMPTY] = Rational(-10) 523 | self.U[Index.CONF_1_1_a, Index.STACK2_EMPTY] = Rational(-10) 524 | self.U[Index.CONF_BOT_1_b, Index.STACK2_EMPTY] = Rational(-10) 525 | self.U[Index.CONF_0_1_b, Index.STACK2_EMPTY] = Rational(-10) 526 | self.U[Index.CONF_1_1_b, Index.STACK2_EMPTY] = Rational(-10) 527 | 528 | self.U[Index.STACK1_ZERO, Index.STACK1_EMPTY] = Rational(-3) 529 | self.U[Index.STACK2_ZERO, Index.STACK2_EMPTY] = Rational(-3) 530 | 531 | def initialize_transition_function(self): 532 | # transition function 533 | for i in range(Index.STACK1_PUSH_0, Index.STACK1_NOOP + 1): 534 | for j in range(Index.CONF_BOT_BOT_EOS, Index.CONF_1_1_b + 1): 535 | # As soon as the configuration has been determined, reset 536 | # the information about what action should be taken, 537 | # so that only the correct action is considered (by overwriting 538 | # the -10's with 0's below). 539 | self.U[i, j] = Rational(-10) 540 | for i in range(Index.STACK2_PUSH_0, Index.STACK2_NOOP + 1): 541 | for j in range(Index.CONF_BOT_BOT_EOS, Index.CONF_1_1_b + 1): 542 | self.U[i, j] = Rational(-10) 543 | 544 | def make_action_detector(self): # noqa: C901 545 | self.initialize_transition_function() 546 | 547 | # This is the transition matrix 548 | for sym in self.Σ: 549 | if sym == EOS: 550 | # If we read the EOS symbol, we don't do anything. 551 | # We set those manually, since EOS-transitions are not in the PDA. 552 | for action in [Index.STACK1_NOOP, Index.STACK2_NOOP]: 553 | for γ_top_1, γ_top_2 in product(self.pda.Γ_1, self.pda.Γ_2): 554 | self.U[action, conf2idx(γ_top_1, γ_top_2, EOS)] = Rational(0) 555 | continue 556 | 557 | for γ_top_1, γ_top_2 in self.pda.δ[sym]: 558 | # This implements the *unweighted* transition function, i.e., the 559 | # controller of the LM 560 | (action_1, γ_new_1), (action_2, γ_new_2), _ = self.pda.δ[sym][ 561 | (γ_top_1, γ_top_2) 562 | ] 563 | 564 | conf = conf2idx(γ_top_1, γ_top_2, sym) 565 | 566 | if action_1 == Action.PUSH and γ_new_1 == Sym("0"): 567 | self.U[Index.STACK1_PUSH_0, conf] = Rational(0) 568 | elif action_1 == Action.PUSH and γ_new_1 == Sym("1"): 569 | self.U[Index.STACK1_PUSH_1, conf] = Rational(0) 570 | elif action_1 == Action.POP and γ_new_1 == Sym("0"): 571 | self.U[Index.STACK1_POP_0, conf] = Rational(0) 572 | elif action_1 == Action.POP and γ_new_1 == Sym("1"): 573 | self.U[Index.STACK1_POP_1, conf] = Rational(0) 574 | elif action_1 == Action.NOOP: 575 | self.U[Index.STACK1_NOOP, conf] = Rational(0) 576 | else: 577 | raise ValueError( 578 | "Unknown action: action: {}, γ_new: {}.".format( 579 | action_1, γ_new_1 580 | ) 581 | ) 582 | 583 | if action_2 == Action.PUSH and γ_new_2 == Sym("0"): 584 | self.U[Index.STACK2_PUSH_0, conf] = Rational(0) 585 | elif action_2 == Action.PUSH and γ_new_2 == Sym("1"): 586 | self.U[Index.STACK2_PUSH_1, conf] = Rational(0) 587 | elif action_2 == Action.POP and γ_new_2 == Sym("0"): 588 | self.U[Index.STACK2_POP_0, conf] = Rational(0) 589 | elif action_2 == Action.POP and γ_new_2 == Sym("1"): 590 | self.U[Index.STACK2_POP_1, conf] = Rational(0) 591 | elif action_2 == Action.NOOP: 592 | self.U[Index.STACK2_NOOP, conf] = Rational(0) 593 | else: 594 | raise ValueError( 595 | "Unknown action: action: {}, γ_new: {}.".format( 596 | action_2, γ_new_2 597 | ) 598 | ) 599 | 600 | def make_action_executor(self): 601 | # PHASE 3: we execute all possible actions 602 | # These two take whatever is on the stack so far (primarily it would be on 603 | # stack 1, but by phase 4, it has been moved to stack 3) and move it 604 | # down by dividing by 10. 605 | self.U[Index.STACK1_PUSH_0, Index.BUFFER21] = Rational("1/10") 606 | self.U[Index.STACK1_PUSH_1, Index.BUFFER21] = Rational("1/10") 607 | 608 | self.U[Index.STACK2_PUSH_0, Index.BUFFER22] = Rational("1/10") 609 | self.U[Index.STACK2_PUSH_1, Index.BUFFER22] = Rational("1/10") 610 | 611 | # These two take whatever is on the stack and move it up by multiplying by 10. 612 | self.U[Index.STACK1_POP_0, Index.BUFFER21] = Rational(10) 613 | self.U[Index.STACK1_POP_1, Index.BUFFER21] = Rational(10) 614 | 615 | self.U[Index.STACK2_POP_0, Index.BUFFER22] = Rational(10) 616 | self.U[Index.STACK2_POP_1, Index.BUFFER22] = Rational(10) 617 | 618 | # Copy the last buffer into the NOOP cells, but only if none of the other 619 | # actions are active (handled in `make_action_detector`). 620 | self.U[Index.STACK1_NOOP, Index.BUFFER21] = Rational(1) 621 | self.U[Index.STACK2_NOOP, Index.BUFFER22] = Rational(1) 622 | 623 | def make_action_committer(self): 624 | # PHASE 4: The final phase---commit the action from the individual 625 | # action cells to the stack. 626 | # Only one of the four entries PUSH_0, PUSH_1, POP_0, POP_1 627 | # will be "active" (non-zero), the rest will be 0. 628 | 629 | # This submatrix encodes the copying of the actions from the ``actions'' part 630 | # of the hidden state to the part of the hidden state corresponding to stack 1. 631 | 632 | self.U[Index.STACK1, Index.STACK1_PUSH_0] = Rational(1) 633 | self.U[Index.STACK1, Index.STACK1_PUSH_1] = Rational(1) 634 | self.U[Index.STACK2, Index.STACK2_PUSH_0] = Rational(1) 635 | self.U[Index.STACK2, Index.STACK2_PUSH_1] = Rational(1) 636 | 637 | self.U[Index.STACK1, Index.STACK1_POP_0] = Rational(1) 638 | self.U[Index.STACK1, Index.STACK1_POP_1] = Rational(1) 639 | self.U[Index.STACK2, Index.STACK2_POP_0] = Rational(1) 640 | self.U[Index.STACK2, Index.STACK2_POP_1] = Rational(1) 641 | 642 | self.U[Index.STACK1, Index.STACK1_NOOP] = Rational(1) 643 | self.U[Index.STACK2, Index.STACK2_NOOP] = Rational(1) 644 | 645 | def make_acceptor(self): 646 | # PHASE 5: acceptor 647 | # This is the final state of the RNN. 648 | # It accepts if the stack is empty (encoded in the computation component) 649 | # and the input is empty, i.e., the previous input was EOS. 650 | 651 | # Checks the emptiness of the computation component 652 | self.U[Index.ACCEPT, Index.STACK1_PUSH_0] = Rational(-10) 653 | self.U[Index.ACCEPT, Index.STACK1_PUSH_1] = Rational(-10) 654 | self.U[Index.ACCEPT, Index.STACK1_POP_0] = Rational(-10) 655 | self.U[Index.ACCEPT, Index.STACK1_POP_1] = Rational(-10) 656 | self.U[Index.ACCEPT, Index.STACK1_NOOP] = Rational(-10) 657 | self.U[Index.ACCEPT, Index.STACK2_PUSH_0] = Rational(-10) 658 | self.U[Index.ACCEPT, Index.STACK2_PUSH_1] = Rational(-10) 659 | self.U[Index.ACCEPT, Index.STACK2_POP_0] = Rational(-10) 660 | self.U[Index.ACCEPT, Index.STACK2_POP_1] = Rational(-10) 661 | self.U[Index.ACCEPT, Index.STACK2_NOOP] = Rational(-10) 662 | 663 | # Checks that the input was EOS 664 | self.U[Index.ACCEPT, Index.CONF_BOT_BOT_a] = Rational(-10) 665 | self.U[Index.ACCEPT, Index.CONF_BOT_0_a] = Rational(-10) 666 | self.U[Index.ACCEPT, Index.CONF_BOT_1_a] = Rational(-10) 667 | self.U[Index.ACCEPT, Index.CONF_BOT_BOT_b] = Rational(-10) 668 | self.U[Index.ACCEPT, Index.CONF_BOT_0_b] = Rational(-10) 669 | self.U[Index.ACCEPT, Index.CONF_BOT_1_b] = Rational(-10) 670 | self.U[Index.ACCEPT, Index.CONF_0_BOT_a] = Rational(-10) 671 | self.U[Index.ACCEPT, Index.CONF_0_0_a] = Rational(-10) 672 | self.U[Index.ACCEPT, Index.CONF_0_1_a] = Rational(-10) 673 | self.U[Index.ACCEPT, Index.CONF_0_BOT_b] = Rational(-10) 674 | self.U[Index.ACCEPT, Index.CONF_0_0_b] = Rational(-10) 675 | self.U[Index.ACCEPT, Index.CONF_0_1_b] = Rational(-10) 676 | self.U[Index.ACCEPT, Index.CONF_1_BOT_a] = Rational(-10) 677 | self.U[Index.ACCEPT, Index.CONF_1_0_a] = Rational(-10) 678 | self.U[Index.ACCEPT, Index.CONF_1_1_a] = Rational(-10) 679 | self.U[Index.ACCEPT, Index.CONF_1_BOT_b] = Rational(-10) 680 | self.U[Index.ACCEPT, Index.CONF_1_0_b] = Rational(-10) 681 | self.U[Index.ACCEPT, Index.CONF_1_1_b] = Rational(-10) 682 | 683 | def make_U(self): 684 | """Constructs the recurrence matrix U of the Siegelmann RNN.""" 685 | 686 | # ORCHESTRATION 687 | self.make_buffer_transitions() 688 | self.make_phase_transitions() 689 | 690 | # PHASE 1 691 | self.make_stack_detector() 692 | 693 | # PHASE 2 694 | self.make_configuration_detector() 695 | 696 | # PHASE 3 697 | self.make_action_detector() 698 | self.make_action_executor() 699 | 700 | # PHASE 4 701 | self.make_action_committer() 702 | self.make_acceptor() 703 | 704 | def make_V(self): 705 | """Constructs the emission matrix V of the Siegelmann RNN.""" 706 | # Enables setting the hidden state to 1 at the indices 707 | # corresponding to all possible combinations of the stack configurations 708 | # (top of the stack) and the incomping input symbol. 709 | # For input symbol a 710 | self.V[Index.CONF_BOT_BOT_a, sym2idx[Sym("a")]] = Rational(1) 711 | self.V[Index.CONF_BOT_0_a, sym2idx[Sym("a")]] = Rational(1) 712 | self.V[Index.CONF_BOT_1_a, sym2idx[Sym("a")]] = Rational(1) 713 | self.V[Index.CONF_0_BOT_a, sym2idx[Sym("a")]] = Rational(1) 714 | self.V[Index.CONF_0_0_a, sym2idx[Sym("a")]] = Rational(1) 715 | self.V[Index.CONF_0_1_a, sym2idx[Sym("a")]] = Rational(1) 716 | self.V[Index.CONF_1_BOT_a, sym2idx[Sym("a")]] = Rational(1) 717 | self.V[Index.CONF_1_0_a, sym2idx[Sym("a")]] = Rational(1) 718 | self.V[Index.CONF_1_1_a, sym2idx[Sym("a")]] = Rational(1) 719 | # For input symbol b 720 | self.V[Index.CONF_BOT_BOT_b, sym2idx[Sym("b")]] = Rational(1) 721 | self.V[Index.CONF_BOT_0_b, sym2idx[Sym("b")]] = Rational(1) 722 | self.V[Index.CONF_BOT_1_b, sym2idx[Sym("b")]] = Rational(1) 723 | self.V[Index.CONF_0_BOT_b, sym2idx[Sym("b")]] = Rational(1) 724 | self.V[Index.CONF_0_0_b, sym2idx[Sym("b")]] = Rational(1) 725 | self.V[Index.CONF_0_1_b, sym2idx[Sym("b")]] = Rational(1) 726 | self.V[Index.CONF_1_BOT_b, sym2idx[Sym("b")]] = Rational(1) 727 | self.V[Index.CONF_1_0_b, sym2idx[Sym("b")]] = Rational(1) 728 | self.V[Index.CONF_1_1_b, sym2idx[Sym("b")]] = Rational(1) 729 | # For input symbol EOS 730 | self.V[Index.CONF_BOT_BOT_EOS, sym2idx[EOS]] = Rational(1) 731 | self.V[Index.CONF_BOT_0_EOS, sym2idx[EOS]] = Rational(1) 732 | self.V[Index.CONF_BOT_1_EOS, sym2idx[EOS]] = Rational(1) 733 | self.V[Index.CONF_0_BOT_EOS, sym2idx[EOS]] = Rational(1) 734 | self.V[Index.CONF_0_0_EOS, sym2idx[EOS]] = Rational(1) 735 | self.V[Index.CONF_0_1_EOS, sym2idx[EOS]] = Rational(1) 736 | self.V[Index.CONF_1_BOT_EOS, sym2idx[EOS]] = Rational(1) 737 | self.V[Index.CONF_1_0_EOS, sym2idx[EOS]] = Rational(1) 738 | self.V[Index.CONF_1_1_EOS, sym2idx[EOS]] = Rational(1) 739 | 740 | def make_b(self): 741 | """Constructs the bias vector b of the Siegelmann RNN.""" 742 | 743 | self.b[Index.STACK1_EMPTY, 0] = Rational(1) 744 | self.b[Index.STACK2_EMPTY, 0] = Rational(1) 745 | self.b[Index.STACK1_ZERO, 0] = Rational(3) 746 | self.b[Index.STACK2_ZERO, 0] = Rational(3) 747 | self.b[Index.STACK1_ONE, 0] = Rational(-2) 748 | self.b[Index.STACK2_ONE, 0] = Rational(-2) 749 | 750 | # This provides the base of the quantities that get added to 751 | # the previous encoding of the stack after it is divided by 10. 752 | # Add the encoding of the new top of the stack 753 | self.b[Index.STACK1_PUSH_0, 0] = Rational("1/10") 754 | self.b[Index.STACK1_PUSH_1, 0] = Rational("3/10") 755 | 756 | self.b[Index.STACK2_PUSH_0, 0] = Rational("1/10") 757 | self.b[Index.STACK2_PUSH_1, 0] = Rational("3/10") 758 | 759 | # Remove the top encoding of the stack, after it has been multiplied by 10. 760 | self.b[Index.STACK1_POP_0, 0] = Rational(-1) 761 | self.b[Index.STACK1_POP_1, 0] = Rational(-3) 762 | 763 | self.b[Index.STACK2_POP_0, 0] = Rational(-1) 764 | self.b[Index.STACK2_POP_1, 0] = Rational(-3) 765 | 766 | self.b[Index.STACK1_NOOP, 0] = Rational(0) 767 | self.b[Index.STACK2_NOOP, 0] = Rational(0) 768 | 769 | # This zeroes out the configurations which are not active 770 | # even though some of the "triggering" cells in the hidden state are active. 771 | self.b[Index.CONF_BOT_BOT_a, 0] = Rational(-2) 772 | self.b[Index.CONF_BOT_0_a, 0] = Rational(-2) 773 | self.b[Index.CONF_BOT_1_a, 0] = Rational(-2) 774 | self.b[Index.CONF_BOT_BOT_b, 0] = Rational(-2) 775 | self.b[Index.CONF_BOT_0_b, 0] = Rational(-2) 776 | self.b[Index.CONF_BOT_1_b, 0] = Rational(-2) 777 | self.b[Index.CONF_BOT_BOT_EOS, 0] = Rational(-2) 778 | self.b[Index.CONF_BOT_0_EOS, 0] = Rational(-2) 779 | self.b[Index.CONF_BOT_1_EOS, 0] = Rational(-2) 780 | self.b[Index.CONF_0_BOT_a, 0] = Rational(-2) 781 | self.b[Index.CONF_0_0_a, 0] = Rational(-2) 782 | self.b[Index.CONF_0_1_a, 0] = Rational(-2) 783 | self.b[Index.CONF_0_BOT_b, 0] = Rational(-2) 784 | self.b[Index.CONF_0_0_b, 0] = Rational(-2) 785 | self.b[Index.CONF_0_1_b, 0] = Rational(-2) 786 | self.b[Index.CONF_0_BOT_EOS, 0] = Rational(-2) 787 | self.b[Index.CONF_0_0_EOS, 0] = Rational(-2) 788 | self.b[Index.CONF_0_1_EOS, 0] = Rational(-2) 789 | self.b[Index.CONF_1_BOT_a, 0] = Rational(-2) 790 | self.b[Index.CONF_1_0_a, 0] = Rational(-2) 791 | self.b[Index.CONF_1_1_a, 0] = Rational(-2) 792 | self.b[Index.CONF_1_BOT_b, 0] = Rational(-2) 793 | self.b[Index.CONF_1_0_b, 0] = Rational(-2) 794 | self.b[Index.CONF_1_1_b, 0] = Rational(-2) 795 | self.b[Index.CONF_1_BOT_EOS, 0] = Rational(-2) 796 | self.b[Index.CONF_1_0_EOS, 0] = Rational(-2) 797 | self.b[Index.CONF_1_1_EOS, 0] = Rational(-2) 798 | 799 | self.b[Index.ACCEPT, 0] = Rational(1) 800 | 801 | def make_emission_params(self): 802 | for ii in range(9): 803 | self.bw[ii] = Rational(-1) 804 | 805 | # Combines the two stack configurations into a one-hot encoding of a common 806 | # configuration. 807 | self.W[EmissionIndex.CONF_BOTBOT, Index.STACK1_EMPTY] = Rational(1) 808 | self.W[EmissionIndex.CONF_BOT0, Index.STACK1_EMPTY] = Rational(1) 809 | self.W[EmissionIndex.CONF_BOT1, Index.STACK1_EMPTY] = Rational(1) 810 | self.W[EmissionIndex.CONF_0BOT, Index.STACK1_ZERO] = Rational(1) 811 | self.W[EmissionIndex.CONF_00, Index.STACK1_ZERO] = Rational(1) 812 | self.W[EmissionIndex.CONF_01, Index.STACK1_ZERO] = Rational(1) 813 | self.W[EmissionIndex.CONF_1BOT, Index.STACK1_ONE] = Rational(1) 814 | self.W[EmissionIndex.CONF_10, Index.STACK1_ONE] = Rational(1) 815 | self.W[EmissionIndex.CONF_11, Index.STACK1_ONE] = Rational(1) 816 | self.W[EmissionIndex.CONF_BOTBOT, Index.STACK2_EMPTY] = Rational(1) 817 | self.W[EmissionIndex.CONF_0BOT, Index.STACK2_EMPTY] = Rational(1) 818 | self.W[EmissionIndex.CONF_1BOT, Index.STACK2_EMPTY] = Rational(1) 819 | self.W[EmissionIndex.CONF_BOT0, Index.STACK2_ZERO] = Rational(1) 820 | self.W[EmissionIndex.CONF_00, Index.STACK2_ZERO] = Rational(1) 821 | self.W[EmissionIndex.CONF_10, Index.STACK2_ZERO] = Rational(1) 822 | self.W[EmissionIndex.CONF_BOT1, Index.STACK2_ONE] = Rational(1) 823 | self.W[EmissionIndex.CONF_01, Index.STACK2_ONE] = Rational(1) 824 | self.W[EmissionIndex.CONF_11, Index.STACK2_ONE] = Rational(1) 825 | 826 | # Indexes the relevant (log) probabilities in output matrix based on the 827 | # one-hot representation of the stack configurations. 828 | for a in self.Σ - {EOS}: 829 | d = self.sym2idx[a] 830 | self.E[d, EmissionIndex.CONF_BOTBOT] = log(self.pda.δ[a][(BOT, BOT)][2]) 831 | self.E[d, EmissionIndex.CONF_BOT0] = log(self.pda.δ[a][(BOT, Sym("0"))][2]) 832 | self.E[d, EmissionIndex.CONF_BOT1] = log(self.pda.δ[a][(BOT, Sym("1"))][2]) 833 | self.E[d, EmissionIndex.CONF_0BOT] = log(self.pda.δ[a][(Sym("0"), BOT)][2]) 834 | self.E[d, EmissionIndex.CONF_00] = log( 835 | self.pda.δ[a][(Sym("0"), Sym("0"))][2] 836 | ) 837 | self.E[d, EmissionIndex.CONF_01] = log( 838 | self.pda.δ[a][(Sym("0"), Sym("1"))][2] 839 | ) 840 | self.E[d, EmissionIndex.CONF_1BOT] = log(self.pda.δ[a][(Sym("1"), BOT)][2]) 841 | self.E[d, EmissionIndex.CONF_10] = log( 842 | self.pda.δ[a][(Sym("1"), Sym("0"))][2] 843 | ) 844 | self.E[d, EmissionIndex.CONF_11] = log( 845 | self.pda.δ[a][(Sym("1"), Sym("1"))][2] 846 | ) 847 | 848 | def initial_hidden_state(self) -> Matrix: 849 | """Sets the initial hidden state of the RNN to the zero vector 850 | indicating the first phase.""" 851 | # We start with an zero hidden state in phase 1 852 | h = zeros(self.D, 1, dtype=Rational(0)) 853 | h[Index.PHASE1] = Rational(1) 854 | return h 855 | 856 | def accept(self, y: Union[str, String]) -> Tuple[bool, float]: 857 | """Computes the acceptance probability of a string and whether it is accepted. 858 | 859 | Args: 860 | y (Union[str, String]): The string to be accepted. 861 | 862 | Returns: 863 | Tuple[bool, float]: Whether the string is accepted 864 | and the acceptance probability. 865 | """ 866 | if isinstance(y, str): 867 | y = String(y) 868 | y.y += [EOS] 869 | 870 | # Process the string 871 | h, logp = self.initial_hidden_state(), Rational(0) 872 | for a in y: 873 | h, _acc, _logp = self.step(h, a) 874 | logp += _logp 875 | 876 | return h[Index.ACCEPT] == Rational(1), logp 877 | 878 | def __call__(self, y: Union[str, String]) -> Tuple[bool, float]: 879 | """Computes the acceptance probability of a string and whether it is accepted 880 | by simply calling the `accept` method. 881 | 882 | Args: 883 | y (Union[str, String]): The string to be accepted. 884 | 885 | Returns: 886 | Tuple[bool, float]: Whether the string is accepted 887 | and the acceptance probability. 888 | """ 889 | return self.accept(y) 890 | 891 | def step(self, h: Matrix, a: Sym) -> Tuple[Matrix, bool, float]: 892 | """Performs a single whole step of the Siegelmann RNN composed of the four 893 | sub-steps/phases. 894 | 895 | Args: 896 | h (Matrix): The current hidden state of the RNN. 897 | a (Sym): The current input symbol. 898 | 899 | Returns: 900 | Tuple[Matrix, bool, float]: The new hidden state, whether the string is 901 | accepted and the acceptance probability. 902 | """ 903 | assert a in self.Σ 904 | 905 | # We have four stages in which the hidden state is updated 906 | # 1) Peek 907 | # 2) Combine 908 | # 3) Transition 909 | # 4) Consolidate 910 | 911 | def apply(_h: Matrix) -> Matrix: 912 | """Performs the update (sub-)step of the Siegelmann RNN on the current 913 | hidden state _h.""" 914 | z = self.U * _h + self.V * self.one_hot.col(self.sym2idx[a]) + self.b 915 | 916 | hʼ = zeros(self.D, 1, dtype=Rational(0)) 917 | for i in range(self.D): 918 | hʼ[i] = σ.subs(x, z[i]) 919 | return hʼ 920 | 921 | def apply_emission(_h: Matrix) -> Matrix: 922 | """Applies the MLP projecting the state to a useful one-hot encoding 923 | for the emission probabilities.""" 924 | u = self.W * h + self.bw 925 | # print(u) 926 | 927 | uʼ = zeros(9, 1, dtype=Rational(0)) 928 | for i in range(9): 929 | uʼ[i] = σ.subs(x, u[i]) 930 | 931 | # print(uʼ) 932 | return self.E * uʼ 933 | 934 | # print("\n\n\n\n >>>> START:") 935 | # self.disp(h) 936 | # print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 937 | 938 | # print("\n\n----------------------------\n>>>>> PHASE 1") 939 | h = apply(h) 940 | # print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 941 | # self.disp(h) 942 | # print("\n\n----------------------------\n>>>>> PHASE 2") 943 | h = apply(h) 944 | # print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 945 | # self.disp(h) 946 | logits = apply_emission(h) 947 | # print(logits) 948 | # print("\n\n----------------------------\n>>>>> PHASE 3") 949 | h = apply(h) 950 | # print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 951 | # self.disp(h) 952 | # print("\n\n----------------------------\n>>>>> PHASE 4") 953 | h = apply(h) 954 | # print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 955 | # self.disp(h) 956 | # print(cantor_decode(h[Index.STACK1]), cantor_decode(h[Index.STACK2])) 957 | # self.disp(h) 958 | # print("\n\n\n\n\n") 959 | 960 | return h, h[Index.ACCEPT], logits[self.sym2idx[a]] 961 | --------------------------------------------------------------------------------