├── requirements.txt
├── readme.md
├── deepspeed.yaml
├── config.yaml
├── sentiments.py
├── scripts
    ├── graph_plot.py
    └── graph_plot.svg
├── captions.py
├── utils.py
├── randomwalks.py
├── carps.py
├── ilql.py
└── models.py


/requirements.txt:
--------------------------------------------------------------------------------
 1 | torch==1.12.1+cu113
 2 | transformers==4.21.1
 3 | accelerate==0.12.0
 4 | deepspeed==0.7.0
 5 | datasets==2.4.0
 6 | tokenizers==0.12.1
 7 | tqdm==4.64.0
 8 | wandb==0.13.2
 9 | networkx==2.8.6
10 | 


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | Simplified implementation of [Implicit Language Q Learning (Snell et al. 2022)](https://sea-snell.github.io/ILQL_site/) ([official](https://github.com/Sea-Snell/Implicit-Language-Q-Learning/), [paper](https://arxiv.org/abs/2206.11871))
2 | 
3 | Evaluating on Graph Shortest Path task from [Decision Transformer (Lili Chen et al. 2021)](https://arxiv.org/abs/2106.01345):
4 | 
5 | <img src="scripts/graph_plot.svg" width="100%">
6 | 
7 | where for each random graph, a transformer is trained to find optimal trajectories using only 1000 random walks.
8 | 


--------------------------------------------------------------------------------
/deepspeed.yaml:
--------------------------------------------------------------------------------
 1 | compute_environment: LOCAL_MACHINE
 2 | deepspeed_config:
 3 |   deepspeed_config_file: zero3.json
 4 |   deepspeed_multinode_launcher: standard
 5 |   gradient_accumulation_steps: 1
 6 |   offload_optimizer_device: none
 7 |   offload_param_device: none
 8 |   gradient_clipping: 1.0
 9 |   zero3_init_flag: false
10 |   zero_stage: 3
11 | distributed_type: DEEPSPEED
12 | downcast_bf16: no
13 | fsdp_config: {}
14 | machine_rank: ???
15 | main_process_ip: ???
16 | main_process_port: 1234
17 | main_training_function: main
18 | mixed_precision: bf16
19 | num_machines: ???
20 | num_processes: ???
21 | rdzv_backend: static
22 | same_network: true
23 | use_cpu: false
24 | 


--------------------------------------------------------------------------------
/config.yaml:
--------------------------------------------------------------------------------
 1 | RandomWalks:
 2 |   lr: 0.001
 3 |   opt_betas: [0.9, 0.95]
 4 |   batch_size: 500
 5 |   tau: 0.7
 6 |   gamma: 0.99
 7 |   cql_scale: 0.1
 8 |   awac_scale: 1
 9 |   alpha: 0.1
10 |   steps_for_target_q_sync: 10
11 |   steps_for_eval: 5
12 |   inference_betas: [1]
13 |   n_epochs: 100
14 |   seed: 1000
15 |   n_layers_unfrozen: 0
16 |   two_qs: true
17 |   gptconfig:
18 |     n_embd: 144
19 |     n_layer: 6
20 |     n_head: 1
21 | 
22 | Sentiments:
23 |   lr: 0.0006
24 |   opt_betas: [0.9, 0.95]
25 |   batch_size: 6
26 |   tau: 0.5
27 |   gamma: 0.99
28 |   cql_scale: 0.1
29 |   awac_scale: 1
30 |   alpha: 1
31 |   steps_for_target_q_sync: 40
32 |   steps_for_eval: 100
33 |   inference_betas: [1]
34 |   model: EleutherAI/gpt-j-6B
35 |   n_layers_unfrozen: 0
36 |   two_qs: true
37 |   n_epochs: 1
38 | 
39 | Carps:
40 |   lr: 0.00004
41 |   opt_betas: [0.9, 0.95]
42 |   batch_size: 72
43 |   tau: 0.5
44 |   cql_scale: 0.1
45 |   alpha: 1
46 |   steps_for_target_q_sync: 100
47 |   inference_betas: [0, 1, 2, 4]
48 |   n_epochs: 10
49 |   model: gpt2-large
50 |   n_layers_unfrozen: 2
51 |   max_length: 48
52 |   diff_reward: true
53 | 
54 | Captions:
55 |   lr: 0.0006
56 |   opt_betas: [0.9, 0.95]
57 |   batch_size: 16
58 |   tau: 0.7
59 |   gamma: 0.99
60 |   cql_scale: 0.1
61 |   awac_scale: 1
62 |   alpha: 1
63 |   steps_for_target_q_sync: 50
64 |   steps_for_eval: 100
65 |   inference_betas: [0, 1]
66 |   model: gpt2-large
67 |   n_layers_unfrozen: 0
68 |   two_qs: true
69 |   n_epochs: 1
70 | 


--------------------------------------------------------------------------------
/sentiments.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import numpy as np
 3 | import torch as th
 4 | from torch import tensor
 5 | import torch.nn.functional as F
 6 | from torch.utils.data import TensorDataset
 7 | from functools import partial, reduce
 8 | 
 9 | from transformers import AutoTokenizer, pipeline
10 | from datasets import load_dataset
11 | import wandb
12 | from tqdm import tqdm
13 | import math
14 | 
15 | from utils import batch_map, tohuman, load_tensors
16 | 
17 | def get_reward(sentiment_pipe, texts):
18 |     sentiments = batch_map(lambda batch: sentiment_pipe(batch), texts, bsize=1024, desc='Sentiments')
19 |     return tensor([-s['score'] if s['label'] == 'NEGATIVE' else s['score'] for s in sentiments])
20 | 
21 | class Sentiments:
22 |     def __init__(self, tokenizer: AutoTokenizer, max_length=50, n_samples=64, needs_reward_model=False):
23 |         self.max_length = max_length
24 |         self.tokenizer = tokenizer
25 | 
26 |         if needs_reward_model:
27 |             self.sentiment_pipe = pipeline('sentiment-analysis', 'lvwerra/distilbert-imdb', device=th.device(0))
28 |         else:
29 |             self.sentiment_pipe = None
30 | 
31 |         texts = load_dataset('imdb', split='train+test')['text']
32 |         tensors = load_tensors(
33 |             'sentiments',
34 |             texts=texts,
35 |             reward_model=partial(get_reward, self.sentiment_pipe),
36 |             tokenizer=self.tokenizer,
37 |             max_length=max_length,
38 |             use_cache=True
39 |         )
40 | 
41 |         query = tensor([self.tokenizer.bos_token_id] * n_samples).view(n_samples, 1)
42 |         self.logit_mask = None
43 | 
44 |         self.dataset = TensorDataset(tensors['input_ids'], tensors['attention_mask'], tensors['rewards'])
45 |         self.eval_dataset = TensorDataset(query)
46 | 
47 |     def eval(self, samples, beta):
48 |         reviews = self.tokenizer.batch_decode(samples, skip_special_tokens=True)
49 | 
50 |         rewards = [1-s['score'] if s['label'] == 'NEGATIVE' else s['score'] for s in self.sentiment_pipe(reviews)]
51 |         reward = np.mean(rewards)
52 | 
53 |         rows = list(zip(reviews, rewards))
54 |         print(f'\n{beta=} {reward=:.2f}\n' + '\n'.join([f'[{sent:.2f}] {text}' for text, sent in rows[:8]]))
55 | 
56 |         stats = { f'reward/{beta}': reward,
57 |                   f'responses/{beta}': wandb.Table(columns=['response', 'sentiment'], rows=rows[:32]) }
58 | 
59 |         return reward, stats
60 | 
61 | if __name__ == '__main__':
62 |     import sys
63 |     from rich import print
64 |     tokenizer = AutoTokenizer.from_pretrained(sys.argv[1])
65 |     tokenizer.pad_token = tokenizer.eos_token
66 |     ds = Sentiments(tokenizer, needs_reward_model=True).dataset
67 |     print(f'{next(iter(ds))=}')
68 | 


--------------------------------------------------------------------------------
/scripts/graph_plot.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import sys
 3 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 4 | from ilql import main
 5 | from torch import tensor
 6 | import torch as th
 7 | import networkx as nx
 8 | import numpy as np
 9 | 
10 | optimal_lengths = []
11 | sampled_lengths = []
12 | iql_lengths = []
13 | 
14 | for seed in range(10):
15 |     model, data = main(seed=seed, debug=True)
16 |     model.eval()
17 | 
18 |     g = nx.from_numpy_array(data.adj, create_using=nx.DiGraph)
19 | 
20 |     # optimal
21 |     for start in set(range(data.n_nodes)) - {data.goal}:
22 |         try:
23 |             shortest_path = nx.shortest_path(g, start, data.goal)[:data.walk_size]
24 |             optimal_lengths.append(len(shortest_path)-1)
25 |         except:
26 |             optimal_lengths.append(data.walk_size)
27 | 
28 |     # ilql
29 |     starts = th.arange(1, data.n_nodes).unsqueeze(1).to(model.device)
30 |     paths, _ = model.sample(starts, max_length=data.walk_size, logit_mask=tensor(~data.adj), beta=10) # argmax
31 |     for path in paths:
32 |         length = data.walk_size
33 |         for ind, node in enumerate(path):
34 |             if node == data.goal:
35 |                 length = ind
36 |                 break
37 | 
38 |         iql_lengths.append(length)
39 | 
40 |     # all samples
41 |     for path in data.tensors[0]:
42 |         length = data.walk_size
43 |         for ind, node in enumerate(path):
44 |             if node == data.goal:
45 |                 length = ind
46 |                 break
47 | 
48 |         sampled_lengths.append(length)
49 | # ■ ~
50 | 
51 | from matplotlib import pyplot
52 | import matplotlib
53 | 
54 | fontcolor = '#444'
55 | matplotlib.rcParams['text.color'] = fontcolor
56 | matplotlib.rcParams['axes.labelcolor'] = fontcolor
57 | matplotlib.rcParams['xtick.color'] = fontcolor
58 | matplotlib.rcParams['ytick.color'] = fontcolor
59 | matplotlib.rcParams['xtick.labelcolor'] = fontcolor
60 | matplotlib.rcParams['ytick.labelcolor'] = fontcolor
61 | matplotlib.rcParams['xtick.labelcolor'] = fontcolor
62 | 
63 | matplotlib.rcParams["font.family"] = "Futura"
64 | matplotlib.rcParams["font.size"] = 15
65 | matplotlib.rcParams["xtick.labelsize"] = 20
66 | matplotlib.rcParams["ytick.labelsize"] = 20
67 | matplotlib.rcParams["figure.titlesize"] = 12
68 | matplotlib.rcParams["figure.figsize"] = 15, 8
69 | 
70 | matplotlib.style.use('ggplot')
71 | matplotlib.rcParams['figure.dpi'] = 70
72 | 
73 | ax = pyplot.gca()
74 | ax.set_facecolor('#fff')
75 | ax.grid(color='lightgray', alpha=0.4, axis='y')
76 | ax.tick_params(top=False, labeltop=False, bottom=False, labelbottom=True, left=False, labelleft=True)
77 | 
78 | optimal_hist = np.histogram(optimal_lengths, bins=np.arange(1, data.walk_size+2), density=True)[0]
79 | sampled_hist = np.histogram(sampled_lengths, bins=np.arange(1, data.walk_size+2), density=True)[0]
80 | iql_hist = np.histogram(iql_lengths, bins=np.arange(1, data.walk_size+2), density=True)[0]
81 | 
82 | barsize = 0.36
83 | iql_color = '#99a3fd'
84 | opt_color = '#f2ad48'
85 | random_color='lightgray'
86 | 
87 | pyplot.bar(np.arange(1, data.walk_size+1)-barsize/1.5, optimal_hist, width=barsize, label='shortest path', color=opt_color, zorder=2)
88 | pyplot.bar(np.arange(1, data.walk_size+1), iql_hist, width=barsize, label='ILQL', color=iql_color, zorder=3)
89 | pyplot.bar(np.arange(1, data.walk_size+1)+barsize/1.5, sampled_hist, width=barsize, label='random walk', color=random_color, zorder=1)
90 | 
91 | pyplot.legend(fontsize=16)
92 | pyplot.xticks(np.arange(1, data.walk_size+1), list(np.arange(1, data.walk_size)) + ['∞'])
93 | 
94 | pyplot.xlabel('# of steps to goal', fontsize=22, color=fontcolor, labelpad=20)
95 | pyplot.ylabel('proportion of paths', fontsize=22, color=fontcolor, labelpad=20)
96 | 
97 | pyplot.savefig('scripts/graph_plot.svg')
98 | 


--------------------------------------------------------------------------------
/captions.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import numpy as np
 3 | import torch as th
 4 | from torch import tensor
 5 | import torch.nn.functional as F
 6 | from torch.utils.data import TensorDataset
 7 | 
 8 | import wandb
 9 | import sqlite3
10 | 
11 | from utils import batch_map, tohuman
12 | 
13 | def load_tensors(cache_path, tokenizer, use_cache=True):
14 |     cache_path = f'{cache_path}_tokenizer={tokenizer.name_or_path}.pt'
15 | 
16 |     if use_cache and os.path.exists(cache_path):
17 |         out = th.load(cache_path)
18 |     else:
19 |         conn = sqlite3.connect('sac_public_2022_06_29.sqlite')
20 |         c = conn.cursor()
21 |         c.execute("SELECT prompt, rating FROM ratings "
22 |                   "JOIN images ON images.id=ratings.iid "
23 |                   "JOIN generations ON images.gid=generations.id "
24 |                   "WHERE rating IS NOT NULL;")
25 | 
26 |         prompts, ratings = tuple(map(list, zip(*filter(lambda x: len(x[0]) > 10, c.fetchall()))))
27 | 
28 |         out = tokenizer(prompts, padding=True, return_tensors='pt')
29 |         input_ids, attention_mask = out['input_ids'], out['attention_mask']
30 | 
31 |         # append eos
32 |         input_ids = F.pad(input_ids, (0, 1), value=tokenizer.eos_token_id)
33 |         attention_mask = F.pad(attention_mask, (0, 1), value=0)
34 | 
35 |         # figure out sentences' endings
36 |         diff_padding = th.zeros(input_ids.shape[0], 1, dtype=th.long)
37 |         endings = input_ids.eq(tokenizer.eos_token_id).diff(prepend=diff_padding, dim=-1).nonzero(as_tuple=True)
38 | 
39 |         ratings = tensor(ratings, dtype=th.float32).view(-1, 1)
40 |         ratings = (ratings - ratings.mean()) / (ratings.std() + 1e-100)
41 |         rewards = ratings.repeat(1, input_ids.shape[1])
42 | 
43 |         # zero padding
44 |         rewards[input_ids.eq(tokenizer.pad_token_id).nonzero(as_tuple=True)] = 0
45 |         # refill rewards for the actual eos in case pad == eos
46 |         rewards[endings] = ratings.view(-1)
47 | 
48 |         # prepend bos
49 |         input_ids = F.pad(input_ids, (1, 0), value=tokenizer.eos_token_id)
50 |         attention_mask = F.pad(attention_mask, (1, 0), value=1)
51 | 
52 |         out = {'input_ids': input_ids, 'attention_mask': attention_mask, 'rewards': rewards}
53 | 
54 |         if not os.path.exists(os.path.dirname(cache_path)):
55 |             os.mkdir(os.path.dirname(cache_path))
56 | 
57 |         th.save(out, cache_path)
58 | 
59 |     print(f"Total {tohuman(np.prod(out['input_ids'].shape))} tokens")
60 |     return out
61 | 
62 | class AestheticCaptions(TensorDataset):
63 |     def __init__(self, tokenizer, max_length=77, n_samples=32, batch_size=1, use_cache=True):
64 |         self.max_length = max_length
65 |         self.n_samples = n_samples
66 |         self.batch_size = batch_size
67 |         self.tokenizer = tokenizer
68 | 
69 |         tensors = load_tensors('cache/aesthetic-captions', self.tokenizer, use_cache=use_cache)
70 |         super().__init__(tensors['input_ids'], tensors['attention_mask'], tensors['rewards'])
71 | 
72 |     def eval(self, logs, model, betas=[1]):
73 |         query = tensor([self.tokenizer.eos_token_id] * self.n_samples, device=model.device).view(self.n_samples, 1)
74 | 
75 |         for beta in betas:
76 |             responses = batch_map(
77 |                 lambda batch: model.sample(query, beta=beta, max_length=self.max_length,
78 |                                            eos_token_id=self.tokenizer.eos_token_id)[0],
79 |                 query, bsize=self.batch_size, desc='Generating')
80 | 
81 |             responses = self.tokenizer.batch_decode(responses, skip_special_tokens=True)
82 | 
83 |             print(f'\n{beta=}\n' + '\n'.join([f'{text}' for text in responses[:16]]))
84 | 
85 |             logs.update({f'responses/{beta}': wandb.Table(columns=['response'], data=[[r] for r in responses[:32]])})
86 | 
87 |         return np.inf, {}
88 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import numpy as np
  3 | import torch as th
  4 | from tqdm import tqdm
  5 | import math
  6 | from contextlib import contextmanager
  7 | import torch.nn.functional as F
  8 | from time import time
  9 | import deepspeed
 10 | 
 11 | try:
 12 |     __IPYTHON__
 13 |     run_from_ipython = True
 14 | except NameError:
 15 |     run_from_ipython = False
 16 | 
 17 | def randexclude(rng: np.random.RandomState, n: int, exclude: int) -> int:
 18 |     while True:
 19 |         x = rng.randint(n)
 20 |         if x != exclude:
 21 |             return x
 22 | 
 23 | def tohuman(n: int) -> str:
 24 |     if n > 1e9:
 25 |         return f'{n / 1e9:.1f}B'
 26 |     elif n > 1e6:
 27 |         return f'{n / 1e6:.1f}M'
 28 |     elif n > 1e3:
 29 |         return f'{n / 1e3:.1f}K'
 30 |     return str(n)
 31 | 
 32 | def logvars(name, logs, xs):
 33 |     xs = th.vstack(xs)
 34 |     logs.update({ f'{name}-mean': xs.mean(),
 35 |                   f'{name}-std': xs.std(),
 36 |                   f'{name}-min': xs.min(),
 37 |                   f'{name}-max': xs.max() })
 38 | 
 39 | def batch_map(fn, xs, bsize: int, desc=None):
 40 |     out = []
 41 |     for ind in tqdm(range(math.ceil(len(xs) / bsize)), desc=desc, disable=not desc):
 42 |         batch = xs[ind*bsize:min(len(xs), (ind+1)*bsize)]
 43 |         out.extend(fn(batch))
 44 | 
 45 |     return out
 46 | 
 47 | def load_tensors(name, texts, reward_model, tokenizer, max_length=64, use_cache=True):
 48 |     cache_path = f'cache/{name}_{max_length=}_tokenizer={tokenizer.name_or_path.split("/")[-1]}.pt'
 49 |     if use_cache and os.path.exists(cache_path):
 50 |         tensors = th.load(cache_path)
 51 |     else:
 52 |         tensors = tokenizer(
 53 |             [tokenizer.bos_token + x for x in texts],
 54 |             max_length=max_length,
 55 |             truncation=True,
 56 |             padding=True,
 57 |             return_tensors='pt'
 58 |         )
 59 | 
 60 |         trimmed_texts = tokenizer.batch_decode(tensors['input_ids'], skip_special_tokens=True)
 61 |         rewards = th.as_tensor(reward_model(trimmed_texts))
 62 |         rewards = (rewards - rewards.mean()) / (rewards.std() + 1e-30)
 63 |         rewards = rewards.view(-1, 1).repeat(1, tensors['input_ids'].shape[1])
 64 |         rewards[tensors['attention_mask'].eq(0)] = 0
 65 | 
 66 |         tensors['rewards'] = rewards
 67 |         tensors['attention_mask'] = F.pad(tensors['attention_mask'], (0, 1), value=0)
 68 |         tensors['input_ids'] = F.pad(tensors['input_ids'], (0, 1), value=tokenizer.eos_token_id)
 69 | 
 70 |         if not os.path.exists(os.path.dirname(cache_path)):
 71 |             os.mkdir(os.path.dirname(cache_path))
 72 | 
 73 |         th.save(tensors, cache_path)
 74 | 
 75 |     print(f"{tohuman(np.prod(tensors['input_ids'].shape))} tokens")
 76 |     return tensors
 77 | 
 78 | 
 79 | def isdelim(c: str):
 80 |     return c == '?' or c == '!' or c == '.' or c == ';'
 81 | 
 82 | def pprint(s):
 83 |     trig = False
 84 |     si = 0
 85 |     l = len(s)-1
 86 | 
 87 |     for i in range(len(s)):
 88 |         if i == l:
 89 |             print(s[si:].strip())
 90 | 
 91 |         elif trig or isdelim(s[i]):
 92 |             trig = True
 93 | 
 94 |             if s[i].isspace():
 95 |                 print(s[si:i+1].strip())
 96 |                 si = i + 1
 97 |                 trig = False
 98 | 
 99 | @contextmanager
100 | def timeit(desc='something important'):
101 |     print(f'{desc}...')
102 |     stime = time()
103 |     try:
104 |         yield None
105 |     finally:
106 |         print(f'done with {desc.lower()} in {time() - stime:.1f}s')
107 | 
108 | def check_weights(param):
109 |     if os.environ.get('DEEPSPEED_ZERO_STAGE', '0') == '3':
110 |         with deepspeed.zero.GatheredParameters(param[0].weight, modifier_rank=0):
111 |             if deepspeed.comm.get_rank() == 0:
112 |                 return param[0].weight.sum()
113 |     else:
114 |         return param[0].weight.sum()
115 | 


--------------------------------------------------------------------------------
/randomwalks.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import torch as th
  3 | from torch import tensor
  4 | from torch.utils.data import TensorDataset
  5 | import torch.nn.functional as F
  6 | from utils import randexclude
  7 | import networkx as nx
  8 | 
  9 | # Toy dataset from Decision Transformer (Chen et. al 2021)
 10 | class RandomWalks:
 11 |     def __init__(self, n_nodes=20, max_length=10, n_walks=1000, p_edge=0.1, seed=1002):
 12 |         self.n_nodes = n_nodes
 13 |         self.n_walks = n_walks
 14 |         self.max_length = max_length
 15 |         rng = np.random.RandomState(seed)
 16 | 
 17 |         walks, rewards = [], []
 18 |         while True:
 19 |             self.adj = rng.rand(n_nodes, n_nodes) > (1 - p_edge)
 20 |             np.fill_diagonal(self.adj, 0)
 21 |             if np.all(self.adj.sum(1)): break
 22 | 
 23 |         # terminal state
 24 |         self.adj[0, :] = 0
 25 |         self.adj[0, 0] = 1
 26 | 
 27 |         self.goal = 0
 28 |         for _ in range(n_walks):
 29 |             node = randexclude(rng, n_nodes, self.goal)
 30 |             walk = [node]
 31 | 
 32 |             for istep in range(max_length-1):
 33 |                 node = rng.choice(np.nonzero(self.adj[node])[0])
 34 |                 walk.append(node)
 35 |                 if node == self.goal:
 36 |                     break
 37 | 
 38 |             r = th.zeros(max_length-1)
 39 |             r[:len(walk)-1] = -1 if walk[-1] == self.goal else -100
 40 | 
 41 |             rewards.append(r)
 42 |             walks.append(walk)
 43 | 
 44 |         states = []
 45 |         attention_masks = []
 46 | 
 47 |         for r, walk in zip(rewards, map(th.tensor, walks)):
 48 |             attention_mask = th.zeros(max_length, dtype=int)
 49 |             attention_mask[:len(walk)] = 1
 50 | 
 51 |             attention_masks.append(attention_mask)
 52 |             states.append(F.pad(walk, (0, max_length-len(walk))))
 53 | 
 54 |         self.worstlen = self.max_length
 55 |         self.avglen = sum(map(len, walks)) / self.n_walks
 56 |         self.bestlen = 0
 57 |         g = nx.from_numpy_array(self.adj, create_using=nx.DiGraph)
 58 |         for start in set(range(self.n_nodes)) - {self.goal}:
 59 |             try:
 60 |                 shortest_path = nx.shortest_path(g, start, self.goal)[:self.max_length]
 61 |                 self.bestlen += len(shortest_path)
 62 |             except:
 63 |                 self.bestlen += self.max_length
 64 | 
 65 |         self.bestlen /= self.n_nodes - 1
 66 | 
 67 |         print(f'{self.n_walks} walks of which {(np.array([r[0] for r in rewards])==-1).mean()*100:.0f}% arrived at destination')
 68 | 
 69 |         # disallows selecting unaccessible nodes in a graph
 70 |         self.logit_mask = tensor(~self.adj)
 71 | 
 72 |         self.dataset = TensorDataset(th.stack(states), th.stack(attention_masks), th.stack(rewards))
 73 |         self.eval_dataset = TensorDataset(th.arange(1, self.n_nodes).unsqueeze(1))
 74 | 
 75 |     def render(self):
 76 |         from matplotlib import pyplot
 77 | 
 78 |         g = nx.from_numpy_array(self.adj, create_using=nx.DiGraph)
 79 |         pos = nx.spring_layout(g, seed=7357)
 80 | 
 81 |         pyplot.figure(figsize=(10, 8))
 82 |         nx.draw_networkx_edges(g, pos=pos, alpha=0.5, width=1, edge_color='#d3d3d3')
 83 |         nx.draw_networkx_nodes(g, nodelist=set(range(len(self.adj))) - {self.goal}, pos=pos, node_size=300, node_color='orange')
 84 |         nx.draw_networkx_nodes(g, nodelist=[self.goal], pos=pos, node_size=300, node_color='darkblue')
 85 |         pyplot.show()
 86 | 
 87 |     def eval(self, samples, beta):
 88 |         narrived = 0
 89 |         actlen = 0
 90 |         for node in range(self.n_nodes-1):
 91 |             for istep in range(self.max_length):
 92 |                 if samples[node, istep] == self.goal:
 93 |                     narrived += 1
 94 |                     break
 95 | 
 96 |             actlen += (istep + 1) / (self.n_nodes - 1)
 97 | 
 98 |         current = (self.worstlen - actlen)/(self.worstlen - self.bestlen)
 99 |         average = (self.worstlen - self.avglen)/(self.worstlen - self.bestlen)
100 | 
101 |         stats = { 'actlen': actlen,
102 |                   'avglen': self.avglen,
103 |                   'bestlen': self.bestlen,
104 |                   'worstlen': self.worstlen,
105 |                   'arrived': f'{narrived / (self.n_nodes-1) * 100:.0f}%',
106 |                   'optimal': f'{current*100:.0f}% > {average*100:.0f}%' }
107 | 
108 |         return -actlen, stats
109 | 


--------------------------------------------------------------------------------
/carps.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import numpy as np
  3 | import torch as th
  4 | from torch import tensor
  5 | import torch.nn.functional as F
  6 | from torch.utils.data import Dataset
  7 | from functools import partial, reduce
  8 | from transformers import AutoTokenizer
  9 | from datasets import load_dataset
 10 | from util.carp_util import load_carp, scorer
 11 | import wandb
 12 | 
 13 | tokenizer = AutoTokenizer.from_pretrained('gpt2')
 14 | 
 15 | carp = load_carp(
 16 |     model_type='coop',
 17 |     config_path='ControlledCarp/magiCARP/configs/coop/alignment_coop.yml',
 18 |     ckpt_path='New_Alignment_CoOp_Carp_L/'
 19 | ).to('cuda')
 20 | carp.eval()
 21 | 
 22 | def clean_text(text):
 23 |     return '. '.join(map(
 24 |         lambda x: x.strip(),
 25 |         text.replace(' . ', '. ').replace(' , ', ', ').replace(" '", "'").replace(" n't", "n't").split('. ')
 26 |     ))
 27 | 
 28 | def sizesplit(size: int, xs):
 29 |     for ind in range(len(xs) // size + int((len(xs) % size) > 0)):
 30 |         yield xs[ind*size:min(len(xs), (ind+1)*size)]
 31 | 
 32 | def topk_mask(xs, k):
 33 |     mintop = th.topk(xs, k)[0][:, -1].unsqueeze(-1)
 34 |     return th.where(xs < mintop, -np.inf * th.ones_like(xs, dtype=xs.dtype), xs)
 35 | 
 36 | def tokenize(max_length, diff_reward, offset_reward, review, sample):
 37 |     text = clean_text(sample['text'])
 38 |     tokens = tokenizer.encode(text, return_tensors='pt')[:, :max_length-1]
 39 |     tokens = F.pad(tokens, (0, max_length-tokens.shape[1]-1), value=tokenizer.eos_token_id)
 40 | 
 41 |     if diff_reward:
 42 |         substrings = []
 43 |         newtext = ""
 44 |         for token in tokens[0]:
 45 |             newtext += tokenizer.decode(token)
 46 |             substrings.append(newtext)
 47 | 
 48 |         rewards = carp_score(substrings, review).cpu()
 49 |         rewards = th.hstack((tensor([offset_reward]), rewards)).diff()
 50 |         rewards = th.where(tokens[0] == tokenizer.eos_token_id, 0, rewards)
 51 | 
 52 |     else:
 53 |         r = carp_score(text, review).item()
 54 |         rewards = th.empty(max_length-1)
 55 |         rewards.fill_(r)
 56 |         rewards[tokens[0] == tokenizer.eos_token_id] = 0
 57 | 
 58 |     attn = [1] * max_length
 59 |     attn[-1] = 0
 60 |     sample['text'] = text
 61 |     sample['tokens'] = th.hstack((tensor([[tokenizer.eos_token_id]]), tokens))
 62 |     sample['attention'] = attn
 63 |     sample['rewards'] = rewards
 64 |     return sample
 65 | 
 66 | @th.inference_mode()
 67 | def carp_score(texts, review):
 68 |     return scorer(texts, [review], carp, mode='coop').view(-1)
 69 | 
 70 | @th.inference_mode()
 71 | def sample(model, query=None, n_samples=128, beta=1, max_length=32, temperature=0.8, top_k=20):
 72 |     if query is None:
 73 |         query = tensor([tokenizer.bos_token_id] * n_samples, device=model.device).view(n_samples, 1)
 74 | 
 75 |     for _ in range(max_length):
 76 |         logits, qs, _, vs = model(input_ids=query)
 77 |         logits = logits[:, -1, :]
 78 |         qs = qs[:, -1, :]
 79 |         vs = vs[:, -1, :]
 80 | 
 81 |         adv = qs - vs
 82 |         pi = F.log_softmax(logits, -1)
 83 |         modpi = topk_mask(pi + beta * adv, top_k)
 84 |         ps = F.softmax(modpi / temperature, -1)
 85 | 
 86 |         tokens = th.multinomial(ps, 1)
 87 |         query = th.hstack((query, tokens))
 88 | 
 89 |     return query
 90 | 
 91 | class Carps(Dataset):
 92 |     def __init__(self, review='good', max_length=48, diff_reward=True, n_samples=64):
 93 |         self.review = review
 94 |         self.max_length = max_length
 95 |         self.n_samples = n_samples
 96 | 
 97 |         cache_path = f'stash/carps-{max_length}l-{diff_reward}d.pt'
 98 | 
 99 |         if os.path.exists(cache_path):
100 |             cache = th.load(cache_path)
101 |             self.tokens = cache['tokens']
102 |             self.rewards = cache['rewards']
103 |             self.attention_masks = cache['attention_masks']
104 |             self.validation_queries = cache['validation_queries']
105 |         else:
106 |             ds, valid = load_dataset(
107 |                 'text',
108 |                 data_files={'train': 'roc_train_all.txt', 'valid': 'roc_valid.txt'},
109 |                 split=['train', f'valid[:{n_samples}]'])
110 | 
111 |             if diff_reward:
112 |                 vocab = list(tokenizer.get_vocab().keys())
113 |                 offset = th.hstack([carp_score(words, review) for words in sizesplit(32, vocab)]).mean()
114 |             else:
115 |                 offset = 0
116 | 
117 |             ds = ds.map(partial(tokenize, max_length, diff_reward, offset, review))
118 |             valid = valid.map(partial(tokenize, max_length, diff_reward, offset, review))
119 | 
120 |             self.tokens = th.tensor(ds['tokens']).squeeze(1)
121 |             self.rewards = tensor(ds['rewards'])
122 |             self.attention_masks = tensor(ds['attention'])
123 |             self.validation_queries = tensor(valid['tokens']).squeeze(1)[:n_samples, :6]
124 | 
125 |             th.save({ 'tokens': self.tokens,
126 |                       'rewards': self.rewards,
127 |                       'attention_masks': self.attention_masks,
128 |                       'validation_queries': self.validation_queries }, cache_path)
129 | 
130 |     def __len__(self):
131 |         return self.tokens.shape[0]
132 | 
133 |     def __getitem__(self, ind):
134 |         return self.tokens[ind], self.attention_masks[ind], self.rewards[ind]
135 | 
136 |     def eval(self, logs, model, betas=[1]):
137 |         model.eval()
138 |         queries = self.validation_queries.to(model.device)
139 | 
140 |         for beta in betas:
141 |             responses = sample(model, query=queries, beta=beta, max_length=self.max_length, n_samples=self.n_samples)
142 |             texts = [tokenizer.decode(response[1:]) for response in responses]
143 | 
144 |             rewards = th.hstack([carp_score(ts, self.review) for ts in sizesplit(8, texts)])
145 |             reward = rewards.mean().item()
146 |             rows = list(zip(texts, rewards.tolist()))
147 | 
148 |             print(f'\n{beta=} {reward=:.2f}\n' + '\n'.join([f'[{r:.2f}] {text}' for text, r in rows[:8]]))
149 | 
150 |             logs[f'reward/beta{beta}'] = reward
151 |             logs.update({f'responses/beta{beta}': wandb.Table(columns=['response', 'reward'], rows=rows[:32])})
152 | 
153 |         stats = {'reward': f'{reward:.2f}'}
154 |         model.train()
155 |         return reward, stats
156 | 


--------------------------------------------------------------------------------
/ilql.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import sys
  3 | import yaml
  4 | from time import time
  5 | 
  6 | import torch as th
  7 | from torch import tensor, nn
  8 | import torch.nn.functional as F
  9 | from torch.utils.data import DataLoader
 10 | from transformers import GPT2Config, AutoTokenizer
 11 | from accelerate import Accelerator
 12 | import numpy as np
 13 | 
 14 | import wandb
 15 | from tqdm import tqdm, trange
 16 | from utils import run_from_ipython, timeit, check_weights
 17 | from models import QVModel
 18 | from copy import deepcopy
 19 | import accelerate
 20 | import deepspeed
 21 | 
 22 | th.set_printoptions(sci_mode=False)
 23 | 
 24 | WORLD_SIZE = int(os.environ.get('WORLD_SIZE', 1))
 25 | WORLD_RANK = int(os.environ.get('RANK', 0))
 26 | LOCAL_RANK = int(os.environ.get('LOCAL_RANK', 0))
 27 | 
 28 | def main(**args):
 29 |     task = args['task'] if 'task' in args else 'RandomWalks'
 30 |     config = yaml.safe_load(open('config.yaml'))[task]
 31 |     config.update(args)
 32 | 
 33 |     accelerator = Accelerator(log_with='wandb')
 34 |     accelerator.print(os.environ)
 35 |     device = accelerator.device
 36 | 
 37 |     if WORLD_SIZE > 1:
 38 |         th.distributed.barrier(device_ids=[LOCAL_RANK])
 39 |     else:
 40 |         th.random.manual_seed(1000)
 41 | 
 42 |     if not config.get('debug', False) and accelerator.is_main_process:
 43 |         modelname = config.get('model', '')
 44 |         accelerator.init_trackers(project_name='test-ilql', init_kwargs={
 45 |             'wandb': {'name': f'ilql-{task}-{modelname}',
 46 |                       'mode': 'disabled' if args.get('debug', False) else 'online'}}, config=config)
 47 | 
 48 |         config = wandb.config
 49 | 
 50 |     if task == 'RandomWalks':
 51 |         from randomwalks import RandomWalks
 52 |         data = RandomWalks(seed=config['seed'])
 53 |         gptconfig = GPT2Config(**config['gptconfig'], vocab_size=data.n_nodes)
 54 |         model = QVModel(gptconfig, config)
 55 | 
 56 |     elif task == 'Sentiments':
 57 |         from sentiments import Sentiments
 58 | 
 59 |         with accelerator.main_process_first():
 60 |             tokenizer = AutoTokenizer.from_pretrained(config['model'])
 61 |             tokenizer.pad_token_id = tokenizer.eos_token_id
 62 |             data = Sentiments(tokenizer, needs_reward_model=accelerator.is_main_process)
 63 | 
 64 |         with timeit('init model'):
 65 |             model = QVModel(config['model'], config)
 66 | 
 67 |     elif task == 'Carps':
 68 |         from carps import Carps
 69 |         data = Carps(max_length=config['max_length'], diff_reward=config['diff_reward'])
 70 |         model = QVModel(config['model'], two_qs=config['two_qs']).to(device)
 71 | 
 72 |     elif task == 'Captions':
 73 |         from captions import AestheticCaptions
 74 | 
 75 |         tokenizer = AutoTokenizer.from_pretrained(config['model'])
 76 |         tokenizer.pad_token = tokenizer.eos_token_id
 77 |         with accelerator.main_process_first():
 78 |             data = AestheticCaptions(tokenizer, batch_size=config['batch_size'], n_samples=16)
 79 | 
 80 |         model = QVModel(config['model'], two_qs=config['two_qs']).to(device)
 81 |     else:
 82 |         raise ValueError(f'nonexistent {task=}')
 83 | 
 84 |     if hasattr(model.gpt, 'gpt_neox'):
 85 |         gpt_blocks = list(model.gpt.gpt_neox.layers)[:-config['n_layers_unfrozen']]
 86 |     else:
 87 |         gpt_blocks = list(model.gpt.transformer.h)[:-config['n_layers_unfrozen']]
 88 | 
 89 |     for m in gpt_blocks:
 90 |         m.requires_grad_(False)
 91 | 
 92 |     train_dataloader = DataLoader(data.dataset, batch_size=config['batch_size'])
 93 | 
 94 |     eval_batch_size = max(1, len(data.eval_dataset) // WORLD_SIZE)
 95 |     eval_dataloader = DataLoader(data.eval_dataset, eval_batch_size)
 96 | 
 97 |     opt_cls = (
 98 |         th.optim.AdamW
 99 |         if accelerator.state.deepspeed_plugin is None
100 |         or "optimizer" not in accelerator.state.deepspeed_plugin.deepspeed_config
101 |         else accelerate.utils.DummyOptim
102 |     )
103 |     opt = opt_cls([p for p in model.parameters() if p.requires_grad], lr=config['lr'], betas=config['opt_betas'])
104 | 
105 |     total_steps = int(config['n_epochs'] * (len(data.dataset) // (config['batch_size'] * WORLD_SIZE)))
106 |     n_opt_steps = 0
107 | 
108 |     with timeit('prepare'):
109 |         model, opt, train_dataloader, eval_dataloader = accelerator.prepare(
110 |             model, opt, train_dataloader, eval_dataloader
111 |         )
112 | 
113 |     print(f'{WORLD_RANK=}: {model(**accelerator.unwrap_model(model).dummy_inputs)[0].device}')
114 | 
115 |     model.train()
116 |     tbar = trange(total_steps, disable=not accelerator.is_local_main_process)
117 | 
118 |     for iepoch in range(config['n_epochs']):
119 |         for batch in train_dataloader:
120 |             logs = {}
121 | 
122 |             if n_opt_steps % config['steps_for_eval'] == 0:
123 |                 model.eval()
124 |                 beta = config['inference_betas'][0]
125 | 
126 |                 all_samples = []
127 |                 for tokens in eval_dataloader:
128 |                     tokens = tokens[0].to(device)
129 |                     with th.no_grad():
130 |                         samples, stats = accelerator.unwrap_model(model).sample(
131 |                             tokens,
132 |                             beta=beta,
133 |                             max_length=data.max_length,
134 |                             logit_mask=data.logit_mask
135 |                         )
136 | 
137 |                     all_samples.append(samples)
138 |                     logs.update(stats)
139 | 
140 |                 samples = accelerator.gather(th.vstack(all_samples))
141 | 
142 |                 if accelerator.is_main_process:
143 |                     reward, stats = data.eval(samples, beta)
144 |                     logs.update(stats)
145 |                     tbar.set_postfix(stats)
146 | 
147 |                 model.train()
148 | 
149 |             for ix in range(len(batch)):
150 |                 batch[ix] = batch[ix].to(device)
151 | 
152 |             batch_time = time()
153 |             forward_time = time()
154 |             loss, stats = model.loss(batch)
155 |             forward_time = time() - forward_time
156 | 
157 |             backward_time = time()
158 |             accelerator.backward(loss)
159 |             backward_time = time() - backward_time
160 | 
161 |             opt.step()
162 |             opt.zero_grad()
163 |             n_opt_steps += 1
164 | 
165 |             batch_time = time() - batch_time
166 |             tokens_per_sec = batch[0].numel() * WORLD_SIZE / batch_time
167 |             tbar.set_description(f'{tokens_per_sec=:.2f} {batch_time=:.2f}')
168 |             tbar.update()
169 | 
170 |             if (n_opt_steps + 1) % config['steps_for_target_q_sync'] == 0:
171 |                 accelerator.unwrap_model(model).sync_target_q_heads()
172 | 
173 |             logs.update(stats)
174 |             logs['target_sum'] = check_weights(accelerator.unwrap_model(model).target_q1_head)
175 |             logs['batch_time'] = batch_time
176 | 
177 |             if not config.get('debug', False):
178 |                 accelerator.log(logs)
179 | 
180 |     return model, data
181 | 
182 | if __name__ == '__main__':
183 |     if os.environ.get('LOCAL_RANK'):
184 |         os.environ['OMPI_COMM_WORLD_LOCAL_RANK'] = os.environ['LOCAL_RANK']
185 | 
186 |     if run_from_ipython:
187 |         args = {'debug': True}
188 |     else:
189 |         # poor man's argparse
190 |         args = {a[2:]: eval(v) for a, v in map(lambda s: s.split('='), sys.argv[1:])}
191 | 
192 |     main(**args)
193 | 


--------------------------------------------------------------------------------
/models.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import torch as th
  3 | import numpy as np
  4 | from torch import tensor, nn
  5 | import torch.nn.functional as F
  6 | import transformers
  7 | from transformers import AutoModelForCausalLM, PretrainedConfig, AutoConfig
  8 | 
  9 | from typing import NamedTuple, Tuple, Union
 10 | from copy import deepcopy
 11 | from collections import defaultdict
 12 | from accelerate.utils import compute_module_sizes
 13 | from itertools import chain
 14 | 
 15 | import accelerate
 16 | import deepspeed
 17 | 
 18 | def topk_mask(xs: th.FloatTensor, k: int):
 19 |     mintop = th.topk(xs, k)[0][:, -1].unsqueeze(-1)
 20 |     return th.where(xs < mintop, -np.inf * th.ones_like(xs, dtype=xs.dtype), xs)
 21 | 
 22 | class QVOutput(Tuple):
 23 |     logits: th.FloatTensor
 24 |     qs: th.FloatTensor
 25 |     target_qs: th.FloatTensor
 26 |     vs: th.FloatTensor
 27 |     past_key_values: Tuple[th.FloatTensor]
 28 | 
 29 | def make_head(n_embd: int, out: int):
 30 |     return nn.Sequential(
 31 |         nn.Linear(n_embd, n_embd * 2),
 32 |         nn.ReLU(),
 33 |         nn.Linear(n_embd * 2, out)
 34 |     )
 35 | 
 36 | class QVModel(nn.Module):
 37 |     def __init__(self, config: Union[PretrainedConfig, str], params):
 38 |         super().__init__()
 39 | 
 40 |         # enable zero3 init within from_pretrained
 41 |         if os.environ.get('DEEPSPEED_ZERO_STAGE', '0') == '3':
 42 |             config_path = os.environ.get('DEEPSPEED_CONFIG_FILE', '')
 43 |             if config_path:
 44 |                 _hfconfig = transformers.deepspeed.HfDeepSpeedConfig(config_path)
 45 | 
 46 |         if isinstance(config, PretrainedConfig):
 47 |             self.gpt = AutoModelForCausalLM.from_config(config)
 48 |         else:
 49 |             self.gpt = AutoModelForCausalLM.from_pretrained(config)
 50 | 
 51 |         if hasattr(self.gpt.config, 'hidden_size'):
 52 |             self.n_embd = self.gpt.config.hidden_size
 53 |         else:
 54 |             self.n_embd = self.gpt.config.n_embd
 55 |         self.vocab_size = self.gpt.config.vocab_size
 56 | 
 57 |         self.v_head = make_head(self.n_embd, 1)
 58 |         self.q1_head = make_head(self.n_embd, self.vocab_size)
 59 |         self.target_q1_head = deepcopy(self.q1_head)
 60 |         self.target_q1_head.requires_grad_(False)
 61 | 
 62 |         self.tau = params['tau']
 63 |         self.alpha = params['alpha']
 64 |         self.gamma = params['gamma']
 65 |         self.awac_scale = params['awac_scale']
 66 |         self.cql_scale = params['cql_scale']
 67 |         self.two_qs = params['two_qs']
 68 | 
 69 |         if self.two_qs:
 70 |             self.q2_head = make_head(self.n_embd, self.vocab_size)
 71 |             self.target_q2_head = deepcopy(self.q2_head)
 72 |             self.target_q2_head.requires_grad_(False)
 73 | 
 74 |     def forward(self, **x):
 75 |         if hasattr(self.gpt, 'gpt_neox'):
 76 |             out = self.gpt.gpt_neox(**x)
 77 |         else:
 78 |             out = self.gpt.transformer(**x)
 79 | 
 80 |         hs = out.last_hidden_state
 81 | 
 82 |         if self.two_qs:
 83 |             qs = (self.q1_head(hs), self.q2_head(hs))
 84 |             target_qs = (self.target_q1_head(hs), self.target_q2_head(hs))
 85 |         else:
 86 |             qs = self.q1_head(hs)
 87 |             target_qs = self.target_q1_head(hs)
 88 | 
 89 |         if hasattr(self.gpt, 'gpt_neox'):
 90 |             logits = self.gpt.embed_out(hs)
 91 |         else:
 92 |             logits = self.gpt.lm_head(hs)
 93 | 
 94 |         return QVOutput((logits, qs, target_qs, self.v_head(hs), out.past_key_values))
 95 | 
 96 |     def loss(self, batch):
 97 |         tokens, attn, rewards = batch
 98 |         actions = tokens[:, 1:, None]
 99 |         isterminal = attn[:, :-1]
100 | 
101 |         logits, qs, target_qs, vs, _ = self(input_ids=tokens, attention_mask=attn)
102 |         bsize, ntokens, dsize = logits.shape
103 | 
104 |         if self.two_qs:
105 |             Q1 = qs[0][:, :-1].gather(-1, actions).squeeze(-1)
106 |             Q2 = qs[1][:, :-1].gather(-1, actions).squeeze(-1)
107 | 
108 |             targetQ1 = target_qs[0][:, :-1].gather(-1, actions).squeeze(-1).detach()
109 |             targetQ2 = target_qs[1][:, :-1].gather(-1, actions).squeeze(-1).detach()
110 |             targetQ = th.minimum(targetQ1, targetQ2)
111 |         else:
112 |             Q = qs[:, :-1].gather(-1, actions).squeeze(-1)
113 |             targetQ = target_qs[:, :-1].gather(-1, actions).squeeze(-1).detach()
114 | 
115 |         n_nonterminal = max(1, isterminal.sum())
116 |         V = vs[:, 1:].squeeze() * isterminal
117 |         Q_ = rewards + self.gamma * V
118 | 
119 |         if self.two_qs:
120 |             loss_q1 = ((Q1 - Q_.detach()) * isterminal).pow(2).sum() / n_nonterminal
121 |             loss_q2 = ((Q2 - Q_.detach()) * isterminal).pow(2).sum() / n_nonterminal
122 |             loss_q = loss_q1 + loss_q2
123 |         else:
124 |             loss_q = ((Q - Q_.detach()) * isterminal).pow(2).sum() / n_nonterminal
125 | 
126 |         loss_v = (((targetQ >= V).int() * self.tau * (targetQ - V).pow(2) + (targetQ < V).int() * (1 - self.tau) * (targetQ - V).pow(2)) * isterminal).sum() / n_nonterminal
127 | 
128 |         if self.two_qs:
129 |             loss_cql_q1 = (F.cross_entropy(qs[0][:, :-1].reshape(-1, dsize), actions.reshape(-1), reduction='none').reshape(bsize, ntokens-1) * isterminal).sum() / n_nonterminal
130 |             loss_cql_q2 = (F.cross_entropy(qs[1][:, :-1].reshape(-1, dsize), actions.reshape(-1), reduction='none').reshape(bsize, ntokens-1) * isterminal).sum() / n_nonterminal
131 |             loss_cql = loss_cql_q1 + loss_cql_q2
132 |         else:
133 |             loss_cql = (F.cross_entropy(qs[:, :-1].reshape(-1, dsize), actions.reshape(-1), reduction='none').reshape(bsize, ntokens-1) * isterminal).sum() / n_nonterminal
134 | 
135 |         loss_awac = (F.cross_entropy(logits[:, :-1].reshape(-1, dsize), actions.reshape(-1), reduction='none').reshape(bsize, ntokens-1) * isterminal).sum() / n_nonterminal
136 | 
137 |         loss = loss_q + loss_v + self.cql_scale * loss_cql + self.awac_scale * loss_awac
138 |         stats = {
139 |             k: v for k, v in locals().items() if k in
140 |             ['loss', 'loss_v', 'loss_q', 'loss_cql', 'loss_awac']
141 |         }
142 | 
143 |         return loss, stats
144 | 
145 |     def _sync_target_q_heads(self, alpha):
146 |         for target_param, copy_param in zip(self.target_q1_head.parameters(), self.q1_head.parameters()):
147 |             target_param.data.copy_((alpha * copy_param.data) + (1.0 - alpha) * target_param.data)
148 | 
149 |         if self.two_qs:
150 |             for target_param, copy_param in zip(self.target_q2_head.parameters(), self.q2_head.parameters()):
151 |                 target_param.data.copy_((alpha * copy_param.data) + (1.0 - alpha) * target_param.data)
152 | 
153 |     def sync_target_q_heads(self):
154 |         if os.environ.get('DEEPSPEED_ZERO_STAGE', '0') == '3':
155 |             params = chain(self.q1_head.parameters(),
156 |                            self.target_q1_head.parameters(),
157 |                            self.q2_head.parameters() if self.two_qs else [],
158 |                            self.target_q2_head.parameters() if self.two_qs else [])
159 | 
160 |             with deepspeed.zero.GatheredParameters(list(params), modifier_rank=0):
161 |                 if deepspeed.comm.get_rank() == 0:
162 |                     self._sync_target_q_heads(self.alpha)
163 |         else:
164 |             self._sync_target_q_heads(self.alpha)
165 | 
166 |     @th.inference_mode()
167 |     def sample(self, query, beta=1, max_length=32, temperature=1, top_k=20, logit_mask=None, logs=True, eos_token_id=50256):
168 |         input = query.clone()
169 |         past_key_values = None
170 |         tensors = defaultdict(list)
171 | 
172 |         finished = th.zeros(input.shape[0], 1, dtype=th.long, device=query.device)
173 | 
174 |         for _ in range(max_length-1):
175 |             logits, _, target_qs, vs, past_key_values = self.forward(input_ids=input, past_key_values=past_key_values)
176 | 
177 |             if self.two_qs:
178 |                 qs = th.minimum(target_qs[0][:, -1], target_qs[1][:, -1])
179 |             else:
180 |                 qs = target_qs[:, -1]
181 | 
182 |             logits = logits[:, -1]
183 | 
184 |             if logit_mask is not None:
185 |                 logits[th.where(logit_mask[input[:, -1]])] = -np.inf
186 | 
187 |             adv = qs - vs[:, -1, :]
188 |             pi = F.log_softmax(logits, -1)
189 |             modpi = topk_mask(pi + beta * adv, top_k)
190 |             ps = F.softmax(modpi / temperature, -1)
191 | 
192 |             tokens = th.multinomial(ps, 1)
193 |             tokens = (1 - finished) * tokens + finished * eos_token_id
194 | 
195 |             query = th.hstack((query, tokens))
196 | 
197 |             input = tokens
198 |             finished = (tokens == eos_token_id).long()
199 | 
200 |             if logs:
201 |                 tensors['qs'].append(qs)
202 |                 tensors['vs'].append(vs)
203 |                 tensors['adv'].append(adv)
204 | 
205 |         stats = {}
206 |         for name, xs in tensors.items():
207 |             xs = th.vstack(xs)
208 |             stats.update({
209 |                 f'{name}-min': xs.min(),
210 |                 f'{name}-max': xs.max(),
211 |                 f'{name}-std': xs.std(),
212 |                 f'{name}-avg': xs.mean(),
213 |             })
214 | 
215 |         return query, stats
216 | 
217 |     @property
218 |     def dummy_inputs(self):
219 |         return {'input_ids': th.ones(1, 1, device=self.gpt.device, dtype=th.long)}
220 | 
221 |     @property
222 |     def device(self):
223 |         return self.gpt.device
224 | 


--------------------------------------------------------------------------------
/scripts/graph_plot.svg:
--------------------------------------------------------------------------------
   1 | <?xml version="1.0" encoding="UTF-8" standalone="no"?>
   2 | <svg
   3 |    width="943.41223pt"
   4 |    height="543.47913pt"
   5 |    viewBox="0 0 943.41223 543.47913"
   6 |    version="1.1"
   7 |    id="svg515"
   8 |    sodipodi:docname="graph_plot.svg"
   9 |    inkscape:version="1.2.1 (9c6d41e, 2022-07-14)"
  10 |    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
  11 |    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
  12 |    xmlns:xlink="http://www.w3.org/1999/xlink"
  13 |    xmlns="http://www.w3.org/2000/svg"
  14 |    xmlns:svg="http://www.w3.org/2000/svg"
  15 |    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
  16 |    xmlns:cc="http://creativecommons.org/ns#"
  17 |    xmlns:dc="http://purl.org/dc/elements/1.1/">
  18 |   <sodipodi:namedview
  19 |      id="namedview517"
  20 |      pagecolor="#ffffff"
  21 |      bordercolor="#999999"
  22 |      borderopacity="1"
  23 |      inkscape:showpageshadow="0"
  24 |      inkscape:pageopacity="0"
  25 |      inkscape:pagecheckerboard="0"
  26 |      inkscape:deskcolor="#d1d1d1"
  27 |      inkscape:document-units="pt"
  28 |      showgrid="false"
  29 |      inkscape:zoom="0.57655246"
  30 |      inkscape:cx="666.02786"
  31 |      inkscape:cy="334.74838"
  32 |      inkscape:window-width="1280"
  33 |      inkscape:window-height="772"
  34 |      inkscape:window-x="1280"
  35 |      inkscape:window-y="0"
  36 |      inkscape:window-maximized="0"
  37 |      inkscape:current-layer="svg515" />
  38 |   <metadata
  39 |      id="metadata2">
  40 |     <rdf:RDF>
  41 |       <cc:Work>
  42 |         <dc:type
  43 |            rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
  44 |         <dc:date>2022-09-04T18:32:28.332967</dc:date>
  45 |         <dc:format>image/svg+xml</dc:format>
  46 |         <dc:creator>
  47 |           <cc:Agent>
  48 |             <dc:title>Matplotlib v3.5.3, https://matplotlib.org/</dc:title>
  49 |           </cc:Agent>
  50 |         </dc:creator>
  51 |       </cc:Work>
  52 |     </rdf:RDF>
  53 |   </metadata>
  54 |   <defs
  55 |      id="defs6">
  56 |     <style
  57 |        type="text/css"
  58 |        id="style4">*{stroke-linejoin: round; stroke-linecap: butt}</style>
  59 |   </defs>
  60 |   <g
  61 |      id="figure_1"
  62 |      transform="translate(-48.130919,-5.2033426)">
  63 |     <g
  64 |        id="patch_1"
  65 |        transform="matrix(0.86028061,0,0,0.94354012,48.130919,5.2033426)">
  66 |       <path
  67 |          d="M 0,0 H 1080 V 576 H 0 Z"
  68 |          style="fill:#ffffff"
  69 |          id="path8"
  70 |          sodipodi:nodetypes="ccccc" />
  71 |     </g>
  72 |     <g
  73 |        id="axes_1"
  74 |        transform="translate(6.5041783,-52.033426)">
  75 |       <g
  76 |          id="patch_2">
  77 |         <path
  78 |            d="M 135,512.64 H 972 V 69.12 H 135 Z"
  79 |            style="fill:#ffffff"
  80 |            id="path11" />
  81 |       </g>
  82 |       <g
  83 |          id="matplotlib.axis_1">
  84 |         <g
  85 |            id="xtick_1">
  86 |           <g
  87 |              id="line2d_1">
  88 |             <path
  89 |                d="M 205.52328,512.64 V 69.12"
  90 |                clip-path="url(#pa228791ba9)"
  91 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
  92 |                id="path14" />
  93 |           </g>
  94 |           <g
  95 |              id="text_1">
  96 |             <!-- 1 -->
  97 |             <g
  98 |                style="fill:#444444"
  99 |                transform="matrix(0.2,0,0,-0.2,199.35141,536.13375)"
 100 |                id="g22">
 101 |               <defs
 102 |                  id="defs18">
 103 |                 <path
 104 |                    id="Futura-Medium-31"
 105 |                    d="M 1734,4144 H 900 l 400,681 H 2463 V 0 h -729 z"
 106 |                    transform="scale(0.015625)" />
 107 |               </defs>
 108 |               <use
 109 |                  xlink:href="#Futura-Medium-31"
 110 |                  id="use20" />
 111 |             </g>
 112 |           </g>
 113 |         </g>
 114 |         <g
 115 |            id="xtick_2">
 116 |           <g
 117 |              id="line2d_2">
 118 |             <path
 119 |                d="M 282.85144,512.64 V 69.12"
 120 |                clip-path="url(#pa228791ba9)"
 121 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
 122 |                id="path26" />
 123 |           </g>
 124 |           <g
 125 |              id="text_2">
 126 |             <!-- 2 -->
 127 |             <g
 128 |                style="fill:#444444"
 129 |                transform="matrix(0.2,0,0,-0.2,276.67957,536.13375)"
 130 |                id="g34">
 131 |               <defs
 132 |                  id="defs30">
 133 |                 <path
 134 |                    id="Futura-Medium-32"
 135 |                    d="M 1603,684 H 3400 V 0 H 159 l 1675,2028 q 247,303 414,525 168,222 255,372 178,294 178,528 0,338 -238,566 -237,228 -587,228 -731,0 -837,-866 H 294 q 175,1525 1540,1525 660,0 1107,-418 450,-419 450,-1038 0,-394 -210,-772 -106,-197 -311,-467 -204,-270 -517,-639 z"
 136 |                    transform="scale(0.015625)" />
 137 |               </defs>
 138 |               <use
 139 |                  xlink:href="#Futura-Medium-32"
 140 |                  id="use32" />
 141 |             </g>
 142 |           </g>
 143 |         </g>
 144 |         <g
 145 |            id="xtick_3">
 146 |           <g
 147 |              id="line2d_3">
 148 |             <path
 149 |                d="M 360.1796,512.64 V 69.12"
 150 |                clip-path="url(#pa228791ba9)"
 151 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
 152 |                id="path38" />
 153 |           </g>
 154 |           <g
 155 |              id="text_3">
 156 |             <!-- 3 -->
 157 |             <g
 158 |                style="fill:#444444"
 159 |                transform="matrix(0.2,0,0,-0.2,354.00773,536.13375)"
 160 |                id="g46">
 161 |               <defs
 162 |                  id="defs42">
 163 |                 <path
 164 |                    id="Futura-Medium-33"
 165 |                    d="m 1656,2241 v 622 q 463,0 622,103 285,181 285,581 0,309 -207,503 -203,197 -528,197 -303,0 -469,-147 -159,-150 -237,-506 H 397 q 112,681 462,998 350,317 979,317 634,0 1034,-375 400,-371 400,-959 0,-662 -525,-1031 600,-353 600,-1147 Q 3347,747 2909,334 2472,-78 1788,-78 916,-78 494,563 319,831 272,1306 h 703 q 56,-350 273,-536 218,-186 568,-186 353,0 584,235 231,240 231,609 0,488 -347,694 -203,119 -628,119 z"
 166 |                    transform="scale(0.015625)" />
 167 |               </defs>
 168 |               <use
 169 |                  xlink:href="#Futura-Medium-33"
 170 |                  id="use44" />
 171 |             </g>
 172 |           </g>
 173 |         </g>
 174 |         <g
 175 |            id="xtick_4">
 176 |           <g
 177 |              id="line2d_4">
 178 |             <path
 179 |                d="M 437.50776,512.64 V 69.12"
 180 |                clip-path="url(#pa228791ba9)"
 181 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
 182 |                id="path50" />
 183 |           </g>
 184 |           <g
 185 |              id="text_4">
 186 |             <!-- 4 -->
 187 |             <g
 188 |                style="fill:#444444"
 189 |                transform="matrix(0.2,0,0,-0.2,431.33589,536.13375)"
 190 |                id="g58">
 191 |               <defs
 192 |                  id="defs54">
 193 |                 <path
 194 |                    id="Futura-Medium-34"
 195 |                    d="m 3156,1347 h 513 V 684 H 3156 V 0 H 2428 V 684 H 59 l 3097,4500 z m -728,0 V 2963 L 1316,1347 Z"
 196 |                    transform="scale(0.015625)" />
 197 |               </defs>
 198 |               <use
 199 |                  xlink:href="#Futura-Medium-34"
 200 |                  id="use56" />
 201 |             </g>
 202 |           </g>
 203 |         </g>
 204 |         <g
 205 |            id="xtick_5">
 206 |           <g
 207 |              id="line2d_5">
 208 |             <path
 209 |                d="M 514.83592,512.64 V 69.12"
 210 |                clip-path="url(#pa228791ba9)"
 211 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
 212 |                id="path62" />
 213 |           </g>
 214 |           <g
 215 |              id="text_5">
 216 |             <!-- 5 -->
 217 |             <g
 218 |                style="fill:#444444"
 219 |                transform="matrix(0.2,0,0,-0.2,508.66404,536.13375)"
 220 |                id="g70">
 221 |               <defs
 222 |                  id="defs66">
 223 |                 <path
 224 |                    id="Futura-Medium-35"
 225 |                    d="M 3366,4144 H 1878 l -237,-800 q 53,6 101,9 49,3 89,3 703,0 1153,-475 454,-472 454,-1209 Q 3438,909 2925,415 2413,-78 1634,-78 675,-78 122,822 l 550,466 q 228,-388 459,-547 232,-157 566,-157 437,0 734,307 297,315 297,765 0,463 -294,757 -293,303 -734,303 -491,0 -847,-428 H 613 l 759,2537 h 1994 z"
 226 |                    transform="scale(0.015625)" />
 227 |               </defs>
 228 |               <use
 229 |                  xlink:href="#Futura-Medium-35"
 230 |                  id="use68" />
 231 |             </g>
 232 |           </g>
 233 |         </g>
 234 |         <g
 235 |            id="xtick_6">
 236 |           <g
 237 |              id="line2d_6">
 238 |             <path
 239 |                d="M 592.16408,512.64 V 69.12"
 240 |                clip-path="url(#pa228791ba9)"
 241 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
 242 |                id="path74" />
 243 |           </g>
 244 |           <g
 245 |              id="text_6">
 246 |             <!-- 6 -->
 247 |             <g
 248 |                style="fill:#444444"
 249 |                transform="matrix(0.2,0,0,-0.2,585.99221,536.13375)"
 250 |                id="g82">
 251 |               <defs
 252 |                  id="defs78">
 253 |                 <path
 254 |                    id="Futura-Medium-36"
 255 |                    d="M 2513,4994 3047,4594 1894,3025 q 94,13 159,19 66,6 103,6 588,0 1016,-450 425,-444 425,-1069 Q 3597,869 3122,394 2650,-78 1988,-78 1331,-78 838,388 353,853 353,1491 q 0,572 472,1212 z m 375,-3510 q 0,385 -266,650 -269,269 -647,269 -381,0 -647,-266 -265,-265 -265,-643 0,-378 265,-644 266,-266 650,-266 375,0 641,266 269,269 269,634 z"
 256 |                    transform="scale(0.015625)" />
 257 |               </defs>
 258 |               <use
 259 |                  xlink:href="#Futura-Medium-36"
 260 |                  id="use80" />
 261 |             </g>
 262 |           </g>
 263 |         </g>
 264 |         <g
 265 |            id="xtick_7">
 266 |           <g
 267 |              id="line2d_7">
 268 |             <path
 269 |                d="M 669.49224,512.64 V 69.12"
 270 |                clip-path="url(#pa228791ba9)"
 271 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
 272 |                id="path86" />
 273 |           </g>
 274 |           <g
 275 |              id="text_7">
 276 |             <!-- 7 -->
 277 |             <g
 278 |                style="fill:#444444"
 279 |                transform="matrix(0.2,0,0,-0.2,663.32036,536.13375)"
 280 |                id="g94">
 281 |               <defs
 282 |                  id="defs90">
 283 |                 <path
 284 |                    id="Futura-Medium-37"
 285 |                    d="M 2713,4144 H 331 v 681 H 3928 L 884,-166 306,200 Z"
 286 |                    transform="scale(0.015625)" />
 287 |               </defs>
 288 |               <use
 289 |                  xlink:href="#Futura-Medium-37"
 290 |                  id="use92" />
 291 |             </g>
 292 |           </g>
 293 |         </g>
 294 |         <g
 295 |            id="xtick_8">
 296 |           <g
 297 |              id="line2d_8">
 298 |             <path
 299 |                d="M 746.8204,512.64 V 69.12"
 300 |                clip-path="url(#pa228791ba9)"
 301 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
 302 |                id="path98" />
 303 |           </g>
 304 |           <g
 305 |              id="text_8">
 306 |             <!-- 8 -->
 307 |             <g
 308 |                style="fill:#444444"
 309 |                transform="matrix(0.2,0,0,-0.2,740.64852,536.13375)"
 310 |                id="g106">
 311 |               <defs
 312 |                  id="defs102">
 313 |                 <path
 314 |                    id="Futura-Medium-38"
 315 |                    d="m 1113,2509 q -532,375 -532,1022 0,588 403,982 407,393 1010,393 600,0 1000,-393 400,-391 400,-985 0,-622 -528,-1015 290,-157 456,-457 169,-300 169,-675 Q 3491,741 3072,331 2653,-78 2000,-78 1331,-78 897,334 459,747 459,1372 q 0,812 654,1137 z m 153,1025 q 0,-309 209,-518 213,-213 513,-213 296,0 508,212 213,213 213,510 0,297 -213,509 -212,213 -508,213 -297,0 -510,-213 -212,-212 -212,-500 z m -91,-2137 q 0,-334 231,-575 235,-238 569,-238 331,0 566,235 234,237 234,569 0,334 -235,568 -234,235 -565,235 -328,0 -566,-235 -234,-234 -234,-559 z"
 316 |                    transform="scale(0.015625)" />
 317 |               </defs>
 318 |               <use
 319 |                  xlink:href="#Futura-Medium-38"
 320 |                  id="use104" />
 321 |             </g>
 322 |           </g>
 323 |         </g>
 324 |         <g
 325 |            id="xtick_9">
 326 |           <g
 327 |              id="line2d_9">
 328 |             <path
 329 |                d="M 824.14856,512.64 V 69.12"
 330 |                clip-path="url(#pa228791ba9)"
 331 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
 332 |                id="path110" />
 333 |           </g>
 334 |           <g
 335 |              id="text_9">
 336 |             <!-- 9 -->
 337 |             <g
 338 |                style="fill:#444444"
 339 |                transform="matrix(0.2,0,0,-0.2,817.97668,536.13375)"
 340 |                id="g118">
 341 |               <defs
 342 |                  id="defs114">
 343 |                 <path
 344 |                    id="Futura-Medium-39"
 345 |                    d="M 1438,-166 903,231 2056,1803 q -93,-12 -159,-19 -66,-6 -106,-6 -585,0 -1013,447 -425,444 -425,1069 0,665 472,1137 475,475 1138,475 659,0 1146,-468 488,-466 488,-1100 0,-563 -475,-1216 z m -375,3513 q 0,-391 265,-656 269,-269 647,-269 378,0 644,266 269,268 269,646 0,379 -269,647 -266,266 -647,266 -375,0 -641,-266 -268,-268 -268,-634 z"
 346 |                    transform="scale(0.015625)" />
 347 |               </defs>
 348 |               <use
 349 |                  xlink:href="#Futura-Medium-39"
 350 |                  id="use116" />
 351 |             </g>
 352 |           </g>
 353 |         </g>
 354 |         <g
 355 |            id="xtick_10">
 356 |           <g
 357 |              id="line2d_10">
 358 |             <path
 359 |                d="M 901.47672,512.64 V 69.12"
 360 |                clip-path="url(#pa228791ba9)"
 361 |                style="fill:none;stroke:#ffffff;stroke-width:0.8;stroke-linecap:square"
 362 |                id="path122" />
 363 |           </g>
 364 |           <g
 365 |              id="text_10">
 366 |             <!-- ∞ -->
 367 |             <g
 368 |                style="fill:#444444"
 369 |                transform="matrix(0.2,0,0,-0.2,893.93766,536.13375)"
 370 |                id="g130">
 371 |               <defs
 372 |                  id="defs126">
 373 |                 <path
 374 |                    id="Futura-Medium-221e"
 375 |                    d="m 2431,1988 q -509,-669 -1162,-669 -503,0 -822,319 -319,321 -319,831 0,494 322,815 322,322 819,322 637,0 1131,-665 481,665 1163,665 506,0 818,-322 319,-318 319,-825 0,-500 -319,-821 -312,-322 -809,-322 -663,0 -1141,672 z m 341,475 q 347,-532 700,-532 219,0 366,150 146,157 146,382 0,218 -150,375 -143,153 -365,153 -350,0 -697,-528 z m -716,0 q -340,528 -703,528 -219,0 -362,-153 -147,-157 -147,-375 0,-222 147,-379 150,-153 368,-153 354,0 697,532 z"
 376 |                    transform="scale(0.015625)" />
 377 |               </defs>
 378 |               <use
 379 |                  xlink:href="#Futura-Medium-221e"
 380 |                  id="use128" />
 381 |             </g>
 382 |           </g>
 383 |         </g>
 384 |         <g
 385 |            id="text_11">
 386 |           <!-- # of steps to goal -->
 387 |           <g
 388 |              style="fill:#444444"
 389 |              transform="matrix(0.22,0,0,-0.22,466.31984,579.74406)"
 390 |              id="g183">
 391 |             <defs
 392 |                id="defs145">
 393 |               <path
 394 |                  id="Futura-Medium-23"
 395 |                  d="m 3834,3469 v -550 h -759 l -112,-897 h 693 V 1472 H 2894 L 2709,-22 2159,50 2338,1472 H 1409 L 1222,-22 666,50 844,1484 H 131 v 538 h 791 l 112,897 H 313 v 550 h 790 l 172,1400 547,-72 -166,-1328 h 922 l 175,1400 547,-72 -166,-1328 z M 2509,2919 h -918 l -113,-897 h 925 z"
 396 |                  transform="scale(0.015625)" />
 397 |               <path
 398 |                  id="Futura-Medium-20"
 399 |                  transform="scale(0.015625)"
 400 |                  d="" />
 401 |               <path
 402 |                  id="Futura-Medium-6f"
 403 |                  d="m 300,1544 q 0,659 472,1121 472,463 1150,463 681,0 1156,-465 469,-466 469,-1144 Q 3547,834 3075,372 2600,-88 1909,-88 1225,-88 763,381 300,844 300,1544 Z m 719,-13 q 0,-456 244,-722 250,-268 659,-268 412,0 659,265 247,266 247,710 0,443 -247,709 -250,269 -659,269 -403,0 -653,-269 -250,-269 -250,-694 z"
 404 |                  transform="scale(0.015625)" />
 405 |               <path
 406 |                  id="Futura-Medium-66"
 407 |                  d="M 1172,2388 V 0 H 466 V 2388 H 216 v 656 h 250 v 1175 q 0,575 200,812 275,332 800,332 187,0 472,-110 v -719 l -72,38 q -228,116 -375,116 -188,0 -253,-135 -66,-131 -66,-506 V 3044 h 766 v -656 z"
 408 |                  transform="scale(0.015625)" />
 409 |               <path
 410 |                  id="Futura-Medium-73"
 411 |                  d="m 2213,2525 -582,-309 q -137,281 -340,281 -97,0 -166,-64 -69,-64 -69,-164 0,-175 407,-347 559,-241 752,-444 194,-203 194,-547 0,-440 -325,-737 Q 1769,-88 1322,-88 556,-88 238,659 l 600,279 q 125,-219 190,-279 128,-118 306,-118 357,0 357,325 0,187 -275,350 -107,53 -213,103 -106,50 -215,103 -307,150 -432,300 -159,191 -159,491 0,396 272,656 278,259 675,259 584,0 869,-603 z"
 412 |                  transform="scale(0.015625)" />
 413 |               <path
 414 |                  id="Futura-Medium-74"
 415 |                  d="M 1169,2388 V 0 H 466 V 2388 H 166 v 656 h 300 v 1115 h 703 V 3044 h 547 v -656 z"
 416 |                  transform="scale(0.015625)" />
 417 |               <path
 418 |                  id="Futura-Medium-65"
 419 |                  d="M 3216,1356 H 1034 q 29,-375 244,-597 216,-218 553,-218 263,0 435,125 168,125 384,462 L 3244,797 Q 3106,563 2953,395 2800,228 2625,120 2450,13 2247,-37 2044,-88 1806,-88 1125,-88 713,350 300,791 300,1519 q 0,722 400,1169 403,440 1069,440 672,0 1062,-428 388,-425 388,-1178 z m -722,575 q -147,563 -710,563 -128,0 -241,-39 -112,-39 -204,-113 -92,-73 -158,-176 -65,-103 -100,-235 z"
 420 |                  transform="scale(0.015625)" />
 421 |               <path
 422 |                  id="Futura-Medium-70"
 423 |                  d="M 1147,-1656 H 444 v 4700 h 703 v -331 q 416,415 944,415 628,0 1034,-462 413,-460 413,-1160 0,-684 -410,-1140 -406,-454 -1025,-454 -534,0 -956,429 z m 1672,3159 q 0,438 -238,713 -240,278 -606,278 -387,0 -628,-269 -241,-269 -241,-706 0,-428 241,-706 237,-272 625,-272 366,0 603,275 244,275 244,687 z"
 424 |                  transform="scale(0.015625)" />
 425 |               <path
 426 |                  id="Futura-Medium-67"
 427 |                  d="m 3388,72 q 0,-216 -15,-380 -14,-164 -39,-289 -75,-344 -293,-591 -413,-475 -1135,-475 -609,0 -1003,329 -406,337 -469,934 h 707 q 40,-225 134,-347 219,-284 638,-284 771,0 771,947 V 341 Q 2266,-88 1719,-88 1097,-88 700,363 300,819 300,1503 q 0,666 372,1131 400,494 1056,494 575,0 956,-428 v 344 h 704 z m -675,1444 q 0,443 -238,709 -241,272 -616,272 -400,0 -631,-297 -209,-266 -209,-687 0,-416 209,-682 228,-290 631,-290 404,0 638,293 216,266 216,682 z"
 428 |                  transform="scale(0.015625)" />
 429 |               <path
 430 |                  id="Futura-Medium-61"
 431 |                  d="m 2688,3044 h 706 V 0 H 2688 V 319 Q 2253,-88 1753,-88 1122,-88 709,369 300,834 300,1531 q 0,685 409,1141 410,456 1025,456 532,0 954,-437 z M 1019,1531 q 0,-437 234,-712 241,-278 606,-278 391,0 632,268 240,279 240,707 0,428 -240,706 -241,272 -625,272 -363,0 -607,-275 -240,-278 -240,-688 z"
 432 |                  transform="scale(0.015625)" />
 433 |               <path
 434 |                  id="Futura-Medium-6c"
 435 |                  d="M 1147,5278 V 0 H 444 v 5278 z"
 436 |                  transform="scale(0.015625)" />
 437 |             </defs>
 438 |             <use
 439 |                xlink:href="#Futura-Medium-23"
 440 |                id="use147" />
 441 |             <use
 442 |                xlink:href="#Futura-Medium-20"
 443 |                x="61.71875"
 444 |                id="use149" />
 445 |             <use
 446 |                xlink:href="#Futura-Medium-6f"
 447 |                x="92.578125"
 448 |                id="use151" />
 449 |             <use
 450 |                xlink:href="#Futura-Medium-66"
 451 |                x="152.68555"
 452 |                id="use153" />
 453 |             <use
 454 |                xlink:href="#Futura-Medium-20"
 455 |                x="182.37305"
 456 |                id="use155" />
 457 |             <use
 458 |                xlink:href="#Futura-Medium-73"
 459 |                x="213.23242"
 460 |                id="use157" />
 461 |             <use
 462 |                xlink:href="#Futura-Medium-74"
 463 |                x="253.80859"
 464 |                id="use159" />
 465 |             <use
 466 |                xlink:href="#Futura-Medium-65"
 467 |                x="282.02734"
 468 |                id="use161" />
 469 |             <use
 470 |                xlink:href="#Futura-Medium-70"
 471 |                x="337.34961"
 472 |                id="use163" />
 473 |             <use
 474 |                xlink:href="#Futura-Medium-73"
 475 |                x="397.31055"
 476 |                id="use165" />
 477 |             <use
 478 |                xlink:href="#Futura-Medium-20"
 479 |                x="437.88672"
 480 |                id="use167" />
 481 |             <use
 482 |                xlink:href="#Futura-Medium-74"
 483 |                x="468.74609"
 484 |                id="use169" />
 485 |             <use
 486 |                xlink:href="#Futura-Medium-6f"
 487 |                x="496.71484"
 488 |                id="use171" />
 489 |             <use
 490 |                xlink:href="#Futura-Medium-20"
 491 |                x="556.82227"
 492 |                id="use173" />
 493 |             <use
 494 |                xlink:href="#Futura-Medium-67"
 495 |                x="587.68164"
 496 |                id="use175" />
 497 |             <use
 498 |                xlink:href="#Futura-Medium-6f"
 499 |                x="647.54492"
 500 |                id="use177" />
 501 |             <use
 502 |                xlink:href="#Futura-Medium-61"
 503 |                x="707.65234"
 504 |                id="use179" />
 505 |             <use
 506 |                xlink:href="#Futura-Medium-6c"
 507 |                x="767.61328"
 508 |                id="use181" />
 509 |           </g>
 510 |         </g>
 511 |       </g>
 512 |       <g
 513 |          id="matplotlib.axis_2">
 514 |         <g
 515 |            id="ytick_1">
 516 |           <g
 517 |              id="line2d_11">
 518 |             <path
 519 |                d="M 135,512.64 H 972"
 520 |                clip-path="url(#pa228791ba9)"
 521 |                style="fill:none;stroke:#d3d3d3;stroke-width:0.8;stroke-linecap:square;stroke-opacity:0.4"
 522 |                id="path187" />
 523 |           </g>
 524 |           <g
 525 |              id="text_12">
 526 |             <!-- 0.0 -->
 527 |             <g
 528 |                style="fill:#444444"
 529 |                transform="matrix(0.2,0,0,-0.2,97.140625,520.88688)"
 530 |                id="g200">
 531 |               <defs
 532 |                  id="defs192">
 533 |                 <path
 534 |                    id="Futura-Medium-30"
 535 |                    d="m 272,2444 q 0,1153 537,1840 482,622 1166,622 684,0 1166,-622 537,-687 537,-1871 Q 3678,1231 3141,544 2659,-78 1978,-78 1297,-78 809,544 272,1231 272,2444 Z m 716,-3 q 0,-803 278,-1335 278,-522 709,-522 428,0 709,522 282,519 282,1310 0,790 -282,1312 -281,519 -709,519 -425,0 -706,-519 Q 988,3213 988,2441 Z"
 536 |                    transform="scale(0.015625)" />
 537 |                 <path
 538 |                    id="Futura-Medium-2e"
 539 |                    d="m 553,350 q 0,175 128,303 128,128 307,128 178,0 306,-128 Q 1422,525 1422,347 1422,166 1294,38 1169,-88 988,-88 803,-88 678,37 553,163 553,350 Z"
 540 |                    transform="scale(0.015625)" />
 541 |               </defs>
 542 |               <use
 543 |                  xlink:href="#Futura-Medium-30"
 544 |                  id="use194" />
 545 |               <use
 546 |                  xlink:href="#Futura-Medium-2e"
 547 |                  x="61.71875"
 548 |                  id="use196" />
 549 |               <use
 550 |                  xlink:href="#Futura-Medium-30"
 551 |                  x="92.578125"
 552 |                  id="use198" />
 553 |             </g>
 554 |           </g>
 555 |         </g>
 556 |         <g
 557 |            id="ytick_2">
 558 |           <g
 559 |              id="line2d_12">
 560 |             <path
 561 |                d="M 135,434.93581 H 972"
 562 |                clip-path="url(#pa228791ba9)"
 563 |                style="fill:none;stroke:#d3d3d3;stroke-width:0.8;stroke-linecap:square;stroke-opacity:0.4"
 564 |                id="path204" />
 565 |           </g>
 566 |           <g
 567 |              id="text_13">
 568 |             <!-- 0.1 -->
 569 |             <g
 570 |                style="fill:#444444"
 571 |                transform="matrix(0.2,0,0,-0.2,98.740625,443.18268)"
 572 |                id="g213">
 573 |               <use
 574 |                  xlink:href="#Futura-Medium-30"
 575 |                  id="use207" />
 576 |               <use
 577 |                  xlink:href="#Futura-Medium-2e"
 578 |                  x="61.71875"
 579 |                  id="use209" />
 580 |               <use
 581 |                  xlink:href="#Futura-Medium-31"
 582 |                  x="84.578125"
 583 |                  id="use211" />
 584 |             </g>
 585 |           </g>
 586 |         </g>
 587 |         <g
 588 |            id="ytick_3">
 589 |           <g
 590 |              id="line2d_13">
 591 |             <path
 592 |                d="M 135,357.23161 H 972"
 593 |                clip-path="url(#pa228791ba9)"
 594 |                style="fill:none;stroke:#d3d3d3;stroke-width:0.8;stroke-linecap:square;stroke-opacity:0.4"
 595 |                id="path217" />
 596 |           </g>
 597 |           <g
 598 |              id="text_14">
 599 |             <!-- 0.2 -->
 600 |             <g
 601 |                style="fill:#444444"
 602 |                transform="matrix(0.2,0,0,-0.2,97.140625,365.47849)"
 603 |                id="g226">
 604 |               <use
 605 |                  xlink:href="#Futura-Medium-30"
 606 |                  id="use220" />
 607 |               <use
 608 |                  xlink:href="#Futura-Medium-2e"
 609 |                  x="61.71875"
 610 |                  id="use222" />
 611 |               <use
 612 |                  xlink:href="#Futura-Medium-32"
 613 |                  x="92.578125"
 614 |                  id="use224" />
 615 |             </g>
 616 |           </g>
 617 |         </g>
 618 |         <g
 619 |            id="ytick_4">
 620 |           <g
 621 |              id="line2d_14">
 622 |             <path
 623 |                d="M 135,279.52742 H 972"
 624 |                clip-path="url(#pa228791ba9)"
 625 |                style="fill:none;stroke:#d3d3d3;stroke-width:0.8;stroke-linecap:square;stroke-opacity:0.4"
 626 |                id="path230" />
 627 |           </g>
 628 |           <g
 629 |              id="text_15">
 630 |             <!-- 0.3 -->
 631 |             <g
 632 |                style="fill:#444444"
 633 |                transform="matrix(0.2,0,0,-0.2,97.140625,287.77429)"
 634 |                id="g239">
 635 |               <use
 636 |                  xlink:href="#Futura-Medium-30"
 637 |                  id="use233" />
 638 |               <use
 639 |                  xlink:href="#Futura-Medium-2e"
 640 |                  x="61.71875"
 641 |                  id="use235" />
 642 |               <use
 643 |                  xlink:href="#Futura-Medium-33"
 644 |                  x="92.578125"
 645 |                  id="use237" />
 646 |             </g>
 647 |           </g>
 648 |         </g>
 649 |         <g
 650 |            id="ytick_5">
 651 |           <g
 652 |              id="line2d_15">
 653 |             <path
 654 |                d="M 135,201.82322 H 972"
 655 |                clip-path="url(#pa228791ba9)"
 656 |                style="fill:none;stroke:#d3d3d3;stroke-width:0.8;stroke-linecap:square;stroke-opacity:0.4"
 657 |                id="path243" />
 658 |           </g>
 659 |           <g
 660 |              id="text_16">
 661 |             <!-- 0.4 -->
 662 |             <g
 663 |                style="fill:#444444"
 664 |                transform="matrix(0.2,0,0,-0.2,97.140625,210.0701)"
 665 |                id="g252">
 666 |               <use
 667 |                  xlink:href="#Futura-Medium-30"
 668 |                  id="use246" />
 669 |               <use
 670 |                  xlink:href="#Futura-Medium-2e"
 671 |                  x="61.71875"
 672 |                  id="use248" />
 673 |               <use
 674 |                  xlink:href="#Futura-Medium-34"
 675 |                  x="92.578125"
 676 |                  id="use250" />
 677 |             </g>
 678 |           </g>
 679 |         </g>
 680 |         <g
 681 |            id="ytick_6">
 682 |           <g
 683 |              id="line2d_16">
 684 |             <path
 685 |                d="M 135,124.11903 H 972"
 686 |                clip-path="url(#pa228791ba9)"
 687 |                style="fill:none;stroke:#d3d3d3;stroke-width:0.8;stroke-linecap:square;stroke-opacity:0.4"
 688 |                id="path256" />
 689 |           </g>
 690 |           <g
 691 |              id="text_17">
 692 |             <!-- 0.5 -->
 693 |             <g
 694 |                style="fill:#444444"
 695 |                transform="matrix(0.2,0,0,-0.2,97.140625,132.3659)"
 696 |                id="g265">
 697 |               <use
 698 |                  xlink:href="#Futura-Medium-30"
 699 |                  id="use259" />
 700 |               <use
 701 |                  xlink:href="#Futura-Medium-2e"
 702 |                  x="61.71875"
 703 |                  id="use261" />
 704 |               <use
 705 |                  xlink:href="#Futura-Medium-35"
 706 |                  x="92.578125"
 707 |                  id="use263" />
 708 |             </g>
 709 |           </g>
 710 |         </g>
 711 |         <g
 712 |            id="text_18">
 713 |           <!-- proportion of paths -->
 714 |           <g
 715 |              style="fill:#444444"
 716 |              transform="matrix(0,-0.22,-0.22,0,71.448125,387.80203)"
 717 |              id="g313">
 718 |             <defs
 719 |                id="defs273">
 720 |               <path
 721 |                  id="Futura-Medium-72"
 722 |                  d="m 444,3044 h 703 v -272 q 194,203 344,278 153,78 362,78 278,0 581,-181 l -321,-644 q -200,144 -391,144 -575,0 -575,-869 V 0 H 444 Z"
 723 |                  transform="scale(0.015625)" />
 724 |               <path
 725 |                  id="Futura-Medium-69"
 726 |                  d="M 1147,3044 V 0 H 444 V 3044 Z M 338,4309 q 0,185 134,319 134,135 322,135 190,0 325,-135 134,-131 134,-322 0,-190 -134,-325 -131,-134 -322,-134 -191,0 -325,134 -134,135 -134,328 z"
 727 |                  transform="scale(0.015625)" />
 728 |               <path
 729 |                  id="Futura-Medium-6e"
 730 |                  d="m 444,3044 h 706 v -281 q 369,365 831,365 532,0 828,-334 257,-285 257,-928 V 0 h -707 v 1700 q 0,450 -125,622 -121,175 -443,175 -350,0 -497,-231 -144,-228 -144,-797 V 0 H 444 Z"
 731 |                  transform="scale(0.015625)" />
 732 |               <path
 733 |                  id="Futura-Medium-68"
 734 |                  d="m 444,5278 h 703 V 2791 q 375,337 828,337 516,0 825,-334 263,-291 263,-928 V 0 h -704 v 1800 q 0,366 -130,530 -129,164 -416,164 -369,0 -519,-228 -147,-232 -147,-794 V 0 H 444 Z"
 735 |                  transform="scale(0.015625)" />
 736 |             </defs>
 737 |             <use
 738 |                xlink:href="#Futura-Medium-70"
 739 |                id="use275" />
 740 |             <use
 741 |                xlink:href="#Futura-Medium-72"
 742 |                x="59.960938"
 743 |                id="use277" />
 744 |             <use
 745 |                xlink:href="#Futura-Medium-6f"
 746 |                x="96.488281"
 747 |                id="use279" />
 748 |             <use
 749 |                xlink:href="#Futura-Medium-70"
 750 |                x="156.5957"
 751 |                id="use281" />
 752 |             <use
 753 |                xlink:href="#Futura-Medium-6f"
 754 |                x="216.55664"
 755 |                id="use283" />
 756 |             <use
 757 |                xlink:href="#Futura-Medium-72"
 758 |                x="276.66406"
 759 |                id="use285" />
 760 |             <use
 761 |                xlink:href="#Futura-Medium-74"
 762 |                x="317.69141"
 763 |                id="use287" />
 764 |             <use
 765 |                xlink:href="#Futura-Medium-69"
 766 |                x="345.03516"
 767 |                id="use289" />
 768 |             <use
 769 |                xlink:href="#Futura-Medium-6f"
 770 |                x="369.9375"
 771 |                id="use291" />
 772 |             <use
 773 |                xlink:href="#Futura-Medium-6e"
 774 |                x="430.04492"
 775 |                id="use293" />
 776 |             <use
 777 |                xlink:href="#Futura-Medium-20"
 778 |                x="484.87891"
 779 |                id="use295" />
 780 |             <use
 781 |                xlink:href="#Futura-Medium-6f"
 782 |                x="515.73828"
 783 |                id="use297" />
 784 |             <use
 785 |                xlink:href="#Futura-Medium-66"
 786 |                x="575.8457"
 787 |                id="use299" />
 788 |             <use
 789 |                xlink:href="#Futura-Medium-20"
 790 |                x="605.5332"
 791 |                id="use301" />
 792 |             <use
 793 |                xlink:href="#Futura-Medium-70"
 794 |                x="636.39258"
 795 |                id="use303" />
 796 |             <use
 797 |                xlink:href="#Futura-Medium-61"
 798 |                x="696.35352"
 799 |                id="use305" />
 800 |             <use
 801 |                xlink:href="#Futura-Medium-74"
 802 |                x="756.31445"
 803 |                id="use307" />
 804 |             <use
 805 |                xlink:href="#Futura-Medium-68"
 806 |                x="785.6582"
 807 |                id="use309" />
 808 |             <use
 809 |                xlink:href="#Futura-Medium-73"
 810 |                x="840.49219"
 811 |                id="use311" />
 812 |           </g>
 813 |         </g>
 814 |       </g>
 815 |       <g
 816 |          id="patch_3">
 817 |         <path
 818 |            d="m 210.16297,512.64 h 27.83814 v -54.93687 h -27.83814 z"
 819 |            clip-path="url(#pa228791ba9)"
 820 |            style="fill:#d3d3d3"
 821 |            id="path317" />
 822 |       </g>
 823 |       <g
 824 |          id="patch_4">
 825 |         <path
 826 |            d="m 287.49113,512.64 h 27.83814 v -63.40662 h -27.83814 z"
 827 |            clip-path="url(#pa228791ba9)"
 828 |            style="fill:#d3d3d3"
 829 |            id="path320" />
 830 |       </g>
 831 |       <g
 832 |          id="patch_5">
 833 |         <path
 834 |            d="m 364.81929,512.64 h 27.83814 v -51.75099 h -27.83814 z"
 835 |            clip-path="url(#pa228791ba9)"
 836 |            style="fill:#d3d3d3"
 837 |            id="path323" />
 838 |       </g>
 839 |       <g
 840 |          id="patch_6">
 841 |         <path
 842 |            d="m 442.14745,512.64 h 27.83814 v -44.67991 h -27.83814 z"
 843 |            clip-path="url(#pa228791ba9)"
 844 |            style="fill:#d3d3d3"
 845 |            id="path326" />
 846 |       </g>
 847 |       <g
 848 |          id="patch_7">
 849 |         <path
 850 |            d="m 519.47561,512.64 h 27.83814 v -38.07505 h -27.83814 z"
 851 |            clip-path="url(#pa228791ba9)"
 852 |            style="fill:#d3d3d3"
 853 |            id="path329" />
 854 |       </g>
 855 |       <g
 856 |          id="patch_8">
 857 |         <path
 858 |            d="m 596.80377,512.64 h 27.83814 v -30.53775 h -27.83814 z"
 859 |            clip-path="url(#pa228791ba9)"
 860 |            style="fill:#d3d3d3"
 861 |            id="path332" />
 862 |       </g>
 863 |       <g
 864 |          id="patch_9">
 865 |         <path
 866 |            d="m 674.13193,512.64 h 27.83814 v -24.94305 h -27.83814 z"
 867 |            clip-path="url(#pa228791ba9)"
 868 |            style="fill:#d3d3d3"
 869 |            id="path335" />
 870 |       </g>
 871 |       <g
 872 |          id="patch_10">
 873 |         <path
 874 |            d="m 751.46009,512.64 h 27.83814 v -25.40927 h -27.83814 z"
 875 |            clip-path="url(#pa228791ba9)"
 876 |            style="fill:#d3d3d3"
 877 |            id="path338" />
 878 |       </g>
 879 |       <g
 880 |          id="patch_11">
 881 |         <path
 882 |            d="m 828.78825,512.64 h 27.83814 v -20.90243 h -27.83814 z"
 883 |            clip-path="url(#pa228791ba9)"
 884 |            style="fill:#d3d3d3"
 885 |            id="path341" />
 886 |       </g>
 887 |       <g
 888 |          id="patch_12">
 889 |         <path
 890 |            d="m 906.11641,512.64 h 27.83814 V 90.24 h -27.83814 z"
 891 |            clip-path="url(#pa228791ba9)"
 892 |            style="fill:#d3d3d3"
 893 |            id="path344" />
 894 |       </g>
 895 |       <g
 896 |          id="patch_13">
 897 |         <path
 898 |            d="m 173.04545,512.64 h 27.83814 v -98.15267 h -27.83814 z"
 899 |            clip-path="url(#pa228791ba9)"
 900 |            style="fill:#f2ad48"
 901 |            id="path347" />
 902 |       </g>
 903 |       <g
 904 |          id="patch_14">
 905 |         <path
 906 |            d="m 250.37361,512.64 h 27.83814 V 349.05222 h -27.83814 z"
 907 |            clip-path="url(#pa228791ba9)"
 908 |            style="fill:#f2ad48"
 909 |            id="path350" />
 910 |       </g>
 911 |       <g
 912 |          id="patch_15">
 913 |         <path
 914 |            d="m 327.70177,512.64 h 27.83814 V 328.60375 h -27.83814 z"
 915 |            clip-path="url(#pa228791ba9)"
 916 |            style="fill:#f2ad48"
 917 |            id="path353" />
 918 |       </g>
 919 |       <g
 920 |          id="patch_16">
 921 |         <path
 922 |            d="m 405.02993,512.64 h 27.83814 V 377.68008 h -27.83814 z"
 923 |            clip-path="url(#pa228791ba9)"
 924 |            style="fill:#f2ad48"
 925 |            id="path356" />
 926 |       </g>
 927 |       <g
 928 |          id="patch_17">
 929 |         <path
 930 |            d="m 482.35809,512.64 h 27.83814 V 406.30795 h -27.83814 z"
 931 |            clip-path="url(#pa228791ba9)"
 932 |            style="fill:#f2ad48"
 933 |            id="path359" />
 934 |       </g>
 935 |       <g
 936 |          id="patch_18">
 937 |         <path
 938 |            d="m 559.68625,512.64 h 27.83814 v -61.34542 h -27.83814 z"
 939 |            clip-path="url(#pa228791ba9)"
 940 |            style="fill:#f2ad48"
 941 |            id="path362" />
 942 |       </g>
 943 |       <g
 944 |          id="patch_19">
 945 |         <path
 946 |            d="m 637.01441,512.64 h 27.83814 v -16.35878 h -27.83814 z"
 947 |            clip-path="url(#pa228791ba9)"
 948 |            style="fill:#f2ad48"
 949 |            id="path365" />
 950 |       </g>
 951 |       <g
 952 |          id="patch_20">
 953 |         <path
 954 |            d="m 714.34257,512.64 h 27.83814 v 0 z"
 955 |            clip-path="url(#pa228791ba9)"
 956 |            style="fill:#f2ad48"
 957 |            id="path368" />
 958 |       </g>
 959 |       <g
 960 |          id="patch_21">
 961 |         <path
 962 |            d="m 791.67073,512.64 h 27.83814 v 0 z"
 963 |            clip-path="url(#pa228791ba9)"
 964 |            style="fill:#f2ad48"
 965 |            id="path371" />
 966 |       </g>
 967 |       <g
 968 |          id="patch_22">
 969 |         <path
 970 |            d="m 868.99889,512.64 h 27.83814 v -12.26908 h -27.83814 z"
 971 |            clip-path="url(#pa228791ba9)"
 972 |            style="fill:#f2ad48"
 973 |            id="path374" />
 974 |       </g>
 975 |       <g
 976 |          id="patch_23">
 977 |         <path
 978 |            d="M 135,512.64 V 69.12"
 979 |            style="fill:none;stroke:#ffffff;stroke-linecap:square;stroke-linejoin:miter"
 980 |            id="path377" />
 981 |       </g>
 982 |       <g
 983 |          id="patch_24">
 984 |         <path
 985 |            d="M 972,512.64 V 69.12"
 986 |            style="fill:none;stroke:#ffffff;stroke-linecap:square;stroke-linejoin:miter"
 987 |            id="path380" />
 988 |       </g>
 989 |       <g
 990 |          id="patch_25">
 991 |         <path
 992 |            d="M 135,512.64 H 972"
 993 |            style="fill:none;stroke:#ffffff;stroke-linecap:square;stroke-linejoin:miter"
 994 |            id="path383" />
 995 |       </g>
 996 |       <g
 997 |          id="patch_26">
 998 |         <path
 999 |            d="M 135,69.12 H 972"
1000 |            style="fill:none;stroke:#ffffff;stroke-linecap:square;stroke-linejoin:miter"
1001 |            id="path386" />
1002 |       </g>
1003 |       <g
1004 |          id="patch_27">
1005 |         <path
1006 |            d="m 191.60421,512.64 h 27.83814 v -94.06297 h -27.83814 z"
1007 |            clip-path="url(#pa228791ba9)"
1008 |            style="fill:#99a3fd"
1009 |            id="path389" />
1010 |       </g>
1011 |       <g
1012 |          id="patch_28">
1013 |         <path
1014 |            d="m 268.93237,512.64 h 27.83814 V 369.50069 h -27.83814 z"
1015 |            clip-path="url(#pa228791ba9)"
1016 |            style="fill:#99a3fd"
1017 |            id="path392" />
1018 |       </g>
1019 |       <g
1020 |          id="patch_29">
1021 |         <path
1022 |            d="m 346.26053,512.64 h 27.83814 V 349.05222 h -27.83814 z"
1023 |            clip-path="url(#pa228791ba9)"
1024 |            style="fill:#99a3fd"
1025 |            id="path395" />
1026 |       </g>
1027 |       <g
1028 |          id="patch_30">
1029 |         <path
1030 |            d="m 423.58869,512.64 h 27.83814 V 377.68008 h -27.83814 z"
1031 |            clip-path="url(#pa228791ba9)"
1032 |            style="fill:#99a3fd"
1033 |            id="path398" />
1034 |       </g>
1035 |       <g
1036 |          id="patch_31">
1037 |         <path
1038 |            d="m 500.91685,512.64 h 27.83814 V 398.12856 h -27.83814 z"
1039 |            clip-path="url(#pa228791ba9)"
1040 |            style="fill:#99a3fd"
1041 |            id="path401" />
1042 |       </g>
1043 |       <g
1044 |          id="patch_32">
1045 |         <path
1046 |            d="m 578.24501,512.64 h 27.83814 v -57.25572 h -27.83814 z"
1047 |            clip-path="url(#pa228791ba9)"
1048 |            style="fill:#99a3fd"
1049 |            id="path404" />
1050 |       </g>
1051 |       <g
1052 |          id="patch_33">
1053 |         <path
1054 |            d="m 655.57317,512.64 h 27.83814 v -24.53817 h -27.83814 z"
1055 |            clip-path="url(#pa228791ba9)"
1056 |            style="fill:#99a3fd"
1057 |            id="path407" />
1058 |       </g>
1059 |       <g
1060 |          id="patch_34">
1061 |         <path
1062 |            d="m 732.90133,512.64 h 27.83814 v -4.08969 h -27.83814 z"
1063 |            clip-path="url(#pa228791ba9)"
1064 |            style="fill:#99a3fd"
1065 |            id="path410" />
1066 |       </g>
1067 |       <g
1068 |          id="patch_35">
1069 |         <path
1070 |            d="m 810.22949,512.64 h 27.83814 v 0 z"
1071 |            clip-path="url(#pa228791ba9)"
1072 |            style="fill:#99a3fd"
1073 |            id="path413" />
1074 |       </g>
1075 |       <g
1076 |          id="patch_36">
1077 |         <path
1078 |            d="m 887.55765,512.64 h 27.83814 v -40.89694 h -27.83814 z"
1079 |            clip-path="url(#pa228791ba9)"
1080 |            style="fill:#99a3fd"
1081 |            id="path416" />
1082 |       </g>
1083 |       <g
1084 |          id="legend_1">
1085 |         <g
1086 |            id="patch_37">
1087 |           <path
1088 |              d="m 146.2,157.925 h 145.9325 q 3.2,0 3.2,-3.2 V 80.32 q 0,-3.2 -3.2,-3.2 H 146.2 q -3.2,0 -3.2,3.2 v 74.405 q 0,3.2 3.2,3.2 z"
1089 |              style="opacity:0.8;fill:#e5e5e5;stroke:#cccccc;stroke-width:0.5;stroke-linejoin:miter"
1090 |              id="path419" />
1091 |         </g>
1092 |         <g
1093 |            id="patch_38">
1094 |           <path
1095 |              d="m 149.4,96.715 h 32 v -11.2 h -32 z"
1096 |              style="fill:#f2ad48"
1097 |              id="path422" />
1098 |         </g>
1099 |         <g
1100 |            id="text_19">
1101 |           <!-- shortest path -->
1102 |           <g
1103 |              style="fill:#444444"
1104 |              transform="matrix(0.16,0,0,-0.16,194.2,96.715)"
1105 |              id="g451">
1106 |             <use
1107 |                xlink:href="#Futura-Medium-73"
1108 |                id="use425" />
1109 |             <use
1110 |                xlink:href="#Futura-Medium-68"
1111 |                x="40.576172"
1112 |                id="use427" />
1113 |             <use
1114 |                xlink:href="#Futura-Medium-6f"
1115 |                x="95.410156"
1116 |                id="use429" />
1117 |             <use
1118 |                xlink:href="#Futura-Medium-72"
1119 |                x="155.51758"
1120 |                id="use431" />
1121 |             <use
1122 |                xlink:href="#Futura-Medium-74"
1123 |                x="196.54492"
1124 |                id="use433" />
1125 |             <use
1126 |                xlink:href="#Futura-Medium-65"
1127 |                x="224.76367"
1128 |                id="use435" />
1129 |             <use
1130 |                xlink:href="#Futura-Medium-73"
1131 |                x="280.08594"
1132 |                id="use437" />
1133 |             <use
1134 |                xlink:href="#Futura-Medium-74"
1135 |                x="320.66211"
1136 |                id="use439" />
1137 |             <use
1138 |                xlink:href="#Futura-Medium-20"
1139 |                x="348.00586"
1140 |                id="use441" />
1141 |             <use
1142 |                xlink:href="#Futura-Medium-70"
1143 |                x="378.86523"
1144 |                id="use443" />
1145 |             <use
1146 |                xlink:href="#Futura-Medium-61"
1147 |                x="438.82617"
1148 |                id="use445" />
1149 |             <use
1150 |                xlink:href="#Futura-Medium-74"
1151 |                x="498.78711"
1152 |                id="use447" />
1153 |             <use
1154 |                xlink:href="#Futura-Medium-68"
1155 |                x="528.13086"
1156 |                id="use449" />
1157 |           </g>
1158 |         </g>
1159 |         <g
1160 |            id="patch_39">
1161 |           <path
1162 |              d="m 149.4,122.05 h 32 v -11.2 h -32 z"
1163 |              style="fill:#99a3fd"
1164 |              id="path454" />
1165 |         </g>
1166 |         <g
1167 |            id="text_20">
1168 |           <!-- ILQL -->
1169 |           <g
1170 |              style="fill:#444444"
1171 |              transform="matrix(0.16,0,0,-0.16,194.2,122.05)"
1172 |              id="g470">
1173 |             <defs
1174 |                id="defs460">
1175 |               <path
1176 |                  id="Futura-Medium-49"
1177 |                  d="M 1306,4825 V 0 H 578 v 4825 z"
1178 |                  transform="scale(0.015625)" />
1179 |               <path
1180 |                  id="Futura-Medium-4c"
1181 |                  d="M 1306,4825 V 684 H 2725 V 0 H 578 v 4825 z"
1182 |                  transform="scale(0.015625)" />
1183 |               <path
1184 |                  id="Futura-Medium-51"
1185 |                  d="M 5488,-88 H 4572 L 4194,300 Q 3591,-88 2850,-88 q -959,0 -1703,651 -809,715 -809,1828 0,1090 765,1818 744,707 1756,707 919,0 1647,-619 875,-744 875,-1941 0,-937 -690,-1665 z m -2004,1997 700,-696 q 463,481 463,1190 0,778 -516,1303 -515,525 -1281,525 -775,0 -1275,-525 -503,-522 -503,-1322 0,-781 519,-1278 540,-512 1234,-512 472,0 853,215 L 2544,1909 Z"
1186 |                  transform="scale(0.015625)" />
1187 |             </defs>
1188 |             <use
1189 |                xlink:href="#Futura-Medium-49"
1190 |                id="use462" />
1191 |             <use
1192 |                xlink:href="#Futura-Medium-4c"
1193 |                x="29.492188"
1194 |                id="use464" />
1195 |             <use
1196 |                xlink:href="#Futura-Medium-51"
1197 |                x="73.828125"
1198 |                id="use466" />
1199 |             <use
1200 |                xlink:href="#Futura-Medium-4c"
1201 |                x="163.18359"
1202 |                id="use468" />
1203 |           </g>
1204 |         </g>
1205 |         <g
1206 |            id="patch_40">
1207 |           <path
1208 |              d="m 149.4,147.385 h 32 v -11.2 h -32 z"
1209 |              style="fill:#d3d3d3"
1210 |              id="path473" />
1211 |         </g>
1212 |         <g
1213 |            id="text_21">
1214 |           <!-- random walk -->
1215 |           <g
1216 |              style="fill:#444444"
1217 |              transform="matrix(0.16,0,0,-0.16,194.2,147.385)"
1218 |              id="g504">
1219 |             <defs
1220 |                id="defs480">
1221 |               <path
1222 |                  id="Futura-Medium-64"
1223 |                  d="m 2688,5278 h 706 V 0 H 2688 V 319 Q 2272,-88 1747,-88 1122,-88 709,369 300,834 300,1531 q 0,682 409,1138 407,459 1022,459 535,0 957,-437 z M 1019,1531 q 0,-437 234,-712 241,-278 606,-278 391,0 632,268 240,279 240,707 0,428 -240,706 -241,272 -625,272 -363,0 -607,-275 -240,-278 -240,-688 z"
1224 |                  transform="scale(0.015625)" />
1225 |               <path
1226 |                  id="Futura-Medium-6d"
1227 |                  d="m 444,3044 h 703 v -281 q 203,212 347,287 153,78 384,78 516,0 816,-450 331,450 897,450 1028,0 1028,-1247 V 0 h -706 v 1691 q 0,437 -107,618 -109,185 -359,185 -291,0 -424,-219 -132,-219 -132,-703 V 0 h -707 v 1681 q 0,813 -468,813 -297,0 -433,-222 -136,-222 -136,-700 V 0 H 444 Z"
1228 |                  transform="scale(0.015625)" />
1229 |               <path
1230 |                  id="Futura-Medium-77"
1231 |                  d="m 825,3044 797,-1738 859,2016 866,-2016 787,1738 h 788 L 3328,-228 2481,1731 1625,-228 34,3044 Z"
1232 |                  transform="scale(0.015625)" />
1233 |               <path
1234 |                  id="Futura-Medium-6b"
1235 |                  d="M 1147,5278 V 2059 l 991,985 h 943 L 1759,1766 3178,0 H 2266 L 1259,1284 1147,1172 V 0 H 444 v 5278 z"
1236 |                  transform="scale(0.015625)" />
1237 |             </defs>
1238 |             <use
1239 |                xlink:href="#Futura-Medium-72"
1240 |                id="use482" />
1241 |             <use
1242 |                xlink:href="#Futura-Medium-61"
1243 |                x="36.777344"
1244 |                id="use484" />
1245 |             <use
1246 |                xlink:href="#Futura-Medium-6e"
1247 |                x="96.738281"
1248 |                id="use486" />
1249 |             <use
1250 |                xlink:href="#Futura-Medium-64"
1251 |                x="151.57227"
1252 |                id="use488" />
1253 |             <use
1254 |                xlink:href="#Futura-Medium-6f"
1255 |                x="211.5332"
1256 |                id="use490" />
1257 |             <use
1258 |                xlink:href="#Futura-Medium-6d"
1259 |                x="271.64062"
1260 |                id="use492" />
1261 |             <use
1262 |                xlink:href="#Futura-Medium-20"
1263 |                x="350.74219"
1264 |                id="use494" />
1265 |             <use
1266 |                xlink:href="#Futura-Medium-77"
1267 |                x="381.60156"
1268 |                id="use496" />
1269 |             <use
1270 |                xlink:href="#Futura-Medium-61"
1271 |                x="457.36914"
1272 |                id="use498" />
1273 |             <use
1274 |                xlink:href="#Futura-Medium-6c"
1275 |                x="517.33008"
1276 |                id="use500" />
1277 |             <use
1278 |                xlink:href="#Futura-Medium-6b"
1279 |                x="542.23242"
1280 |                id="use502" />
1281 |           </g>
1282 |         </g>
1283 |       </g>
1284 |     </g>
1285 |   </g>
1286 |   <defs
1287 |      id="defs513">
1288 |     <clipPath
1289 |        id="pa228791ba9">
1290 |       <rect
1291 |          x="135"
1292 |          y="69.120003"
1293 |          width="837"
1294 |          height="443.51999"
1295 |          id="rect510" />
1296 |     </clipPath>
1297 |   </defs>
1298 | </svg>
1299 | 


--------------------------------------------------------------------------------