<\/div>/, `
${renderedComponent}
`);
31 | const gaId = process.env.GA_ID;
32 | console.warn('Google analytics ID is', gaId);
33 | if (gaId) {
34 | let GA_SCRIPT = `
35 |
36 | `;
43 | GA_SCRIPT = GA_SCRIPT.replace(/__GA_CODE_HERE__/g, gaId);
44 | fullHtml = fullHtml.replace('', `${GA_SCRIPT}`);
45 | }
46 | process.stdout.write(fullHtml);
47 | });
48 |
--------------------------------------------------------------------------------
/python_code/hash_chapter1_reimplementation_test.py:
--------------------------------------------------------------------------------
1 | import random
2 | import argparse
3 |
4 | import hash_chapter1_reimpl_js
5 | import hash_chapter1_impl
6 | import build_autogenerated_chapter1_hash
7 |
8 |
9 | def get_implementation(is_broken, impl):
10 | if impl == "js":
11 | module = hash_chapter1_reimpl_js
12 | elif impl == "py_ref":
13 | module = hash_chapter1_impl
14 | elif impl == "py_extracted":
15 | module = build_autogenerated_chapter1_hash
16 | else:
17 | assert False
18 |
19 | return (module.create_new_broken if is_broken else module.create_new, module.has_key)
20 |
21 |
22 | def run(test_implementation, is_broken, n_inserts):
23 | MAX_VAL = 5000
24 | ref_create_new, ref_has_key = get_implementation(is_broken, "py_ref")
25 | test_create_new, test_has_key = get_implementation(is_broken, test_implementation)
26 |
27 | numbers = list(set(random.randint(-MAX_VAL, MAX_VAL) for _ in range(n_inserts)))
28 |
29 | ref_keys = ref_create_new(numbers)
30 | test_keys = test_create_new(numbers)
31 |
32 | for number in numbers:
33 | if not is_broken:
34 | assert ref_has_key(ref_keys, number)
35 | assert test_has_key(test_keys, number)
36 | else:
37 | assert ref_has_key(ref_keys, number) == test_has_key(test_keys, number)
38 |
39 | for i in range(n_inserts * 3):
40 | number = random.randint(-MAX_VAL, MAX_VAL)
41 | assert ref_has_key(ref_keys, number) == test_has_key(test_keys, number)
42 |
43 |
44 | if __name__ == "__main__":
45 | parser = argparse.ArgumentParser(description='Stress-test chapter1 reimplementation')
46 | parser.add_argument('--is-broken', action='store_true')
47 | parser.add_argument('--test-implementation', choices=['py_extracted', 'js'], required=True)
48 | parser.add_argument('--num-inserts', type=int, default=500)
49 | args = parser.parse_args()
50 |
51 | run(test_implementation=args.test_implementation,
52 | is_broken=args.is_broken,
53 | n_inserts=args.num_inserts)
54 |
--------------------------------------------------------------------------------
/python_code/dictinfo33.py:
--------------------------------------------------------------------------------
1 | from ctypes import Structure, c_ulong, POINTER, cast, addressof, py_object, c_long, c_void_p
2 | from common import get_object_field_or_null, EMPTY, DUMMY
3 |
4 |
5 | class PyDictKeyEntry(Structure):
6 | _fields_ = [
7 | ('me_hash', c_long),
8 | ('me_key', py_object),
9 | ('me_value', py_object),
10 | ]
11 |
12 |
13 | class PyDictKeysObject(Structure):
14 | _fields_ = [
15 | ('dk_refcnt', c_long),
16 | ('dk_size', c_long),
17 | ('dict_lookup_func', POINTER(c_void_p)),
18 | ('dk_usable', c_long),
19 | ('dk_entries', PyDictKeyEntry),
20 | ]
21 |
22 |
23 | class PyDictObject(Structure):
24 | _fields_ = [
25 | ('ob_refcnt', c_ulong),
26 | ('ob_type', c_ulong),
27 | ('ma_used', c_long),
28 | ('ma_keys', POINTER(PyDictKeysObject)),
29 |
30 | # Not actually a void*, split tables are not supported right now
31 | ('ma_values', POINTER(c_void_p))
32 | ]
33 |
34 |
35 | def dictobject(d):
36 | return cast(id(d), POINTER(PyDictObject)).contents
37 |
38 |
39 | d = {0: 0}
40 | del d[0]
41 | dummy_internal = dictobject(d).ma_keys.contents.dk_entries.me_key
42 | del d
43 |
44 |
45 | def usable_fraction(size):
46 | return (size * 2 + 1) // 3
47 |
48 |
49 | def dump_py_dict(d):
50 | do = dictobject(d)
51 |
52 | keys = []
53 | hashes = []
54 | values = []
55 |
56 | size = do.ma_keys.contents.dk_size
57 | entries = cast(addressof(do.ma_keys.contents.dk_entries), POINTER(PyDictKeyEntry))
58 | for i in range(size):
59 | key = get_object_field_or_null(entries[i], 'me_key')
60 | keys.append(key if key is not dummy_internal else DUMMY)
61 |
62 | for i, key in enumerate(keys):
63 | if key is EMPTY:
64 | hashes.append(EMPTY)
65 | values.append(EMPTY)
66 | else:
67 | hashes.append(entries[i].me_hash)
68 | values.append(get_object_field_or_null(entries[i], 'me_value'))
69 |
70 | return hashes, keys, values, usable_fraction(do.ma_keys.contents.dk_size) - do.ma_keys.contents.dk_usable, do.ma_used
71 |
--------------------------------------------------------------------------------
/python_code/hash_chapter2_impl.py:
--------------------------------------------------------------------------------
1 | from common import DUMMY, EMPTY
2 |
3 |
4 | def create_new(from_keys):
5 | n = len(from_keys)
6 | hash_codes = [EMPTY for i in range(2 * n)]
7 | keys = [EMPTY for i in range(2 * n)]
8 |
9 | for key in from_keys:
10 | hash_code = hash(key)
11 | idx = hash_code % len(keys)
12 |
13 | while keys[idx] is not EMPTY:
14 | if hash_codes[idx] == hash_code and keys[idx] == key:
15 | break
16 | idx = (idx + 1) % len(keys)
17 |
18 | hash_codes[idx] = hash_code
19 | keys[idx] = key
20 |
21 | return hash_codes, keys
22 |
23 |
24 | def insert(hash_codes, keys, key):
25 | hash_code = hash(key)
26 | idx = hash_code % len(keys)
27 |
28 | while hash_codes[idx] is not EMPTY:
29 | if hash_codes[idx] == hash_code and keys[idx] == key:
30 | return
31 | idx = (idx + 1) % len(keys)
32 |
33 | hash_codes[idx] = hash_code
34 | keys[idx] = key
35 |
36 |
37 | def remove(hash_codes, keys, key):
38 | hash_code = hash(key)
39 | idx = hash_code % len(keys)
40 |
41 | while hash_codes[idx] is not EMPTY:
42 | if hash_codes[idx] == hash_code and keys[idx] == key:
43 | keys[idx] = DUMMY
44 | return
45 | idx = (idx + 1) % len(keys)
46 |
47 | raise KeyError()
48 |
49 |
50 | def has_key(hash_codes, keys, key):
51 | hash_code = hash(key)
52 | idx = hash_code % len(keys)
53 | while hash_codes[idx] is not EMPTY:
54 | if hash_codes[idx] == hash_code and keys[idx] == key:
55 | return True
56 | idx = (idx + 1) % len(keys)
57 | return False
58 |
59 |
60 | def resize(hash_codes, keys):
61 | new_hash_codes = [EMPTY for i in range(len(hash_codes) * 2)]
62 | new_keys = [EMPTY for i in range(len(keys) * 2)]
63 | for hash_code, key in zip(hash_codes, keys):
64 | if key is EMPTY or key is DUMMY:
65 | continue
66 | idx = hash_code % len(new_keys)
67 | while new_hash_codes[idx] is not EMPTY:
68 | idx = (idx + 1) % len(new_keys)
69 | new_hash_codes[idx] = hash_code
70 | new_keys[idx] = key
71 |
72 | return new_hash_codes, new_keys
73 |
--------------------------------------------------------------------------------
/patches/python32_debug.diff:
--------------------------------------------------------------------------------
1 | diff --git a/Objects/dictobject.c b/Objects/dictobject.c
2 | index c10bfccdce..3734a08281 100644
3 | --- a/Objects/dictobject.c
4 | +++ b/Objects/dictobject.c
5 | @@ -321,6 +321,8 @@ lookdict(PyDictObject *mp, PyObject *key, register Py_hash_t hash)
6 | PyObject *startkey;
7 |
8 | i = (size_t)hash & mask;
9 | + fprintf(stderr, "lookdict hash = %ld\n", hash);
10 | + fprintf(stderr, "initial i = %zu\n", i);
11 | ep = &ep0[i];
12 | if (ep->me_key == NULL || ep->me_key == key)
13 | return ep;
14 | @@ -355,7 +357,9 @@ lookdict(PyDictObject *mp, PyObject *key, register Py_hash_t hash)
15 | least likely outcome, so test for that last. */
16 | for (perturb = hash; ; perturb >>= PERTURB_SHIFT) {
17 | i = (i << 2) + i + perturb + 1;
18 | + fprintf(stderr, "next i = %zu perturb = %zu\n", i, perturb);
19 | ep = &ep0[i & mask];
20 | + fprintf(stderr, "next i & mask = %zu perturb = %zu\n", i & mask, perturb);
21 | if (ep->me_key == NULL)
22 | return freeslot == NULL ? ep : freeslot;
23 | if (ep->me_key == key)
24 | @@ -648,6 +652,7 @@ dictresize(PyDictObject *mp, Py_ssize_t minused)
25 | }
26 | }
27 | else {
28 | + fprintf(stderr, "PyMem_NEW branch");
29 | newtable = PyMem_NEW(PyDictEntry, newsize);
30 | if (newtable == NULL) {
31 | PyErr_NoMemory();
32 | @@ -693,6 +698,7 @@ PyObject *
33 | _PyDict_NewPresized(Py_ssize_t minused)
34 | {
35 | PyObject *op = PyDict_New();
36 | + fprintf(stderr, "_PyDict_NewPresized() %p %d\n", op, (int)minused);
37 |
38 | if (minused>5 && op != NULL && dictresize((PyDictObject *)op, minused) == -1) {
39 | Py_DECREF(op);
40 | diff --git a/Objects/longobject.c b/Objects/longobject.c
41 | index e2a4ef9c5e..7d72c88417 100644
42 | --- a/Objects/longobject.c
43 | +++ b/Objects/longobject.c
44 | @@ -2611,6 +2611,7 @@ long_hash(PyLongObject *v)
45 | sign = -1;
46 | i = -(i);
47 | }
48 | + fprintf(stderr, "i = %ld\n", i);
49 | while (--i >= 0) {
50 | /* Here x is a quantity in the range [0, _PyHASH_MODULUS); we
51 | want to compute x * 2**PyLong_SHIFT + v->ob_digit[i] modulo
52 |
--------------------------------------------------------------------------------
/webpack.common.js:
--------------------------------------------------------------------------------
1 | const webpack = require('webpack');
2 | const path = require('path');
3 | const MiniCssExtractPlugin = require('mini-css-extract-plugin');
4 | const CleanWebpackPlugin = require('clean-webpack-plugin');
5 | const CopyWebpackPlugin = require('copy-webpack-plugin');
6 | const {RawSource} = require('webpack-sources');
7 | const {exec} = require('child_process');
8 | const fs = require('fs');
9 |
10 | module.exports = {
11 | entry: './src/index.js',
12 | output: {
13 | filename: '[name].[contenthash].js',
14 | path: path.resolve(__dirname, 'dist'),
15 | },
16 | module: {
17 | rules: [
18 | {
19 | test: /\.js$/,
20 | use: {
21 | loader: 'babel-loader',
22 | /* presets come from .babelrc */
23 | options: {
24 | plugins: ['lodash'],
25 | },
26 | },
27 | },
28 | {
29 | test: /\.(png|jpg|gif|svg)$/i,
30 | use: [
31 | {
32 | loader: 'url-loader',
33 | options: {
34 | limit: 8192,
35 | },
36 | },
37 | ],
38 | },
39 | {
40 | test: /\.css$/,
41 | use: [MiniCssExtractPlugin.loader, 'css-loader'],
42 | },
43 | ],
44 | },
45 | plugins: [
46 | new CleanWebpackPlugin(['dist']),
47 | new MiniCssExtractPlugin({
48 | filename: '[name].[contenthash].css',
49 | }),
50 | new CopyWebpackPlugin([{from: 'src/og/og.png', to: 'og.png'}]),
51 | ],
52 | watchOptions: {
53 | // does not work properly, ssr/mustache/etc is a mess now
54 | ignored: /\.html$/,
55 | },
56 | optimization: {
57 | runtimeChunk: 'single',
58 | splitChunks: {
59 | cacheGroups: {
60 | vendor: {
61 | test: /[\\/]node_modules[\\/]/,
62 | name: 'vendors',
63 | chunks: 'all',
64 | },
65 | },
66 | },
67 | },
68 | };
69 |
--------------------------------------------------------------------------------
/python_code/js_reimpl_common.py:
--------------------------------------------------------------------------------
1 | import socket
2 | import json
3 |
4 | from common import DUMMY, EMPTY
5 |
6 | none_info = {
7 | "type": "None",
8 | "hash": str(hash(None))
9 | }
10 |
11 |
12 | def dump_simple_py_obj(obj):
13 | if obj is DUMMY:
14 | return {
15 | "type": "DUMMY"
16 | }
17 | elif obj is EMPTY:
18 | return None
19 | elif obj is None:
20 | return none_info
21 | elif isinstance(obj, int):
22 | return {
23 | 'type': 'int',
24 | 'value': str(obj)
25 | }
26 | return obj
27 |
28 |
29 | def dump_pairs(pairs):
30 | res = []
31 | for k, v in pairs:
32 | res.append([dump_simple_py_obj(k), dump_simple_py_obj(v)])
33 |
34 | return res
35 |
36 |
37 | def dump_array(array):
38 | return list(map(dump_simple_py_obj, array))
39 |
40 |
41 | def parse_array(array):
42 | return list(map(parse_simple_py_obj, array))
43 |
44 |
45 | def parse_simple_py_obj(obj):
46 | if isinstance(obj, dict):
47 | assert obj["type"] in ["DUMMY", "None", "int"]
48 | if obj["type"] == "DUMMY":
49 | return DUMMY
50 | if obj["type"] == "None":
51 | return None
52 | return int(obj["value"])
53 | elif obj is None:
54 | return EMPTY
55 | return obj
56 |
57 |
58 | sock = None
59 | sockfile = None
60 |
61 |
62 | def _init_sock_stuff():
63 | global sock
64 | global sockfile
65 |
66 | # TODO: unhardcode?
67 | SOCK_FILENAME = 'pynode.sock'
68 |
69 | if sock is None:
70 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
71 | sock.connect(SOCK_FILENAME)
72 | sockfile = sock.makefile('r')
73 |
74 | return sock, sockfile
75 |
76 |
77 | def run_op_chapter1_chapter2(chapter, hash_codes, keys, op, **kwargs):
78 | _init_sock_stuff()
79 |
80 | for name in kwargs:
81 | if name != 'array':
82 | kwargs[name] = dump_simple_py_obj(kwargs[name])
83 | else:
84 | kwargs[name] = dump_array(kwargs[name])
85 |
86 | data = {
87 | "dict": chapter,
88 | "op": op,
89 | "args": kwargs,
90 | "hashCodes": dump_array(hash_codes) if hash_codes is not None else None,
91 | "keys": dump_array(keys) if keys is not None else None,
92 | }
93 |
94 | sock.send(bytes(json.dumps(data) + "\n", 'UTF-8'))
95 | response = json.loads(sockfile.readline())
96 |
97 | if "exception" in response and response["exception"]:
98 | raise KeyError()
99 |
100 | if 'result' in response and response['result'] is not None:
101 | # TODO: this is pretty hacky
102 | return response["result"]
103 | elif "hashCodes" in response:
104 | return parse_array(response["hashCodes"]), parse_array(response["keys"])
105 | else:
106 | return parse_array(response["keys"])
107 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | ### Firebase
2 | public/
3 |
4 | ### PYTHON
5 | #
6 | # Byte-compiled / optimized / DLL files
7 | __pycache__/
8 | *.py[cod]
9 | *$py.class
10 |
11 | # C extensions
12 | *.so
13 |
14 | # Distribution / packaging
15 | .Python
16 | build/
17 | develop-eggs/
18 | dist/
19 | downloads/
20 | eggs/
21 | .eggs/
22 | lib/
23 | lib64/
24 | parts/
25 | sdist/
26 | var/
27 | wheels/
28 | *.egg-info/
29 | .installed.cfg
30 | *.egg
31 | MANIFEST
32 |
33 | # PyInstaller
34 | # Usually these files are written by a python script from a template
35 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
36 | *.manifest
37 | *.spec
38 |
39 | # Installer logs
40 | pip-log.txt
41 | pip-delete-this-directory.txt
42 |
43 | # Unit test / coverage reports
44 | htmlcov/
45 | .tox/
46 | .coverage
47 | .coverage.*
48 | .cache
49 | nosetests.xml
50 | coverage.xml
51 | *.cover
52 | .hypothesis/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | .static_storage/
61 | .media/
62 | local_settings.py
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | target/
76 |
77 | # Jupyter Notebook
78 | .ipynb_checkpoints
79 |
80 | # pyenv
81 | .python-version
82 |
83 | # celery beat schedule file
84 | celerybeat-schedule
85 |
86 | # SageMath parsed files
87 | *.sage.py
88 |
89 | # Environments
90 | .env
91 | .venv
92 | env/
93 | venv/
94 | ENV/
95 | env.bak/
96 | venv.bak/
97 |
98 | # Spyder project settings
99 | .spyderproject
100 | .spyproject
101 |
102 | # Rope project settings
103 | .ropeproject
104 |
105 | # mkdocs documentation
106 | /site
107 |
108 | # mypy
109 | .mypy_cache/
110 |
111 |
112 | ### NODE
113 | # Logs
114 | logs
115 | *.log
116 | npm-debug.log*
117 | yarn-debug.log*
118 | yarn-error.log*
119 |
120 | # Runtime data
121 | pids
122 | *.pid
123 | *.seed
124 | *.pid.lock
125 |
126 | # Directory for instrumented libs generated by jscoverage/JSCover
127 | lib-cov
128 |
129 | # Coverage directory used by tools like istanbul
130 | coverage
131 |
132 | # nyc test coverage
133 | .nyc_output
134 |
135 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
136 | .grunt
137 |
138 | # Bower dependency directory (https://bower.io/)
139 | bower_components
140 |
141 | # node-waf configuration
142 | .lock-wscript
143 |
144 | # Compiled binary addons (https://nodejs.org/api/addons.html)
145 | build/Release
146 |
147 | # Dependency directories
148 | node_modules/
149 | jspm_packages/
150 |
151 | # Typescript v1 declaration files
152 | typings/
153 |
154 | # Optional npm cache directory
155 | .npm
156 |
157 | # Optional eslint cache
158 | .eslintcache
159 |
160 | # Optional REPL history
161 | .node_repl_history
162 |
163 | # Output of 'npm pack'
164 | *.tgz
165 |
166 | # Yarn Integrity file
167 | .yarn-integrity
168 |
169 | # dotenv environment variables file
170 | .env
171 |
172 | # next.js build output
173 | .next
174 |
175 | # vim custom added
176 | *.swp
177 | *.swo
178 |
179 |
--------------------------------------------------------------------------------
/stress_test_python.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -e -o pipefail
3 |
4 | NUM_INSERTS=200
5 | NUM_INSERTS_SMALLER=100
6 |
7 | eval "`pyenv init -`"
8 |
9 | pyenv shell 3.2.6
10 |
11 | for kv in {numbers,all}; do
12 | echo "DICT 3.2: kv = ${kv}, num_inserts = $NUM_INSERTS"
13 | for is in {0,9,-1}; do
14 | echo " initial size = ${is}"
15 | for reimpl in {dict32_reimpl_py_extracted,dict_actual,dict32_reimpl_py,dict32_reimpl_js}; do
16 | echo " Implementation: $reimpl"
17 | python3 python_code/dict32_reimplementation_test_v2.py --reference-implementation dict_actual --test-implementation $reimpl --no-extra-getitem-checks --num-inserts $NUM_INSERTS --kv all --initial-size $is
18 | done
19 | done
20 | done
21 |
22 | # TODO: merge with previous loop to remove copy&paste
23 | for kv in {numbers,all}; do
24 | echo "HASH from chapter 3 (w/o recycling): kv = ${kv}, num_inserts = $NUM_INSERTS"
25 | for is in {0,9,-1}; do
26 | echo " initial size = ${is}"
27 | for reimpl in {almost_python_dict_no_recycling_py_simpler,almost_python_dict_no_recycling_py_extracted,almost_python_dict_no_recycling_js}; do
28 | echo " Implementation: $reimpl"
29 | python3 python_code/dict32_reimplementation_test_v2.py --reference-implementation almost_python_dict_no_recycling_py --test-implementation $reimpl --no-extra-getitem-checks --num-inserts $NUM_INSERTS --kv all --initial-size $is
30 | done
31 | done
32 | done
33 |
34 |
35 | # TODO: merge with previous loop to remove copy&paste
36 | for kv in {numbers,all}; do
37 | echo "HASH from chapter 3 (w/ recycling): kv = ${kv}, num_inserts = $NUM_INSERTS"
38 | for is in {0,9,-1}; do
39 | echo " initial size = ${is}"
40 | for reimpl in {almost_python_dict_recycling_py_extracted,almost_python_dict_recycling_js}; do
41 | echo " Implementation: $reimpl"
42 | python3 python_code/dict32_reimplementation_test_v2.py --reference-implementation almost_python_dict_recycling_py --test-implementation $reimpl --no-extra-getitem-checks --num-inserts $NUM_INSERTS --kv all --initial-size $is
43 | done
44 | done
45 | done
46 |
47 | for kv in {numbers,all}; do
48 | echo "HASH from chapter 2: kv = ${kv}, num_inserts = $NUM_INSERTS"
49 | for is in {5,10,20,-1}; do
50 | echo " initial size = ${is}"
51 | for reimpl in {js_reimpl,py_extracted}; do
52 | echo " Implementation: $reimpl"
53 | python3 python_code/hash_chapter2_reimplementation_test.py --test-implementation py_extracted --num-inserts $NUM_INSERTS --initial-size $is --kv $kv
54 | done;
55 | done;
56 | done;
57 |
58 | echo "HASH from chapter 1: num_inserts = $NUM_INSERTS_SMALLER"
59 | for reimpl in {js,py_extracted}; do
60 | echo " Implementation: $reimpl"
61 | python3 python_code/hash_chapter1_reimplementation_test.py --test-implementation $reimpl --num-inserts $NUM_INSERTS_SMALLER
62 | done;
63 |
64 | echo "Linear search from chapter 1: size = $NUM_INSERTS_SMALLER"
65 | for reimpl in {js,py_extracted}; do
66 | echo " Implementation: $reimpl"
67 | python3 python_code/chapter1_linear_search_reimplementation_test.py --test-implementation $reimpl --size $NUM_INSERTS_SMALLER
68 | done;
69 |
70 | echo "Testing probing visualization from chapter4"
71 | python3 python_code/chapter4_probing_python_reimplementation_test.py
72 |
--------------------------------------------------------------------------------
/python_code/common.py:
--------------------------------------------------------------------------------
1 | import random
2 | import string
3 |
4 |
5 | class EmptyValueClass(object):
6 | def __str__(self):
7 | return "EMPTY"
8 |
9 | def __repr__(self):
10 | return "
"
11 |
12 |
13 | class DummyValueClass(object):
14 | def __str__(self):
15 | return ""
16 |
17 | def __repr__(self):
18 | return ""
19 |
20 |
21 | EMPTY = EmptyValueClass()
22 | DUMMY = DummyValueClass()
23 |
24 |
25 | def get_object_field_or_null(obj, field_name):
26 | try:
27 | return getattr(obj, field_name)
28 | except ValueError:
29 | return EMPTY
30 |
31 |
32 | def get_object_field_or_none(obj, field_name):
33 | try:
34 | return getattr(obj, field_name)
35 | except ValueError:
36 | return None
37 |
38 |
39 | def generate_random_string(str_len=5):
40 | # FROM: https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python
41 | return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(str_len))
42 |
43 |
44 | _unicode_chars = string.ascii_uppercase + string.digits + "йцукенгшщзхъфывапролджэячсмитьбю"
45 |
46 |
47 | def generate_random_unicode(str_len):
48 | # FROM: https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python
49 | return ''.join(random.choice(_unicode_chars) for _ in range(str_len))
50 |
51 |
52 | class IntKeyValueFactory(object):
53 | def __init__(self, n_inserts):
54 | self.n_inserts = n_inserts
55 | self._insert_count = 0
56 | self._key_range = list(range(n_inserts))
57 |
58 | def generate_key(self):
59 | return random.choice(self._key_range)
60 |
61 | def generate_value(self):
62 | self._insert_count += 1
63 | return self._insert_count
64 |
65 |
66 | class AllKeyValueFactory(object):
67 | def __init__(self, n_inserts, int_chance=0.1, long_chance=0.1, len0_chance=0.01, len1_chance=0.1, len2_chance=0.3, len3_chance=0.2, len_random_chance=0.17):
68 | self.int_pbf = int_chance
69 | self.long_pbf = self.int_pbf + long_chance
70 | self.len0_pbf = self.int_pbf + len0_chance
71 | self.len1_pbf = self.len0_pbf + len1_chance
72 | self.len2_pbf = self.len1_pbf + len2_chance
73 | self.len3_pbf = self.len2_pbf + len3_chance
74 | self.len_random_pbf = self.len3_pbf + len_random_chance
75 | assert 0.0 <= self.len3_pbf <= 1.0
76 |
77 | half_range = int(n_inserts / 2)
78 | self._int_range = [i - half_range for i in range(2 * half_range)]
79 |
80 | def _generate_obj(self):
81 | r = random.random()
82 | if r <= self.int_pbf:
83 | return random.choice(self._int_range)
84 | if r <= self.long_pbf:
85 | sign = "-" if random.random() < 0.5 else ""
86 | first_digit = random.choice("123456789")
87 | return sign + first_digit + ''.join(random.choice("0123456789") for _ in range(random.randint(20, 50)))
88 | if r <= self.len0_pbf:
89 | return ""
90 | if r <= self.len1_pbf:
91 | return generate_random_unicode(1)
92 | if r <= self.len2_pbf:
93 | return generate_random_unicode(2)
94 | if r <= self.len3_pbf:
95 | return generate_random_unicode(3)
96 | if r <= self.len_random_pbf:
97 | return generate_random_unicode(random.randint(4, 25))
98 | return None
99 |
100 | def generate_key(self):
101 | return self._generate_obj()
102 |
103 | def generate_value(self):
104 | return self._generate_obj()
105 |
--------------------------------------------------------------------------------
/python_code/dict_reimplementation.py:
--------------------------------------------------------------------------------
1 | from common import DUMMY, EMPTY
2 | from dict_reimpl_common import BaseDictImpl, Slot
3 | from operator import attrgetter
4 |
5 |
6 | class PyDictReimplementationBase(BaseDictImpl):
7 | START_SIZE = 8
8 | PERTURB_SHIFT = 5
9 |
10 | def __init__(self, pairs=None):
11 | BaseDictImpl.__init__(self)
12 | start_size = self.find_nearest_size(len(pairs)) if pairs else self.START_SIZE
13 | self.slots = [Slot() for _ in range(start_size)]
14 | if pairs:
15 | for k, v in pairs:
16 | self[k] = v
17 |
18 | def __setitem__(self, key, value):
19 | hash_code = hash(key)
20 | perturb = self.signed_to_unsigned(hash_code)
21 | idx = hash_code % len(self.slots)
22 | target_idx = None
23 | while self.slots[idx].key is not EMPTY:
24 | if self.slots[idx].hash_code == hash_code and self.slots[idx].key == key:
25 | target_idx = idx
26 | break
27 | if target_idx is None and self.slots[idx].key is DUMMY:
28 | target_idx = idx
29 |
30 | idx = (idx * 5 + perturb + 1) % len(self.slots)
31 | perturb >>= self.PERTURB_SHIFT
32 |
33 | if target_idx is None:
34 | target_idx = idx
35 |
36 | if self.slots[target_idx].key is EMPTY:
37 | self.used += 1
38 | self.fill += 1
39 | elif self.slots[target_idx].key is DUMMY:
40 | self.used += 1
41 |
42 | self.slots[target_idx] = Slot(hash_code, key, value)
43 | if self.fill * 3 >= len(self.slots) * 2:
44 | self.resize()
45 |
46 | def __delitem__(self, key):
47 | idx = self.lookdict(key)
48 |
49 | self.used -= 1
50 | self.slots[idx].key = DUMMY
51 | self.slots[idx].value = EMPTY
52 |
53 | def __getitem__(self, key):
54 | idx = self.lookdict(key)
55 |
56 | return self.slots[idx].value
57 |
58 | @staticmethod
59 | def signed_to_unsigned(hash_code):
60 | return 2**64 + hash_code if hash_code < 0 else hash_code
61 |
62 | def lookdict(self, key):
63 | hash_code = hash(key)
64 | perturb = self.signed_to_unsigned(hash_code)
65 |
66 | idx = hash_code % len(self.slots)
67 | while self.slots[idx].key is not EMPTY:
68 | if self.slots[idx].hash_code == hash_code and self.slots[idx].key == key:
69 | return idx
70 |
71 | idx = (idx * 5 + perturb + 1) % len(self.slots)
72 | perturb >>= self.PERTURB_SHIFT
73 |
74 | raise KeyError()
75 |
76 | def resize(self):
77 | old_slots = self.slots
78 | new_size = self.find_nearest_size(self._next_size())
79 | self.slots = [Slot() for _ in range(new_size)]
80 | self.fill = self.used
81 | for slot in old_slots:
82 | if slot.key is not EMPTY and slot.key is not DUMMY:
83 | perturb = self.signed_to_unsigned(slot.hash_code)
84 | idx = slot.hash_code % len(self.slots)
85 | while self.slots[idx].key is not EMPTY:
86 | idx = (idx * 5 + perturb + 1) % len(self.slots)
87 | perturb >>= self.PERTURB_SHIFT
88 |
89 | self.slots[idx] = Slot(slot.hash_code, slot.key, slot.value)
90 |
91 |
92 | class PyDictReimplementation32(PyDictReimplementationBase):
93 | def _next_size(self):
94 | return self.used * (4 if self.used <= 50000 else 2)
95 |
96 |
97 | def dump_reimpl_dict(d):
98 | def extract_fields(field_name):
99 | return list(map(attrgetter(field_name), d.slots))
100 | return extract_fields('hash_code'), extract_fields('key'), extract_fields('value'), d.fill, d.used
101 |
--------------------------------------------------------------------------------
/patches/smooth-scrollbar+8.3.1.patch:
--------------------------------------------------------------------------------
1 | patch-package
2 | --- a/node_modules/smooth-scrollbar/events/touch.js
3 | +++ b/node_modules/smooth-scrollbar/events/touch.js
4 | @@ -1,7 +1,7 @@
5 | import { eventScope, TouchRecord, } from '../utils/';
6 | var activeScrollbar;
7 | export function touchHandler(scrollbar) {
8 | - var MIN_EAING_MOMENTUM = 50;
9 | + var MIN_EAING_MOMENTUM = 3;
10 | var EASING_MULTIPLIER = /Android/.test(navigator.userAgent) ? 3 : 2;
11 | var target = scrollbar.options.delegateTo || scrollbar.containerEl;
12 | var touchRecord = new TouchRecord();
13 | --- a/node_modules/smooth-scrollbar/geometry/update.js
14 | +++ b/node_modules/smooth-scrollbar/geometry/update.js
15 | @@ -4,6 +4,9 @@ export function update(scrollbar) {
16 | x: Math.max(newSize.content.width - newSize.container.width, 0),
17 | y: Math.max(newSize.content.height - newSize.container.height, 0),
18 | };
19 | + // hack for a weird chrome on windows bug
20 | + if (limit.x <= 2) limit.x = 0;
21 | + if (limit.y <= 2) limit.y = 0;
22 | // metrics
23 | var containerBounding = scrollbar.containerEl.getBoundingClientRect();
24 | var bounding = {
25 | --- a/node_modules/smooth-scrollbar/scrollbar.js
26 | +++ b/node_modules/smooth-scrollbar/scrollbar.js
27 | @@ -322,6 +322,10 @@ var Scrollbar = /** @class */ (function () {
28 | if (limit.x === 0 && limit.y === 0) {
29 | this._updateDebounced();
30 | }
31 | + if (Math.abs(deltaY) > Math.abs(deltaX)) {
32 | + if (deltaY > 0 && offset.y === limit.y) return true;
33 | + if (deltaY < 0 && offset.y === 0) return true;
34 | + }
35 | var destX = clamp(deltaX + offset.x, 0, limit.x);
36 | var destY = clamp(deltaY + offset.y, 0, limit.y);
37 | var res = true;
38 | --- a/node_modules/smooth-scrollbar/track/track.js
39 | +++ b/node_modules/smooth-scrollbar/track/track.js
40 | @@ -41,8 +41,9 @@ var ScrollbarTrack = /** @class */ (function () {
41 | this.element.classList.remove('show');
42 | };
43 | ScrollbarTrack.prototype.update = function (scrollOffset, containerSize, pageSize) {
44 | + // -2 is a hack for a weird chrome on windows bug
45 | setStyle(this.element, {
46 | - display: pageSize <= containerSize ? 'none' : 'block',
47 | + display: pageSize - 2 <= containerSize ? 'none' : 'block',
48 | });
49 | this.thumb.update(scrollOffset, containerSize, pageSize);
50 | };
51 | deleted file mode 100644
52 | --- a/node_modules/smooth-scrollbar/track/track.js.map
53 | +++ /dev/null
54 | @@ -1 +0,0 @@
55 | -{"version":3,"file":"track.js","sourceRoot":"","sources":["../src/track/track.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EACL,QAAQ,GACT,MAAM,WAAW,CAAC;AAEnB;IAUE,wBACE,SAAyB,EACzB,YAAwB;QAAxB,6BAAA,EAAA,gBAAwB;QAT1B;;WAEG;QACM,YAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAEzC,aAAQ,GAAG,KAAK,CAAC;QAMvB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,qCAAmC,SAAW,CAAC;QAExE,IAAI,CAAC,KAAK,GAAG,IAAI,cAAc,CAC7B,SAAS,EACT,YAAY,CACb,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;;;OAIG;IACH,iCAAQ,GAAR,UAAS,kBAA+B;QACtC,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,6BAAI,GAAJ;QACE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,6BAAI,GAAJ;QACE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACnB,MAAM,CAAC;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,+BAAM,GAAN,UACE,YAAoB,EACpB,aAAqB,EACrB,QAAgB;QAEhB,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE;YACrB,OAAO,EAAE,QAAQ,IAAI,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;SACtD,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;IAC3D,CAAC;IACH,qBAAC;AAAD,CAAC,AApED,IAoEC"}
56 | \ No newline at end of file
57 |
--------------------------------------------------------------------------------
/python_code/js_reimplementation_interface.py:
--------------------------------------------------------------------------------
1 | import socket
2 | import json
3 |
4 | from common import DUMMY, EMPTY
5 | from js_reimpl_common import dump_simple_py_obj, parse_simple_py_obj, dump_pairs
6 | from dict_reimpl_common import Slot
7 |
8 |
9 | class JsImplBase(object):
10 | # TODO: unhardcode?
11 | SOCK_FILENAME = 'pynode.sock'
12 |
13 | def __init__(self, pairs=None):
14 | pairs = pairs or []
15 |
16 | self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
17 | self.sock.connect(self.SOCK_FILENAME)
18 | self.sockfile = self.sock.makefile('r')
19 |
20 | self.slots = None
21 | self.fill = None
22 | self.used = None
23 |
24 | self.run_op("__init__", pairs=pairs)
25 |
26 | def __del__(self):
27 | self.sock.close()
28 |
29 | def dump_slots(self):
30 | def dump_slot(slot):
31 | key = dump_simple_py_obj(slot.key)
32 | value = dump_simple_py_obj(slot.value)
33 |
34 | hash_code = slot.hash_code
35 | if hash_code is EMPTY:
36 | hash_code = None
37 |
38 | return {
39 | "hashCode": str(hash_code) if hash_code is not None else None,
40 | "key": key,
41 | "value": value,
42 | }
43 |
44 | if self.slots is None:
45 | return None
46 |
47 | return list(map(dump_slot, self.slots))
48 |
49 | def restore_slots(self, slots):
50 | def restore_slot(slot):
51 | key = parse_simple_py_obj(slot["key"])
52 | value = parse_simple_py_obj(slot["value"])
53 | assert value is not DUMMY
54 |
55 | hash_code = int(slot["hashCode"]) if slot["hashCode"] is not None else None
56 | if hash_code is None:
57 | hash_code = EMPTY
58 |
59 | return Slot(hash_code, key, value)
60 |
61 | self.slots = list(map(restore_slot, slots))
62 |
63 | def run_op(self, op, **kwargs):
64 | for name in kwargs:
65 | if name != 'pairs':
66 | kwargs[name] = dump_simple_py_obj(kwargs[name])
67 | else:
68 | kwargs[name] = dump_pairs(kwargs[name])
69 |
70 | data = {
71 | "dict": self.dict_type,
72 | "op": op,
73 | "args": kwargs,
74 | "self": {
75 | "slots": self.dump_slots(),
76 | "used": self.used,
77 | "fill": self.fill
78 | }
79 | }
80 |
81 | # pprint(("<< sending", data, op, kwargs))
82 | self.sock.send(bytes(json.dumps(data) + "\n", 'UTF-8'))
83 | response = json.loads(self.sockfile.readline())
84 | # pprint((">> receiving", response))
85 |
86 | self.restore_slots(response["self"]["slots"])
87 | self.fill = response["self"]["fill"]
88 | self.used = response["self"]["used"]
89 | if response["exception"]:
90 | raise KeyError("whatever")
91 |
92 | return parse_simple_py_obj(response["result"])
93 |
94 |
95 | class Dict32JsImpl(JsImplBase):
96 | dict_type = "dict32"
97 |
98 | def __setitem__(self, key, value):
99 | return self.run_op("__setitem__", key=key, value=value)
100 |
101 | def __delitem__(self, key):
102 | return self.run_op("__delitem__", key=key)
103 |
104 | def __getitem__(self, key):
105 | return self.run_op("__getitem__", key=key)
106 |
107 |
108 | class AlmostPythonDictBaseJsImpl(JsImplBase):
109 | dict_type = "almost_python_dict"
110 |
111 | def __delitem__(self, key):
112 | return self.run_op("__delitem__", key=key)
113 |
114 | def __getitem__(self, key):
115 | return self.run_op("__getitem__", key=key)
116 |
117 |
118 | class AlmostPythonDictRecyclingJsImpl(AlmostPythonDictBaseJsImpl):
119 | def __setitem__(self, key, value):
120 | return self.run_op("__setitem__recycling", key=key, value=value)
121 |
122 |
123 | class AlmostPythonDictNoRecyclingJsImpl(AlmostPythonDictBaseJsImpl):
124 | def __setitem__(self, key, value):
125 | return self.run_op("__setitem__no_recycling", key=key, value=value)
126 |
--------------------------------------------------------------------------------
/scripts/extractPythonCode.js:
--------------------------------------------------------------------------------
1 | import 'ignore-styles';
2 |
3 | import {
4 | DICT32_INIT,
5 | DICT32_SETITEM,
6 | DICT32_RESIZE_CODE,
7 | _DICT32_GETITEM_ONLY,
8 | _DICT32_DELITEM_ONLY,
9 | DICT32_LOOKDICT,
10 | STATICMETHOD_SIGNED_TO_UNSIGNED,
11 | PROBING_PYTHON_CODE,
12 | } from '../src/chapter4_real_python_dict';
13 |
14 | import {
15 | HASH_CLASS_INIT_CODE,
16 | HASH_CLASS_SETITEM_RECYCLING_CODE,
17 | HASH_CLASS_SETITEM_SIMPLIFIED_CODE,
18 | _HASH_CLASS_GETITEM_ONLY,
19 | _HASH_CLASS_DELITEM_ONLY,
20 | HASH_CLASS_LOOKDICT,
21 | HASH_CLASS_RESIZE_CODE,
22 | FIND_NEAREST_SIZE_CODE_STRING,
23 | SLOT_CLASS_CODE_STRING,
24 | } from '../src/chapter3_hash_class';
25 |
26 | import {
27 | HASH_CREATE_NEW_CODE,
28 | HASH_SEARCH_CODE,
29 | HASH_REMOVE_CODE,
30 | HASH_RESIZE_CODE,
31 | HASH_INSERT_CODE,
32 | } from '../src/chapter2_hash_table_functions';
33 |
34 | import {
35 | SIMPLIFIED_INSERT_ALL_BROKEN_CODE,
36 | SIMPLIFIED_INSERT_ALL_CODE,
37 | SIMPLIFIED_SEARCH_CODE,
38 | SIMPLE_LIST_SEARCH,
39 | } from '../src/chapter1_simplified_hash';
40 |
41 | import fs from 'fs';
42 | import * as path from 'path';
43 |
44 | function extractCodeLines(codeWithBpAndLevels) {
45 | return codeWithBpAndLevels.map(([line, bp, level]) => line);
46 | }
47 |
48 | function outputCode(filename, headers, importedCode, indent4 = true) {
49 | let allLines = [];
50 | for (let part of importedCode) {
51 | let lines;
52 | if (typeof part !== 'string') {
53 | lines = extractCodeLines(part);
54 | } else {
55 | lines = part.split('\n');
56 | }
57 |
58 | allLines.push(...lines);
59 | if (lines[lines.length - 1] !== '') {
60 | allLines.push('');
61 | }
62 | }
63 | const joinedLines = allLines.map(line => (line.length > 0 && indent4 ? ' ' + line : line)).join('\n');
64 | fs.writeFileSync(filename, headers.join('\n') + '\n' + joinedLines);
65 | }
66 |
67 | const commonImports = `import sys
68 | import os
69 | sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'python_code'))
70 | from common import DUMMY, EMPTY
71 |
72 | `;
73 |
74 | const dict32def = `
75 | class Dict32Extracted(object):`;
76 |
77 | const DIR = 'build';
78 |
79 | outputCode(
80 | path.join(DIR, 'dict32js_extracted.py'),
81 | [commonImports, SLOT_CLASS_CODE_STRING, dict32def],
82 | [
83 | DICT32_INIT,
84 | FIND_NEAREST_SIZE_CODE_STRING,
85 | STATICMETHOD_SIGNED_TO_UNSIGNED,
86 | DICT32_SETITEM,
87 | DICT32_RESIZE_CODE,
88 | _DICT32_GETITEM_ONLY,
89 | _DICT32_DELITEM_ONLY,
90 | DICT32_LOOKDICT,
91 | ]
92 | );
93 |
94 | const hashClassRecyclingDef = `
95 | class HashClassRecyclingExtracted(object):`;
96 |
97 | outputCode(
98 | path.join(DIR, 'hash_class_recycling_extracted.py'),
99 | [commonImports, SLOT_CLASS_CODE_STRING, hashClassRecyclingDef],
100 | [
101 | HASH_CLASS_INIT_CODE,
102 | FIND_NEAREST_SIZE_CODE_STRING,
103 | HASH_CLASS_SETITEM_RECYCLING_CODE,
104 | HASH_CLASS_RESIZE_CODE,
105 | _HASH_CLASS_GETITEM_ONLY,
106 | _HASH_CLASS_DELITEM_ONLY,
107 | HASH_CLASS_LOOKDICT,
108 | ]
109 | );
110 |
111 | const hashClassNoRecyclingDef = `
112 | class HashClassNoRecyclingExtracted(object):`;
113 |
114 | outputCode(
115 | path.join(DIR, 'hash_class_no_recycling_extracted.py'),
116 | [commonImports, SLOT_CLASS_CODE_STRING, hashClassNoRecyclingDef],
117 | [
118 | HASH_CLASS_INIT_CODE,
119 | FIND_NEAREST_SIZE_CODE_STRING,
120 | HASH_CLASS_SETITEM_SIMPLIFIED_CODE,
121 | HASH_CLASS_RESIZE_CODE,
122 | _HASH_CLASS_GETITEM_ONLY,
123 | _HASH_CLASS_DELITEM_ONLY,
124 | HASH_CLASS_LOOKDICT,
125 | ]
126 | );
127 |
128 | outputCode(
129 | path.join(DIR, 'hash_chapter2_extracted.py'),
130 | [commonImports],
131 | [HASH_CREATE_NEW_CODE, HASH_SEARCH_CODE, HASH_REMOVE_CODE, HASH_RESIZE_CODE, HASH_INSERT_CODE],
132 | false
133 | );
134 |
135 | outputCode(
136 | path.join(DIR, 'hash_chapter1_extracted.py'),
137 | [commonImports],
138 | [SIMPLIFIED_INSERT_ALL_CODE, SIMPLIFIED_INSERT_ALL_BROKEN_CODE, SIMPLIFIED_SEARCH_CODE, SIMPLE_LIST_SEARCH],
139 | false
140 | );
141 |
142 | outputCode(path.join(DIR, 'chapter4_probing_python_code.py'), [commonImports], [PROBING_PYTHON_CODE], false);
143 |
--------------------------------------------------------------------------------
/python_code/hash_chapter3_class_impl.py:
--------------------------------------------------------------------------------
1 | from common import DUMMY, EMPTY
2 | from dict_reimpl_common import BaseDictImpl, Slot
3 |
4 |
5 | class AlmostPythonDictBase(BaseDictImpl):
6 | START_SIZE = 8
7 |
8 | def __init__(self, pairs=None):
9 | BaseDictImpl.__init__(self)
10 | self._keys_set = set()
11 | if pairs:
12 | for k, v in pairs:
13 | self[k] = v
14 |
15 | def lookdict(self, key):
16 | hash_code = hash(key)
17 |
18 | idx = hash_code % len(self.slots)
19 | while self.slots[idx].key is not EMPTY:
20 | if self.slots[idx].hash_code == hash_code and self.slots[idx].key == key:
21 | return idx
22 |
23 | idx = (idx + 1) % len(self.slots)
24 |
25 | raise KeyError()
26 |
27 | def __getitem__(self, key):
28 | idx = self.lookdict(key)
29 |
30 | return self.slots[idx].value
31 |
32 | def __delitem__(self, key):
33 | idx = self.lookdict(key)
34 |
35 | self.used -= 1
36 | self.slots[idx].key = DUMMY
37 | self.slots[idx].value = EMPTY
38 | self._keys_set.remove(key)
39 |
40 | def resize(self):
41 | old_slots = self.slots
42 | new_size = self.find_nearest_size(2 * self.used)
43 | self.slots = [Slot() for _ in range(new_size)]
44 |
45 | for slot in old_slots:
46 | if slot.key is not EMPTY and slot.key is not DUMMY:
47 | idx = slot.hash_code % len(self.slots)
48 | while self.slots[idx].key is not EMPTY:
49 | idx = (idx + 1) % len(self.slots)
50 |
51 | self.slots[idx] = Slot(slot.hash_code, slot.key, slot.value)
52 |
53 | self.fill = self.used
54 |
55 | def keys(self):
56 | return self._keys_set
57 |
58 | def __len__(self):
59 | return len(self.keys())
60 |
61 |
62 | class AlmostPythonDictImplementationRecycling(AlmostPythonDictBase):
63 | def __setitem__(self, key, value):
64 | hash_code = hash(key)
65 | idx = hash_code % len(self.slots)
66 | target_idx = None
67 | while self.slots[idx].key is not EMPTY:
68 | if self.slots[idx].hash_code == hash_code and self.slots[idx].key == key:
69 | target_idx = idx
70 | break
71 | if target_idx is None and self.slots[idx].key is DUMMY:
72 | target_idx = idx
73 |
74 | idx = (idx + 1) % len(self.slots)
75 |
76 | if target_idx is None:
77 | target_idx = idx
78 |
79 | if self.slots[target_idx].key is EMPTY:
80 | self.used += 1
81 | self.fill += 1
82 | elif self.slots[target_idx].key is DUMMY:
83 | self.used += 1
84 |
85 | self.slots[target_idx] = Slot(hash_code, key, value)
86 |
87 | if self.fill * 3 >= len(self.slots) * 2:
88 | self.resize()
89 |
90 | self._keys_set.add(key)
91 |
92 |
93 | class AlmostPythonDictImplementationNoRecycling(AlmostPythonDictBase):
94 | def __setitem__(self, key, value):
95 | hash_code = hash(key)
96 | idx = hash_code % len(self.slots)
97 | target_idx = None
98 | while self.slots[idx].key is not EMPTY:
99 | if self.slots[idx].hash_code == hash_code and\
100 | self.slots[idx].key == key:
101 | target_idx = idx
102 | break
103 | idx = (idx + 1) % len(self.slots)
104 |
105 | if target_idx is None:
106 | target_idx = idx
107 | if self.slots[target_idx].key is EMPTY:
108 | self.used += 1
109 | self.fill += 1
110 |
111 | self.slots[target_idx] = Slot(hash_code, key, value)
112 | if self.fill * 3 >= len(self.slots) * 2:
113 | self.resize()
114 |
115 | self._keys_set.add(key)
116 |
117 |
118 | class AlmostPythonDictImplementationNoRecyclingSimplerVersion(AlmostPythonDictBase):
119 | def __setitem__(self, key, value):
120 | hash_code = hash(key)
121 | idx = hash_code % len(self.slots)
122 | while self.slots[idx].key is not EMPTY:
123 | if self.slots[idx].hash_code == hash_code and\
124 | self.slots[idx].key == key:
125 | break
126 | idx = (idx + 1) % len(self.slots)
127 |
128 | if self.slots[idx].key is EMPTY:
129 | self.used += 1
130 | self.fill += 1
131 |
132 | self.slots[idx] = Slot(hash_code, key, value)
133 | if self.fill * 3 >= len(self.slots) * 2:
134 | self.resize()
135 |
136 | self._keys_set.add(key)
137 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "inside_python_dict",
3 | "version": "1.0.0",
4 | "description": "Inside python dict - an explorable explanation",
5 | "main": "index.js",
6 | "scripts": {
7 | "jest": "jest --env=node",
8 | "test:pystress": "npm run extractcode && ./stress_test_python.sh",
9 | "test:pyunit": "npm run extractcode && ./unittest_python.sh",
10 | "test": "npm run jest && npm run test:pyunit && npm run test:pystress",
11 | "build:ssr": "./ssr-all.sh",
12 | "update:html": "mkdir -p build && (for i in {chapter1,chapter2,chapter3,chapter4,new_demos}; do mustache src/mustache/$i.json src/page.html.template > src/autogenerated/$i.html; done)",
13 | "start": "webpack-dev-server --config webpack.dev.js --host 0.0.0.0",
14 | "serve": "http-server -p 9090 dist/",
15 | "build": "webpack --config webpack.prod.js",
16 | "babel-node": "npx babel-node --presets '@babel/env'",
17 | "extractcode": "mkdir -p build && npm run babel-node scripts/extractPythonCode.js",
18 | "dictserver": "rm pynode.sock ; npm run babel-node scripts/pyReimplWrapper.js; rm pynode.sock",
19 | "postinstall": "patch-package"
20 | },
21 | "repository": {
22 | "type": "git",
23 | "url": "git+https://github.com/eleweek/inside_python_dict.git"
24 | },
25 | "author": "Alexander Putilin",
26 | "license": "MIT",
27 | "bugs": {
28 | "url": "https://github.com/eleweek/inside_python_dict/issues"
29 | },
30 | "homepage": "https://github.com/eleweek/inside_python_dict#readme",
31 | "prettier": {
32 | "printWidth": 120,
33 | "tabWidth": 4,
34 | "useTabs": false,
35 | "singleQuote": true,
36 | "bracketSpacing": false,
37 | "semi": true,
38 | "trailingComma": "es5"
39 | },
40 | "dependencies": {
41 | "@fortawesome/fontawesome-svg-core": "^1.2.10",
42 | "@fortawesome/free-brands-svg-icons": "^5.6.1",
43 | "@fortawesome/free-solid-svg-icons": "^5.6.1",
44 | "@fortawesome/react-fontawesome": "^0.1.3",
45 | "bignumber.js": "^8.0.1",
46 | "bootstrap": "^4.1.3",
47 | "bowser": "^2.0.0-beta.3",
48 | "classnames": "^2.2.6",
49 | "d3": "^5.7.0",
50 | "d3-selection-multi": "^1.0.1",
51 | "i": "^0.3.6",
52 | "immutable": "^4.0.0-rc.12",
53 | "lodash": "^4.17.11",
54 | "lowlight": "^1.11.0",
55 | "memoize-one": "^4.1.0",
56 | "mobx": "^5.8.0",
57 | "mobx-react": "^5.4.3",
58 | "rc-slider": "^9.7.2",
59 | "react": "^16.6.3",
60 | "react-css-transition-replace": "^3.0.3",
61 | "react-dom": "^16.6.3",
62 | "react-error-boundary": "^1.2.3",
63 | "react-input-autosize": "^2.2.1",
64 | "react-popper": "^1.3.2",
65 | "react-router-dom": "^5.2.0",
66 | "react-smooth-scrollbar": "^8.0.6",
67 | "react-stickynode": "^2.1.0",
68 | "rehype": "^7.0.0",
69 | "smooth-scrollbar": "8.3.1"
70 | },
71 | "devDependencies": {
72 | "@babel/cli": "^7.2.0",
73 | "@babel/core": "^7.2.2",
74 | "@babel/node": "^7.2.2",
75 | "@babel/plugin-proposal-class-properties": "^7.2.1",
76 | "@babel/plugin-proposal-decorators": "^7.2.2",
77 | "@babel/plugin-proposal-export-namespace-from": "^7.2.0",
78 | "@babel/plugin-proposal-function-sent": "^7.2.0",
79 | "@babel/plugin-proposal-json-strings": "^7.2.0",
80 | "@babel/plugin-proposal-numeric-separator": "^7.2.0",
81 | "@babel/plugin-proposal-object-rest-spread": "^7.2.0",
82 | "@babel/plugin-proposal-optional-chaining": "^7.2.0",
83 | "@babel/plugin-proposal-throw-expressions": "^7.2.0",
84 | "@babel/plugin-syntax-dynamic-import": "^7.2.0",
85 | "@babel/plugin-syntax-import-meta": "^7.2.0",
86 | "@babel/plugin-transform-destructuring": "^7.2.0",
87 | "@babel/preset-env": "^7.2.0",
88 | "@babel/preset-react": "^7.0.0",
89 | "babel-core": "^7.0.0-bridge.0",
90 | "babel-jest": "^23.4.2",
91 | "babel-loader": "^8.0.0",
92 | "babel-plugin-lodash": "^3.3.4",
93 | "clean-webpack-plugin": "^1.0.0",
94 | "copy-webpack-plugin": "^5.1.1",
95 | "css-loader": "^2.0.1",
96 | "dotenv": "^6.2.0",
97 | "html-webpack-plugin": "^3.2.0",
98 | "http-server": "^0.11.1",
99 | "husky": "^1.2.1",
100 | "ignore-styles": "^5.0.1",
101 | "jest": "^23.6.0",
102 | "mini-css-extract-plugin": "^0.5.0",
103 | "mustache": "^3.0.1",
104 | "npm": "^6.5.0",
105 | "patch-package": "^5.1.1",
106 | "prettier": "^1.15.3",
107 | "pretty-quick": "^1.8.0",
108 | "split": "^1.0.1",
109 | "style-loader": "^0.23.1",
110 | "unminified-webpack-plugin": "^2.0.0",
111 | "url-loader": "^4.1.1",
112 | "webpack": "^4.27.1",
113 | "webpack-bundle-analyzer": "^3.0.3",
114 | "webpack-cli": "^3.1.2",
115 | "webpack-dev-server": "^3.1.10",
116 | "webpack-merge": "^4.1.5"
117 | },
118 | "husky": {
119 | "hooks": {
120 | "pre-commit": "pretty-quick --staged"
121 | }
122 | }
123 | }
124 |
--------------------------------------------------------------------------------
/python_code/hash_chapter3_class_impl_test.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from common import DUMMY, EMPTY
3 | from hash_chapter3_class_impl import AlmostPythonDictImplementationRecycling, AlmostPythonDictImplementationNoRecycling
4 |
5 |
6 | class HashDictImplementationTest(unittest.TestCase):
7 | def test_handcrafted(self):
8 | d = AlmostPythonDictImplementationRecycling()
9 | self.assertEqual(len(d.slots), 8)
10 |
11 | def assert_contains(i, h, k, v):
12 | self.assertEqual(d.slots[i].hash_code, h)
13 | self.assertEqual(d.slots[i].key, k)
14 | self.assertEqual(d.slots[i].value, v)
15 |
16 | d[""] = 1
17 | d[17] = 2
18 | d[18] = 3
19 | self.assertEqual(d[""], 1)
20 | self.assertEqual(d[17], 2)
21 | self.assertEqual(d[18], 3)
22 |
23 | assert_contains(0, 0, "", 1)
24 | assert_contains(1, 17, 17, 2)
25 | assert_contains(2, 18, 18, 3)
26 |
27 | self.assertEqual(d.fill, 3)
28 | self.assertEqual(d.used, 3)
29 |
30 | with self.assertRaises(KeyError):
31 | del d[1]
32 |
33 | del d[17]
34 | assert_contains(1, 17, DUMMY, EMPTY)
35 |
36 | self.assertEqual(d.fill, 3)
37 | self.assertEqual(d.used, 2)
38 | # hash("abcd") % 8 == 0
39 |
40 | # py 3.2 hash()
41 | d["abcd"] = 4
42 | self.assertEqual(d["abcd"], 4)
43 | assert_contains(1, -2835746963027601024, "abcd", 4)
44 | self.assertEqual(d.fill, 3)
45 | self.assertEqual(d.used, 3)
46 |
47 | d["abcd"] = 5
48 | self.assertEqual(d["abcd"], 5)
49 | assert_contains(1, -2835746963027601024, "abcd", 5)
50 | self.assertEqual(d.fill, 3)
51 | self.assertEqual(d.used, 3)
52 |
53 | del d["abcd"]
54 | with self.assertRaises(KeyError):
55 | d["abcd"]
56 |
57 | d[15] = 6
58 | d[14] = 7
59 |
60 | assert_contains(7, 15, 15, 6)
61 | assert_contains(6, 14, 14, 7)
62 |
63 | self.assertEqual(len(d.slots), 8)
64 | self.assertEqual(d.fill, 5)
65 | self.assertEqual(d.used, 4)
66 | d[13] = 8
67 | self.assertEqual(len(d.slots), 16)
68 | self.assertEqual(d.fill, 5)
69 | self.assertEqual(d.used, 5)
70 |
71 | assert_contains(0, 0, "", 1)
72 | assert_contains(2, 18, 18, 3)
73 | assert_contains(13, 13, 13, 8)
74 | assert_contains(14, 14, 14, 7)
75 | assert_contains(15, 15, 15, 6)
76 |
77 | def test_handcrafted_simple_setitem(self):
78 | d = AlmostPythonDictImplementationNoRecycling()
79 | self.assertEqual(len(d.slots), 8)
80 |
81 | def assert_contains(i, h, k, v):
82 | self.assertEqual(d.slots[i].hash_code, h)
83 | self.assertEqual(d.slots[i].key, k)
84 | self.assertEqual(d.slots[i].value, v)
85 |
86 | d[""] = 1
87 | d[17] = 2
88 | d[18] = 3
89 | self.assertEqual(d[""], 1)
90 | self.assertEqual(d[17], 2)
91 | self.assertEqual(d[18], 3)
92 |
93 | assert_contains(0, 0, "", 1)
94 | assert_contains(1, 17, 17, 2)
95 | assert_contains(2, 18, 18, 3)
96 |
97 | self.assertEqual(d.fill, 3)
98 | self.assertEqual(d.used, 3)
99 |
100 | with self.assertRaises(KeyError):
101 | del d[1]
102 |
103 | del d[17]
104 | assert_contains(1, 17, DUMMY, EMPTY)
105 |
106 | self.assertEqual(d.fill, 3)
107 | self.assertEqual(d.used, 2)
108 | # hash("abcd") % 8 == 0
109 |
110 | # py 3.2 hash()
111 | d["abcd"] = 4
112 | self.assertEqual(d["abcd"], 4)
113 | assert_contains(3, -2835746963027601024, "abcd", 4)
114 | self.assertEqual(d.fill, 4)
115 | self.assertEqual(d.used, 3)
116 |
117 | d["abcd"] = 5
118 | self.assertEqual(d["abcd"], 5)
119 | assert_contains(3, -2835746963027601024, "abcd", 5)
120 | self.assertEqual(d.fill, 4)
121 | self.assertEqual(d.used, 3)
122 |
123 | del d["abcd"]
124 | with self.assertRaises(KeyError):
125 | d["abcd"]
126 |
127 | self.assertEqual(len(d.slots), 8)
128 | self.assertEqual(d.fill, 4)
129 | self.assertEqual(d.used, 2)
130 |
131 | d[15] = 6
132 | self.assertEqual(len(d.slots), 8)
133 | self.assertEqual(d.fill, 5)
134 | self.assertEqual(d.used, 3)
135 | assert_contains(7, 15, 15, 6)
136 |
137 | d[13] = 8
138 | self.assertEqual(len(d.slots), 16)
139 | self.assertEqual(d.fill, 4)
140 | self.assertEqual(d.used, 4)
141 |
142 | assert_contains(0, 0, "", 1)
143 | assert_contains(2, 18, 18, 3)
144 | assert_contains(13, 13, 13, 8)
145 | assert_contains(15, 15, 15, 6)
146 |
147 |
148 | def main():
149 | unittest.main()
150 |
151 |
152 | if __name__ == "__main__":
153 | main()
154 |
--------------------------------------------------------------------------------
/python_code/hash_chapter2_reimplementation_test.py:
--------------------------------------------------------------------------------
1 | import random
2 | import argparse
3 |
4 | from common import DUMMY, EMPTY, AllKeyValueFactory, IntKeyValueFactory
5 |
6 | import hash_chapter2_reimpl_js
7 | import hash_chapter2_impl
8 | import build_autogenerated_chapter2
9 |
10 | TEST_IMPLEMENTATIONS = {
11 | 'js_reimpl': hash_chapter2_reimpl_js,
12 | 'py_extracted': build_autogenerated_chapter2
13 | }
14 |
15 |
16 | def verify_same(ref_hash_codes, ref_keys, hash_codes, keys):
17 | if (ref_hash_codes, ref_keys) != (hash_codes, keys):
18 | print("ORIG SIZES", len(ref_hash_codes), len(ref_keys))
19 | print("NEW SIZES", len(hash_codes), len(keys))
20 | if len(ref_hash_codes) == len(hash_codes) == len(ref_keys) == len(keys):
21 | size = len(hash_codes)
22 | print("NEW | ORIG")
23 | for i in range(size):
24 | if ref_hash_codes[i] is not EMPTY or hash_codes[i] is not EMPTY:
25 | print(i, " " * 3,
26 | ref_hash_codes[i], ref_keys[i], " " * 3,
27 | hash_codes[i], keys[i], " " * 3)
28 |
29 | assert ref_hash_codes == hash_codes and ref_keys == keys
30 |
31 |
32 | def run(ref_impl, test_impl, n_inserts, key_value_factory, initial_state, extra_checks, verbose):
33 | SINGLE_REMOVE_CHANCE = 0.3
34 |
35 | ref_hash_codes, ref_keys = ref_impl.create_new(initial_state)
36 | test_hash_codes, test_keys = test_impl.create_new(initial_state)
37 |
38 | def vs():
39 | verify_same(ref_hash_codes, ref_keys, test_hash_codes, test_keys)
40 |
41 | vs()
42 |
43 | if verbose:
44 | print("Starting test")
45 |
46 | for i in range(n_inserts):
47 | key_to_insert = key_value_factory.generate_key()
48 |
49 | existing_keys = set([k for k in ref_keys if k is not DUMMY and k is not EMPTY])
50 | fill = sum(1 for k in ref_keys if k is not EMPTY)
51 | if existing_keys and random.random() < SINGLE_REMOVE_CHANCE:
52 | key_to_remove = random.choice(list(existing_keys))
53 | assert ref_impl.has_key(ref_hash_codes, ref_keys, key_to_remove)
54 | assert test_impl.has_key(test_hash_codes, test_keys, key_to_remove)
55 |
56 | ref_impl.remove(ref_hash_codes, ref_keys, key_to_remove)
57 | test_impl.remove(test_hash_codes, test_keys, key_to_remove)
58 | existing_keys.remove(key_to_remove)
59 |
60 | assert not ref_impl.has_key(ref_hash_codes, ref_keys, key_to_remove)
61 | assert not test_impl.has_key(test_hash_codes, test_keys, key_to_remove)
62 |
63 | is_key_present = ref_impl.has_key(ref_hash_codes, ref_keys, key_to_insert)
64 | assert (key_to_insert in existing_keys) == is_key_present
65 |
66 | if not is_key_present:
67 | if verbose:
68 | print("Inserting {}".format(key_to_insert))
69 | assert not test_impl.has_key(test_hash_codes, test_keys, key_to_insert)
70 | else:
71 | if verbose:
72 | print("Re-Inserting {}".format(key_to_insert))
73 |
74 | ref_impl.insert(ref_hash_codes, ref_keys, key_to_insert)
75 | test_impl.insert(test_hash_codes, test_keys, key_to_insert)
76 | vs()
77 | assert test_impl.has_key(test_hash_codes, test_keys, key_to_insert)
78 | assert ref_impl.has_key(ref_hash_codes, ref_keys, key_to_insert)
79 |
80 | if fill / len(ref_keys) > 0.66:
81 | ref_hash_codes, ref_keys = ref_impl.resize(ref_hash_codes, ref_keys)
82 | test_hash_codes, test_keys = test_impl.resize(test_hash_codes, test_keys)
83 | vs()
84 |
85 | if extra_checks:
86 | for k in existing_keys:
87 | assert test_impl.has_key(test_hash_codes, test_keys, k)
88 | assert ref_impl.has_key(ref_hash_codes, ref_keys, k)
89 |
90 |
91 | if __name__ == "__main__":
92 | parser = argparse.ArgumentParser(description='Stress-test chapter2 reimplementation')
93 | parser.add_argument('--test-implementation', choices=TEST_IMPLEMENTATIONS.keys(), required=True)
94 | parser.add_argument('--num-inserts', type=int, default=500)
95 | parser.add_argument('--forever', action='store_true')
96 | parser.add_argument('--kv', choices=["numbers", "all"], required=True)
97 | parser.add_argument('--initial-size', type=int, default=-1)
98 | parser.add_argument('--extra-getitem-checks', action='store_true', default=False)
99 | parser.add_argument('--verbose', action='store_true', default=False)
100 | args = parser.parse_args()
101 |
102 | if args.kv == "numbers":
103 | kv_factory = IntKeyValueFactory(args.num_inserts)
104 | elif args.kv == "all":
105 | kv_factory = AllKeyValueFactory(args.num_inserts)
106 |
107 | def test_iteration():
108 | initial_size = args.initial_size if args.initial_size >= 0 else random.randint(0, 100)
109 | initial_state = [kv_factory.generate_key() for _ in range(initial_size)]
110 | run(hash_chapter2_impl,
111 | TEST_IMPLEMENTATIONS[args.test_implementation],
112 | n_inserts=args.num_inserts,
113 | key_value_factory=kv_factory,
114 | initial_state=initial_state,
115 | extra_checks=args.extra_getitem_checks,
116 | verbose=args.verbose)
117 |
118 | if args.forever:
119 | while True:
120 | test_iteration()
121 | else:
122 | test_iteration()
123 |
--------------------------------------------------------------------------------
/python_code/hash_chapter2_impl_test.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from hash_chapter2_impl import create_new, has_key, insert, remove, resize, DUMMY
3 | from common import generate_random_string
4 |
5 |
6 | class MyHashTest(unittest.TestCase):
7 | def test_handcrafted(self):
8 | expected_len = 6
9 | hashes, keys = create_new([42, 43, 12])
10 |
11 | self.assertEqual(len(hashes), expected_len)
12 | self.assertEqual(len(keys), expected_len)
13 | insert(hashes, keys, 42)
14 |
15 | self.assertEqual(hashes[42 % expected_len], 42)
16 | self.assertEqual(keys[42 % expected_len], 42)
17 |
18 | self.assertEqual(hashes[43 % expected_len], 43)
19 | self.assertEqual(keys[43 % expected_len], 43)
20 |
21 | self.assertEqual(hashes[42 % expected_len], 42)
22 | self.assertEqual(keys[42 % expected_len], 42)
23 |
24 | self.assertEqual(hashes[12 % expected_len], 42)
25 | self.assertEqual(keys[12 % expected_len], 42)
26 | self.assertEqual(hashes[12 % expected_len + 1], 43)
27 | self.assertEqual(keys[12 % expected_len + 1], 43)
28 | self.assertEqual(hashes[12 % expected_len + 2], 12)
29 | self.assertEqual(keys[12 % expected_len + 2], 12)
30 |
31 | self.assertTrue(has_key(hashes, keys, 42))
32 | self.assertTrue(has_key(hashes, keys, 43))
33 | self.assertTrue(has_key(hashes, keys, 12))
34 | self.assertFalse(has_key(hashes, keys, 45))
35 |
36 | # table: [42, 43, 12, None, None, None]
37 | insert(hashes, keys, "") # hash("") == 0
38 | self.assertEqual(hashes[3], 0)
39 | self.assertEqual(keys[3], "")
40 |
41 | self.assertTrue(has_key(hashes, keys, ""))
42 | self.assertTrue(has_key(hashes, keys, 42))
43 |
44 | insert(hashes, keys, "aba") # hash("aba") % 6 == 5
45 | self.assertEqual(hashes[5], hash("aba"))
46 | self.assertEqual(keys[5], "aba")
47 |
48 | self.assertTrue(has_key(hashes, keys, 12))
49 | remove(hashes, keys, 12)
50 | self.assertFalse(has_key(hashes, keys, 12))
51 |
52 | self.assertEqual(hashes[12 % expected_len], 42)
53 | self.assertEqual(keys[12 % expected_len], 42)
54 |
55 | self.assertEqual(keys[12 % expected_len + 2], DUMMY)
56 |
57 | with self.assertRaises(KeyError):
58 | remove(hashes, keys, 12)
59 | with self.assertRaises(KeyError):
60 | remove(hashes, keys, 45)
61 |
62 | self.assertFalse(has_key(hashes, keys, 12))
63 | self.assertFalse(has_key(hashes, keys, 45))
64 | self.assertTrue(has_key(hashes, keys, 42))
65 | self.assertTrue(has_key(hashes, keys, 43))
66 | self.assertTrue(has_key(hashes, keys, ""))
67 | self.assertTrue(has_key(hashes, keys, "aba"))
68 |
69 | insert(hashes, keys, "abg")
70 | self.assertTrue(has_key(hashes, keys, "abg"))
71 | self.assertEqual(hashes[4], hash("abg"))
72 | self.assertEqual(keys[4], "abg")
73 | hashes, keys = resize(hashes, keys)
74 |
75 | self.assertTrue(has_key(hashes, keys, 42))
76 | self.assertTrue(has_key(hashes, keys, 43))
77 | self.assertTrue(has_key(hashes, keys, ""))
78 | self.assertTrue(has_key(hashes, keys, "aba"))
79 | self.assertTrue(has_key(hashes, keys, "abg"))
80 |
81 | self.assertFalse(has_key(hashes, keys, 12))
82 | self.assertFalse(has_key(hashes, keys, 45))
83 |
84 | self.assertEqual(hashes[6], 42)
85 | self.assertEqual(keys[6], 42)
86 | self.assertEqual(hashes[7], 43)
87 | self.assertEqual(keys[7], 43)
88 |
89 | self.assertEqual(hashes[0], 0)
90 | self.assertEqual(keys[0], "")
91 | for h in hashes:
92 | self.assertTrue(h != 12)
93 |
94 | self.assertEqual(hashes[5], hash("aba"))
95 | self.assertEqual(keys[5], "aba")
96 |
97 | self.assertEqual(hashes[11], hash("abg"))
98 | self.assertEqual(keys[11], "abg")
99 |
100 | def test_all(self):
101 | n = 10
102 | initial_keys = [generate_random_string() for _ in range(n)]
103 | more_keys = [generate_random_string() for _ in range(n // 3)]
104 | myhashes, mykeys = create_new(initial_keys)
105 |
106 | for key in more_keys:
107 | insert(myhashes, mykeys, key)
108 | insert(myhashes, mykeys, key)
109 |
110 | existing_keys = initial_keys + more_keys
111 | for key in existing_keys:
112 | self.assertTrue(has_key(myhashes, mykeys, key))
113 |
114 | myhashes, mykeys = resize(myhashes, mykeys)
115 |
116 | for key in existing_keys:
117 | self.assertTrue(has_key(myhashes, mykeys, key))
118 |
119 | missing_keys = [generate_random_string() for _ in range(3 * n)]
120 | for key in set(missing_keys) - set(existing_keys):
121 | self.assertFalse(has_key(myhashes, mykeys, key))
122 | with self.assertRaises(KeyError):
123 | remove(myhashes, mykeys, key)
124 |
125 | for key in existing_keys:
126 | self.assertTrue(has_key(myhashes, mykeys, key))
127 | remove(myhashes, mykeys, key)
128 | self.assertFalse(has_key(myhashes, mykeys, key))
129 |
130 | for key in more_keys:
131 | self.assertFalse(has_key(myhashes, mykeys, key))
132 | insert(myhashes, mykeys, key)
133 | self.assertTrue(has_key(myhashes, mykeys, key))
134 | remove(myhashes, mykeys, key)
135 | self.assertFalse(has_key(myhashes, mykeys, key))
136 |
137 |
138 | def main():
139 | unittest.main()
140 |
141 |
142 | if __name__ == "__main__":
143 | main()
144 |
--------------------------------------------------------------------------------
/src/player.css:
--------------------------------------------------------------------------------
1 | .player-header {
2 | position: relative; /* for .player-buttons position:absolute to work */
3 | display: flex;
4 | align-items: center;
5 | height: 35px;
6 | width: 100%;
7 | }
8 |
9 | .player-header-desktop {
10 | max-width: 1300px;
11 | margin-left: auto;
12 | margin-right: auto;
13 | }
14 |
15 | .player-main {
16 | max-width: 1300px;
17 | margin-left: auto;
18 | margin-right: auto;
19 | }
20 |
21 | a.player-title {
22 | font-weight: 700;
23 | margin-left: 10px;
24 | text-decoration: none;
25 | color: black;
26 | }
27 |
28 | .player-buttons {
29 | display: flex;
30 | align-items: center;
31 | font-family: 'IBM Plex Mono', monospace;
32 | }
33 |
34 | .player-buttons-mobile {
35 | margin-left: 30px;
36 | }
37 |
38 | .slider-mobile-extra {
39 | position: absolute !important;
40 | bottom: 36px !important;
41 | width: calc(100% - 20px) !important;
42 | }
43 |
44 | .mobile-short-explanation {
45 | font-size: 14px;
46 |
47 | margin-top: 0px;
48 |
49 | padding-left: 5px;
50 | padding-right: 5px;
51 | }
52 |
53 | .mobile-header-title {
54 | margin-left: 10px;
55 | margin-right: 10px;
56 | margin-top: 10px;
57 | margin-bottom: -5px;
58 |
59 | font-weight: 700;
60 | font-size: 14px;
61 | }
62 |
63 | .player-button {
64 | display: flex;
65 | cursor: pointer;
66 | height: 24px;
67 | align-items: center;
68 | justify-content: center;
69 | }
70 |
71 | .player-next {
72 | margin-left: 5px;
73 | }
74 |
75 | .player-prev {
76 | margin-right: 5px;
77 | }
78 |
79 | .player-counters {
80 | display: flex;
81 | justify-content: center;
82 | min-width: 65px;
83 | }
84 |
85 | .player-play-button {
86 | width: 24px;
87 | margin-right: 20px;
88 | }
89 |
90 | .player-theory-button {
91 | margin-left: auto;
92 | cursor: pointer;
93 | color: #416287;
94 | margin-right: 10px;
95 | }
96 |
97 | .player-theory-button.player-button-active,
98 | a.player-title:hover {
99 | color: #d1750c;
100 | }
101 |
102 | .rc-slider-handle-click-focused {
103 | /* box-shadow: 0 0 0 1px #a0b1c3 !important;*/
104 | box-shadow: unset !important;
105 | }
106 |
107 | .rc-slider-handle:focus {
108 | /* box-shadow: 0 0 0 3px #a0b1c3 !important;*/
109 | box-shadow: unset !important;
110 | }
111 |
112 | .slider-transition {
113 | transition: all 0.1s linear 0s;
114 | }
115 |
116 | .player-slider-wrapper {
117 | padding-left: 10px;
118 | padding-right: 10px;
119 | }
120 |
121 | .player-state-vis-wrapper {
122 | padding-left: 10px;
123 | }
124 |
125 | .player-main {
126 | display: flex;
127 | flex-direction: row;
128 | }
129 |
130 | .player-code-and-visualisation {
131 | flex-grow: 1;
132 | }
133 |
134 | .player-theory {
135 | margin-left: 20px;
136 | min-width: 200px;
137 | margin-right: 2px;
138 | }
139 |
140 | .player-theory-border-wrapper {
141 | border-left: 1px rgba(128, 128, 128, 0.15) solid;
142 | }
143 |
144 | .player-theory-inner {
145 | padding-left: 10px;
146 | padding-right: 8px;
147 | padding-bottom: 15px;
148 | }
149 |
150 | .player-theory h1 {
151 | font-size: 24px;
152 | margin-bottom: 10px;
153 | margin-top: 16px;
154 | }
155 |
156 | .player-theory h2 {
157 | font-size: 16px;
158 | margin-bottom: 6px;
159 | margin-top: 16px;
160 | }
161 |
162 | .player-theory p {
163 | font-size: 16px;
164 | line-height: 1.15;
165 | margin-bottom: 4px;
166 | }
167 |
168 | /* hack to make the last couple of lines not hidden */
169 | .player-theory-inner > :last-child {
170 | padding-bottom: 35px;
171 | }
172 |
173 | /*.player-theory .scrollbar-track-y {
174 | left: 0px !important;
175 | right: none !important;
176 | }*/
177 |
178 | .player-inputs-outer {
179 | max-width: 1300px;
180 | margin-left: auto;
181 | margin-right: auto;
182 |
183 | margin-top: 7px;
184 | margin-bottom: 7px;
185 | }
186 |
187 | .player-inputs-inner {
188 | margin-left: 10px;
189 | }
190 |
191 | .player-input-label {
192 | min-width: 100px;
193 | padding-right: 10px;
194 | }
195 |
196 | .player-input {
197 | height: 35px;
198 | min-width: 150px;
199 | width: 35%;
200 | padding-left: 5px;
201 | padding-right: 5px;
202 | /* border: 0.5px solid #416287 !important; */
203 | /*border: 1px solid #e5e6f1 !important;*/
204 | border: none;
205 | border-radius: 3px;
206 | margin-bottom: 3px;
207 | background-color: #f4f4f7;
208 | }
209 |
210 | .player-input:focus {
211 | /* border: 1px solid #416287 !important;*/
212 | outline: none !important;
213 | }
214 |
215 | .player-input-error:focus {
216 | outline: none !important;
217 | }
218 |
219 | .player-input-error {
220 | background-color: #fdeded;
221 | }
222 |
223 | .player-input-wrapper {
224 | display: flex;
225 | flex-direction: row;
226 | align-items: center;
227 | }
228 |
229 | .player-input-comment {
230 | margin-left: 10px;
231 | }
232 |
233 | @media screen and (min-width: 890px) {
234 | .player-buttons-desktop {
235 | position: absolute;
236 | transform: translateX(-50%);
237 | left: 50%;
238 | }
239 | }
240 |
241 | @media screen and (max-width: 891px) {
242 | .player-buttons-desktop {
243 | padding-left: 20px;
244 | }
245 | }
246 |
247 | @media screen and (max-width: 480px) {
248 | .player-header {
249 | height: 50px;
250 | }
251 | }
252 |
253 | /* Hacks to not display arrows for number inputs */
254 | input::-webkit-outer-spin-button,
255 | input::-webkit-inner-spin-button {
256 | /* display: none; <- Crashes Chrome on hover */
257 | -webkit-appearance: none;
258 | margin: 0; /* <-- Apparently some margin are still there even though it's hidden */
259 | }
260 |
261 | input[type='number'] {
262 | -moz-appearance: textfield; /* Firefox */
263 | }
264 |
--------------------------------------------------------------------------------
/src/mainpage.css:
--------------------------------------------------------------------------------
1 | .frontpage {
2 | padding-left: 10px;
3 | padding-right: 10px;
4 | max-width: 1300px;
5 | margin-left: auto;
6 | margin-right: auto;
7 | }
8 |
9 | .frontpage .header {
10 | font-family: 'IBM Plex Sans', sans-serif;
11 | line-height: 1.2;
12 | display: flex;
13 | align-items: center;
14 | }
15 |
16 | .frontpage .header > .title {
17 | font-size: 70px;
18 | line-height: 1.3;
19 | font-weight: 700;
20 | }
21 |
22 | .frontpage .header > .definition {
23 | display: flex;
24 | flex-direction: column;
25 | padding-top: 14px;
26 | font-size: 21px;
27 | line-height: 1.2;
28 | margin-left: auto;
29 | }
30 |
31 | .frontpage h1 {
32 | font-size: 60px;
33 | font-weight: 400;
34 |
35 | line-height: 1.2;
36 | margin-bottom: 8px;
37 | }
38 |
39 | .frontpage .section {
40 | margin-top: 90px;
41 | }
42 |
43 | .frontpage .panes-container {
44 | margin-top: 15px;
45 | display: flex;
46 | flex-direction: row;
47 | }
48 |
49 | .frontpage .pane {
50 | display: flex;
51 |
52 | border-radius: 7px;
53 | margin-bottom: 10px;
54 | text-decoration: none !important;
55 |
56 | color: white !important;
57 |
58 | transition: opacity 0.3s ease;
59 | }
60 |
61 | .frontpage .pane:hover {
62 | opacity: 0.9;
63 | }
64 |
65 | .frontpage .pane h2 {
66 | margin-top: 6px;
67 | margin-left: 10px;
68 | font-weight: 100;
69 | font-size: 60px;
70 | line-height: 1.2;
71 | }
72 |
73 | .frontpage .pane .vis-wrapper {
74 | padding-left: 10px;
75 | }
76 |
77 | .frontpage .sorts-collection {
78 | display: flex;
79 | flex-direction: row;
80 | height: 370px;
81 | width: 100%;
82 | }
83 |
84 | .frontpage .bubble-sort {
85 | background-color: #8700ff;
86 | margin-right: 10px;
87 | height: 370px;
88 | }
89 |
90 | .frontpage .quick-sort {
91 | background-color: #e3444a;
92 | height: 370px;
93 | }
94 |
95 | .frontpage .sort-1 {
96 | width: 50%;
97 | }
98 |
99 | .frontpage .sort-2 {
100 | width: 50%;
101 | }
102 |
103 | .frontpage .simplified-hash-collisions {
104 | background-color: #4d9e38;
105 | width: 100%;
106 | margin-right: 10px;
107 | }
108 |
109 | .frontpage .simplified-hash-search {
110 | background-color: #f45c26;
111 | height: 50%;
112 | }
113 |
114 | .frontpage .simplified-hash-create {
115 | background-color: #8700ff;
116 | height: 50%;
117 | }
118 |
119 | .frontpage .simplified-hash-collection {
120 | display: flex;
121 | flex-direction: row;
122 | width: 100%;
123 | height: 370px;
124 | }
125 |
126 | .frontpage .simplified-hash-collection-left {
127 | display: flex;
128 | width: 50%;
129 | }
130 |
131 | .frontpage .simplified-hash-collection-right {
132 | display: flex;
133 | flex-direction: column;
134 | width: 50%;
135 | }
136 |
137 | .frontpage .hash-collection {
138 | display: flex;
139 | flex-direction: column;
140 | }
141 |
142 | .frontpage .hash-collection-top {
143 | display: flex;
144 | flex-direction: row;
145 | height: 180px;
146 | }
147 |
148 | .frontpage .hash-collection-bottom {
149 | display: flex;
150 | flex-direction: row;
151 | height: 180px;
152 | }
153 |
154 | .frontpage .hash-create {
155 | width: 58%;
156 | background-color: #f5c451;
157 | color: black !important;
158 | margin-right: 10px;
159 | }
160 |
161 | .frontpage .hash-search {
162 | width: 42%;
163 | background-color: #3faaef;
164 | }
165 |
166 | .frontpage .hash-remove {
167 | width: 42%;
168 | background-color: #1f7a78;
169 | margin-right: 10px;
170 | }
171 |
172 | .frontpage .hash-resize {
173 | width: 58%;
174 | background-color: #e3444a;
175 | }
176 |
177 | @media screen and (max-width: 890px) {
178 | .frontpage {
179 | padding-left: 10px;
180 | padding-right: 10px;
181 | margin-left: auto;
182 | margin-right: auto;
183 | }
184 |
185 | .frontpage .header {
186 | flex-direction: column;
187 | align-items: flex-start;
188 | }
189 |
190 | .frontpage .header > .title {
191 | font-size: 26px;
192 | line-height: 1.3;
193 | font-weight: 700;
194 | }
195 |
196 | .frontpage .header > .definition {
197 | flex-direction: row;
198 |
199 | padding-top: 5px;
200 | font-size: 12px;
201 | line-height: 1.2;
202 | padding-left: 0px;
203 | margin-left: 0px;
204 | }
205 |
206 | .frontpage .header > .definition > .definition-1:after {
207 | content: '\00a0';
208 | }
209 |
210 | .frontpage h1 {
211 | font-size: 26px;
212 | font-weight: 400;
213 |
214 | line-height: 1.2;
215 | }
216 |
217 | .frontpage .section {
218 | margin-top: 45px;
219 | }
220 |
221 | .frontpage .panes-container {
222 | margin-top: 5px;
223 | display: flex;
224 | flex-direction: row;
225 | }
226 |
227 | .frontpage .pane {
228 | display: flex;
229 |
230 | border-radius: 5px;
231 | margin-bottom: 5px;
232 | text-decoration: none !important;
233 |
234 | color: white !important;
235 | }
236 |
237 | .frontpage .pane h2 {
238 | font-weight: 200;
239 |
240 | margin-top: 6px;
241 | margin-left: 10px;
242 | font-size: 34px;
243 | }
244 |
245 | .frontpage .hash-collection {
246 | height: 300px;
247 | }
248 |
249 | .frontpage .hash-collection-top {
250 | width: 100%;
251 | height: 50%;
252 | flex-direction: column;
253 | }
254 |
255 | .frontpage .hash-collection-bottom {
256 | width: 100%;
257 | height: 50%;
258 | flex-direction: column;
259 | }
260 |
261 | .frontpage .hash-collection-bottom > *,
262 | .frontpage .hash-collection-top > * {
263 | height: 50%;
264 | width: 100%;
265 | }
266 |
267 | .frontpage .simplified-hash-collection {
268 | height: 210px;
269 | flex-direction: column;
270 | }
271 |
272 | .frontpage .simplified-hash-collection > * {
273 | width: 100%;
274 | }
275 |
276 | .frontpage .hash-remove {
277 | margin-right: 5px;
278 | }
279 |
280 | .frontpage .hash-remove {
281 | margin-right: 5px;
282 | }
283 |
284 | .frontpage .simplified-hash-collisions {
285 | margin-right: 0px;
286 | height: 70px;
287 | }
288 |
289 | .frontpage .simplified-hash-search,
290 | .frontpage .simplified-hash-create,
291 | .frontpage .bubble-sort,
292 | .frontpage .quick-sort {
293 | margin-right: 0px;
294 | height: 70px;
295 | }
296 |
297 | .frontpage .sort-2,
298 | .frontpage .sort-1 {
299 | width: 100%;
300 | }
301 |
302 | .frontpage .sorts-collection {
303 | height: 140px;
304 | flex-direction: column;
305 | }
306 |
307 | .frontpage .hash-create {
308 | color: black !important;
309 | margin-right: 5px;
310 | }
311 |
312 | a.link {
313 | font-size: 12px;
314 | }
315 | }
316 |
--------------------------------------------------------------------------------
/python_code/dict32_reimplementation_test_v2.py:
--------------------------------------------------------------------------------
1 | import random
2 | import argparse
3 | import json
4 |
5 | from common import EMPTY, AllKeyValueFactory, IntKeyValueFactory
6 | from dictinfo import dump_py_dict
7 | from dict_reimplementation import PyDictReimplementation32, dump_reimpl_dict
8 | from js_reimplementation_interface import Dict32JsImpl, AlmostPythonDictRecyclingJsImpl, AlmostPythonDictNoRecyclingJsImpl
9 | import hash_chapter3_class_impl
10 | import build_autogenerated_chapter3_chapter4
11 |
12 |
13 | def dict_factory(pairs=None):
14 | if not pairs:
15 | return {}
16 |
17 | # quick&dirty
18 | def to_string(x):
19 | return json.dumps(x) if x is not None else "None"
20 | d = eval("{" + ", ".join("{}:{}".format(to_string(k), to_string(v)) for [k, v] in pairs) + "}")
21 | return d
22 |
23 |
24 | IMPLEMENTATIONS = {
25 | "dict_actual": (dict_factory, dump_py_dict),
26 | "dict32_reimpl_py": (PyDictReimplementation32, dump_reimpl_dict),
27 | "dict32_reimpl_js": (Dict32JsImpl, dump_reimpl_dict),
28 |
29 | "dict32_reimpl_py_extracted": (build_autogenerated_chapter3_chapter4.Dict32Extracted, dump_reimpl_dict),
30 |
31 | "almost_python_dict_recycling_py": (hash_chapter3_class_impl.AlmostPythonDictImplementationRecycling, dump_reimpl_dict),
32 | "almost_python_dict_no_recycling_py": (hash_chapter3_class_impl.AlmostPythonDictImplementationNoRecycling, dump_reimpl_dict),
33 | "almost_python_dict_no_recycling_py_simpler": (hash_chapter3_class_impl.AlmostPythonDictImplementationNoRecyclingSimplerVersion, dump_reimpl_dict),
34 | "almost_python_dict_recycling_js": (AlmostPythonDictRecyclingJsImpl, dump_reimpl_dict),
35 | "almost_python_dict_no_recycling_js": (AlmostPythonDictNoRecyclingJsImpl, dump_reimpl_dict),
36 |
37 | "almost_python_dict_recycling_py_extracted": (build_autogenerated_chapter3_chapter4.HashClassRecyclingExtracted, dump_reimpl_dict),
38 | "almost_python_dict_no_recycling_py_extracted": (build_autogenerated_chapter3_chapter4.HashClassNoRecyclingExtracted, dump_reimpl_dict),
39 | }
40 |
41 |
42 | def verify_same(d, dump_d_func, dreimpl, dump_dreimpl_func):
43 | dump_d = dump_d_func(d)
44 | dump_reimpl = dump_dreimpl_func(dreimpl)
45 |
46 | if dump_d != dump_reimpl:
47 | hashes_orig, keys_orig, values_orig, fill_orig, used_orig = dump_d
48 | hashes_new, keys_new, values_new, fill_new, used_new = dump_reimpl
49 | print("ORIG SIZE", len(hashes_orig))
50 | print("NEW SIZE", len(hashes_new))
51 | print("ORIG fill/used: ", fill_orig, used_orig)
52 | print("NEW fill/used: ", fill_new, used_new)
53 | if len(hashes_orig) == len(hashes_new):
54 | size = len(hashes_orig)
55 | print("NEW | ORIG")
56 | for i in range(size):
57 | if hashes_new[i] is not EMPTY or hashes_orig[i] is not EMPTY:
58 | print(i, " " * 3,
59 | hashes_new[i], keys_new[i], values_new[i], " " * 3,
60 | hashes_orig[i], keys_orig[i], values_orig[i])
61 |
62 | assert dump_d == dump_reimpl
63 |
64 |
65 | def run(ref_impl_factory, ref_impl_dump, test_impl_factory, test_impl_dump, n_inserts, extra_checks, key_value_factory, initial_state, verbose):
66 | SINGLE_REMOVE_CHANCE = 0.3
67 | MASS_REMOVE_CHANCE = 0.002
68 | MASS_REMOVE_COEFF = 0.8
69 |
70 | removed = set()
71 |
72 | if initial_state:
73 | d = ref_impl_factory(initial_state)
74 | else:
75 | d = ref_impl_factory()
76 |
77 | if initial_state:
78 | dreimpl = test_impl_factory(initial_state)
79 | else:
80 | dreimpl = test_impl_factory()
81 |
82 | if verbose:
83 | print("Starting test")
84 |
85 | for i in range(n_inserts):
86 | should_remove = (random.random() < SINGLE_REMOVE_CHANCE)
87 | if should_remove and d and d.keys(): # TODO: ugly, written while on a plane
88 | to_remove = random.choice(list(d.keys()))
89 | if verbose:
90 | print("Removing {}".format(to_remove))
91 | del d[to_remove]
92 | del dreimpl[to_remove]
93 | if verbose:
94 | print(d)
95 | verify_same(d, ref_impl_dump, dreimpl, test_impl_dump)
96 | removed.add(to_remove)
97 |
98 | should_mass_remove = (random.random() < MASS_REMOVE_CHANCE)
99 | if should_mass_remove and len(d) > 10:
100 | to_remove_list = random.sample(list(d.keys()), int(MASS_REMOVE_COEFF * len(d)))
101 | if verbose:
102 | print("Mass-Removing {} elements".format(len(to_remove_list)))
103 | for k in to_remove_list:
104 | del d[k]
105 | del dreimpl[k]
106 | removed.add(k)
107 |
108 | if extra_checks:
109 | for k in d.keys():
110 | assert d[k] == dreimpl[k]
111 |
112 | for r in removed:
113 | try:
114 | dreimpl[r]
115 | assert False
116 | except KeyError:
117 | pass
118 |
119 | key_to_insert = key_value_factory.generate_key()
120 | value_to_insert = key_value_factory.generate_value()
121 | _keys_set = getattr(d, '_keys_set', None)
122 | # TODO: ugly code written on a plane
123 | # TODO: properly implement in/not in when I land
124 | if _keys_set is not None:
125 | key_present = key_to_insert in _keys_set
126 | else:
127 | key_present = key_to_insert in d
128 |
129 | if not key_present:
130 | if verbose:
131 | print("Inserting ({key}, {value})".format(key=key_to_insert, value=value_to_insert))
132 | try:
133 | dreimpl[key_to_insert]
134 | assert False
135 | except KeyError:
136 | pass
137 | else:
138 | if verbose:
139 | print("Replacing ({key}, {value1}) with ({key}, {value2})".format(key=key_to_insert, value1=d[key_to_insert], value2=value_to_insert))
140 | removed.discard(key_to_insert)
141 | d[key_to_insert] = value_to_insert
142 | dreimpl[key_to_insert] = value_to_insert
143 | if verbose:
144 | print(d)
145 | verify_same(d, ref_impl_dump, dreimpl, test_impl_dump)
146 | assert dreimpl[key_to_insert] == value_to_insert
147 |
148 |
149 | if __name__ == "__main__":
150 | parser = argparse.ArgumentParser(description='Stress-test dict-like reimplementations')
151 | parser.add_argument('--reference-implementation', choices=IMPLEMENTATIONS.keys(), required=True)
152 | parser.add_argument('--test-implementation', choices=IMPLEMENTATIONS.keys(), required=True)
153 | parser.add_argument('--no-extra-getitem-checks', dest='extra_checks', action='store_false')
154 | parser.add_argument('--num-inserts', type=int, default=500)
155 | parser.add_argument('--forever', action='store_true')
156 | parser.add_argument('--kv', choices=["numbers", "all"], required=True)
157 | parser.add_argument('--initial-size', type=int, default=-1)
158 | parser.add_argument('--verbose', action='store_true')
159 | args = parser.parse_args()
160 |
161 | if args.kv == "numbers":
162 | kv_factory = IntKeyValueFactory(args.num_inserts)
163 | elif args.kv == "all":
164 | kv_factory = AllKeyValueFactory(args.num_inserts)
165 |
166 | ref_impl = IMPLEMENTATIONS[args.reference_implementation]
167 | test_impl = IMPLEMENTATIONS[args.test_implementation]
168 |
169 | def test_iteration():
170 | initial_size = args.initial_size if args.initial_size >= 0 else random.randint(0, 100)
171 | initial_state = [(kv_factory.generate_key(), kv_factory.generate_value()) for _ in range(initial_size)]
172 | run(*(ref_impl + test_impl),
173 | n_inserts=args.num_inserts,
174 | extra_checks=args.extra_checks,
175 | key_value_factory=kv_factory,
176 | initial_state=initial_state,
177 | verbose=args.verbose)
178 |
179 | if args.forever:
180 | while True:
181 | test_iteration()
182 | else:
183 | test_iteration()
184 |
--------------------------------------------------------------------------------
/src/theory.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import {ProbingVisualization, GenerateProbingLinks} from './probing_visualization';
3 | import {HashExamples} from './chapter2_hash_table_functions';
4 |
5 | export function CollisionsTheory() {
6 | return (
7 | <>
8 | Коллизии в хеш-таблицах
9 |
10 | Хеш-таблица — структура данных, хранящая пары «ключ-значение» или только ключ. Позволяет быстро найти
11 | ключ и, если есть, связанное с ним значение.
12 |
13 | Как найти нужный элемент с помощью линейного поиска
14 |
15 | Последовательно перебирать все элементы списка, пока не найдется требуемый. В худшем случае — придется
16 | проверить все элементы. А в среднем — половину.
17 |
18 | Как найти нужный элемент хеш-таблицы
19 |
20 | По ключу понять, с какого места таблицы искать. И начать поиск рядом с нужным элементом. В среднем такой
21 | поиск займет пару итераций.{' '}
22 |
23 | Пример из реальной жизни
24 |
25 | Представьте, что вы ищете книгу в библиотеке. Если книги стоят в случайном порядке, то придется
26 | перебирать все по одной, пока не найдете нужную. А если книги стоят в алфавитном порядке, то
27 | библиотекарь может пойти сразу к нужной полке и перебрать несколько книг, а не всю библиотеку.{' '}
28 |
29 | Устройство простейшей хеш-таблицы
30 |
31 | Простейшая хеш-таблица — это массив из ключей. Индекс ключа вычисляется по самому ключу. Однако у разных
32 | ключей индекс может совпасть. Такие ситуации называются коллизиями.{' '}
33 |
34 |
35 | В следующей визуализации мы разберемся, как решать коллизии и создадим простейшую хеш-таблицу без потери
36 | данных.
37 |
38 |
39 | );
40 | }
41 |
42 | const {links: probingLinks} = new GenerateProbingLinks().run(8, '', 'i+1');
43 |
44 | function ProbingVisualizationAndDescription() {
45 | return (
46 | <>
47 | {' '}
48 | Алгоритм разрешения коллизий
50 |
51 | Если текущая ячейка занята, то мы проверим следующую. Если занята и она, то проверим следующую за ней. И
52 | так до тех пор, пока не найдем свободную ячейку.
53 |
54 |
55 | Таблицы с таким способом разрешения коллизий называются хеш-таблицами{' '}
56 |
61 | с открытой адресацией
62 |
63 | .{' '}
64 |
65 |
66 | Есть и другой способ разрешения коллизий —{' '}
67 |
72 | метод цепочек
73 |
74 | . В хеш-таблицах с цепочками каждая ячейка является односвязным списком. Такие ячейки позволяют хранить
75 | сразу несколько элементов.
76 |
77 |
78 | );
79 | }
80 |
81 | export function SimplifiedHashTheory(props) {
82 | return (
83 | <>
84 | Простейшие хеш-таблицы
85 |
86 | Хеш-таблица — структура данных, хранящая пары «ключ-значение» или только ключ. Позволяет быстро найти
87 | ключ и, если есть, связанное с ним значение.
88 |
89 |
90 | Простейшая хеш-таблица — это массив из ключей. Индекс ключа вычисляется по самому ключу. Однако у разных
91 | ключей индекс может совпасть. Такие ситуации называются коллизиями.{' '}
92 |
93 | Производительность и расход памяти
95 |
96 | Чем больше свободного места в хеш-таблице, тем меньше коллизий. В пустой таблице не будет ни одной
97 | коллизии, а в почти полной — они будут почти наверняка.{' '}
98 |
99 |
100 | Однако чем больше свободного места, тем больше расходуется памяти. Поэтому при использовании хеш-таблиц
101 | стараются достичь баланса. Таблица должна быть не слишком пустой, но и не слишком заполненной. Проблемы
102 | с производительностью начиют быть заметны при заполненности на две трети.
103 |
104 | На нашей визуализации размер таблицы выбран так, чтобы таблица была заполнена наполовину.
105 | Создание хеш-таблицы
106 |
107 | Для создания хеш-таблицы мы последовательно вставляем ключи один за другим. При возникновении коллизии
108 | переходим к соседней ячейке. Вставляем ключ в первую свободную ячейку.
109 |
110 | Поиск в хеш-таблице
111 |
112 | Поиск ключа аналогичен вставке. Вычисляем индекс и ищем пустой слот, как при разрешении коллизий. Если
113 | по пути мы не нашли ключа, то его нет в таблице.
114 |
115 |
116 | );
117 | }
118 |
119 | export function HashTheory(props) {
120 | return (
121 | <>
122 | Хеш-таблицы с открытой адресацией
123 |
124 | Хеш-таблица — структура данных, хранящая пары «ключ-значение» или только ключ. Позволяет быстро найти
125 | ключ и, если есть, связанное с ним значение.{' '}
126 |
127 |
128 | Хеш-таблица с открытой адресацией представляет себе массив из ячеек, в каждой из которых может быть
129 | записан ключ. Индекс ключа вычисляется по самому ключу с помощью хеш-функции. У разных ключей индекс
130 | может совпасть. Такие ситуации называются коллизиями.
131 |
132 |
133 | Задача хеш-функции — по объекту вычислить число по заранее заданному алгоритму. Вычисленное число
134 | называется хеш-кодом. Алгоритм хеш-фукнции должен помогать равномерно «раскидать» значения по
135 | хеш-таблице. Поэтому у двух похожих значений (например "hello" и "hello!")
136 | может оказаться совсем разный хеш-код.
137 |
138 | Примеры значений хеш-функций
139 |
140 | В Python есть встроенная хеш-функция hash(). В нашем коде мы используем именно ее.
141 |
142 | Создание хеш-таблицы
144 |
145 | Для создания хеш-таблицы мы последовательно вставляем ключи один за другим. При возникновении коллизии
146 | переходим к соседней ячейке. Вставляем ключ в первую свободную ячейку.
147 |
148 | Поиск
149 |
150 | Поиск ключа аналогичен вставке. Вычисляем индекс и ищем пустой слот, как при разрешении коллизий. Если
151 | по пути мы не нашли ключа, то его нет в таблице.
152 |
153 | Удаление
154 |
155 | При удалении ключа мы перезаписываем его специальным значением-плейсхолдером DUMMY. Если мы
156 | оставим слот пустым, то при поиске другого ключа алгоритм разрешения коллизий может остановиться раньше
157 | времени.
158 |
159 | Расширение
160 |
161 | Если при работе хеш-таблицы она заполняется, то ее нужно расширить. Однако вычисленные индексы элементов
162 | зависят от размера таблицы. Это значит, что если изменится размер — изменятся и индексы. Для того, чтобы
163 | вставленные ключи можно было снова найти, мы создаем новую таблицу и вставляем в нее ключи из старой.
164 |
165 |
166 | );
167 | }
168 |
169 | export function BubbleSortTheory() {
170 | return (
171 | <>
172 | Сортировка пузырьком
173 |
174 | Сортировка пузырьком — самая простая для понимания и реализации сортировка. Алгоритм по очереди
175 | просматривает все элементы списка{`\u00a0`}— сравнивает текущий элемент со следующим и при необходимости
176 | меняет их местами. Таким образом, алгоритм проходит по списку, пока все элементы не будут упорядочены.
177 |
178 | Пузырьковая сортировка является простейшей и эффективна лишь для небольших списков.
179 |
180 | );
181 | }
182 |
183 | export function QuickSortTheory() {
184 | return (
185 | <>
186 | Быстрая сортировка
187 | Быстрая сортировка — одна из самых быстрых сортировок. Основана на принципе «Разделяй и властвуй».
188 |
189 | На каждой итерации алгоритма выбирается элемент-делитель. Делитель может быть любым элементом массива. В
190 | нашей реализации выбирается элемент посередине.{' '}
191 |
192 |
193 | Затем элементы распределяются относительно делителя так, чтобы слева были все меньшие элементы, а справа
194 | {`\u00a0`}— все большие. Таким образом массив становится несколько более упорядоченным и делитель
195 | оказывается на нужном месте.
196 |
197 | После этого алгоритм повторяется рекурсивно для левой и правой половин.
198 |
199 | );
200 | }
201 |
--------------------------------------------------------------------------------
/src/styles.css:
--------------------------------------------------------------------------------
1 | /* FROM: https://stackoverflow.com/questions/12502234/how-to-prevent-webkit-text-rendering-change-during-css-transition */
2 | @media screen and (-webkit-min-device-pixel-ratio: 2) {
3 | body {
4 | -webkit-font-smoothing: subpixel-antialiased;
5 | }
6 | }
7 |
8 | body #root {
9 | color: black;
10 | font-family: 'IBM Plex Sans';
11 | }
12 |
13 | pre {
14 | font-family: 'IBM Plex Mono';
15 | }
16 |
17 | .app-container,
18 | .footer-container {
19 | padding-right: 30px;
20 | padding-left: 30px;
21 | }
22 |
23 | .app-container {
24 | padding-top: 20px;
25 | }
26 |
27 | .footer-container {
28 | margin-top: 40px;
29 | padding-bottom: 30px;
30 | }
31 |
32 | .footer-list {
33 | display: flex;
34 | flex-direction: row;
35 | flex-wrap: wrap;
36 | }
37 |
38 | .footer-list-item {
39 | padding-right: 30px;
40 | padding-bottom: 8px;
41 | }
42 |
43 | .chapter > .subcontainer {
44 | max-width: 768px;
45 | }
46 |
47 | .my-full-width {
48 | max-width: none !important;
49 | }
50 |
51 | h1,
52 | h2,
53 | h3,
54 | h4,
55 | h5,
56 | h6 {
57 | font-family: 'IBM Plex Sans', sans-serif;
58 | font-weight: 700;
59 | }
60 |
61 | code,
62 | .parsable-input,
63 | .invalid-feedback.invalid-feedback-block-parsable-input {
64 | font-family: 'IBM Plex Mono', monospace;
65 | }
66 |
67 | p,
68 | a,
69 | li,
70 | blockquote {
71 | font-family: 'IBM Plex Sans', sans-serif;
72 | font-weight: 400;
73 | }
74 |
75 | .parsable-input,
76 | .parsable-input-autosize {
77 | /*font-size: 14px;*/
78 | }
79 |
80 | div.hash-example-instance {
81 | padding-bottom: 10px;
82 | }
83 |
84 | .invalid-feedback.invalid-feedback-block-parsable-input {
85 | padding-left: 8px;
86 | font-size: 14px;
87 | white-space: pre;
88 | margin-top: 0px;
89 | background-color: rgba(220, 53, 69, 0.25); /* from box-shadow color of input in bootstrap */
90 | }
91 |
92 | .parsable-input-with-error-div {
93 | background-color: #fff;
94 | }
95 |
96 | .parsable-input-inline {
97 | display: inline-block;
98 | }
99 |
100 | .parsable-input-block {
101 | min-width: 300px;
102 | }
103 |
104 | .array-vis,
105 | .hash-vis-wrapper {
106 | width: 100%;
107 | position: relative;
108 | }
109 |
110 | .hash-vis-wrapper {
111 | display: flex;
112 | }
113 |
114 | .hash-vis {
115 | transform-style: preserve-3d;
116 | }
117 |
118 | .tetris-labels {
119 | display: flex;
120 | flex-direction: column;
121 | }
122 |
123 | .tetris-label-div {
124 | display: flex;
125 | align-items: center;
126 | }
127 |
128 | .tetris-label {
129 | /* make it align to the right */
130 | margin-left: auto;
131 | margin-right: 10px;
132 | }
133 |
134 | .box {
135 | display: flex;
136 | font-family: 'IBM Plex Mono', monospace;
137 | position: absolute;
138 | opacity: 1;
139 |
140 | justify-content: center;
141 | align-items: center;
142 | text-align: center;
143 | }
144 |
145 | .box-content {
146 | word-wrap: break-word;
147 | min-width: 0;
148 | z-index: 2;
149 | transform: translateZ(2px);
150 | position: relative;
151 | }
152 |
153 | .box-content-extra-type {
154 | color: grey;
155 | }
156 |
157 | .box-empty {
158 | background: #f4f4f7;
159 | z-index: -1;
160 | }
161 |
162 | .box-full {
163 | background: #e5e6f1;
164 | }
165 |
166 | .box-animated {
167 | transition-property: opacity, transform;
168 | transition-duration: 0.5s;
169 | transition-timing-function: ease;
170 | }
171 |
172 | .active-box-selection {
173 | background: rgba(252, 233, 168);
174 | width: 40px;
175 | height: 40px;
176 | z-index: 2;
177 | position: absolute;
178 | }
179 |
180 | .active-box-selection-1 {
181 | border: none;
182 | }
183 |
184 | .active-box-selection-2 {
185 | /* border-style: dashed; */
186 | border: none;
187 | }
188 |
189 | .active-box-selection-animated {
190 | transition-property: opacity, transform;
191 | transition-timing-function: ease;
192 | }
193 |
194 | .box.box-created,
195 | .box.box-removing,
196 | .box.box-removed {
197 | opacity: 0;
198 | }
199 |
200 | .box.box-removed,
201 | .box.box-created {
202 | transition: none;
203 | }
204 |
205 | .highlight {
206 | background-color: #fcf8e3;
207 | }
208 |
209 | /*pre > code,
210 | .code-explanation,
211 | .code-explanation > code {
212 | font-size: 12px !important;
213 | }*/
214 |
215 | .code-block {
216 | /*line-height: 1.15;*/
217 | margin-bottom: 15px;
218 | }
219 |
220 | .code-block-with-annotations-scrollbar-container .scrollbar-track-y {
221 | left: 0px !important;
222 | right: none !important;
223 | }
224 |
225 | .scrollbar-track-y {
226 | width: 4px !important;
227 | }
228 |
229 | .scrollbar-thumb-y {
230 | width: 4px !important;
231 | }
232 |
233 | .scrollbar-track-x {
234 | height: 4px !important;
235 | }
236 |
237 | .scrollbar-thumb-x {
238 | height: 4px !important;
239 | }
240 |
241 | .visualized-code > .code-block-row {
242 | margin-top: 20px;
243 | }
244 |
245 | .code-block-with-annotations {
246 | line-height: 1.15;
247 | padding-top: 10px;
248 | padding-right: 10px;
249 | padding-bottom: 7px;
250 | padding-left: 10px;
251 | white-space: nowrap;
252 | }
253 |
254 | pre.code-line-container {
255 | margin: 0px;
256 | display: inline;
257 | }
258 |
259 | .line-with-annotation {
260 | background: white;
261 | }
262 |
263 | pre {
264 | font-size: 100%;
265 | overflow: hidden;
266 | }
267 |
268 | .line-with-annotation.current-line-highlight,
269 | .line-with-annotation.current-line-highlight:hover,
270 | .mobile-short-explanation {
271 | background-color: rgba(252, 233, 168);
272 | }
273 |
274 | .cross-fade-leave {
275 | opacity: 1;
276 | }
277 |
278 | .cross-fade-leave.cross-fade-leave-active {
279 | opacity: 0;
280 | transition: opacity 0.2s ease-in;
281 | }
282 |
283 | .cross-fade-enter {
284 | opacity: 0;
285 | }
286 | .cross-fade-enter.cross-fade-enter-active {
287 | opacity: 1;
288 | transition: opacity 0.2s ease-in;
289 | }
290 |
291 | .cross-fade-height {
292 | transition: height 0.5s ease-in-out;
293 | }
294 |
295 | .fc-inline {
296 | display: inline-block !important;
297 | width: 75px !important;
298 | line-height: 1 !important;
299 | margin-left: 5px;
300 | margin-right: 5px;
301 | }
302 |
303 | div.visualized-code {
304 | margin-top: 25px;
305 | margin-bottom: 35px;
306 | padding-left: 18px;
307 | }
308 |
309 | .hl-left {
310 | border-left: solid 4px #999999;
311 | }
312 |
313 | .tetris {
314 | display: flex;
315 | padding-bottom: 5px;
316 | }
317 |
318 | /* supposedly creates another layer */
319 | .fix-animation {
320 | transform: translateZ(0);
321 | will-change: transform;
322 | }
323 |
324 | .tetris-row {
325 | display: flex;
326 | margin-bottom: 10px;
327 | }
328 |
329 | blockquote.blockquote {
330 | font-size: 1rem !important;
331 | background: #f9f9f9;
332 | border-left: 5px solid #ccc;
333 | padding: 0.5em 10px;
334 | margin-top: 5px;
335 | }
336 |
337 | .inline-block {
338 | display: inline-block;
339 | }
340 |
341 | .slider-controls {
342 | margin-bottom: 10px;
343 | }
344 |
345 | .slider-controls-button-short {
346 | width: 40px;
347 | }
348 |
349 | .button-without-scbl-hideable {
350 | width: 100px;
351 | }
352 |
353 | @media (max-width: 600px) {
354 | .button-with-scbl-hideable {
355 | width: 40px;
356 | }
357 |
358 | .scbl-hideable {
359 | display: none;
360 | }
361 |
362 | .input-toolbar-button-label {
363 | display: none;
364 | }
365 | }
366 |
367 | @media (min-width: 601px) {
368 | .button-with-scbl-hideable {
369 | width: 100px;
370 | }
371 |
372 | .scbl-hideable {
373 | }
374 |
375 | .input-toolbar-button-label {
376 | }
377 | }
378 |
379 | .my-sticky-outer-outer-wrapper {
380 | margin-bottom: 12px;
381 | }
382 |
383 | .my-sticky-wrapper {
384 | padding-top: 5px;
385 | }
386 |
387 | .sticky-outer-wrapper .my-sticky-wrapper {
388 | margin-left: -30px;
389 | padding-left: 30px;
390 | margin-right: -30px;
391 | padding-right: 30px;
392 | }
393 |
394 | .sticky-outer-wrapper.active .my-sticky-wrapper {
395 | animation: 0.5s stickied-animation 0s ease;
396 | box-shadow: 0 5px 7px rgba(0, 0, 0, 0.19), 0 3px 3px rgba(0, 0, 0, 0.23);
397 | background: white;
398 | }
399 |
400 | .sticky-outer-wrapper:not(.active) .my-sticky-wrapper {
401 | border-top: 1px solid #dee2e6;
402 | border-bottom: 1px solid #dee2e6;
403 | }
404 |
405 | @keyframes stickied-animation {
406 | 0% {
407 | transform: translateX(0px);
408 | }
409 | 14% {
410 | transform: translateX(10px);
411 | }
412 | 28% {
413 | transform: translateX(-10px);
414 | }
415 | 42% {
416 | transform: translateX(-5px);
417 | }
418 | 56% {
419 | transform: translateX(5px);
420 | }
421 | 70% {
422 | transform: translateX(-2px);
423 | }
424 | 85% {
425 | transform: translateX(2px);
426 | }
427 | 100% {
428 | transform: translateX(0px);
429 | }
430 | }
431 |
432 | .row-block-input-toolbar {
433 | background: white;
434 | }
435 |
436 | .row-block-input-toolbar .col-input {
437 | margin-bottom: 5px;
438 | }
439 |
440 | .row-block-input-toolbar .col-buttons {
441 | margin-bottom: 5px;
442 | }
443 |
444 | .force-stick-to-top {
445 | position: sticky !important;
446 | top: 5px;
447 | z-index: 1000;
448 | }
449 |
450 | .badge-undo-redo-count {
451 | font-family: 'IBM Plex Mono', monospace;
452 | min-width: 22px;
453 | }
454 |
455 | .div-p {
456 | margin-bottom: 16px;
457 | }
458 |
459 | .toc-a:hover {
460 | text-decoration: none !important;
461 | }
462 |
463 | .toc-a:hover .toc-title {
464 | text-decoration: underline !important;
465 | }
466 |
467 | .line-with-annotation {
468 | width: 100%;
469 | display: flex;
470 | align-items: center;
471 | }
472 |
473 | .code-line-container,
474 | .code-explanation {
475 | flex-shrink: 0;
476 | }
477 |
478 | .dynamic-p {
479 | margin-bottom: 0;
480 | }
481 |
482 | .dynamic-p-inner-wrapper {
483 | transition-property: background-color;
484 | transition-duration: 750ms;
485 | transition-function: ease;
486 | }
487 |
488 | footer a.link {
489 | color: rgba(0, 0, 0);
490 | border: none;
491 | }
492 |
493 | footer a.link:hover {
494 | text-decoration: none;
495 | color: #b03000;
496 | border: none;
497 | }
498 |
499 | footer {
500 | margin-top: 60px;
501 | margin-bottom: 10px;
502 | }
503 |
504 | .hash-examples {
505 | margin-top: 8px;
506 | }
507 |
508 | @media screen and (min-width: 481px) {
509 | .line-with-annotation:hover {
510 | background: #e9ecef;
511 | width: 100%;
512 | }
513 | }
514 |
--------------------------------------------------------------------------------
/scripts/pyReimplWrapper.js:
--------------------------------------------------------------------------------
1 | const net = require('net');
2 | const split = require('split');
3 | import 'ignore-styles';
4 |
5 | import {BigNumber} from 'bignumber.js';
6 | import {DUMMY, EMPTY, None} from '../src/hash_impl_common';
7 | import {Dict32} from '../src/chapter4_real_python_dict';
8 | import {GenerateProbingLinks} from '../src/probing_visualization.js';
9 | import {AlmostPythonDict} from '../src/chapter3_hash_class';
10 | import {Ops as Chapter2Ops} from '../src/chapter2_hash_table_functions';
11 | import {Ops as Chapter1Ops} from '../src/chapter1_simplified_hash';
12 | import {Slot} from '../src/chapter3_and_4_common';
13 | import {List as ImmutableList} from 'immutable';
14 |
15 | function parseSimplePyObj(obj) {
16 | if (obj === null || typeof obj === 'string') {
17 | return obj;
18 | } else if (typeof obj === 'object' && obj.type === 'None') {
19 | let res = None;
20 | // TODO FIXME: this does not support multiple clients
21 | res._hashCode = obj.hash;
22 | return res;
23 | } else if (typeof obj === 'object' && obj.type === 'DUMMY') {
24 | return DUMMY;
25 | } else if (typeof obj === 'object' && obj.type === 'EMPTY') {
26 | return EMPTY;
27 | } else if (typeof obj === 'object' && obj.type === 'int') {
28 | return BigNumber(obj.value);
29 | } else {
30 | throw new Error(`Unknown obj ${JSON.stringify(obj)}`);
31 | }
32 | }
33 |
34 | function parseArray(array) {
35 | return array.map(parseSimplePyObj);
36 | }
37 |
38 | function dumpArray(array) {
39 | return array.map(dumpSimplePyObj);
40 | }
41 |
42 | function parsePairs(pairs) {
43 | return pairs.map(([k, v]) => [parseSimplePyObj(k), parseSimplePyObj(v)]);
44 | }
45 |
46 | function dumpSimplePyObj(obj) {
47 | if (obj === DUMMY) {
48 | return {
49 | type: 'DUMMY',
50 | };
51 | } else if (obj === None) {
52 | return {
53 | type: 'None',
54 | };
55 | } else if (BigNumber.isBigNumber(obj)) {
56 | return {
57 | type: 'int',
58 | value: obj.toString(),
59 | };
60 | } else {
61 | return obj;
62 | }
63 | }
64 |
65 | function restorePyDictState(state) {
66 | let {pySelf} = Dict32.__init__();
67 | if (state.slots != null) {
68 | pySelf = pySelf.set(
69 | 'slots',
70 | new ImmutableList(
71 | state.slots.map(slot => {
72 | let key = parseSimplePyObj(slot.key);
73 | let value = parseSimplePyObj(slot.value);
74 |
75 | return Slot({
76 | pyHashCode: slot.hashCode ? new BigNumber(slot.hashCode) : null,
77 | key: key,
78 | value: value,
79 | });
80 | })
81 | )
82 | );
83 | } else {
84 | pySelf = pySelf.set('slots', null);
85 | }
86 | pySelf = pySelf.set('used', state.used);
87 | pySelf = pySelf.set('fill', state.fill);
88 |
89 | return pySelf;
90 | }
91 |
92 | function dumpPyDictState(pySelf) {
93 | let data = {};
94 |
95 | data.slots = pySelf.get('slots').map(slot => {
96 | return {
97 | hashCode: slot.pyHashCode != null ? slot.pyHashCode.toString() : null,
98 | key: dumpSimplePyObj(slot.key),
99 | value: dumpSimplePyObj(slot.value),
100 | };
101 | });
102 | data.used = pySelf.get('used');
103 | data.fill = pySelf.get('fill');
104 |
105 | return data;
106 | }
107 |
108 | function dict32RunOp(pySelf, op, key, value, pairs) {
109 | switch (op) {
110 | case '__init__':
111 | pySelf = Dict32.__init__(pairs).pySelf;
112 | return {pySelf};
113 | case '__getitem__': {
114 | const {result, isException} = Dict32.__getitem__(pySelf, key);
115 | return {pySelf, result, isException};
116 | }
117 | case '__setitem__': {
118 | ({pySelf} = Dict32.__setitem__(pySelf, key, value));
119 | return {pySelf};
120 | }
121 | case '__delitem__': {
122 | let isException;
123 | ({pySelf, isException} = Dict32.__delitem__(pySelf, key));
124 | return {pySelf, isException};
125 | }
126 | default:
127 | throw new Error('Unknown op: ' + op);
128 | }
129 | }
130 |
131 | function almostPyDictRunOp(pySelf, op, key, value, pairs) {
132 | switch (op) {
133 | case '__init__':
134 | pySelf = AlmostPythonDict.__init__(pairs).pySelf;
135 | return {pySelf};
136 | case '__getitem__': {
137 | const {result, isException} = AlmostPythonDict.__getitem__(pySelf, key);
138 | return {pySelf, result, isException};
139 | }
140 | case '__setitem__recycling': {
141 | ({pySelf} = AlmostPythonDict.__setitem__recycling(pySelf, key, value));
142 | return {pySelf};
143 | }
144 | case '__setitem__no_recycling': {
145 | ({pySelf} = AlmostPythonDict.__setitem__no_recycling(pySelf, key, value));
146 | return {pySelf};
147 | }
148 | case '__delitem__': {
149 | let isException;
150 | ({pySelf, isException} = AlmostPythonDict.__delitem__(pySelf, key));
151 | return {pySelf, isException};
152 | }
153 | default:
154 | throw new Error('Unknown op: ' + op);
155 | }
156 | }
157 |
158 | function chapter1run(keys, op, key, numbers) {
159 | switch (op) {
160 | case 'create_new':
161 | ({keys} = Chapter1Ops.createNew(numbers));
162 | return {keys};
163 | case 'create_new_broken':
164 | ({keys} = Chapter1Ops.createNewBroken(numbers));
165 | return {keys};
166 | case 'has_key': {
167 | let result;
168 | ({keys, result} = Chapter1Ops.hasKey(keys, key));
169 | return {keys, result};
170 | }
171 | case 'linear_search': {
172 | let {result} = Chapter1Ops.linearSearch(numbers, key);
173 | return {result};
174 | }
175 | default:
176 | throw new Error('Unknown op: ' + op);
177 | }
178 | }
179 |
180 | function chapter2run(hashCodes, keys, op, key, array) {
181 | switch (op) {
182 | case 'create_new':
183 | ({hashCodes, keys} = Chapter2Ops.createNew(array));
184 | return {hashCodes, keys};
185 | case 'insert':
186 | ({hashCodes, keys} = Chapter2Ops.insert(hashCodes, keys, key));
187 | return {hashCodes, keys};
188 | case 'remove': {
189 | let isException;
190 | ({hashCodes, keys, isException} = Chapter2Ops.remove(hashCodes, keys, key));
191 | return {hashCodes, keys, isException};
192 | }
193 | case 'has_key': {
194 | let result;
195 | ({hashCodes, keys, result} = Chapter2Ops.hasKey(hashCodes, keys, key));
196 | return {hashCodes, keys, result};
197 | }
198 | case 'resize':
199 | ({hashCodes, keys} = Chapter2Ops.resize(hashCodes, keys));
200 | return {hashCodes, keys};
201 | default:
202 | throw new Error('Unknown op: ' + op);
203 | }
204 | }
205 |
206 | const server = net.createServer(c => {
207 | console.log('Client connected');
208 |
209 | c.on('end', () => {
210 | console.log('Client disconnected');
211 | });
212 |
213 | c.pipe(split()).on('data', line => {
214 | console.log('Received line of length ' + line.length);
215 | if (!line) return;
216 |
217 | const data = JSON.parse(line);
218 | const dictType = data.dict;
219 | const op = data.op;
220 | let {key, value, pairs, array} = data.args;
221 | if (key !== undefined) {
222 | key = parseSimplePyObj(key);
223 | }
224 | if (value !== undefined) {
225 | value = parseSimplePyObj(value);
226 | }
227 | if (pairs !== undefined) {
228 | pairs = parsePairs(pairs);
229 | }
230 | if (array !== undefined) {
231 | array = parseArray(array);
232 | }
233 |
234 | console.log(op, data.args);
235 |
236 | let isException, result;
237 | let response;
238 |
239 | if (dictType === 'dict32' || dictType === 'almost_python_dict') {
240 | let pySelf = restorePyDictState(data.self);
241 | if (dictType === 'dict32') {
242 | ({pySelf, isException, result} = dict32RunOp(pySelf, op, key, value, pairs));
243 | } else if (dictType === 'almost_python_dict') {
244 | ({pySelf, isException, result} = almostPyDictRunOp(pySelf, op, key, value, pairs));
245 | } else {
246 | throw new Error('Unknown dict type');
247 | }
248 |
249 | response = {
250 | exception: isException || false,
251 | result: result !== undefined ? dumpSimplePyObj(result) : null,
252 | self: dumpPyDictState(pySelf),
253 | };
254 | } else if (dictType === 'chapter2') {
255 | let hashCodes = data.hashCodes != null ? new ImmutableList(parseArray(data.hashCodes)) : undefined;
256 | let keys = data.keys != null ? new ImmutableList(parseArray(data.keys)) : undefined;
257 | ({hashCodes, keys, isException, result} = chapter2run(hashCodes, keys, op, key, array));
258 | response = {
259 | exception: isException || false,
260 | result: result !== undefined ? result : null,
261 | hashCodes: dumpArray(hashCodes),
262 | keys: dumpArray(keys),
263 | };
264 | } else if (dictType === 'chapter1') {
265 | let keys = data.keys != null ? new ImmutableList(parseArray(data.keys)) : undefined;
266 | ({keys, result} = chapter1run(keys, op, key, array));
267 | response = {
268 | result: result !== undefined ? result : null,
269 | keys: keys !== undefined ? dumpArray(keys) : null,
270 | };
271 | } else if (dictType === 'pythonProbing') {
272 | let g = new GenerateProbingLinks();
273 | const result = g.run(data.args.slotsCount, key, 'python');
274 | response = {
275 | result,
276 | };
277 | } else {
278 | throw new Error('Unknown dict type');
279 | }
280 |
281 | c.write(JSON.stringify(response) + '\n');
282 | });
283 | });
284 |
285 | server.on('error', err => {
286 | throw err;
287 | });
288 |
289 | server.on('listening', () => {
290 | console.log(`Listening`);
291 | });
292 |
293 | server.listen('pynode.sock', () => {
294 | console.log('Starting listening...');
295 | });
296 |
--------------------------------------------------------------------------------
/src/hash_impl_common.js:
--------------------------------------------------------------------------------
1 | import {BigNumber} from 'bignumber.js';
2 | import _ from 'lodash';
3 |
4 | export function repr(obj, allowNull = false) {
5 | let res;
6 | if (typeof obj === 'number') {
7 | res = ['int', obj];
8 | } else if (typeof obj === 'string') {
9 | res = ['string', obj];
10 | } else if (isNone(obj)) {
11 | res = ['none'];
12 | } else if (isDummy(obj)) {
13 | res = ['dummy'];
14 | } else if (BigNumber.isBigNumber(obj)) {
15 | res = ['bignumber.js', obj.toFixed()];
16 | } else if (obj === null && allowNull) {
17 | res = ['jsnull', obj];
18 | } else {
19 | throw new Error(`Unknown key: ${JSON.stringify(obj)}`);
20 | }
21 |
22 | return JSON.stringify(res);
23 | }
24 |
25 | export function displayStr(obj, quoteString = true) {
26 | if (typeof obj === 'number' || isNone(obj) || isDummy(obj)) {
27 | return obj.toString();
28 | } else if (BigNumber.isBigNumber(obj)) {
29 | return obj.toFixed();
30 | } else if (typeof obj === 'string') {
31 | if (quoteString) {
32 | return JSON.stringify(obj);
33 | } else {
34 | return obj;
35 | }
36 | } else {
37 | throw new Error(`Unknown key: ${JSON.stringify(obj)}`);
38 | }
39 | }
40 |
41 | class Int64 {
42 | SIZE = 64;
43 | JS_NUM_MAX_SIZE = 32;
44 |
45 | constructor(jsNumInt32 = 0) {
46 | this.JS_NUM_MAX_SIZE = 32;
47 |
48 | this.data = [];
49 | let signBit = jsNumInt32 >= 0 ? 0 : 1;
50 |
51 | for (let i = 0; i < this.JS_NUM_MAX_SIZE; ++i) {
52 | let bit = jsNumInt32 & (1 << i) ? 1 : 0;
53 | this.data.push(bit);
54 | }
55 |
56 | for (let i = this.JS_NUM_MAX_SIZE; i < this.SIZE; ++i) {
57 | this.data.push(signBit);
58 | }
59 | }
60 |
61 | xor(other) {
62 | for (let i = 0; i < this.SIZE; ++i) {
63 | this.data[i] ^= other.data[i];
64 | }
65 |
66 | return this;
67 | }
68 |
69 | sign() {
70 | return this.data[this.data.length - 1] == 1 ? -1 : 1;
71 | }
72 |
73 | inc() {
74 | this.data[0] += 1;
75 | this._carryOverAll();
76 | }
77 |
78 | eq(other) {
79 | for (let i = 0; i < this.SIZE; ++i) {
80 | if (this.data[i] != other.data[i]) {
81 | return false;
82 | }
83 | }
84 | return true;
85 | }
86 |
87 | add(other) {
88 | let carry = 0;
89 | for (let i = 0; i < this.SIZE; ++i) {
90 | this.data[i] += other.data[i] + carry;
91 | carry = (this.data[i] / 2) | 0;
92 | this.data[i] %= 2;
93 | }
94 |
95 | return this;
96 | }
97 |
98 | complement() {
99 | for (let i = 0; i < this.SIZE; ++i) {
100 | this.data[i] = this.data[i] == 0 ? 1 : 0;
101 | }
102 |
103 | return this;
104 | }
105 |
106 | mul(other) {
107 | let newData = [];
108 |
109 | for (let i = 0; i < this.SIZE; ++i) {
110 | newData[i] = 0;
111 | }
112 |
113 | for (let i = 0; i < this.SIZE; ++i) {
114 | if (this.data[i] === 0) {
115 | continue;
116 | }
117 |
118 | for (let j = 0; j < this.SIZE - i; ++j) {
119 | newData[i + j] += this.data[i] * other.data[j];
120 | }
121 | }
122 |
123 | this.data = newData;
124 | this._carryOverAll();
125 |
126 | return this;
127 | }
128 |
129 | toNumber() {
130 | let res = 0;
131 | for (let i = 0; i < 32; ++i) {
132 | if (this.data[i]) {
133 | res |= 1 << i;
134 | }
135 | }
136 |
137 | return res;
138 | }
139 |
140 | toStringDestroying() {
141 | let sign = this.sign();
142 | if (sign < 0) {
143 | this.complement().inc();
144 | }
145 |
146 | let decPower = [1];
147 | let decRes = [0];
148 | for (let i = 0; i < this.SIZE; ++i) {
149 | let carry = 0;
150 | if (this.data[i]) {
151 | for (let j = 0; j < decPower.length; ++j) {
152 | if (j >= decRes.length) decRes.push(0);
153 |
154 | decRes[j] += decPower[j] + carry;
155 | carry = (decRes[j] / 10) | 0;
156 | decRes[j] %= 10;
157 | }
158 | }
159 | if (carry) {
160 | decRes.push(carry);
161 | }
162 |
163 | carry = 0;
164 | for (let j = 0; j < decPower.length; ++j) {
165 | decPower[j] = decPower[j] * 2 + carry;
166 | carry = (decPower[j] / 10) | 0;
167 | decPower[j] %= 10;
168 | }
169 | if (carry) {
170 | decPower.push(carry);
171 | }
172 | }
173 |
174 | this.data = null;
175 |
176 | let res = [];
177 | if (sign < 0) res.push('-');
178 | for (let j = decRes.length - 1; j >= 0; j--) {
179 | res.push(String.fromCharCode('0'.charCodeAt(0) + decRes[j]));
180 | }
181 |
182 | return res.join('');
183 | }
184 |
185 | _carryOverAll() {
186 | let carry = 0;
187 | for (let i = 0; i < this.SIZE; ++i) {
188 | this.data[i] += carry;
189 | carry = (this.data[i] / 2) | 0;
190 | this.data[i] %= 2;
191 | }
192 | }
193 | }
194 |
195 | function pyHashStringAndUnicode(s) {
196 | let res = new Int64(s.charCodeAt(0) << 7);
197 | let magic = new Int64(1000003);
198 |
199 | for (let i = 0; i < s.length; ++i) {
200 | res.mul(magic).xor(new Int64(s.charCodeAt(i)));
201 | }
202 |
203 | res.xor(new Int64(s.length));
204 |
205 | if (res.eq(new Int64(-1))) {
206 | return '-2';
207 | } else {
208 | return res.toStringDestroying();
209 | }
210 | }
211 |
212 | export function pyHashString(s) {
213 | let sUtf8 = unescape(encodeURIComponent(s));
214 | return pyHashStringAndUnicode(sUtf8);
215 | }
216 |
217 | export function pyHashUnicode(s) {
218 | return pyHashStringAndUnicode(s);
219 | }
220 |
221 | export function pyHashLong(num) {
222 | const twoToPyLong_SHIFT = BigNumber(2).pow(30);
223 | const BASE = twoToPyLong_SHIFT;
224 | const _PyHASH_MODULUS = BigNumber(2)
225 | .pow(61)
226 | .minus(1);
227 |
228 | let x = BigNumber(0);
229 | let sign = 1;
230 | if (num.lt(0)) {
231 | sign = -1;
232 | num = num.negated();
233 | }
234 |
235 | let digits = [];
236 | while (num.gt(0)) {
237 | const d = num.mod(BASE);
238 | num = num.idiv(BASE);
239 | digits.push(d);
240 | }
241 |
242 | for (const d of digits.reverse()) {
243 | x = x
244 | .times(twoToPyLong_SHIFT)
245 | .plus(d)
246 | .modulo(_PyHASH_MODULUS);
247 | }
248 |
249 | if (sign < 0) {
250 | x = x.negated();
251 | }
252 | if (x.gte(BigNumber(2).pow(63))) {
253 | x = BigNumber(2)
254 | .pow(64)
255 | .minus(x);
256 | }
257 |
258 | if (x.eq(-1)) {
259 | x = BigNumber(-2);
260 | }
261 |
262 | return x;
263 | }
264 |
265 | export function pyHash(o) {
266 | if (typeof o === 'string') {
267 | return BigNumber(pyHashUnicode(o));
268 | } else if (typeof o === 'number') {
269 | // TODO:
270 | // throw new Error(`Plain JS numbers are not supported, use BigNumber`)
271 | return pyHashLong(BigNumber(o));
272 | } else if (BigNumber.isBigNumber(o)) {
273 | return pyHashLong(o);
274 | } else if (isNone(o)) {
275 | // TODO: for None hash seems to be always different
276 | return BigNumber(o._hashCode);
277 | } else {
278 | throw new Error('pyHash called with an object of unknown type: ' + JSON.stringify(o));
279 | }
280 | }
281 |
282 | export class BreakpointFunction {
283 | constructor() {
284 | this._breakpoints = [];
285 | this._extraBpContext = null;
286 | }
287 |
288 | addBP(point) {
289 | let bp = {
290 | point: point,
291 | };
292 |
293 | for (let key in this) {
294 | if (key[0] !== '_') {
295 | const value = this[key];
296 | if (value !== undefined && typeof value !== 'function') {
297 | bp[key] = value;
298 | }
299 | }
300 | }
301 |
302 | if (this._extraBpContext) {
303 | for (let key in this._extraBpContext) {
304 | bp[key] = this._extraBpContext[key];
305 | }
306 | }
307 |
308 | this._breakpoints.push(bp);
309 | }
310 |
311 | setExtraBpContext(extraBpContext) {
312 | this._extraBpContext = extraBpContext;
313 | }
314 |
315 | getBreakpoints() {
316 | return this._breakpoints;
317 | }
318 | }
319 |
320 | export function joinBreakpoints(breakpointsList, prefixes) {
321 | const bps = [];
322 | const maxTime = Math.max(...breakpointsList.map(l => l.length));
323 | for (let i = 0; i < maxTime; ++i) {
324 | const newBp = {};
325 | for (const [breakpoints, prefix] of _.zip(breakpointsList, prefixes)) {
326 | const bp = breakpoints[Math.min(i, breakpoints.length - 1)];
327 | for (let key in bp) {
328 | newBp[prefix + '_' + key] = bp[key];
329 | }
330 | }
331 | bps.push(newBp);
332 | }
333 | console.log(bps);
334 | return bps;
335 | }
336 |
337 | export function computeIdx(hashCodeBig, len) {
338 | return +hashCodeBig
339 | .mod(len)
340 | .plus(len)
341 | .mod(len)
342 | .toString();
343 | }
344 |
345 | export class HashBreakpointFunction extends BreakpointFunction {
346 | computeIdx(hashCodeBig, len) {
347 | return computeIdx(hashCodeBig, len);
348 | }
349 | }
350 |
351 | class DummyClass {
352 | toString() {
353 | return 'DUMMY';
354 | }
355 | }
356 |
357 | class NoneClass {
358 | _hashCode = '-9223372036581563745';
359 |
360 | toString() {
361 | return 'None';
362 | }
363 | }
364 |
365 | export const DUMMY = new DummyClass();
366 | export const None = new NoneClass();
367 |
368 | export function isNone(o) {
369 | return o instanceof NoneClass;
370 | }
371 |
372 | export function isDummy(o) {
373 | return o instanceof DummyClass;
374 | }
375 |
376 | export function EQ(o1, o2) {
377 | if (BigNumber.isBigNumber(o1) && (BigNumber.isBigNumber(o2) || typeof o2 === 'number')) {
378 | return o1.eq(o2);
379 | } else if (BigNumber.isBigNumber(o2) && typeof o1 === 'number') {
380 | return o2.eq(o1);
381 | }
382 |
383 | return o1 === o2;
384 | }
385 |
386 | function signedToUnsigned(num) {
387 | if (num.lt(0)) {
388 | return num.plus(BigNumber(2).pow(64));
389 | } else {
390 | return num;
391 | }
392 | }
393 |
394 | export function computePerturb(hashCode) {
395 | return signedToUnsigned(hashCode);
396 | }
397 |
398 | export function nextIdxPerturb(idx, perturb, size) {
399 | return +BigNumber(5 * idx + 1)
400 | .plus(perturb)
401 | .mod(size)
402 | .toString();
403 | }
404 |
405 | export function perturbShift(perturb) {
406 | return perturb.idiv(BigNumber(2).pow(5)); // >>= 5
407 | }
408 |
--------------------------------------------------------------------------------
/src/py_obj_parsing.js:
--------------------------------------------------------------------------------
1 | import {None, isNone, EQ} from './hash_impl_common';
2 |
3 | import {BigNumber} from 'bignumber.js';
4 |
5 | class PyParsingError extends Error {
6 | constructor(text, pos) {
7 | // super(`${text} (at position ${pos})`);
8 | const isStr = typeof text === 'string';
9 | const textEn = isStr ? text : text.en;
10 | super(textEn);
11 | this.pos = pos;
12 | this.text = isStr ? {en: textEn} : text;
13 | }
14 | }
15 |
16 | const digitsMinusPlus = '-+0123456789';
17 | const minusPlus = '-+';
18 |
19 | // TODO: add mode for validating stuff: e.g. parseString() should throw on `"string contents" stuff after`
20 | export class PyObjParser {
21 | constructor(literal) {
22 | this.s = literal;
23 | this.pos = 0;
24 | }
25 |
26 | skipWhitespace() {
27 | while (this.pos < this.s.length && /\s/.test(this.s[this.pos])) {
28 | this.pos++;
29 | }
30 | }
31 |
32 | current() {
33 | return this.s[this.pos];
34 | }
35 |
36 | next() {
37 | return this.s[this.pos + 1];
38 | }
39 |
40 | isWhiteSpaceOrEol(c) {
41 | return c == null || /\s/.test(c);
42 | }
43 |
44 | isCurrentWhitespaceOrEol() {
45 | return this.isWhiteSpaceOrEol(this.current());
46 | }
47 |
48 | consume(expectedChar) {
49 | const c = this.current();
50 | if (c == null) {
51 | this.throwErr(
52 | `Encountered unexpected EOL, expected ${expectedChar}`,
53 | `Неожиданный конец данных, ожидается ${expectedChar}`
54 | );
55 | }
56 |
57 | if (c !== expectedChar) {
58 | this.throwErr(
59 | `Expected \`${expectedChar}\`, got \`${c}\``,
60 | `Ожидается \`${expectedChar}\` вместо \`${c}\``
61 | );
62 | }
63 | this.pos++;
64 | }
65 |
66 | consumeWS(expectedChar) {
67 | this.skipWhitespace();
68 | this.consume(expectedChar);
69 | }
70 |
71 | maybeConsume(expectedChar) {
72 | if (this.current() === expectedChar) {
73 | this.consume(expectedChar);
74 | }
75 | }
76 |
77 | maybeConsumeWS(expectedChar) {
78 | this.skipWhitespace();
79 | this.maybeConsume(expectedChar);
80 | }
81 |
82 | throwErr(textEn, textRu, pos) {
83 | // TODO FIXME: pos computation looks way too complicated
84 | let posToInclude = pos != null ? pos : this.pos;
85 | posToInclude = Math.min(posToInclude, this.s.length - 1);
86 | if (posToInclude < 0) posToInclude = 0;
87 | throw new PyParsingError({en: textEn, ru: textRu}, posToInclude);
88 | }
89 |
90 | _parseStringOrNumberOrNone(allowedSeparators, fromDict, allowNonesInError) {
91 | // TODO: The whole None parsing and error reporting for unwrapped strings
92 | // TODO: is a bit of a mess
93 | if (this.isNextNone(allowedSeparators)) {
94 | return this._parseNoneOrThrowUnknownIdentifier(allowedSeparators);
95 | }
96 | return this._parseStringOrNumber(allowedSeparators, fromDict, allowNonesInError);
97 | }
98 |
99 | _parseStringOrNumber(allowedSeparators, fromDict = true, allowNonesInError = false) {
100 | this.skipWhitespace();
101 | let startPos = this.pos;
102 | const c = this.current();
103 | if (fromDict) {
104 | if (c === '{' || c === '[') {
105 | this.throwErr(
106 | 'Nested lists and dictionaries are not supported. Only strings and ints are.',
107 | 'Вложенные списки и словари не поддерживаются'
108 | );
109 | }
110 | if (c == null) {
111 | this.throwErr('Dict literal added abruptly - expected value', 'Неожиданный конец словаря');
112 | }
113 | }
114 |
115 | if (digitsMinusPlus.includes(c)) {
116 | return {res: this.parseNumber(allowedSeparators), startPos};
117 | } else if (`"'`.includes(c)) {
118 | return {res: this.parseString(), startPos};
119 | } else {
120 | this.throwErr(
121 | `Expected value - string, integer${
122 | allowNonesInError ? ' or None' : ''
123 | }. If you wanted a string, wrap it in quotes`,
124 | `Ожидается строка или целое${allowNonesInError ? ' или None' : ''}. Строки должны быть в кавычках.`
125 | );
126 | }
127 | }
128 |
129 | parseDict(minSize = null) {
130 | const allowedSeparators = ',:}';
131 | const c = this.current();
132 |
133 | this.consumeWS('{');
134 | let res = [];
135 | this.skipWhitespace();
136 | while (this.current() !== '}') {
137 | if (this.current() == null) {
138 | this.throwErr(
139 | 'Dict literal ended abruptly - no closing }',
140 | 'Неожиданный конец словаря, нет закрывающей }'
141 | );
142 | }
143 | let key = this._parseStringOrNumberOrNone(allowedSeparators).res;
144 | this.consumeWS(':');
145 | let value = this._parseStringOrNumberOrNone(allowedSeparators).res;
146 | res.push([key, value]);
147 |
148 | this.skipWhitespace();
149 | if (this.current() !== '}' && this.current() != null) this.consume(',');
150 | }
151 | this.consumeWS('}');
152 | if (minSize != null) {
153 | if (res.length < minSize) {
154 | if (minSize > 1) {
155 | this.throwErr(`There should be at least ${minSize} pairs`, `Должно быть не меньше ${minSize} пар`);
156 | } else {
157 | this.throwErr(`The data cannot be empty`, 'Пустые данные');
158 | }
159 | }
160 | }
161 | return res;
162 | }
163 |
164 | parseList(allowDuplicates = true, minSize = null, extraValueValidator) {
165 | const allowedSeparators = ',]';
166 | const c = this.current();
167 |
168 | console.log('parseList', c, this.s);
169 | this.maybeConsumeWS('[');
170 | let res = [];
171 | this.skipWhitespace();
172 | while (this.current() !== ']') {
173 | // if (this.current() == null) {
174 | // this.throwErr('List literal ended abruptly - no closing ]');
175 | // }
176 | if (this.current() == null) {
177 | break;
178 | }
179 | let {res: val, startPos: valStartPos} = this._parseStringOrNumberOrNone(allowedSeparators);
180 | if (!allowDuplicates) {
181 | for (let existingVal of res) {
182 | if (EQ(val, existingVal)) {
183 | this.throwErr(
184 | 'Duplicates are not allowed in this list',
185 | 'В списке не должно быть дублированных значений'
186 | );
187 | }
188 | }
189 | }
190 | if (extraValueValidator) {
191 | const error = extraValueValidator(val);
192 | if (error) {
193 | this.throwErr(error.en, error.ru, valStartPos);
194 | }
195 | }
196 | res.push(val);
197 | this.skipWhitespace();
198 | if (this.current() !== ']' && this.current() != null) this.maybeConsume(',');
199 | this.skipWhitespace();
200 | }
201 | this.maybeConsumeWS(']');
202 | if (minSize != null) {
203 | if (res.length < minSize) {
204 | if (minSize > 1) {
205 | this.throwErr(
206 | `In this chapter, the list need to have length at least ${minSize}`,
207 | `Список должен быть длиной не меньше ${minSize}`
208 | );
209 | } else {
210 | this.throwErr(`In this chapter, the list cannot be empty`, 'Список не может быть пустым');
211 | }
212 | }
213 | }
214 | return res;
215 | }
216 |
217 | parseNumber(allowedSeparators = '') {
218 | this.skipWhitespace();
219 | if (this.current() == null) {
220 | this.throwErr("Number can't be empty", 'Число не может быть пустым');
221 | }
222 |
223 | const originalPos = this.pos;
224 | while (digitsMinusPlus.includes(this.current())) {
225 | this.pos++;
226 | }
227 |
228 | if (this.current() === '.') {
229 | this.throwErr('Floats are not supported', 'Флоаты пока не поддерживаются');
230 | }
231 |
232 | if (this.current() === 'e') {
233 | this.throwErr('Floats in scientific notation are not supported', 'Флоаты пока не поддерживаются');
234 | }
235 |
236 | const nonDecimalErrorStringEn = 'Non-decimal bases are not supported';
237 | const nonDecimalErrorStringRu = 'Поддерживаются только числа в десятичной системе счисления';
238 | if (this.current() === 'x') {
239 | this.throwErr(nonDecimalErrorStringEn, nonDecimalErrorStringRu);
240 | }
241 | if (!this.isCurrentWhitespaceOrEol() && (!allowedSeparators || !allowedSeparators.includes(this.current()))) {
242 | // TODO: a bit more descriptive? and a bit less hacky?
243 | this.throwErr('Invalid syntax: number with non-digit characters', 'В числе должны быть только цифры');
244 | }
245 |
246 | const num = this.s.slice(originalPos, this.pos);
247 | if (num[0] === '0' && num.length > 1) {
248 | this.throwErr(nonDecimalErrorStringEn, nonDecimalErrorStringRu);
249 | }
250 | // TODO: python parses numbers like ++1, -+--1, etc properly
251 | if (isNaN(+num)) {
252 | this.throwErr('Invalid number', 'Невалидное число', originalPos);
253 | }
254 | return BigNumber(num);
255 | }
256 |
257 | parseString() {
258 | // TODO: handle escape characters
259 | // TODO: handle/throw an error on triple-quoted strings
260 | this.skipWhitespace();
261 | const c = this.current();
262 | if (c !== "'" && c !== '"') {
263 | this.throwErr(
264 | 'String must be wrapped in quotation characters (either `\'` or `"`)',
265 | 'Строки должны быть в кавычках'
266 | );
267 | }
268 | const quote = c;
269 | this.consume(quote);
270 |
271 | const originalPos = this.pos;
272 | let res = [];
273 | while (this.current() != null && this.current() !== quote) {
274 | if (this.current() === '\\') {
275 | if (this.next() !== '\\' && this.next() !== '"') {
276 | this.throwErr(
277 | 'The only supported escape sequences are for \\\\ and \\"',
278 | 'Такие escape-последовательности не поддерживаются',
279 | this.pos + 1
280 | );
281 | }
282 | res.push(this.next());
283 | this.pos += 2;
284 | } else {
285 | res.push(this.current());
286 | this.pos++;
287 | }
288 | }
289 | this.consume(quote);
290 | return res.join('');
291 | }
292 |
293 | isNextNone(allowedSeparators = '') {
294 | this.skipWhitespace();
295 | return (
296 | this.s.slice(this.pos, this.pos + 4) === 'None' &&
297 | (this.isWhiteSpaceOrEol(this.s[this.pos + 4]) || allowedSeparators.includes(this.s[this.pos + 4]))
298 | );
299 | }
300 |
301 | // Quite hacky
302 | _parseNoneOrThrowUnknownIdentifier(allowedSeparators) {
303 | this.skipWhitespace();
304 | if (this.isNextNone(allowedSeparators)) {
305 | const startPos = this.pos;
306 | this.pos += 4;
307 | return {res: None, startPos};
308 | }
309 | this.throwErr(
310 | 'Unknown identifier (if you wanted a string, wrap it in quotation marks - `"` or `\'`)',
311 | 'Строки должны быть в кавычках'
312 | );
313 | }
314 |
315 | checkTrailingChars() {
316 | this.skipWhitespace();
317 | if (this.pos < this.s.length) {
318 | this.throwErr('Trailing characters', 'Лишние символы');
319 | }
320 | }
321 | }
322 |
323 | function _checkTrailingChars(parser, parseFunc) {
324 | const res = parseFunc();
325 | parser.checkTrailingChars();
326 | return res;
327 | }
328 |
329 | export function parsePyString(s) {
330 | let parser = new PyObjParser(s);
331 | return _checkTrailingChars(parser, () => parser.parseString());
332 | }
333 |
334 | export function parsePyNumber(s) {
335 | let parser = new PyObjParser(s);
336 | return _checkTrailingChars(parser, () => parser.parseNumber());
337 | }
338 |
339 | export function parsePyDict(s, minSize = null) {
340 | let parser = new PyObjParser(s);
341 | return _checkTrailingChars(parser, () => parser.parseDict(minSize));
342 | }
343 |
344 | export function parsePyList(s, allowDuplicates = true, minSize = null, extraValueValidator) {
345 | let parser = new PyObjParser(s);
346 | return _checkTrailingChars(parser, () => parser.parseList(allowDuplicates, minSize, extraValueValidator));
347 | }
348 |
349 | export function parsePyStringOrNumber(s) {
350 | let parser = new PyObjParser(s);
351 | return _checkTrailingChars(parser, () => parser._parseStringOrNumber(null, false).res);
352 | }
353 |
354 | export function parsePyStringOrNumberOrNone(s) {
355 | let parser = new PyObjParser(s);
356 | return _checkTrailingChars(parser, () => parser._parseStringOrNumberOrNone(null, false, true).res);
357 | }
358 |
359 | // TODO: Dump functions are very hacky right now
360 |
361 | export function dumpSimplePyObj(o) {
362 | if (isNone(o)) {
363 | return 'None';
364 | }
365 | if (BigNumber.isBigNumber(o)) {
366 | return o.toString();
367 | }
368 | return JSON.stringify(o);
369 | }
370 |
371 | export function dumpPyList(l) {
372 | let strItems = [];
373 | for (let item of l) {
374 | strItems.push(dumpSimplePyObj(item));
375 | }
376 | return '[' + strItems.join(', ') + ']';
377 | }
378 |
379 | export function dumpPyDict(d) {
380 | let strItems = [];
381 | for (let [k, v] of d) {
382 | strItems.push(`${dumpSimplePyObj(k)}: ${dumpSimplePyObj(v)}`);
383 | }
384 | return '{' + strItems.join(', ') + '}';
385 | }
386 |
--------------------------------------------------------------------------------
/src/py_obj_parsing.test.js:
--------------------------------------------------------------------------------
1 | import {BigNumber} from 'bignumber.js';
2 | import {
3 | parsePyString,
4 | parsePyNumber,
5 | parsePyStringOrNumber,
6 | parsePyDict,
7 | parsePyList,
8 | dumpPyList,
9 | dumpPyDict,
10 | PyObjParser,
11 | } from './py_obj_parsing';
12 | import {None, isNone} from './hash_impl_common';
13 |
14 | test('Parsing empty strings', () => {
15 | expect(parsePyString('""')).toEqual('');
16 | expect(parsePyString("''")).toEqual('');
17 | });
18 |
19 | test('Parsing non-empty strings', () => {
20 | expect(parsePyString('"aba"')).toEqual('aba');
21 | expect(parsePyString(' "aba"')).toEqual('aba');
22 | expect(parsePyString(' "aba" ')).toEqual('aba');
23 | expect(parsePyString('"aba" ')).toEqual('aba');
24 |
25 | expect(parsePyString("'aba'")).toEqual('aba');
26 | expect(parsePyString(" 'aba'")).toEqual('aba');
27 | expect(parsePyString(" 'aba' ")).toEqual('aba');
28 | expect(parsePyString("'aba' ")).toEqual('aba');
29 |
30 | expect(parsePyString('"aba caba"')).toEqual('aba caba');
31 | expect(parsePyString("'aba caba'")).toEqual('aba caba');
32 |
33 | expect(parsePyString('"aba caba "')).toEqual('aba caba ');
34 | expect(parsePyString("' aba caba'")).toEqual(' aba caba');
35 | expect(parsePyString("' aba caba '")).toEqual(' aba caba ');
36 | expect(parsePyString("'aba caba '")).toEqual('aba caba ');
37 |
38 | expect(parsePyString("\"'''\"")).toEqual("'''");
39 | expect(() => parsePyString('aba caba')).toThrowError(/String must be wrapped.*quot/);
40 | expect(() => parsePyString("'aba caba")).toThrowError(/EOL/);
41 | });
42 |
43 | test('Parsing escaped strings', () => {
44 | expect(parsePyString('"\\\\"')).toEqual('\\');
45 | expect(parsePyString('"\\\\ \\""')).toEqual('\\ "');
46 | expect(() => parsePyString('"\\n"')).toThrow(/escape sequences/);
47 | expect(() => parsePyString('"ababab\\"')).toThrow(/EOL/);
48 | });
49 |
50 | test('Parsing regular numbers', () => {
51 | expect(parsePyNumber('0')).toEqual(BigNumber(0));
52 | expect(parsePyNumber('1')).toEqual(BigNumber(1));
53 | expect(parsePyNumber('-1')).toEqual(BigNumber(-1));
54 | expect(parsePyNumber('+1')).toEqual(BigNumber(1));
55 |
56 | expect(parsePyNumber(' 0 ')).toEqual(BigNumber(0));
57 | expect(parsePyNumber(' 1 ')).toEqual(BigNumber(1));
58 | expect(parsePyNumber(' -1 ')).toEqual(BigNumber(-1));
59 | expect(parsePyNumber(' +1 ')).toEqual(BigNumber(1));
60 |
61 | expect(parsePyNumber(' +1 ')).toEqual(BigNumber(1));
62 |
63 | expect(parsePyNumber('+123132')).toEqual(BigNumber(123132));
64 | expect(parsePyNumber('123132')).toEqual(BigNumber(123132));
65 | expect(parsePyNumber('+131')).toEqual(BigNumber(131));
66 | expect(parsePyNumber('-131')).toEqual(BigNumber(-131));
67 | expect(parsePyNumber('-123132')).toEqual(BigNumber(-123132));
68 | });
69 |
70 | test('Parsing numbers: reject floats and non-decimals', () => {
71 | expect(() => parsePyNumber('+1.')).toThrowError(/Floats.*not supported/);
72 | expect(() => parsePyNumber('1.')).toThrowError(/Floats.*not supported/);
73 | expect(() => parsePyNumber('1.2')).toThrowError(/Floats.*not supported/);
74 | expect(() => parsePyNumber('1.22323')).toThrowError(/Floats.*not supported/);
75 | expect(() => parsePyNumber('1e5')).toThrowError(/Floats.*not supported/);
76 | // The next one is a bit questionable, because it is not really a number
77 | expect(() => parsePyNumber('1e')).toThrowError(/Floats.*not supported/);
78 |
79 | expect(() => parsePyNumber('0777')).toThrowError(/Non-decimal/);
80 | expect(() => parsePyNumber('07')).toThrowError(/Non-decimal/);
81 | expect(() => parsePyNumber('0x777')).toThrowError(/Non-decimal/);
82 | expect(() => parsePyNumber('0x777')).toThrowError(/Non-decimal/);
83 |
84 | // again, it is not expected to properly validate non-decimals
85 | expect(() => parsePyNumber('0x777dsfdsf')).toThrowError(/Non-decimal/);
86 | });
87 |
88 | test('Parsing numbers: reject non-numbers', () => {
89 | expect(() => parsePyNumber('')).toThrowError(/Number can't be empty/);
90 | expect(() => parsePyNumber(' ')).toThrowError(/Number can't be empty/);
91 | expect(() => parsePyNumber('a')).toThrowError(/Invalid syntax/);
92 | expect(() => parsePyNumber(' a ')).toThrowError(/Invalid syntax/);
93 | expect(() => parsePyNumber('ababab')).toThrowError(/Invalid syntax/);
94 | expect(() => parsePyNumber(' a bababba')).toThrowError(/Invalid syntax/);
95 | expect(() => parsePyNumber('123abc')).toThrowError(/Invalid syntax/);
96 | expect(() => parsePyNumber(' 123a ')).toThrowError(/Invalid syntax/);
97 |
98 | // Techically, a number in python, but isn't considered one by the parser right now
99 | expect(() => parsePyNumber('--1')).toThrowError(/Invalid number/);
100 | });
101 |
102 | test('Parsing py strings or numbers', () => {
103 | expect(parsePyStringOrNumber(' "aba" ')).toEqual('aba');
104 | expect(parsePyStringOrNumber(' 17 ')).toEqual(BigNumber(17));
105 | expect(parsePyStringOrNumber(' "17" ')).toEqual('17');
106 | });
107 |
108 | test('Parsing dicts: empty dict', () => {
109 | const empty = [];
110 | expect(parsePyDict('{}')).toEqual(empty);
111 | expect(parsePyDict('{ }')).toEqual(empty);
112 | expect(parsePyDict(' { }')).toEqual(empty);
113 | expect(parsePyDict(' { } ')).toEqual(empty);
114 | expect(parsePyDict('{ } ')).toEqual(empty);
115 | expect(parsePyDict('{} ')).toEqual(empty);
116 | expect(parsePyDict(' {} ')).toEqual(empty);
117 | });
118 |
119 | test('Parsing dicts: just ints', () => {
120 | expect(parsePyDict(' {1:2, 2: 3,4: 5,6:7 }')).toEqual([
121 | [BigNumber(1), BigNumber(2)],
122 | [BigNumber(2), BigNumber(3)],
123 | [BigNumber(4), BigNumber(5)],
124 | [BigNumber(6), BigNumber(7)],
125 | ]);
126 | expect(parsePyDict('{ 1:2,2: 3,4: 5,6:7}')).toEqual([
127 | [BigNumber(1), BigNumber(2)],
128 | [BigNumber(2), BigNumber(3)],
129 | [BigNumber(4), BigNumber(5)],
130 | [BigNumber(6), BigNumber(7)],
131 | ]);
132 |
133 | const m12 = [[BigNumber(1), BigNumber(2)]];
134 | expect(parsePyDict('{1:2}')).toEqual(m12);
135 | expect(parsePyDict(' {1:2}')).toEqual(m12);
136 | expect(parsePyDict(' {1:2} ')).toEqual(m12);
137 | expect(parsePyDict('{1:2} ')).toEqual(m12);
138 | });
139 |
140 | test('Parsing dicts: just strings', () => {
141 | const e = [['a', 'b'], ['b', 'c'], ['d', 'e'], ['f', 'g']];
142 | expect(parsePyDict(" {'a':'b', 'b': 'c','d': 'e','f':'g' }")).toEqual(e);
143 | expect(parsePyDict("{ 'a':\"b\",\"b\": 'c','d': 'e','f':'g'}")).toEqual(e);
144 | });
145 |
146 | test('Parsing dicts: mixed strings and ints', () => {
147 | expect(parsePyDict(" {'a':2, 3: 'c','d': 4,5:'g' }")).toEqual([
148 | ['a', BigNumber(2)],
149 | [BigNumber(3), 'c'],
150 | ['d', BigNumber(4)],
151 | [BigNumber(5), 'g'],
152 | ]);
153 | });
154 |
155 | test('Parsing dicts: mixed strings, ints and Nones with repeated keys', () => {
156 | expect(parsePyDict(" {'a':2, 3: 'c','d': 4,5:'g' , 'a': 'b', 5: 'f' } ")).toEqual([
157 | ['a', BigNumber(2)],
158 | [BigNumber(3), 'c'],
159 | ['d', BigNumber(4)],
160 | [BigNumber(5), 'g'],
161 | ['a', 'b'],
162 | [BigNumber(5), 'f'],
163 | ]);
164 | expect(
165 | parsePyDict(
166 | " {'a':2, 3: 'c', None: 'abc', 'd': 4,5:'g' , 'a': 'b', 5: 'f' , 'a': None, None : 42 } "
167 | )
168 | ).toEqual([
169 | ['a', BigNumber(2)],
170 | [BigNumber(3), 'c'],
171 | [None, 'abc'],
172 | ['d', BigNumber(4)],
173 | [BigNumber(5), 'g'],
174 | ['a', 'b'],
175 | [BigNumber(5), 'f'],
176 | ['a', None],
177 | [None, BigNumber(42)],
178 | ]);
179 | });
180 |
181 | test('Parsing dicts: malformed dicts', () => {
182 | // TODO: more of this?
183 | expect(() => parsePyDict(' {')).toThrowError(/abrupt/);
184 | expect(() => parsePyDict(' { ')).toThrowError(/abrupt/);
185 | expect(() => parsePyDict(' } ')).toThrowError(/Expected.*{/);
186 | expect(() => parsePyDict('a')).toThrowError(/Expected.*{/);
187 | expect(() => parsePyDict("{'a':5")).toThrowError(/abrupt/);
188 | expect(() => parsePyDict("{'a':5")).toThrowError(/abrupt/);
189 | expect(() => parsePyDict("{'a',5")).toThrowError(/Expected.*:/);
190 | expect(() => parsePyDict("{'a':5e}")).toThrowError(/Floats/);
191 | expect(() => parsePyDict("{'a': 'b' 5: 6")).toThrowError(/Expected.*,/);
192 | expect(() => parsePyDict("{'a':5} fd fds")).toThrowError(/Trailing/);
193 | });
194 |
195 | test('Parsing lists: empty list', () => {
196 | expect(parsePyList('[]')).toEqual([]);
197 | expect(parsePyList('[ ]')).toEqual([]);
198 | expect(parsePyList(' [ ]')).toEqual([]);
199 | expect(parsePyList(' [ ] ')).toEqual([]);
200 | expect(parsePyList('[ ] ')).toEqual([]);
201 | expect(parsePyList('[] ')).toEqual([]);
202 | expect(parsePyList(' [] ')).toEqual([]);
203 | });
204 |
205 | test('Parsing lists: just ints', () => {
206 | expect(parsePyList(' [1,2, 2, 3,4, 5,6,7 ]')).toEqual([
207 | BigNumber(1),
208 | BigNumber(2),
209 | BigNumber(2),
210 | BigNumber(3),
211 | BigNumber(4),
212 | BigNumber(5),
213 | BigNumber(6),
214 | BigNumber(7),
215 | ]);
216 | expect(parsePyList('[ 1,2,2, 3,4, 5,6,7]')).toEqual([
217 | BigNumber(1),
218 | BigNumber(2),
219 | BigNumber(2),
220 | BigNumber(3),
221 | BigNumber(4),
222 | BigNumber(5),
223 | BigNumber(6),
224 | BigNumber(7),
225 | ]);
226 |
227 | expect(parsePyList('[1,2]')).toEqual([BigNumber(1), BigNumber(2)]);
228 | expect(parsePyList(' [1,2]')).toEqual([BigNumber(1), BigNumber(2)]);
229 | expect(parsePyList(' [1,2] ')).toEqual([BigNumber(1), BigNumber(2)]);
230 | expect(parsePyList('[1,2] ')).toEqual([BigNumber(1), BigNumber(2)]);
231 | });
232 |
233 | test('Parsing lists: just strings', () => {
234 | expect(parsePyList(" ['a','b', 'b', 'c','d', 'e','f','g' ]")).toEqual([
235 | 'a',
236 | 'b',
237 | 'b',
238 | 'c',
239 | 'd',
240 | 'e',
241 | 'f',
242 | 'g',
243 | ]);
244 | expect(parsePyList("[ 'a',\"b\",\"b\", 'c','d', 'e','f','g']")).toEqual([
245 | 'a',
246 | 'b',
247 | 'b',
248 | 'c',
249 | 'd',
250 | 'e',
251 | 'f',
252 | 'g',
253 | ]);
254 | });
255 |
256 | test('Parsing lists: mixed strings and ints', () => {
257 | expect(parsePyList(" ['a',2, 3, 'c','d', 4,5,'g' ]")).toEqual([
258 | 'a',
259 | BigNumber(2),
260 | BigNumber(3),
261 | 'c',
262 | 'd',
263 | BigNumber(4),
264 | BigNumber(5),
265 | 'g',
266 | ]);
267 | });
268 |
269 | test('Parsing lists: mixed strings, ints and Nones with repeated values', () => {
270 | expect(parsePyList(" ['a',2, 3, 'c' ,'d', 4,5,'g' , 'a', 'b', 5, 'f' ] ")).toEqual([
271 | 'a',
272 | BigNumber(2),
273 | BigNumber(3),
274 | 'c',
275 | 'd',
276 | BigNumber(4),
277 | BigNumber(5),
278 | 'g',
279 | 'a',
280 | 'b',
281 | BigNumber(5),
282 | 'f',
283 | ]);
284 | expect(
285 | parsePyList(
286 | " ['a',2, None , 3, 'c','d', None , 4,5,'g' , None,None,None , 'a', 'b', 5, 'f' ] "
287 | )
288 | ).toEqual([
289 | 'a',
290 | BigNumber(2),
291 | None,
292 | BigNumber(3),
293 | 'c',
294 | 'd',
295 | None,
296 | BigNumber(4),
297 | BigNumber(5),
298 | 'g',
299 | None,
300 | None,
301 | None,
302 | 'a',
303 | 'b',
304 | BigNumber(5),
305 | 'f',
306 | ]);
307 | });
308 |
309 | test('Parsing lists: malformed lists', () => {
310 | // TODO: more of this?
311 | expect(() => parsePyList(' [')).toThrowError(/abrupt/);
312 | expect(() => parsePyList(' [ ')).toThrowError(/abrupt/);
313 | expect(() => parsePyList(' ] ')).toThrowError(/Expected.*\[/);
314 | expect(() => parsePyList(' [5 5] ')).toThrowError(/Expected.*,/);
315 | expect(() => parsePyList('a')).toThrowError(/Expected.*\[/);
316 | expect(() => parsePyList("['a',5")).toThrowError(/abrupt/);
317 | expect(() => parsePyList("['a',5e]")).toThrowError(/Floats/);
318 | expect(() => parsePyList("['a',5] fdsfds")).toThrowError(/Trailing/);
319 | });
320 |
321 | test('Parsing None', () => {
322 | const parseNone = s => {
323 | let p = new PyObjParser(s);
324 | return p._parseNoneOrThrowUnknownIdentifier().res;
325 | };
326 |
327 | expect(isNone(parseNone('None'))).toBe(true);
328 | expect(isNone(parseNone(' None'))).toBe(true);
329 | expect(isNone(parseNone('None '))).toBe(true);
330 | expect(isNone(parseNone(' None '))).toBe(true);
331 | expect(() => parseNone(' Nonenone ')).toThrowError(/Unknown identifier/);
332 | expect(() => parseNone('Nonee')).toThrowError(/Unknown identifier/);
333 | });
334 |
335 | test('Dumping lists', () => {
336 | expect(dumpPyList([])).toEqual('[]');
337 | expect(dumpPyList([1, 1, 2, 3, 5])).toEqual('[1, 1, 2, 3, 5]');
338 | expect(dumpPyList(['abc', 'def', 2, 3, 5])).toEqual('["abc", "def", 2, 3, 5]');
339 | expect(dumpPyList([None, 'abc', None, 'def', 2, 3, 5])).toEqual('[None, "abc", None, "def", 2, 3, 5]');
340 | });
341 |
342 | test('Dumping dicts', () => {
343 | expect(dumpPyDict(new Map())).toEqual('{}');
344 | expect(
345 | dumpPyDict([
346 | [BigNumber(1), BigNumber(2)],
347 | [BigNumber(2), BigNumber(3)],
348 | [BigNumber(3), BigNumber(4)],
349 | [BigNumber(5), BigNumber(9)],
350 | ])
351 | ).toEqual('{1: 2, 2: 3, 3: 4, 5: 9}');
352 | expect(
353 | dumpPyDict([
354 | ['abc', BigNumber(4)],
355 | ['def', 'fgh'],
356 | [BigNumber(2), BigNumber(9)],
357 | [BigNumber(3), 'ar'],
358 | [BigNumber(5), ''],
359 | ])
360 | ).toEqual('{"abc": 4, "def": "fgh", 2: 9, 3: "ar", 5: ""}');
361 | expect(
362 | dumpPyDict([
363 | [None, BigNumber(3)],
364 | ['abc', BigNumber(4)],
365 | ['def', 'fgh'],
366 | [BigNumber(2), BigNumber(9)],
367 | [BigNumber(3), 'ar'],
368 | [BigNumber(5), ''],
369 | [None, 'abc'],
370 | ['abc', BigNumber(5)],
371 | ])
372 | ).toEqual('{None: 3, "abc": 4, "def": "fgh", 2: 9, 3: "ar", 5: "", None: "abc", "abc": 5}');
373 | });
374 |
--------------------------------------------------------------------------------
/src/probing_visualization.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import {Map as ImmutableMap, List as ImmutableList, Set as ImmutableSet} from 'immutable';
3 | import {
4 | BreakpointFunction,
5 | pyHash,
6 | computeIdx,
7 | computePerturb,
8 | perturbShift,
9 | nextIdxPerturb,
10 | displayStr,
11 | } from './hash_impl_common';
12 |
13 | import {isDefinedSmallBoxScreen} from './util';
14 |
15 | const d3 = Object.assign(
16 | {},
17 | require('d3-selection'),
18 | require('d3-interpolate'),
19 | require('d3-shape'),
20 | require('d3-transition'),
21 | require('d3-array')
22 | );
23 |
24 | const DEFAULT_PROBING_BOX_GEOMETRY = {
25 | boxSize: 40,
26 | boxSpacing: 8,
27 | };
28 |
29 | const SMALLER_PROBING_BOX_GEOMETRY = {
30 | boxSize: 30,
31 | boxSpacing: 6,
32 | };
33 |
34 | const TINY_PROBING_BOX_GEOMETRY = {
35 | boxSize: 30,
36 | boxSpacing: 1,
37 | };
38 |
39 | class ProbingVisualizationImpl extends React.Component {
40 | TRANSITION_TIME = 500;
41 | TOP_SPACE = 66;
42 | BOTTOM_SPACE = 66;
43 |
44 | transitionId = null;
45 | lastTransitionId = 0;
46 |
47 | constructor(props) {
48 | super(props);
49 |
50 | this.state = {
51 | firstRender: true,
52 | bpIdx: props.bpIdx,
53 | breakpoints: props.breakpoints,
54 | transitionRunning: false,
55 | transitionToBpIdx: null,
56 | boxSize: null,
57 | boxSpacing: null,
58 | };
59 | }
60 |
61 | setRef = node => {
62 | this.gChild = node;
63 | };
64 |
65 | shouldComponentUpdate(nextProps, nextState) {
66 | let waitForTransition = false;
67 | let shouldUpdate = false;
68 | if (this.state.boxSize !== nextState.boxSize || this.state.boxSize == null || nextState.boxSize == null)
69 | return true;
70 |
71 | /*if (nextProps.boxSize !== this.props.boxSize) {
72 | shouldUpdate = true;
73 | waitForTransition = true;
74 | } else */
75 | if (nextProps.breakpoints !== nextState.breakpoints) {
76 | waitForTransition = true;
77 | shouldUpdate = true;
78 | } else if (
79 | nextProps.bpIdx !== nextState.bpIdx &&
80 | (nextState.transitionToBpIdx == null || nextProps.bpIdx !== nextState.transitionToBpIdx)
81 | ) {
82 | shouldUpdate = true;
83 | waitForTransition =
84 | nextState.transitionToBpIdx != null &&
85 | ((nextState.bpIdx > nextState.transitionToBpIdx && nextProps.bpIdx > nextState.transitionToBpIdx) ||
86 | (nextState.bpIdx < nextState.transitionToBpIdx && nextProps.bpIdx < nextState.transitionToBpIdx));
87 | }
88 |
89 | return shouldUpdate && (!nextState.transitionRunning || !waitForTransition);
90 | }
91 |
92 | render() {
93 | const computedHeight =
94 | this.state.boxSize +
95 | this.TOP_SPACE +
96 | this.BOTTOM_SPACE +
97 | 10 +
98 | 30 +
99 | 25; /* TODO FIXME: this is all a bunch of hacks because repeatedAdj can make patterns overlap TOP_SPACE / BOTTOM_SPACE */
100 |
101 | let {fixedHeight, adjustTop} = this.props;
102 | const height = fixedHeight ? fixedHeight : computedHeight;
103 | adjustTop = adjustTop || 0;
104 |
105 | return (
106 |
107 |
108 |
109 | {['blue', 'green'].map(color => (
110 |
121 |
123 | ))}
124 |
125 |
127 |