19 | | Number of realizations | 20 |
20 | | Average number of ticks node 0 | 7009.15 |
21 | | Average number of ticks node 1 | 6998.15 |
22 | | Average number of ticks node 2 | 257677.55 |
23 | | Average number of ticks node 3 | 261423.6 |
24 |
25 |
26 | ### Preprocessing
27 |
28 | Market opens at 8AM which corresponds to a timestamp of 28800. This timestamp
29 | has been substracted to all timestamps to have a realizations that starts at
30 | time 0.
31 |
32 | Please note that as markets closes at 10PM, the end time of our
33 | substracted realizations is 50400.
34 |
35 |
--------------------------------------------------------------------------------
/hawkes/bund/bund.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/X-DataInitiative/tick-datasets/9d959b6e53e17145e93e9849ff1f9f6d2de8ae51/hawkes/bund/bund.npz
--------------------------------------------------------------------------------
/lib/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/X-DataInitiative/tick-datasets/9d959b6e53e17145e93e9849ff1f9f6d2de8ae51/lib/__init__.py
--------------------------------------------------------------------------------
/lib/binary.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | import numpy as np
4 | from sklearn.datasets import load_svmlight_file
5 |
6 | from lib.dataset_analysis import features_characteristics, print_characteristics
7 |
8 | # name, path, n_observations, n_features
9 | binary_datasets = [
10 | ('Adult Train', '../binary/adult/adult.trn.bz2', 32561, 123),
11 | ('Adult Test', '../binary/adult/adult.tst.bz2', 16281, 123),
12 | ('Covtype Train', '../binary/covtype/covtype.trn.bz2', 581012, 54),
13 | ('ijcnn1 Train', '../binary/ijcnn1/ijcnn1.trn.bz2', 49990, 22),
14 | ('ijcnn1 Test', '../binary/ijcnn1/ijcnn1.tst.bz2', 91701, 22),
15 | ('Reuters Train', '../binary/reuters/reuters.trn.bz2', 7770, 8315),
16 | ('Reuters Test', '../binary/reuters/reuters.tst.bz2', 3299, 8315),
17 | ('KDD 2010 Train', '../binary/kdd2010/kdd2010.trn.bz2',
18 | 19264097, 1129522),
19 | ('KDD 2010 Test', '../binary/kdd2010/kdd2010.tst.bz2',
20 | 748401, 1163024),
21 | ]
22 |
23 |
24 | def iterate_binary_dataset_path():
25 | for name, path, *args in binary_datasets:
26 | path = os.path.join(os.path.dirname(__file__), path)
27 | yield name, path
28 |
29 |
30 | def iterate_binary_dataset():
31 | for name, path, n_observations, n_features in binary_datasets:
32 | path = os.path.join(os.path.dirname(__file__), path)
33 | x, y = load_svmlight_file(path)
34 | yield name, x, y, n_observations, n_features
35 |
36 |
37 | def describe_binary_datasets():
38 | for name, x, y, *args in iterate_binary_dataset():
39 | n_rows = x.shape[0]
40 | positive_ratio = np.sum(y == 1) / n_rows
41 |
42 | characteristics = features_characteristics(x)
43 | characteristics['Class balancing'] = '{:.3g}% positive samples'.format(
44 | positive_ratio * 100)
45 |
46 | print('\n{:}'.format(name))
47 | print_characteristics(characteristics, html=False)
48 |
--------------------------------------------------------------------------------
/lib/compression.py:
--------------------------------------------------------------------------------
1 | import bz2
2 | import os
3 | from shutil import copyfileobj
4 | from itertools import chain
5 |
6 | from lib.binary import iterate_binary_dataset_path
7 | from lib.regression import iterate_regression_dataset_path
8 |
9 |
10 | def get_compressed_file_path(decompressed_file_path):
11 | if decompressed_file_path[-4:] == '.bz2':
12 | return decompressed_file_path
13 | return '{:}.bz2'.format(decompressed_file_path)
14 |
15 |
16 | def get_decompressed_file_path(compressed_file_path):
17 | return compressed_file_path.replace('.bz2', '')
18 |
19 |
20 | def compress_file(decompressed_file_path):
21 | compressed_file_path = get_compressed_file_path(decompressed_file_path)
22 | with open(decompressed_file_path, 'rb') as input:
23 | with bz2.BZ2File(compressed_file_path, 'wb') as output:
24 | copyfileobj(input, output)
25 |
26 |
27 | def decompress_file(compressed_file_path):
28 | decompressed_file_path = get_decompressed_file_path(compressed_file_path)
29 | with bz2.BZ2File(compressed_file_path, 'rb') as input:
30 | with open(decompressed_file_path, 'wb') as output:
31 | copyfileobj(input, output)
32 |
33 |
34 | all_datasets_path = chain(
35 | iterate_binary_dataset_path(),
36 | iterate_regression_dataset_path()
37 | )
38 |
39 |
40 | def compress_all_files(replace=False):
41 | for name, path in all_datasets_path:
42 | path = get_decompressed_file_path(path)
43 | compressed_path = get_compressed_file_path(path)
44 |
45 | if os.path.exists(compressed_path) and replace is False:
46 | print('%s dataset already exists at %s'
47 | % (name, compressed_path))
48 | else:
49 | print('compressing', name)
50 | compress_file(path)
51 |
52 |
53 | def decompress_all_files(replace=False):
54 | for name, path in all_datasets_path:
55 | path = get_compressed_file_path(path)
56 | decompressed_path = get_decompressed_file_path(path)
57 |
58 | if os.path.exists(decompressed_path) and replace is False:
59 | print('%s dataset already exists at %s'
60 | % (name, decompressed_path))
61 | else:
62 | print('decompressing', name)
63 | decompress_file(path)
64 |
--------------------------------------------------------------------------------
/lib/dataset_analysis.py:
--------------------------------------------------------------------------------
1 | from collections import OrderedDict
2 |
3 |
4 | def features_characteristics(features_matrix):
5 | n_rows = features_matrix.shape[0]
6 | n_columns = features_matrix.shape[1]
7 |
8 | sparsity = features_matrix.getnnz() / (n_rows * n_columns)
9 |
10 | characteristics = OrderedDict()
11 | characteristics['Number of observations'] = n_rows
12 | characteristics['Number of features'] = n_columns
13 | characteristics['Sparsity'] = '{:.3g}%'.format(sparsity * 100)
14 | return characteristics
15 |
16 |
17 | def print_characteristics(characteristics, html=False):
18 | if html is False:
19 | for k, v in characteristics.items():
20 | print("{:<25} {:}".format(k, v))
21 | else:
22 | html_data = ''
23 | html_data += '