├── LICENSE ├── README.md ├── docs ├── README_Satori.md ├── README_architecture.md ├── README_dataset.md ├── README_single_machine.md └── figs │ └── system.png ├── driver ├── __init__.py ├── dataset.py ├── drivers │ ├── __init__.py │ ├── base.py │ ├── ddp.py │ └── singleproc.py ├── main.py ├── models.py └── parser.py ├── examples ├── example_Satori_batch_1_node.slurm ├── example_Satori_batch_2_nodes.slurm ├── example_Satori_interactive.sh └── example_single_machine.sh ├── fast_sampler ├── concurrentqueue.h ├── fast_sampler.cpp ├── parallel-hashmap │ ├── .appveyor.yml │ ├── .gitattributes │ ├── .github │ │ └── FUNDING.yml │ ├── .gitignore │ ├── .travis.yml │ ├── CMakeLists.txt │ ├── LICENSE │ ├── README.md │ ├── benchmark │ │ ├── BENCHMARK.md │ │ ├── Makefile │ │ ├── bench.cc │ │ ├── bench.py │ │ ├── charts-template.html │ │ ├── js │ │ │ ├── examples.css │ │ │ ├── jquery.canvaswrapper.js │ │ │ ├── jquery.colorhelpers.js │ │ │ ├── jquery.flot.axislabels.js │ │ │ ├── jquery.flot.browser.js │ │ │ ├── jquery.flot.categories.js │ │ │ ├── jquery.flot.composeImages.js │ │ │ ├── jquery.flot.crosshair.js │ │ │ ├── jquery.flot.drawSeries.js │ │ │ ├── jquery.flot.errorbars.js │ │ │ ├── jquery.flot.fillbetween.js │ │ │ ├── jquery.flot.flatdata.js │ │ │ ├── jquery.flot.hover.js │ │ │ ├── jquery.flot.image.js │ │ │ ├── jquery.flot.js │ │ │ ├── jquery.flot.legend.js │ │ │ ├── jquery.flot.logaxis.js │ │ │ ├── jquery.flot.navigate.js │ │ │ ├── jquery.flot.pie.js │ │ │ ├── jquery.flot.resize.js │ │ │ ├── jquery.flot.saturated.js │ │ │ ├── jquery.flot.selection.js │ │ │ ├── jquery.flot.stack.js │ │ │ ├── jquery.flot.symbol.js │ │ │ ├── jquery.flot.threshold.js │ │ │ ├── jquery.flot.time.js │ │ │ ├── jquery.flot.touch.js │ │ │ ├── jquery.flot.touchNavigate.js │ │ │ ├── jquery.flot.uiConstants.js │ │ │ ├── jquery.js │ │ │ └── jquery.mousewheel.js │ │ ├── make_chart_data.py │ │ └── results │ │ │ ├── output_flat_par │ │ │ ├── output_flat_par_mutex_4 │ │ │ ├── output_flat_par_mutex_5 │ │ │ ├── output_mt_par_only │ │ │ ├── output_mt_stl_flat_par │ │ │ ├── output_mt_stl_flat_par_run2 │ │ │ ├── output_stl_flat │ │ │ ├── output_stl_flat_par │ │ │ └── output_various_N │ ├── cmake │ │ ├── CMakeLists.txt.in │ │ ├── DetectVersion.cmake │ │ ├── DownloadGTest.cmake │ │ ├── helpers.cmake │ │ └── phmap.cmake │ ├── css │ │ ├── bootstrap-responsive.min.css │ │ ├── bootstrap.min.css │ │ ├── colors.css │ │ └── style.css │ ├── examples │ │ ├── allmaps.cc │ │ ├── basic.cc │ │ ├── bench.cc │ │ ├── btree.cc │ │ ├── btree_fwd.h │ │ ├── dump_load.cc │ │ ├── dump_nested.cc │ │ ├── emplace.cc │ │ ├── f1.cc │ │ ├── f2.cc │ │ ├── hash_std.cc │ │ ├── hash_std.h │ │ ├── hash_value.cc │ │ ├── hash_value.h │ │ ├── insert_bench.cc │ │ ├── knucleotide-input.txt │ │ ├── knucleotide-input0.txt │ │ ├── knucleotide.cc │ │ ├── matt.cc │ │ └── serialize.cc │ ├── html │ │ ├── Makefile │ │ ├── bench_results │ │ │ └── martinus_mod │ │ │ │ ├── InsertManyInt.html │ │ │ │ ├── Lookup.html │ │ │ │ └── index2.html │ │ ├── css │ │ │ ├── bootstrap-responsive.min.css │ │ │ ├── bootstrap.min.css │ │ │ ├── colors.css │ │ │ └── style.css │ │ ├── diagrams │ │ │ ├── closed_hashing │ │ │ ├── closed_hashing.svg │ │ │ ├── index_computation │ │ │ └── index_computation.svg │ │ ├── img │ │ │ ├── closed_hashing.png │ │ │ ├── flat_mem_usage.gif │ │ │ ├── flat_mem_usage.png │ │ │ ├── flat_par_mutex_4.PNG │ │ │ ├── flat_par_mutex_5.PNG │ │ │ ├── flat_par_mutex_5_speed.PNG │ │ │ ├── flat_par_mutex_6_speed.PNG │ │ │ ├── flat_par_speed.PNG │ │ │ ├── flat_peak.gif │ │ │ ├── flat_peak.png │ │ │ ├── hashtable_benchmarks.PNG │ │ │ ├── idx_computation_cost.PNG │ │ │ ├── index_computation.png │ │ │ ├── lock_various_sizes.PNG │ │ │ ├── mt_stl_flat_par_both.PNG │ │ │ ├── mt_stl_flat_par_both_run2.PNG │ │ │ ├── mt_stl_flat_par_mem.PNG │ │ │ ├── mt_stl_flat_par_mem_run2.PNG │ │ │ ├── mt_stl_flat_par_mem_run2_zoomed.PNG │ │ │ ├── mt_stl_flat_par_mem_zoomed.PNG │ │ │ ├── mt_stl_flat_par_speed.PNG │ │ │ ├── mt_stl_flat_par_speed_run2.PNG │ │ │ ├── no_preselection.PNG │ │ │ ├── node_mem_usage.gif │ │ │ ├── node_mem_usage.png │ │ │ ├── node_peak.gif │ │ │ ├── node_peak.png │ │ │ ├── par_align_test.png │ │ │ ├── par_mt_memory.PNG │ │ │ ├── par_mt_speed.PNG │ │ │ ├── parallel_flat_peak.gif │ │ │ ├── parallel_flat_peak.png │ │ │ ├── parallel_node_peak.gif │ │ │ ├── parallel_node_peak.png │ │ │ ├── phash.png │ │ │ ├── phmap_logo.png │ │ │ ├── spp_flat_par_both.png │ │ │ ├── stl_flat_both.PNG │ │ │ ├── stl_flat_mem.PNG │ │ │ ├── stl_flat_par_both.PNG │ │ │ ├── stl_flat_par_mem.PNG │ │ │ ├── stl_flat_par_mem_zoomed.PNG │ │ │ ├── stl_flat_par_speed.PNG │ │ │ └── stl_flat_speed.PNG │ │ ├── includes.hs │ │ ├── latex_macros │ │ ├── parallel_hashmap.html │ │ ├── parallel_hashmap.md │ │ ├── parallel_hashmap.pdf │ │ ├── template.html │ │ └── template.latex │ ├── index.html │ ├── parallel_hashmap │ │ ├── btree.h │ │ ├── conanfile.py │ │ ├── meminfo.h │ │ ├── phmap.h │ │ ├── phmap_base.h │ │ ├── phmap_bits.h │ │ ├── phmap_config.h │ │ ├── phmap_dump.h │ │ ├── phmap_fwd_decl.h │ │ └── phmap_utils.h │ ├── phmap.natvis │ ├── phmap_lldb.py │ └── tests │ │ ├── btree_test.cc │ │ ├── btree_test.h │ │ ├── compressed_tuple_test.cc │ │ ├── container_memory_test.cc │ │ ├── dump_load_test.cc │ │ ├── flat_hash_map_test.cc │ │ ├── flat_hash_set_test.cc │ │ ├── hash_generator_testing.h │ │ ├── hash_policy_testing.h │ │ ├── hash_policy_testing_test.cc │ │ ├── hashtable_debug.h │ │ ├── node_hash_map_test.cc │ │ ├── node_hash_policy_test.cc │ │ ├── node_hash_set_test.cc │ │ ├── parallel_flat_hash_map_mutex_test.cc │ │ ├── parallel_flat_hash_map_test.cc │ │ ├── parallel_flat_hash_set_test.cc │ │ ├── parallel_hash_map_test.cc │ │ ├── parallel_node_hash_map_test.cc │ │ ├── parallel_node_hash_set_test.cc │ │ ├── raw_hash_set_allocator_test.cc │ │ ├── raw_hash_set_test.cc │ │ ├── test_instance_tracker.h │ │ ├── tracked.h │ │ ├── unordered_map_constructor_test.h │ │ ├── unordered_map_lookup_test.h │ │ ├── unordered_map_members_test.h │ │ ├── unordered_map_modifiers_test.h │ │ ├── unordered_set_constructor_test.h │ │ ├── unordered_set_lookup_test.h │ │ ├── unordered_set_members_test.h │ │ └── unordered_set_modifiers_test.h ├── sample_cpu.hpp ├── sample_cpu_pyg.hpp ├── setup.py └── utils.hpp └── fast_trainer ├── __init__.py ├── concepts.py ├── distributed.py ├── monkeypatch.py ├── samplers.py ├── shufflers.py ├── test.py ├── train.py ├── transferers.py └── utils.py /README.md: -------------------------------------------------------------------------------- 1 | SALIENT 2 | ======= 3 | 4 | SALIENT is a distributed multi-GPU training and inference system for graph neural networks (GNNs), targeting at massive graphs. It effectively mitigates (and with sufficient memory bandwidth, nearly eliminates) the bottleneck of a PyTorch workflow on mini-batch generation and transfer, rendering the overall training/inference time dominated by GPU computation. The system is described in the paper [Accelerating Training and Inference of Graph Neural Networks with Fast Sampling and Pipelining](https://arxiv.org/abs/2110.08450) published at MLSys 2022. This repository contains the codes of SALIENT. 5 | 6 | ![Illustration of mini-batch progress per training epoch](docs/figs/system.png) 7 | 8 | > **UPDATE:** SALIENT has a successor [SALIENT++](https://github.com/MITIBMxGraph/SALIENT_plusplus), which distributes the node feature matrix across machines. SALIENT does not rely on graph partitioning tools if the main memory can store the entire matrix. 9 | 10 | ## Setup and Example Usage 11 | 12 | SALIENT can be run on a machine with one or multiple GPUs or on a GPU cluster. 13 | 14 | * See this [README](docs/README_single_machine.md) for instructions to setup the system on a GPU machine. 15 | * See this [README](docs/README_Satori.md) for instructions to setup the system on the Satori cluster. 16 | 17 | Pointers of example usage of SALIENT are given in these instructions. 18 | 19 | ## Extension 20 | 21 | SALIENT is designed to be fully compatible with PyG. In particular, defining a GNN architecture is done the same way as writing a usual PyG code. 22 | 23 | * See this [README](docs/README_architecture.md) for instructions to add a GNN architecture. 24 | * See this [README](docs/README_dataset.md) for instructions to add a dataset. 25 | 26 | ## Artifacts 27 | 28 | While this repository is being maintained, a frozen version that reproduces the key results in the paper can be found in a separate [artifact repository](https://github.com/MITIBMxGraph/SALIENT_artifact). 29 | 30 | ## Reference 31 | 32 | Please cite this paper if you use the codes in your work: 33 | 34 | ``` 35 | @INPROCEEDINGS{Kaler2022, 36 | AUTHOR = {Tim Kaler and Nickolas Stathas and Anne Ouyang and Alexandros-Stavros Iliopoulos and Tao B. Schardl and Charles E. Leiserson and Jie Chen}, 37 | TITLE = {Accelerating Training and Inference of Graph Neural Networks with Fast Sampling and Pipelining}, 38 | BOOKTITLE = {Proceedings of Machine Learning and Systems 4}, 39 | YEAR = {2022}, 40 | } 41 | ``` 42 | 43 | ## Contributors 44 | 45 | SALIENT is developed by the xGraph team at the MIT-IBM Watson AI Lab. 46 | 47 | 48 | ## Acknowledgements 49 | This research was sponsored by MIT-IBM Watson AI Lab and in part by the United States Air Force Research Laboratory and the United States Air Force Artificial Intelligence Accelerator and was accomplished under Cooperative Agreement Number FA8750-19-2-1000. The views and conclusions contained in this document are those of the authors and should not be interpreted as representing the official policies, either expressed or implied, of the United States Air Force or the U.S. Government. The U.S. Government is authorized to reproduce and distribute reprints for Government purposes notwithstanding any copyright notation herein. 50 | -------------------------------------------------------------------------------- /docs/README_architecture.md: -------------------------------------------------------------------------------- 1 | # Adding a GNN Architecture 2 | 3 | The following files are involved with GNN architectures. 4 | 5 | `driver/models.py` defines architectures. Note that the `inference()` method inside each GNN class is supposed to perform layer-wise inference, which is less favored compared with batch-wise inference. Hence, it does not need to be implemented. The layer-wise inference code generally differs across architectures (because it needs to match the `forward()` method), while layer-wise inference code is one for all (which has already been implemented). 6 | 7 | `driver/main.py` involves architectures in two places. At the top, architectures are imported from `.models`. In the function `get_model_type()`, architectures are enumerated. -------------------------------------------------------------------------------- /docs/README_dataset.md: -------------------------------------------------------------------------------- 1 | # Adding a Dataset 2 | 3 | The following files are involved with datasets. 4 | 5 | `driver/dataset.py` defines a class `FastDataset`, where the loading of a dataset is handled by the class method `from_path()`. Typically, for an OGB (or PyG) dataset, it is first loaded by the corresponding API. Then, certain processing is done, such as converting the node feature matrix to row-major order and half-precision, and symmetrizing the graph adjacency matrix. To expedite the loading of the dataset in subsequent uses, we store pertinent data in `.pt` files so that subsequent uses do not need to perform the transformation again. 6 | 7 | `driver/main.py` uses the class `FastDataset` in the function `get_dataset()`. -------------------------------------------------------------------------------- /docs/README_single_machine.md: -------------------------------------------------------------------------------- 1 | # Setup Instructions on a GPU Machine 2 | 3 | ### 1. Install Conda 4 | 5 | Follow instructions on the [Conda user guide](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html). For example, to install Miniconda on a linux machine: 6 | 7 | ```bash 8 | wget https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh 9 | bash Miniconda3-py38_4.10.3-Linux-x86_64.sh 10 | ``` 11 | 12 | Or, to install Anaconda: 13 | 14 | ```bash 15 | wget https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh 16 | bash Anaconda3-2021.05-Linux-x86_64.sh 17 | ``` 18 | 19 | SALIENT has been tested on Python 3.8.10. 20 | 21 | It is highly recommended to create a new environment and do the subsequent steps therein. Example with a new environment called `salient`: 22 | 23 | ```bash 24 | conda create -n salient python=3.8 -y 25 | conda activate salient 26 | ``` 27 | 28 | ### 2. Install PyTorch 29 | 30 | Follow instructions on the [PyTorch homepage](https://pytorch.org). For example, to install on a linux machine with CUDA 11.3: 31 | 32 | ```bash 33 | conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch 34 | ``` 35 | 36 | SALIENT has been tested on PyTorch 1.10.0. 37 | 38 | ### 3. Install PyTorch-Geometric (PyG) 39 | 40 | Follow instructions on the [PyG Github page](https://github.com/pyg-team/pytorch_geometric). For example, with PyTorch >= 1.8.0, it suffices to do the following: 41 | 42 | ```bash 43 | conda install pyg -c pyg -c conda-forge 44 | ``` 45 | 46 | SALIENT has been tested on PyG 2.0.2. 47 | 48 | ### 4. Install Latest PyTorch-Sparse 49 | 50 | PyTorch-Sparse is usually installed together with PyG. We made a slight modification to support faster data transfer. See this [pull request](https://github.com/rusty1s/pytorch_sparse/pull/195) for detail. Currently, this change has been merged but not yet distributed. Hence, one should uninstall the prior version of PyTorch-Sparse and install the latest one from source: 51 | 52 | ```bash 53 | pip uninstall torch_sparse 54 | export FORCE_CUDA=1 55 | pip install git+git://github.com/rusty1s/pytorch_sparse.git@master 56 | ``` 57 | 58 | Note: Compilation requires nvcc >= 11. 59 | 60 | ### 5. Install OGB 61 | 62 | ```bash 63 | pip install ogb 64 | ``` 65 | 66 | SALIENT has been tested on OGB 1.3.2. 67 | 68 | ### 6. Install SALIENT's fast_sampler 69 | 70 | Go to the folder `fast_sampler` and install: 71 | 72 | ```bash 73 | cd fast_sampler 74 | python setup.py install 75 | cd .. 76 | ``` 77 | 78 | To check that it is properly installed, start python and type: 79 | 80 | ```python 81 | >>> import torch 82 | >>> import fast_sampler 83 | >>> help(fast_sampler) 84 | ``` 85 | 86 | One should see information of the package. 87 | 88 | Note: Compilation requires a c++ compiler that supports c++17 (e.g., gcc >= 8). 89 | 90 | ### 7. Install Other Dependencies 91 | 92 | ```bash 93 | conda install prettytable -c conda-forge 94 | ``` 95 | 96 | ### 8. Try an Example 97 | 98 | Congratulations! SALIENT has been installed. The script `example_single_machine.sh` under the folder `examples` contains examples to use SALIENT. Read it with care, edit as appropriate (e.g., set the correct `SALIENT_ROOT` and `DATASET_ROOT`), and run under this folder: 99 | 100 | ```bash 101 | ./example_single_machine.sh 102 | ``` 103 | 104 | It may take several minutes to run all examples therein. 105 | 106 | #### 8.1 Tips on Datasets 107 | 108 | Create a folder somewhere to store the datasets. Pass the folder path to `--dataset_root`. The first time an OGB dataset is used, it will be automatically downloaded to that folder (which may take some time depending on size). 109 | 110 | Alternatively, to pre-download an OGB dataset before trying the examples, start python and type: 111 | 112 | ```python 113 | >>> name = # type dataset name here, such as 'ogbn-arxiv' 114 | >>> root = # type dataset root here, such as '/Users/username/dataset' 115 | >>> from ogb.nodeproppred import PygNodePropPredDataset 116 | >>> dataset = PygNodePropPredDataset(name=name, root=root) 117 | ``` 118 | 119 | Note: When an OGB dataset is used the first time, SALIENT will process it after downloading and will store the processed data under a `processed` subfolder of that dataset. Subsequent uses of SALIENT will directly load the processed data. 120 | 121 | #### 8.2 Tips on Command Line Arguments 122 | 123 | To see all command-line arguments of SALIENT, set `PYTHONPATH` to be the root of SALIENT and type 124 | 125 | ```bash 126 | python -m driver.main --help 127 | ``` -------------------------------------------------------------------------------- /docs/figs/system.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/docs/figs/system.png -------------------------------------------------------------------------------- /driver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/driver/__init__.py -------------------------------------------------------------------------------- /driver/dataset.py: -------------------------------------------------------------------------------- 1 | from typing import Mapping, NamedTuple, Any 2 | from pathlib import Path 3 | import torch 4 | from torch_sparse import SparseTensor 5 | from ogb.nodeproppred import PygNodePropPredDataset 6 | 7 | from fast_sampler import to_row_major 8 | 9 | 10 | def get_sparse_tensor(edge_index, num_nodes=None, return_e_id=False): 11 | if isinstance(edge_index, SparseTensor): 12 | adj_t = edge_index 13 | if return_e_id: 14 | value = torch.arange(adj_t.nnz()) 15 | adj_t = adj_t.set_value(value, layout='coo') 16 | return adj_t 17 | 18 | if num_nodes is None: 19 | num_nodes = int(edge_index.max()) + 1 20 | 21 | value = torch.arange(edge_index.size(1)) if return_e_id else None 22 | return SparseTensor(row=edge_index[0], col=edge_index[1], 23 | value=value, 24 | sparse_sizes=(num_nodes, num_nodes)).t() 25 | 26 | 27 | class FastDataset(NamedTuple): 28 | name: str 29 | x: torch.Tensor 30 | y: torch.Tensor 31 | rowptr: torch.Tensor 32 | col: torch.Tensor 33 | split_idx: Mapping[str, torch.Tensor] 34 | meta_info: Mapping[str, Any] 35 | 36 | @classmethod 37 | def from_ogb(self, name: str, root='dataset'): 38 | print('Obtaining dataset from ogb...') 39 | return self.process_ogb(PygNodePropPredDataset(name=name, root=root)) 40 | 41 | @classmethod 42 | def process_ogb(self, dataset): 43 | print('Converting to fast dataset format...') 44 | data = dataset.data 45 | x = to_row_major(data.x).to(torch.float16) 46 | y = data.y.squeeze() 47 | 48 | if y.is_floating_point(): 49 | y = y.nan_to_num_(-1) 50 | y = y.long() 51 | 52 | adj_t = get_sparse_tensor(data.edge_index, num_nodes=x.size(0)) 53 | rowptr, col, _ = adj_t.to_symmetric().csr() 54 | return self(name=dataset.name, x=x, y=y, 55 | rowptr=rowptr, col=col, 56 | split_idx=dataset.get_idx_split(), 57 | meta_info=dataset.meta_info.to_dict()) 58 | 59 | @classmethod 60 | def from_path(self, _path, name): 61 | path = Path(_path).joinpath('_'.join(name.split('-')), 'processed') 62 | if not (path.exists() and path.is_dir()): 63 | print(f'First time preprocessing {name}; may take some time...') 64 | dataset = self.from_ogb(name, root=_path) 65 | print(f'Saving processed data...') 66 | dataset.save(_path, name) 67 | return dataset 68 | else: 69 | return self.from_path_if_exists(_path, name) 70 | 71 | @classmethod 72 | def from_path_if_exists(self, _path, name): 73 | path = Path(_path).joinpath('_'.join(name.split('-')), 'processed') 74 | assert path.exists() and path.is_dir() 75 | data = { 76 | field: torch.load(path.joinpath(field + '.pt')) 77 | for field in self._fields 78 | } 79 | data['y'] = data['y'].long() 80 | data['x'] = data['x'].to(torch.float16) 81 | assert data['name'] == name 82 | return self(**data) 83 | 84 | def save(self, _path, name): 85 | path = Path(_path).joinpath('_'.join(name.split('-')), 'processed') 86 | # path.mkdir() 87 | for i, field in enumerate(self._fields): 88 | torch.save(self[i], path.joinpath(field + '.pt')) 89 | 90 | def adj_t(self): 91 | num_nodes = self.x.size(0) 92 | return SparseTensor(rowptr=self.rowptr, col=self.col, 93 | sparse_sizes=(num_nodes, num_nodes), 94 | is_sorted=True, trust_data=True) 95 | 96 | def share_memory_(self): 97 | self.x.share_memory_() 98 | self.y.share_memory_() 99 | self.rowptr.share_memory_() 100 | self.col.share_memory_() 101 | 102 | for v in self.split_idx.values(): 103 | v.share_memory_() 104 | 105 | @property 106 | def num_features(self): 107 | return self.x.size(1) 108 | 109 | @property 110 | def num_classes(self): 111 | return int(self.meta_info['num classes']) 112 | -------------------------------------------------------------------------------- /driver/drivers/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BaseDriver 2 | from .ddp import DDPDriver 3 | from .singleproc import SingleProcDriver 4 | -------------------------------------------------------------------------------- /driver/drivers/ddp.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from typing import NamedTuple 4 | import torch 5 | import torch.distributed as dist 6 | from torch.nn.parallel import DistributedDataParallel 7 | 8 | from . import BaseDriver 9 | from fast_trainer.shufflers import DistributedShuffler 10 | 11 | 12 | def set_master(addr: str, port=1884): 13 | os.environ['MASTER_ADDR'] = addr 14 | os.environ['MASTER_PORT'] = str(port) 15 | 16 | 17 | class DDPConfig(NamedTuple): 18 | node_num: int 19 | num_devices_per_node: int 20 | total_num_nodes: int 21 | 22 | @property 23 | def world_size(self): 24 | return self.total_num_nodes * self.num_devices_per_node 25 | 26 | 27 | def get_ddp_config(job_dir: str, total_num_nodes: int, 28 | num_devices_per_node: int): 29 | assert total_num_nodes > 0 30 | assert num_devices_per_node > 0 31 | 32 | my_node_name = str(os.environ['SLURMD_NODENAME']) 33 | print(f'{my_node_name} starting', flush=True) 34 | 35 | print("DDP dir is " + str(job_dir), flush=True) 36 | while True: 37 | node_list = os.listdir(job_dir) 38 | print("Node list is " + str(node_list), flush=True) 39 | print("Length is " + str(len(node_list)) + " and waiting for " 40 | + str(total_num_nodes), flush=True) 41 | if len(node_list) == total_num_nodes: 42 | break 43 | time.sleep(1) 44 | 45 | node_list = sorted(node_list) 46 | 47 | for i in range(len(node_list)): 48 | if my_node_name == node_list[i]: 49 | node_num = i 50 | break 51 | else: 52 | raise ValueError(f'Unable to find {my_node_name} in {node_list}') 53 | 54 | set_master(node_list[0]) 55 | 56 | return DDPConfig(node_num=node_num, 57 | num_devices_per_node=num_devices_per_node, 58 | total_num_nodes=len(node_list)) 59 | 60 | 61 | class DDPDriver(BaseDriver): 62 | global_rank: int 63 | ddp_cfg: DDPConfig 64 | 65 | def __init__(self, args, device, dataset, model_type, ddp_cfg: DDPConfig): 66 | assert args.train_type == 'serial' 67 | 68 | self.ddp_cfg = ddp_cfg 69 | self.global_rank = ( 70 | ddp_cfg.node_num * ddp_cfg.num_devices_per_node + device.index) 71 | dist.init_process_group( 72 | 'nccl', rank=self.global_rank, world_size=ddp_cfg.world_size) 73 | 74 | self.orig_model = None 75 | super().__init__(args, [device], dataset, model_type) 76 | 77 | self.train_shuffler = DistributedShuffler( 78 | self.dataset.split_idx['train'], ddp_cfg.world_size) 79 | self.test_shuffler = DistributedShuffler( 80 | self.dataset.split_idx['test'], ddp_cfg.world_size) 81 | self.valid_shuffler = DistributedShuffler( 82 | self.dataset.split_idx['valid'], ddp_cfg.world_size) 83 | self.reset() 84 | 85 | def __del__(self): 86 | dist.destroy_process_group() 87 | 88 | def _reset_model(self): 89 | if self.orig_model is None: 90 | self.orig_model = self.model 91 | self.orig_model.reset_parameters() 92 | self.model = DistributedDataParallel( 93 | self.orig_model, device_ids=[self.main_device]) # , find_unused_parameters=True) 94 | 95 | def get_idx_test(self, name): 96 | if name == 'test': 97 | return self.test_shuffler.get_idx(self.global_rank) 98 | elif name == 'valid': 99 | return self.valid_shuffler.get_idx(self.global_rank) 100 | else: 101 | raise ValueError('invalid test dataset name') 102 | 103 | def get_idx(self, epoch: int): 104 | self.train_shuffler.set_epoch(10000*self.TRIAL_NUM + epoch) 105 | return self.train_shuffler.get_idx(self.global_rank) 106 | 107 | def get_sampler(self, _seed): 108 | return torch.utils.data.distributed.DistributedSampler( 109 | self.dataset.split_idx['train'], 110 | num_replicas=self.ddp_cfg.world_size, 111 | rank=self.global_rank, seed=_seed) 112 | 113 | @property 114 | def my_name(self): 115 | return f'{super().my_name}_{self.ddp_cfg.node_num}_{self.main_device.index}' 116 | 117 | @property 118 | def is_main_proc(self): 119 | return self.global_rank == 0 120 | -------------------------------------------------------------------------------- /driver/drivers/singleproc.py: -------------------------------------------------------------------------------- 1 | from . import BaseDriver 2 | from fast_trainer.shufflers import Shuffler 3 | 4 | 5 | class SingleProcDriver(BaseDriver): 6 | def __init__(self, args, devices, dataset, model_type): 7 | super().__init__(args, devices, dataset, model_type) 8 | 9 | self.train_shuffler = Shuffler(self.dataset.split_idx['train']) 10 | 11 | def get_idx(self, epoch: int): 12 | self.train_shuffler.set_epoch(epoch) 13 | return self.train_shuffler.get_idx() 14 | 15 | def get_idx_test(self, name): 16 | return self.dataset.split_idx[name] 17 | 18 | @property 19 | def is_main_proc(self): 20 | return True 21 | -------------------------------------------------------------------------------- /examples/example_Satori_batch_2_nodes.slurm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | #SBATCH -J example_Satori_batch_2_nodes 4 | #SBATCH -o SALIENT/job_output/%x_%j.err 5 | #SBATCH -e SALIENT/job_output/%x_%j.err 6 | #SBATCH --nodes=1 7 | #SBATCH --gres=gpu:2 8 | #SBATCH --array=1-2 9 | #SBATCH --mem=1T 10 | #SBATCH --exclusive 11 | #SBATCH --time 00:03:00 12 | #SBATCH -p sched_system_all_8 13 | 14 | # Script to run examples in the batch mode on Satori (2 nodes). READ 15 | # ALL INSTRUCTIONS!! 16 | # 17 | # Submit the job using 18 | # 19 | # $ sbatch SALIENT/examples/example_Satori_batch_2_nodes.slurm 20 | # 21 | # Watch the job using 22 | # 23 | # $ squeue 24 | 25 | # Activate environment 26 | HOME2=/nobackup/users/$(whoami) 27 | PYTHON_VIRTUAL_ENVIRONMENT=salient 28 | source $HOME2/anaconda3/etc/profile.d/conda.sh 29 | conda activate $PYTHON_VIRTUAL_ENVIRONMENT 30 | 31 | # Get JOB_NAME (set at the beginning of this script) 32 | JOB_NAME=$SLURM_JOB_NAME 33 | 34 | # The current program hardcodes using the environment variable 35 | # SLURMD_NODENAME to distinguish machines. On Satori, the scheduler 36 | # will set this variable. 37 | 38 | # Set SALIENT root and PYTHONPATH 39 | SALIENT_ROOT=$HOME/SALIENT 40 | export PYTHONPATH=$SALIENT_ROOT 41 | 42 | # Set the data paths 43 | DATASET_ROOT=$HOME2/dataset 44 | OUTPUT_ROOT=$SALIENT_ROOT/job_output 45 | 46 | # Speficy directory for --ddp_dir. It must be an empty dir. For 47 | # example, it is not wise to use $OUTPUT_ROOT/$JOB_NAME, if prior 48 | # results are there 49 | DDP_DIR=$OUTPUT_ROOT/$JOB_NAME/ddp 50 | # 51 | # MANUALLY CREATE DDP_DIR IF NOT EXISTENT, OR CLEAR ALL CONTENTS 52 | # INSIDE IF EXISTENT!! CANNOT DO SO IN THIS SCRIPT, BECAUSE MULTIPLE 53 | # NODES WILL TRY TO CREATE/CLEAR, INTERFERING WITH THE NEXT TOUCH. 54 | # 55 | # Then, under the dir, create one empty file for each node, where the 56 | # file name is the node name 57 | touch $DDP_DIR/$SLURMD_NODENAME 58 | 59 | # Run examples. For the full list of options, see driver/parser.py 60 | # 61 | # Turn on --verbose to see timing statistics 62 | # 63 | # 2 node, 2 GPUs/node, must do ddp 64 | python -m driver.main ogbn-arxiv $JOB_NAME \ 65 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 66 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 67 | --model_name SAGE --test_type batchwise \ 68 | --overwrite_job_dir \ 69 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 2 \ 70 | --ddp_dir $DDP_DIR 71 | -------------------------------------------------------------------------------- /examples/example_Satori_interactive.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to run examples on an interactive node on Satori. For better 4 | # performance, get exclusive access to the onde. READ ALL 5 | # INSTRUCTIONS!! 6 | 7 | # Set JOB_NAME used for this script 8 | JOB_NAME=example_Satori_interactive 9 | 10 | # Set environment variable SLURMD_NODENAME so that os.environ can get 11 | # it. Usually, on a cluster with SLURM scheduler, the scheduler will 12 | # set this variable. In other cases, manually set it like the 13 | # following: 14 | export SLURMD_NODENAME=`hostname` 15 | 16 | # Set SALIENT root and PYTHONPATH 17 | SALIENT_ROOT=$HOME/SALIENT 18 | export PYTHONPATH=$SALIENT_ROOT 19 | 20 | # Set the data paths 21 | DATASET_ROOT=$HOME2/dataset 22 | OUTPUT_ROOT=$SALIENT_ROOT/job_output 23 | 24 | # Run examples. For the full list of options, see driver/parser.py 25 | # 26 | # Turn on --verbose to see timing statistics 27 | # 28 | # 1 node, 1 GPU, no ddp 29 | echo 'Example 1' 30 | echo '' 31 | python -m driver.main ogbn-arxiv $JOB_NAME \ 32 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 33 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 34 | --model_name SAGE --test_type batchwise \ 35 | --overwrite_job_dir \ 36 | --num_workers 30 --max_num_devices_per_node 1 --total_num_nodes 1 37 | echo '' 38 | echo '======================================================================' 39 | echo '' 40 | echo 'Example 2' 41 | echo '' 42 | # 43 | # 1 node, 1 GPU, ddp 44 | python -m driver.main ogbn-arxiv $JOB_NAME \ 45 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 46 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 47 | --model_name SAGE --test_type batchwise \ 48 | --overwrite_job_dir \ 49 | --num_workers 30 --max_num_devices_per_node 1 --total_num_nodes 1 \ 50 | --one_node_ddp 51 | echo '' 52 | echo '======================================================================' 53 | echo '' 54 | echo 'Example 3' 55 | echo '' 56 | # 57 | # 1 node, 2 GPUs, must do ddp 58 | python -m driver.main ogbn-arxiv $JOB_NAME \ 59 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 60 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 61 | --model_name SAGE --test_type batchwise \ 62 | --overwrite_job_dir \ 63 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 64 | --one_node_ddp 65 | echo '' 66 | echo '======================================================================' 67 | echo '' 68 | echo 'Example 4' 69 | echo '' 70 | # 71 | # only run inference 72 | python -m driver.main ogbn-arxiv $JOB_NAME \ 73 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 74 | --trials 1 --do_test_run \ 75 | --do_test_run_filename $OUTPUT_ROOT/$JOB_NAME/model_0_3.pt \ 76 | $OUTPUT_ROOT/$JOB_NAME/model_1_3.pt \ 77 | --model_name SAGE --test_type batchwise \ 78 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 79 | --one_node_ddp 80 | echo '' 81 | echo '======================================================================' 82 | echo '' 83 | echo 'Example 5' 84 | echo '' 85 | # 86 | # ogbn-products 87 | python -m driver.main ogbn-products $JOB_NAME \ 88 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 89 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 90 | --model_name SAGE --test_type batchwise \ 91 | --overwrite_job_dir \ 92 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 93 | --one_node_ddp 94 | echo '' 95 | echo '======================================================================' 96 | echo '' 97 | echo 'Example 6' 98 | echo '' 99 | # 100 | # GIN 101 | python -m driver.main ogbn-arxiv $JOB_NAME \ 102 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 103 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 104 | --model_name GIN --test_type batchwise \ 105 | --overwrite_job_dir \ 106 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 107 | --one_node_ddp 108 | echo '' 109 | echo '======================================================================' 110 | echo '' 111 | echo 'Example 7' 112 | echo '' 113 | # 114 | # GAT 115 | python -m driver.main ogbn-arxiv $JOB_NAME \ 116 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 117 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 118 | --model_name GAT --test_type batchwise \ 119 | --overwrite_job_dir \ 120 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 121 | --one_node_ddp 122 | echo '' 123 | echo '======================================================================' 124 | echo '' 125 | echo 'Example 8' 126 | echo '' 127 | # 128 | # SAGEResInception 129 | python -m driver.main ogbn-arxiv $JOB_NAME \ 130 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 131 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 132 | --model_name SAGEResInception --test_type batchwise \ 133 | --overwrite_job_dir \ 134 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 135 | --one_node_ddp 136 | echo '' 137 | echo '======================================================================' 138 | echo '' 139 | echo 'Example 9' 140 | echo '' 141 | # 142 | # layerwise test 143 | python -m driver.main ogbn-arxiv $JOB_NAME \ 144 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 145 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 146 | --model_name SAGE --test_type layerwise \ 147 | --overwrite_job_dir \ 148 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 149 | --one_node_ddp 150 | -------------------------------------------------------------------------------- /examples/example_single_machine.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to run examples on a single machine. READ ALL INSTRUCTIONS!! 4 | 5 | # Set JOB_NAME used for this script 6 | JOB_NAME=example_single_machine 7 | 8 | # Set environment variable SLURMD_NODENAME so that os.environ can get 9 | # it. Usually, on a cluster with SLURM scheduler, the scheduler will 10 | # set this variable. In other cases, manually set it like the 11 | # following: 12 | export SLURMD_NODENAME=`hostname` 13 | 14 | # Set SALIENT root and PYTHONPATH 15 | SALIENT_ROOT=$HOME/SALIENT 16 | export PYTHONPATH=$SALIENT_ROOT 17 | 18 | # Set the data paths 19 | DATASET_ROOT=$HOME/dataset 20 | OUTPUT_ROOT=$SALIENT_ROOT/job_output 21 | 22 | # Run examples. For the full list of options, see driver/parser.py 23 | # 24 | # Turn on --verbose to see timing statistics 25 | # 26 | # 1 node, 1 GPU, no ddp 27 | echo 'Example 1' 28 | echo '' 29 | python -m driver.main ogbn-arxiv $JOB_NAME \ 30 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 31 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 32 | --model_name SAGE --test_type batchwise \ 33 | --overwrite_job_dir \ 34 | --num_workers 30 --max_num_devices_per_node 1 --total_num_nodes 1 35 | echo '' 36 | echo '======================================================================' 37 | echo '' 38 | echo 'Example 2' 39 | echo '' 40 | # 41 | # 1 node, 1 GPU, ddp 42 | python -m driver.main ogbn-arxiv $JOB_NAME \ 43 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 44 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 45 | --model_name SAGE --test_type batchwise \ 46 | --overwrite_job_dir \ 47 | --num_workers 30 --max_num_devices_per_node 1 --total_num_nodes 1 \ 48 | --one_node_ddp 49 | echo '' 50 | echo '======================================================================' 51 | echo '' 52 | echo 'Example 3' 53 | echo '' 54 | # 55 | # 1 node, 2 GPUs, must do ddp 56 | python -m driver.main ogbn-arxiv $JOB_NAME \ 57 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 58 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 59 | --model_name SAGE --test_type batchwise \ 60 | --overwrite_job_dir \ 61 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 62 | --one_node_ddp 63 | echo '' 64 | echo '======================================================================' 65 | echo '' 66 | echo 'Example 4' 67 | echo '' 68 | # 69 | # only run inference 70 | python -m driver.main ogbn-arxiv $JOB_NAME \ 71 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 72 | --trials 1 --do_test_run \ 73 | --do_test_run_filename $OUTPUT_ROOT/$JOB_NAME/model_0_3.pt \ 74 | $OUTPUT_ROOT/$JOB_NAME/model_1_3.pt \ 75 | --model_name SAGE --test_type batchwise \ 76 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 77 | --one_node_ddp 78 | echo '' 79 | echo '======================================================================' 80 | echo '' 81 | echo 'Example 5' 82 | echo '' 83 | # 84 | # ogbn-products 85 | python -m driver.main ogbn-products $JOB_NAME \ 86 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 87 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 88 | --model_name SAGE --test_type batchwise \ 89 | --overwrite_job_dir \ 90 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 91 | --one_node_ddp 92 | echo '' 93 | echo '======================================================================' 94 | echo '' 95 | echo 'Example 6' 96 | echo '' 97 | # 98 | # GIN 99 | python -m driver.main ogbn-arxiv $JOB_NAME \ 100 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 101 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 102 | --model_name GIN --test_type batchwise \ 103 | --overwrite_job_dir \ 104 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 105 | --one_node_ddp 106 | echo '' 107 | echo '======================================================================' 108 | echo '' 109 | echo 'Example 7' 110 | echo '' 111 | # 112 | # GAT 113 | python -m driver.main ogbn-arxiv $JOB_NAME \ 114 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 115 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 116 | --model_name GAT --test_type batchwise \ 117 | --overwrite_job_dir \ 118 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 119 | --one_node_ddp 120 | echo '' 121 | echo '======================================================================' 122 | echo '' 123 | echo 'Example 8' 124 | echo '' 125 | # 126 | # SAGEResInception 127 | python -m driver.main ogbn-arxiv $JOB_NAME \ 128 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 129 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 130 | --model_name SAGEResInception --test_type batchwise \ 131 | --overwrite_job_dir \ 132 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 133 | --one_node_ddp 134 | echo '' 135 | echo '======================================================================' 136 | echo '' 137 | echo 'Example 9' 138 | echo '' 139 | # 140 | # layerwise test 141 | python -m driver.main ogbn-arxiv $JOB_NAME \ 142 | --dataset_root $DATASET_ROOT --output_root $OUTPUT_ROOT \ 143 | --trials 2 --epochs 3 --test_epoch_frequency 2 \ 144 | --model_name SAGE --test_type layerwise \ 145 | --overwrite_job_dir \ 146 | --num_workers 30 --max_num_devices_per_node 2 --total_num_nodes 1 \ 147 | --one_node_ddp 148 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/.appveyor.yml: -------------------------------------------------------------------------------- 1 | # Use the Appveyor build number for naming the build. 2 | # src: https://www.appveyor.com/docs/build-configuration/#build-versioning 3 | version: '{build}' 4 | # see https://www.appveyor.com/docs/how-to/repository-shallow-clone/ 5 | shallow_clone: true 6 | 7 | platform: x64 8 | 9 | environment: 10 | matrix: 11 | # CMake 12 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 13 | VS: 2017 14 | CONFIG: Release 15 | TEST: OFF 16 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 17 | VS: 2017 18 | CONFIG: Release 19 | TEST: ON 20 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 21 | VS: 2017 22 | CONFIG: Debug 23 | TEST: ON 24 | 25 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 26 | VS: 2015 27 | CONFIG: Release 28 | TEST: OFF 29 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 30 | VS: 2015 31 | CONFIG: Release 32 | TEST: ON 33 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 34 | VS: 2015 35 | CONFIG: Debug 36 | TEST: ON 37 | 38 | matrix: 39 | fast_finish: false 40 | 41 | before_build: 42 | # see https://www.appveyor.com/docs/lang/cpp/#visual-studio-2017 43 | - if "%VS%"=="2017" call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" 44 | - if "%VS%"=="2017" set CMAKE_GENERATOR="Visual Studio 15 2017 Win64" 45 | # see https://www.appveyor.com/docs/lang/cpp/#visual-studio-2015 46 | - if "%VS%"=="2015" call "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 47 | - if "%VS%"=="2015" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64 48 | - if "%VS%"=="2015" set CMAKE_GENERATOR="Visual Studio 14 2015 Win64" 49 | 50 | build_script: 51 | - cmake --version 52 | - cmake -H. -Bbuild -DPHMAP_BUILD_TESTS=%TEST% -DPHMAP_BUILD_EXAMPLES=%TEST% -G %CMAKE_GENERATOR% 53 | - cmake --build build --config %CONFIG% --target ALL_BUILD -- /maxcpucount 54 | 55 | test_script: 56 | - if "%TEST%"=="ON" set CTEST_OUTPUT_ON_FAILURE=1 57 | - if "%TEST%"=="ON" cmake --build build --config %CONFIG% --target RUN_TESTS 58 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/.gitattributes: -------------------------------------------------------------------------------- 1 | * linguist-vendored 2 | *.cc linguist-vendored=false 3 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: greg7mdp 2 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/.gitignore: -------------------------------------------------------------------------------- 1 | VagrantFile 2 | benchmark/build 3 | benchmark/output 4 | benchmark/charts.html 5 | build* 6 | .vagrant 7 | **/.vscode 8 | TAGS 9 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | matrix: 3 | include: 4 | 5 | - os: osx 6 | compiler: gcc 7 | env: COMPILER='g++' CXX_FLAGS='-std=c++11' 8 | 9 | - os: osx 10 | compiler: clang 11 | env: COMPILER='clang++' CXX_FLAGS='-std=c++11' 12 | 13 | # - os: linux 14 | # compiler: gcc 15 | # env: COMPILER='g++' CXX_FLAGS='-std=c++11' 16 | 17 | # - os: linux 18 | # compiler: clang 19 | # env: COMPILER='clang++' CXX_FLAGS='-std=c++11' 20 | 21 | - os: linux 22 | compiler: gcc 23 | addons: 24 | apt: 25 | sources: 26 | - ubuntu-toolchain-r-test 27 | packages: 28 | - g++-5 29 | env: COMPILER='g++-5' CXX_FLAGS='-std=c++11' 30 | 31 | - os: linux 32 | compiler: gcc 33 | addons: 34 | apt: 35 | sources: 36 | - ubuntu-toolchain-r-test 37 | packages: 38 | - g++-6 39 | env: COMPILER='g++-6' CXX_FLAGS='-std=c++11' 40 | 41 | - os: linux 42 | compiler: gcc 43 | addons: 44 | apt: 45 | sources: 46 | - ubuntu-toolchain-r-test 47 | packages: 48 | - g++-7 49 | env: COMPILER='g++-7' CXX_FLAGS='-std=c++11' 50 | 51 | - os: linux 52 | compiler: gcc 53 | addons: 54 | apt: 55 | sources: 56 | - ubuntu-toolchain-r-test 57 | packages: 58 | - g++-8 59 | env: COMPILER='g++-8' CXX_FLAGS='-std=c++14' 60 | 61 | - os: linux 62 | compiler: gcc 63 | addons: 64 | apt: 65 | sources: 66 | - ubuntu-toolchain-r-test 67 | packages: 68 | - g++-8 69 | env: COMPILER='g++-8' CXX_FLAGS='-std=c++17' 70 | 71 | # - os: linux 72 | # compiler: clang 73 | # addons: 74 | # apt: 75 | # sources: 76 | # - llvm-toolchain-trusty-3.9 77 | # packages: 78 | # - clang-3.9 79 | # env: COMPILER='clang++-3.9' CXX_FLAGS='-std=c++11' 80 | 81 | # - os: linux 82 | # compiler: clang 83 | # addons: 84 | # apt: 85 | # sources: 86 | # - llvm-toolchain-trusty-4.0 87 | # packages: 88 | # - clang-4.0 89 | # env: COMPILER='clang++-4.0' CXX_FLAGS='-std=c++11' 90 | 91 | - os: linux 92 | compiler: clang 93 | addons: 94 | apt: 95 | sources: 96 | - llvm-toolchain-trusty-5.0 97 | - ubuntu-toolchain-r-test 98 | packages: 99 | - clang-5.0 100 | - g++-7 101 | env: COMPILER='clang++-5.0' CXX_FLAGS='-std=c++14' 102 | 103 | - os: linux 104 | compiler: clang 105 | addons: 106 | apt: 107 | sources: 108 | - llvm-toolchain-trusty-5.0 109 | - ubuntu-toolchain-r-test 110 | packages: 111 | - clang-5.0 112 | - g++-7 113 | env: COMPILER='clang++-5.0' CXX_FLAGS='-std=c++17' 114 | 115 | dist: trusty 116 | 117 | script: 118 | mkdir build && cd build && cmake -DPHMAP_BUILD_TESTS=ON -DPHMAP_BUILD_EXAMPLES=ON -DCMAKE_CXX_COMPILER=${COMPILER} -DCXXFLAGS="${CXX_FLAGS}" .. && cmake --build . && make test 119 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/BENCHMARK.md: -------------------------------------------------------------------------------- 1 | # parallel-hashmap 2 | 3 | How to run these benchmarks 4 | =========================== 5 | 6 | These bencharks were run on windows using Visual Studio 2017, in a cygwin window with the VC++ 2017 compiler env vars (add something like this in your Cygwin.bat: 7 | 8 | CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" 9 | 10 | Running them on linux would require Makefile changes. 11 | 12 | To build and run the tests, just update the path to the abseil libraries in the makefile, and run make. 13 | 14 | Your charts are now in charts.html. 15 | 16 | 17 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/Makefile: -------------------------------------------------------------------------------- 1 | CXX=CL -EHsc -DNDEBUG -Fo$@ -O2 2 | #CXX=CL -EHsc -g -debug -Zi -Fo$@ 3 | #CXX=g++ -ggdb -O2 -lm -std=c++11 -DNDEBUG 4 | 5 | ABSEIL_LIBS=absl_bad_optional_access.lib absl_bad_variant_access.lib absl_base.lib absl_demangle_internal.lib absl_hash.lib absl_int128.lib absl_internal_bad_any_cast_impl.lib absl_internal_city.lib absl_internal_civil_time.lib absl_internal_debugging_internal.lib absl_internal_graphcycles_internal.lib absl_internal_hashtablez_sampler.lib absl_internal_malloc_internal.lib absl_internal_spinlock_wait.lib absl_internal_strings_internal.lib absl_internal_throw_delegate.lib absl_internal_time_zone.lib absl_optional.lib absl_raw_hash_set.lib absl_stacktrace.lib absl_strings.lib absl_symbolize.lib absl_synchronization.lib absl_time.lib 6 | 7 | PROGS = stl_unordered_map sparsepp phmap abseil_flat abseil_parallel_flat phmap_flat 8 | BUILD_PROGS = $(addprefix build/,$(PROGS)) 9 | SIZE = 100000000 10 | ABSEIL = ../../abseil-cpp 11 | PHMAP_SRC = ../parallel_hashmap 12 | 13 | all: test 14 | 15 | builddir: 16 | @if [ ! -d build ]; then mkdir build; fi 17 | 18 | build/stl_unordered_map: bench.cc Makefile 19 | $(CXX) -DSTL_UNORDERED -I.. bench.cc -o $@ 20 | 21 | build/sparsepp: bench.cc Makefile 22 | $(CXX) -DSPARSEPP -I.. -I../../sparsepp bench.cc -o $@ 23 | 24 | build/phmap: bench.cc Makefile $(PHMAP_SRC)/phmap.h 25 | $(CXX) -DPHMAP -I.. -I$(ABSEIL) bench.cc /MD -o $@ /link /LIBPATH:$(ABSEIL)/build/lib ${ABSEIL_LIBS} 26 | 27 | build/phmap_flat: bench.cc Makefile $(PHMAP_SRC)/phmap.h 28 | $(CXX) -DPHMAP_FLAT -I.. bench.cc /MD -o $@ 29 | 30 | build/abseil_flat: bench.cc Makefile 31 | $(CXX) -DABSEIL_FLAT -I.. -I$(ABSEIL) bench.cc /MD -o $@ /link /LIBPATH:$(ABSEIL)/build/lib ${ABSEIL_LIBS} 32 | 33 | build/abseil_parallel_flat: bench.cc Makefile 34 | $(CXX) -DABSEIL_PARALLEL_FLAT -I.. -I $(ABSEIL) bench.cc /MD -o $@ /link /LIBPATH:$(ABSEIL)/build/lib ${ABSEIL_LIBS} 35 | 36 | build/emplace: emplace.cc Makefile $(PHMAP_SRC)/phmap.h 37 | $(CXX) -DABSEIL_FLAT -I.. -I$(ABSEIL) emplace.cc /MD -o $@ /link /LIBPATH:$(ABSEIL)/build/lib ${ABSEIL_LIBS} 38 | 39 | progs: $(BUILD_PROGS) 40 | 41 | test: builddir progs 42 | -rm -f output 43 | #./build/stl_unordered_map $(SIZE) random >> output 44 | #./build/sparsepp $(SIZE) random >> output 45 | ./build/abseil_flat $(SIZE) random >> output 46 | #./build/phmap_flat $(SIZE) random >> output 47 | ./build/phmap $(SIZE) random >> output 48 | ./build/abseil_parallel_flat $(SIZE) random >> output 49 | python make_chart_data.py < output 50 | 51 | test_cust: 52 | -rm -f output 53 | #./build/abseil_flat $(SIZE) random >> output 54 | #./build/sparsepp $(SIZE) random >> output 55 | ./build/abseil_parallel_flat_5 $(SIZE) random >> output 56 | ./build/abseil_parallel_flat $(SIZE) random >> output 57 | python make_chart_data.py < output 58 | 59 | chart: 60 | python make_chart_data.py < output 61 | 62 | clean: 63 | -rm -fr output build 64 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/bench.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import subprocess 4 | import signal 5 | 6 | programs = [ 7 | 'stl_unordered_map', 8 | 'abseil_flat', 9 | 'abseil_parallel_flat' 10 | ] 11 | 12 | test_size = 1 13 | minkeys = 0 14 | maxkeys = 1000 15 | interval = 100 16 | best_out_of = 1 17 | 18 | if test_size == 2: 19 | multiplier = 1000 * 1000 20 | maxkeys = 500 21 | best_out_of = 3 22 | 23 | elif test_size == 1: 24 | multiplier = 100 * 1000 25 | interval = 200 26 | else: 27 | multiplier = 10 * 1000 28 | 29 | # and use nice/ionice 30 | # and shut down to the console 31 | # and swapoff any swap files/partitions 32 | 33 | outfile = open('output', 'w') 34 | 35 | if len(sys.argv) > 1: 36 | benchtypes = sys.argv[1:] 37 | else: 38 | benchtypes = ('random', 'lookup', 'delete',) 39 | #benchtypes = ( 'lookup', ) 40 | 41 | for benchtype in benchtypes: 42 | nkeys = minkeys * multiplier 43 | while nkeys <= (maxkeys * multiplier): 44 | for program in programs: 45 | fastest_attempt = 1000000 46 | fastest_attempt_data = '' 47 | 48 | for attempt in range(best_out_of): 49 | proc = subprocess.Popen( 50 | ['./build/'+program, str(nkeys), benchtype], stdout=subprocess.PIPE) 51 | 52 | # wait for the program to fill up memory and spit out its "ready" message 53 | try: 54 | runtime = float(proc.stdout.readline().strip()) 55 | except: 56 | runtime = 0 57 | 58 | ps_proc = subprocess.Popen( 59 | ['ps up %d | tail -n1' % proc.pid], shell=True, stdout=subprocess.PIPE) 60 | #nbytes = int(ps_proc.stdout.read().split()[4]) * 1024 61 | # ps_proc.wait() 62 | nbytes = 1000000 63 | 64 | os.kill(proc.pid, signal.SIGKILL) 65 | proc.wait() 66 | 67 | if nbytes and runtime: # otherwise it crashed 68 | line = ','.join( 69 | map(str, [benchtype, nkeys, program, nbytes, "%0.6f" % runtime])) 70 | 71 | if runtime < fastest_attempt: 72 | fastest_attempt = runtime 73 | fastest_attempt_data = line 74 | 75 | if fastest_attempt != 1000000: 76 | print >> outfile, fastest_attempt_data 77 | print fastest_attempt_data 78 | 79 | nkeys += interval * multiplier 80 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/js/examples.css: -------------------------------------------------------------------------------- 1 | * { padding: 0; margin: 0; vertical-align: top; } 2 | 3 | body { 4 | background: url(background.png) repeat-x; 5 | font: 18px "proxima-nova", Helvetica, Arial, sans-serif; 6 | line-height: 1.5; 7 | } 8 | 9 | a { color: #069; } 10 | a:hover { color: #28b; } 11 | 12 | h2 { 13 | margin-top: 15px; 14 | font: normal 32px "omnes-pro", Helvetica, Arial, sans-serif; 15 | } 16 | 17 | h3 { 18 | margin-left: 30px; 19 | font: normal 26px "omnes-pro", Helvetica, Arial, sans-serif; 20 | color: #666; 21 | } 22 | 23 | p { 24 | margin-top: 10px; 25 | } 26 | 27 | button { 28 | font-size: 18px; 29 | padding: 1px 7px; 30 | } 31 | 32 | input { 33 | font-size: 18px; 34 | } 35 | 36 | input[type=checkbox] { 37 | margin: 7px; 38 | } 39 | 40 | #header { 41 | position: relative; 42 | width: 900px; 43 | margin: auto; 44 | } 45 | 46 | #header h2 { 47 | margin-left: 10px; 48 | vertical-align: middle; 49 | font-size: 42px; 50 | font-weight: bold; 51 | text-decoration: none; 52 | color: #000; 53 | } 54 | 55 | #content { 56 | width: 880px; 57 | margin: 0 auto; 58 | padding: 10px; 59 | } 60 | 61 | #footer { 62 | margin-top: 25px; 63 | margin-bottom: 10px; 64 | text-align: center; 65 | font-size: 12px; 66 | color: #999; 67 | } 68 | 69 | .demo-container { 70 | box-sizing: border-box; 71 | width: 850px; 72 | height: 450px; 73 | padding: 20px 15px 15px 15px; 74 | margin: 15px auto 30px auto; 75 | border: 1px solid #ddd; 76 | background: #fff; 77 | background: linear-gradient(#f6f6f6 0, #fff 50px); 78 | background: -o-linear-gradient(#f6f6f6 0, #fff 50px); 79 | background: -ms-linear-gradient(#f6f6f6 0, #fff 50px); 80 | background: -moz-linear-gradient(#f6f6f6 0, #fff 50px); 81 | background: -webkit-linear-gradient(#f6f6f6 0, #fff 50px); 82 | box-shadow: 0 3px 10px rgba(0,0,0,0.15); 83 | -o-box-shadow: 0 3px 10px rgba(0,0,0,0.1); 84 | -ms-box-shadow: 0 3px 10px rgba(0,0,0,0.1); 85 | -moz-box-shadow: 0 3px 10px rgba(0,0,0,0.1); 86 | -webkit-box-shadow: 0 3px 10px rgba(0,0,0,0.1); 87 | -webkit-tap-highlight-color: rgba(0,0,0,0); 88 | -webkit-tap-highlight-color: transparent; 89 | -webkit-touch-callout: none; 90 | -webkit-user-select: none; 91 | -khtml-user-select: none; 92 | -moz-user-select: none; 93 | -ms-user-select: none; 94 | user-select: none; 95 | } 96 | 97 | .demo-placeholder { 98 | width: 100%; 99 | height: 100%; 100 | font-size: 14px; 101 | } 102 | 103 | fieldset { 104 | display: block; 105 | -webkit-margin-start: 2px; 106 | -webkit-margin-end: 2px; 107 | -webkit-padding-before: 0.35em; 108 | -webkit-padding-start: 0.75em; 109 | -webkit-padding-end: 0.75em; 110 | -webkit-padding-after: 0.625em; 111 | min-width: -webkit-min-content; 112 | border-width: 2px; 113 | border-style: groove; 114 | border-color: threedface; 115 | border-image: initial; 116 | padding: 10px; 117 | } 118 | 119 | .legend { 120 | display: block; 121 | -webkit-padding-start: 2px; 122 | -webkit-padding-end: 2px; 123 | border-width: initial; 124 | border-style: none; 125 | border-color: initial; 126 | border-image: initial; 127 | padding-left: 10px; 128 | padding-right: 10px; 129 | padding-top: 10px; 130 | padding-bottom: 10px; 131 | } 132 | 133 | .legendLayer .background { 134 | fill: rgba(255, 255, 255, 0.85); 135 | stroke: rgba(0, 0, 0, 0.85); 136 | stroke-width: 1; 137 | } 138 | 139 | input[type="radio"] { 140 | margin-top: -1px; 141 | vertical-align: middle; 142 | } 143 | 144 | .tickLabel { 145 | line-height: 1.1; 146 | } 147 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/js/jquery.flot.browser.js: -------------------------------------------------------------------------------- 1 | /** ## jquery.flot.browser.js 2 | 3 | This plugin is used to make available some browser-related utility functions. 4 | 5 | ### Methods 6 | */ 7 | 8 | (function ($) { 9 | 'use strict'; 10 | 11 | var browser = { 12 | /** 13 | - getPageXY(e) 14 | 15 | Calculates the pageX and pageY using the screenX, screenY properties of the event 16 | and the scrolling of the page. This is needed because the pageX and pageY 17 | properties of the event are not correct while running tests in Edge. */ 18 | getPageXY: function (e) { 19 | // This code is inspired from https://stackoverflow.com/a/3464890 20 | var doc = document.documentElement, 21 | pageX = e.clientX + (window.pageXOffset || doc.scrollLeft) - (doc.clientLeft || 0), 22 | pageY = e.clientY + (window.pageYOffset || doc.scrollTop) - (doc.clientTop || 0); 23 | return { X: pageX, Y: pageY }; 24 | }, 25 | 26 | /** 27 | - getPixelRatio(context) 28 | 29 | This function returns the current pixel ratio defined by the product of desktop 30 | zoom and page zoom. 31 | Additional info: https://www.html5rocks.com/en/tutorials/canvas/hidpi/ 32 | */ 33 | getPixelRatio: function(context) { 34 | var devicePixelRatio = window.devicePixelRatio || 1, 35 | backingStoreRatio = 36 | context.webkitBackingStorePixelRatio || 37 | context.mozBackingStorePixelRatio || 38 | context.msBackingStorePixelRatio || 39 | context.oBackingStorePixelRatio || 40 | context.backingStorePixelRatio || 1; 41 | return devicePixelRatio / backingStoreRatio; 42 | }, 43 | 44 | /** 45 | - isSafari, isMobileSafari, isOpera, isFirefox, isIE, isEdge, isChrome, isBlink 46 | 47 | This is a collection of functions, used to check if the code is running in a 48 | particular browser or Javascript engine. 49 | */ 50 | isSafari: function() { 51 | // *** https://stackoverflow.com/questions/9847580/how-to-detect-safari-chrome-ie-firefox-and-opera-browser 52 | // Safari 3.0+ "[object HTMLElementConstructor]" 53 | return /constructor/i.test(window.top.HTMLElement) || (function (p) { return p.toString() === "[object SafariRemoteNotification]"; })(!window.top['safari'] || (typeof window.top.safari !== 'undefined' && window.top.safari.pushNotification)); 54 | }, 55 | 56 | isMobileSafari: function() { 57 | //isMobileSafari adapted from https://stackoverflow.com/questions/3007480/determine-if-user-navigated-from-mobile-safari 58 | return navigator.userAgent.match(/(iPod|iPhone|iPad)/) && navigator.userAgent.match(/AppleWebKit/); 59 | }, 60 | 61 | isOpera: function() { 62 | // *** https://stackoverflow.com/questions/9847580/how-to-detect-safari-chrome-ie-firefox-and-opera-browser 63 | //Opera 8.0+ 64 | return (!!window.opr && !!opr.addons) || !!window.opera || navigator.userAgent.indexOf(' OPR/') >= 0; 65 | }, 66 | 67 | isFirefox: function() { 68 | // *** https://stackoverflow.com/questions/9847580/how-to-detect-safari-chrome-ie-firefox-and-opera-browser 69 | // Firefox 1.0+ 70 | return typeof InstallTrigger !== 'undefined'; 71 | }, 72 | 73 | isIE: function() { 74 | // *** https://stackoverflow.com/questions/9847580/how-to-detect-safari-chrome-ie-firefox-and-opera-browser 75 | // Internet Explorer 6-11 76 | return /*@cc_on!@*/false || !!document.documentMode; 77 | }, 78 | 79 | isEdge: function() { 80 | // *** https://stackoverflow.com/questions/9847580/how-to-detect-safari-chrome-ie-firefox-and-opera-browser 81 | // Edge 20+ 82 | return !browser.isIE() && !!window.StyleMedia; 83 | }, 84 | 85 | isChrome: function() { 86 | // *** https://stackoverflow.com/questions/9847580/how-to-detect-safari-chrome-ie-firefox-and-opera-browser 87 | // Chrome 1+ 88 | return !!window.chrome && !!window.chrome.webstore; 89 | }, 90 | 91 | isBlink: function() { 92 | // *** https://stackoverflow.com/questions/9847580/how-to-detect-safari-chrome-ie-firefox-and-opera-browser 93 | return (browser.isChrome() || browser.isOpera()) && !!window.CSS; 94 | } 95 | }; 96 | 97 | $.plot.browser = browser; 98 | })(jQuery); 99 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/js/jquery.flot.flatdata.js: -------------------------------------------------------------------------------- 1 | /* Support for flat 1D data series. 2 | 3 | A 1D flat data series is a data series in the form of a regular 1D array. The 4 | main reason for using a flat data series is that it performs better, consumes 5 | less memory and generates less garbage collection than the regular flot format. 6 | 7 | Example: 8 | 9 | plot.setData([[[0,0], [1,1], [2,2], [3,3]]]); // regular flot format 10 | plot.setData([{flatdata: true, data: [0, 1, 2, 3]}]); // flatdata format 11 | 12 | Set series.flatdata to true to enable this plugin. 13 | 14 | You can use series.start to specify the starting index of the series (default is 0) 15 | You can use series.step to specify the interval between consecutive indexes of the series (default is 1) 16 | */ 17 | 18 | /* global jQuery*/ 19 | 20 | (function ($) { 21 | 'use strict'; 22 | 23 | function process1DRawData(plot, series, data, datapoints) { 24 | if (series.flatdata === true) { 25 | var start = series.start || 0; 26 | var step = typeof series.step === 'number' ? series.step : 1; 27 | datapoints.pointsize = 2; 28 | for (var i = 0, j = 0; i < data.length; i++, j += 2) { 29 | datapoints.points[j] = start + (i * step); 30 | datapoints.points[j + 1] = data[i]; 31 | } 32 | if (datapoints.points !== undefined) { 33 | datapoints.points.length = data.length * 2; 34 | } else { 35 | datapoints.points = []; 36 | } 37 | } 38 | } 39 | 40 | $.plot.plugins.push({ 41 | init: function(plot) { 42 | plot.hooks.processRawData.push(process1DRawData); 43 | }, 44 | name: 'flatdata', 45 | version: '0.0.2' 46 | }); 47 | })(jQuery); 48 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/js/jquery.flot.resize.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /* Flot plugin for automatically redrawing plots as the placeholder resizes. 3 | 4 | Copyright (c) 2007-2014 IOLA and Ole Laursen. 5 | Licensed under the MIT license. 6 | 7 | It works by listening for changes on the placeholder div (through the jQuery 8 | resize event plugin) - if the size changes, it will redraw the plot. 9 | 10 | There are no options. If you need to disable the plugin for some plots, you 11 | can just fix the size of their placeholders. 12 | 13 | */ 14 | 15 | /* Inline dependency: 16 | * jQuery resize event - v1.1 - 3/14/2010 17 | * http://benalman.com/projects/jquery-resize-plugin/ 18 | * 19 | * Copyright (c) 2010 "Cowboy" Ben Alman 20 | * Dual licensed under the MIT and GPL licenses. 21 | * http://benalman.com/about/license/ 22 | */ 23 | (function($,e,t){"$:nomunge";var i=[],n=$.resize=$.extend($.resize,{}),a,r=false,s="setTimeout",u="resize",m=u+"-special-event",o="pendingDelay",l="activeDelay",f="throttleWindow";n[o]=200;n[l]=20;n[f]=true;$.event.special[u]={setup:function(){if(!n[f]&&this[s]){return false}var e=$(this);i.push(this);e.data(m,{w:e.width(),h:e.height()});if(i.length===1){a=t;h()}},teardown:function(){if(!n[f]&&this[s]){return false}var e=$(this);for(var t=i.length-1;t>=0;t--){if(i[t]==this){i.splice(t,1);break}}e.removeData(m);if(!i.length){if(r){cancelAnimationFrame(a)}else{clearTimeout(a)}a=null}},add:function(e){if(!n[f]&&this[s]){return false}var i;function a(e,n,a){var r=$(this),s=r.data(m)||{};s.w=n!==t?n:r.width();s.h=a!==t?a:r.height();i.apply(this,arguments)}if($.isFunction(e)){i=e;return a}else{i=e.handler;e.handler=a}}};function h(t){if(r===true){r=t||1}for(var s=i.length-1;s>=0;s--){var l=$(i[s]);if(l[0]==e||l.is(":visible")){var f=l.width(),c=l.height(),d=l.data(m);if(d&&(f!==d.w||c!==d.h)){l.trigger(u,[d.w=f,d.h=c]);r=t||true}}else{d=l.data(m);d.w=0;d.h=0}}if(a!==null){if(r&&(t==null||t-r<1e3)){a=e.requestAnimationFrame(h)}else{a=setTimeout(h,n[o]);r=false}}}if(!e.requestAnimationFrame){e.requestAnimationFrame=function(){return e.webkitRequestAnimationFrame||e.mozRequestAnimationFrame||e.oRequestAnimationFrame||e.msRequestAnimationFrame||function(t,i){return e.setTimeout(function(){t((new Date).getTime())},n[l])}}()}if(!e.cancelAnimationFrame){e.cancelAnimationFrame=function(){return e.webkitCancelRequestAnimationFrame||e.mozCancelRequestAnimationFrame||e.oCancelRequestAnimationFrame||e.msCancelRequestAnimationFrame||clearTimeout}()}})(jQuery,this); 24 | 25 | /* eslint-enable */ 26 | (function ($) { 27 | var options = { }; // no options 28 | 29 | function init(plot) { 30 | function onResize() { 31 | var placeholder = plot.getPlaceholder(); 32 | 33 | // somebody might have hidden us and we can't plot 34 | // when we don't have the dimensions 35 | if (placeholder.width() === 0 || placeholder.height() === 0) return; 36 | 37 | plot.resize(); 38 | plot.setupGrid(); 39 | plot.draw(); 40 | } 41 | 42 | function bindEvents(plot, eventHolder) { 43 | plot.getPlaceholder().resize(onResize); 44 | } 45 | 46 | function shutdown(plot, eventHolder) { 47 | plot.getPlaceholder().unbind("resize", onResize); 48 | } 49 | 50 | plot.hooks.bindEvents.push(bindEvents); 51 | plot.hooks.shutdown.push(shutdown); 52 | } 53 | 54 | $.plot.plugins.push({ 55 | init: init, 56 | options: options, 57 | name: 'resize', 58 | version: '1.0' 59 | }); 60 | })(jQuery); 61 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/js/jquery.flot.saturated.js: -------------------------------------------------------------------------------- 1 | (function ($) { 2 | 'use strict'; 3 | var saturated = { 4 | saturate: function (a) { 5 | if (a === Infinity) { 6 | return Number.MAX_VALUE; 7 | } 8 | 9 | if (a === -Infinity) { 10 | return -Number.MAX_VALUE; 11 | } 12 | 13 | return a; 14 | }, 15 | delta: function(min, max, noTicks) { 16 | return ((max - min) / noTicks) === Infinity ? (max / noTicks - min / noTicks) : (max - min) / noTicks 17 | }, 18 | multiply: function (a, b) { 19 | return saturated.saturate(a * b); 20 | }, 21 | // returns c * bInt * a. Beahves properly in the case where c is negative 22 | // and bInt * a is bigger that Number.MAX_VALUE (Infinity) 23 | multiplyAdd: function (a, bInt, c) { 24 | if (isFinite(a * bInt)) { 25 | return saturated.saturate(a * bInt + c); 26 | } else { 27 | var result = c; 28 | 29 | for (var i = 0; i < bInt; i++) { 30 | result += a; 31 | } 32 | 33 | return saturated.saturate(result); 34 | } 35 | }, 36 | // round to nearby lower multiple of base 37 | floorInBase: function(n, base) { 38 | return base * Math.floor(n / base); 39 | } 40 | }; 41 | 42 | $.plot.saturated = saturated; 43 | })(jQuery); 44 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/js/jquery.flot.symbol.js: -------------------------------------------------------------------------------- 1 | /* Flot plugin that adds some extra symbols for plotting points. 2 | 3 | Copyright (c) 2007-2014 IOLA and Ole Laursen. 4 | Licensed under the MIT license. 5 | 6 | The symbols are accessed as strings through the standard symbol options: 7 | 8 | series: { 9 | points: { 10 | symbol: "square" // or "diamond", "triangle", "cross", "plus", "ellipse", "rectangle" 11 | } 12 | } 13 | 14 | */ 15 | 16 | (function ($) { 17 | // we normalize the area of each symbol so it is approximately the 18 | // same as a circle of the given radius 19 | 20 | var square = function (ctx, x, y, radius, shadow) { 21 | // pi * r^2 = (2s)^2 => s = r * sqrt(pi)/2 22 | var size = radius * Math.sqrt(Math.PI) / 2; 23 | ctx.rect(x - size, y - size, size + size, size + size); 24 | }, 25 | rectangle = function (ctx, x, y, radius, shadow) { 26 | // pi * r^2 = (2s)^2 => s = r * sqrt(pi)/2 27 | var size = radius * Math.sqrt(Math.PI) / 2; 28 | ctx.rect(x - size, y - size, size + size, size + size); 29 | }, 30 | diamond = function (ctx, x, y, radius, shadow) { 31 | // pi * r^2 = 2s^2 => s = r * sqrt(pi/2) 32 | var size = radius * Math.sqrt(Math.PI / 2); 33 | ctx.moveTo(x - size, y); 34 | ctx.lineTo(x, y - size); 35 | ctx.lineTo(x + size, y); 36 | ctx.lineTo(x, y + size); 37 | ctx.lineTo(x - size, y); 38 | ctx.lineTo(x, y - size); 39 | }, 40 | triangle = function (ctx, x, y, radius, shadow) { 41 | // pi * r^2 = 1/2 * s^2 * sin (pi / 3) => s = r * sqrt(2 * pi / sin(pi / 3)) 42 | var size = radius * Math.sqrt(2 * Math.PI / Math.sin(Math.PI / 3)); 43 | var height = size * Math.sin(Math.PI / 3); 44 | ctx.moveTo(x - size / 2, y + height / 2); 45 | ctx.lineTo(x + size / 2, y + height / 2); 46 | if (!shadow) { 47 | ctx.lineTo(x, y - height / 2); 48 | ctx.lineTo(x - size / 2, y + height / 2); 49 | ctx.lineTo(x + size / 2, y + height / 2); 50 | } 51 | }, 52 | cross = function (ctx, x, y, radius, shadow) { 53 | // pi * r^2 = (2s)^2 => s = r * sqrt(pi)/2 54 | var size = radius * Math.sqrt(Math.PI) / 2; 55 | ctx.moveTo(x - size, y - size); 56 | ctx.lineTo(x + size, y + size); 57 | ctx.moveTo(x - size, y + size); 58 | ctx.lineTo(x + size, y - size); 59 | }, 60 | ellipse = function(ctx, x, y, radius, shadow, fill) { 61 | if (!shadow) { 62 | ctx.moveTo(x + radius, y); 63 | ctx.arc(x, y, radius, 0, Math.PI * 2, false); 64 | } 65 | }, 66 | plus = function (ctx, x, y, radius, shadow) { 67 | var size = radius * Math.sqrt(Math.PI / 2); 68 | ctx.moveTo(x - size, y); 69 | ctx.lineTo(x + size, y); 70 | ctx.moveTo(x, y + size); 71 | ctx.lineTo(x, y - size); 72 | }, 73 | handlers = { 74 | square: square, 75 | rectangle: rectangle, 76 | diamond: diamond, 77 | triangle: triangle, 78 | cross: cross, 79 | ellipse: ellipse, 80 | plus: plus 81 | }; 82 | 83 | square.fill = true; 84 | rectangle.fill = true; 85 | diamond.fill = true; 86 | triangle.fill = true; 87 | ellipse.fill = true; 88 | 89 | function init(plot) { 90 | plot.drawSymbol = handlers; 91 | } 92 | 93 | $.plot.plugins.push({ 94 | init: init, 95 | name: 'symbols', 96 | version: '1.0' 97 | }); 98 | })(jQuery); 99 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/js/jquery.flot.threshold.js: -------------------------------------------------------------------------------- 1 | /* Flot plugin for thresholding data. 2 | 3 | Copyright (c) 2007-2014 IOLA and Ole Laursen. 4 | Licensed under the MIT license. 5 | 6 | The plugin supports these options: 7 | 8 | series: { 9 | threshold: { 10 | below: number 11 | color: colorspec 12 | } 13 | } 14 | 15 | It can also be applied to a single series, like this: 16 | 17 | $.plot( $("#placeholder"), [{ 18 | data: [ ... ], 19 | threshold: { ... } 20 | }]) 21 | 22 | An array can be passed for multiple thresholding, like this: 23 | 24 | threshold: [{ 25 | below: number1 26 | color: color1 27 | },{ 28 | below: number2 29 | color: color2 30 | }] 31 | 32 | These multiple threshold objects can be passed in any order since they are 33 | sorted by the processing function. 34 | 35 | The data points below "below" are drawn with the specified color. This makes 36 | it easy to mark points below 0, e.g. for budget data. 37 | 38 | Internally, the plugin works by splitting the data into two series, above and 39 | below the threshold. The extra series below the threshold will have its label 40 | cleared and the special "originSeries" attribute set to the original series. 41 | You may need to check for this in hover events. 42 | 43 | */ 44 | 45 | (function ($) { 46 | var options = { 47 | series: { threshold: null } // or { below: number, color: color spec} 48 | }; 49 | 50 | function init(plot) { 51 | function thresholdData(plot, s, datapoints, below, color) { 52 | var ps = datapoints.pointsize, i, x, y, p, prevp, 53 | thresholded = $.extend({}, s); // note: shallow copy 54 | 55 | thresholded.datapoints = { points: [], pointsize: ps, format: datapoints.format }; 56 | thresholded.label = null; 57 | thresholded.color = color; 58 | thresholded.threshold = null; 59 | thresholded.originSeries = s; 60 | thresholded.data = []; 61 | 62 | var origpoints = datapoints.points, 63 | addCrossingPoints = s.lines.show; 64 | 65 | var threspoints = []; 66 | var newpoints = []; 67 | var m; 68 | 69 | for (i = 0; i < origpoints.length; i += ps) { 70 | x = origpoints[i]; 71 | y = origpoints[i + 1]; 72 | 73 | prevp = p; 74 | if (y < below) p = threspoints; 75 | else p = newpoints; 76 | 77 | if (addCrossingPoints && prevp !== p && 78 | x !== null && i > 0 && 79 | origpoints[i - ps] != null) { 80 | var interx = x + (below - y) * (x - origpoints[i - ps]) / (y - origpoints[i - ps + 1]); 81 | prevp.push(interx); 82 | prevp.push(below); 83 | for (m = 2; m < ps; ++m) { 84 | prevp.push(origpoints[i + m]); 85 | } 86 | 87 | p.push(null); // start new segment 88 | p.push(null); 89 | for (m = 2; m < ps; ++m) { 90 | p.push(origpoints[i + m]); 91 | } 92 | 93 | p.push(interx); 94 | p.push(below); 95 | for (m = 2; m < ps; ++m) { 96 | p.push(origpoints[i + m]); 97 | } 98 | } 99 | 100 | p.push(x); 101 | p.push(y); 102 | for (m = 2; m < ps; ++m) { 103 | p.push(origpoints[i + m]); 104 | } 105 | } 106 | 107 | datapoints.points = newpoints; 108 | thresholded.datapoints.points = threspoints; 109 | 110 | if (thresholded.datapoints.points.length > 0) { 111 | var origIndex = $.inArray(s, plot.getData()); 112 | // Insert newly-generated series right after original one (to prevent it from becoming top-most) 113 | plot.getData().splice(origIndex + 1, 0, thresholded); 114 | } 115 | 116 | // FIXME: there are probably some edge cases left in bars 117 | } 118 | 119 | function processThresholds(plot, s, datapoints) { 120 | if (!s.threshold) return; 121 | if (s.threshold instanceof Array) { 122 | s.threshold.sort(function(a, b) { 123 | return a.below - b.below; 124 | }); 125 | 126 | $(s.threshold).each(function(i, th) { 127 | thresholdData(plot, s, datapoints, th.below, th.color); 128 | }); 129 | } else { 130 | thresholdData(plot, s, datapoints, s.threshold.below, s.threshold.color); 131 | } 132 | } 133 | 134 | plot.hooks.processDatapoints.push(processThresholds); 135 | } 136 | 137 | $.plot.plugins.push({ 138 | init: init, 139 | options: options, 140 | name: 'threshold', 141 | version: '1.2' 142 | }); 143 | })(jQuery); 144 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/js/jquery.flot.uiConstants.js: -------------------------------------------------------------------------------- 1 | (function ($) { 2 | 'use strict'; 3 | $.plot.uiConstants = { 4 | SNAPPING_CONSTANT: 20, 5 | PANHINT_LENGTH_CONSTANT: 10, 6 | MINOR_TICKS_COUNT_CONSTANT: 4, 7 | TICK_LENGTH_CONSTANT: 10, 8 | ZOOM_DISTANCE_MARGIN: 25 9 | }; 10 | })(jQuery); 11 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/benchmark/js/jquery.mousewheel.js: -------------------------------------------------------------------------------- 1 | // Source: https://github.com/jquery/jquery-mousewheel/blob/a06ef4e1a127795606642c55e22d4f2945edc061/jquery.mousewheel.js 2 | 3 | /*! Copyright (c) 2011 Brandon Aaron (http://brandonaaron.net) 4 | * Licensed under the MIT License (LICENSE.txt). 5 | * 6 | * Thanks to: http://adomas.org/javascript-mouse-wheel/ for some pointers. 7 | * Thanks to: Mathias Bank(http://www.mathias-bank.de) for a scope bug fix. 8 | * Thanks to: Seamus Leahy for adding deltaX and deltaY 9 | * 10 | * Version: 3.0.6 11 | * 12 | * Requires: 1.2.2+ 13 | */ 14 | 15 | (function($) { 16 | 17 | var types = ['DOMMouseScroll', 'mousewheel']; 18 | 19 | if ($.event.fixHooks) { 20 | for ( var i=types.length; i; ) { 21 | $.event.fixHooks[ types[--i] ] = $.event.mouseHooks; 22 | } 23 | } 24 | 25 | $.event.special.mousewheel = { 26 | setup: function() { 27 | if ( this.addEventListener ) { 28 | for ( var i=types.length; i; ) { 29 | this.addEventListener( types[--i], handler, false ); 30 | } 31 | } else { 32 | this.onmousewheel = handler; 33 | } 34 | }, 35 | 36 | teardown: function() { 37 | if ( this.removeEventListener ) { 38 | for ( var i=types.length; i; ) { 39 | this.removeEventListener( types[--i], handler, false ); 40 | } 41 | } else { 42 | this.onmousewheel = null; 43 | } 44 | } 45 | }; 46 | 47 | $.fn.extend({ 48 | mousewheel: function(fn) { 49 | return fn ? this.bind("mousewheel", fn) : this.trigger("mousewheel"); 50 | }, 51 | 52 | unmousewheel: function(fn) { 53 | return this.unbind("mousewheel", fn); 54 | } 55 | }); 56 | 57 | 58 | function handler(event) { 59 | var orgEvent = event || window.event, args = [].slice.call( arguments, 1 ), delta = 0, returnValue = true, deltaX = 0, deltaY = 0; 60 | event = $.event.fix(orgEvent); 61 | event.type = "mousewheel"; 62 | 63 | // Old school scrollwheel delta 64 | if ( orgEvent.wheelDelta ) { delta = orgEvent.wheelDelta/120; } 65 | if ( orgEvent.detail ) { delta = -orgEvent.detail/3; } 66 | 67 | // New school multidimensional scroll (touchpads) deltas 68 | deltaY = delta; 69 | 70 | // Gecko 71 | if ( orgEvent.axis !== undefined && orgEvent.axis === orgEvent.HORIZONTAL_AXIS ) { 72 | deltaY = 0; 73 | deltaX = -1*delta; 74 | } 75 | 76 | // Webkit 77 | if ( orgEvent.wheelDeltaY !== undefined ) { deltaY = orgEvent.wheelDeltaY/120; } 78 | if ( orgEvent.wheelDeltaX !== undefined ) { deltaX = -1*orgEvent.wheelDeltaX/120; } 79 | 80 | // Add event and delta to the front of the arguments 81 | args.unshift(event, delta, deltaX, deltaY); 82 | 83 | return ($.event.dispatch || $.event.handle).apply(this, args); 84 | } 85 | 86 | })(jQuery); 87 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/cmake/CMakeLists.txt.in: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.2) 2 | 3 | project(googletest-download NONE) 4 | 5 | include(ExternalProject) 6 | ExternalProject_Add(googletest 7 | GIT_REPOSITORY https://github.com/google/googletest.git 8 | GIT_TAG master 9 | SOURCE_DIR "${CMAKE_BINARY_DIR}/googletest-src" 10 | BINARY_DIR "${CMAKE_BINARY_DIR}/googletest-build" 11 | CONFIGURE_COMMAND "" 12 | BUILD_COMMAND "" 13 | INSTALL_COMMAND "" 14 | TEST_COMMAND "" 15 | ) -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/cmake/DetectVersion.cmake: -------------------------------------------------------------------------------- 1 | 2 | file(READ "${CMAKE_CURRENT_SOURCE_DIR}/parallel_hashmap/phmap_config.h" _PHMAP_H_CONTENTS) 3 | string(REGEX REPLACE ".*#define PHMAP_VERSION_MAJOR ([0-9]+).*" "\\1" DETECTED_PHMAP_VERSION_MAJOR "${_PHMAP_H_CONTENTS}") 4 | string(REGEX REPLACE ".*#define PHMAP_VERSION_MINOR ([0-9]+).*" "\\1" DETECTED_PHMAP_VERSION_MINOR "${_PHMAP_H_CONTENTS}") 5 | string(REGEX REPLACE ".*#define PHMAP_VERSION_PATCH ([0-9]+).*" "\\1" DETECTED_PHMAP_VERSION_PATCH "${_PHMAP_H_CONTENTS}") 6 | set(DETECTED_PHMAP_VERSION "${DETECTED_PHMAP_VERSION_MAJOR}.${DETECTED_PHMAP_VERSION_MINOR}.${DETECTED_PHMAP_VERSION_PATCH}") 7 | 8 | message(STATUS "Detected PHMAP Version - ${DETECTED_PHMAP_VERSION}") 9 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/cmake/DownloadGTest.cmake: -------------------------------------------------------------------------------- 1 | # Downloads and unpacks googletest at configure time. Based on the instructions 2 | # at https://github.com/google/googletest/tree/master/googletest#incorporating-into-an-existing-cmake-project 3 | 4 | # Download the latest googletest from Github master 5 | configure_file( 6 | ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.in 7 | ${CMAKE_BINARY_DIR}/googletest-download/CMakeLists.txt 8 | ) 9 | 10 | # Configure and build the downloaded googletest source 11 | execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . 12 | RESULT_VARIABLE result 13 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-download ) 14 | 15 | if(result) 16 | message(FATAL_ERROR "CMake step for googletest failed: ${result}") 17 | endif() 18 | 19 | execute_process(COMMAND ${CMAKE_COMMAND} --build . 20 | RESULT_VARIABLE result 21 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-download) 22 | 23 | if(result) 24 | message(FATAL_ERROR "Build step for googletest failed: ${result}") 25 | endif() 26 | 27 | # Prevent overriding the parent project's compiler/linker settings on Windows 28 | set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) 29 | 30 | # Add googletest directly to our build. This defines the gtest and gtest_main 31 | # targets. 32 | add_subdirectory(${CMAKE_BINARY_DIR}/googletest-src 33 | ${CMAKE_BINARY_DIR}/googletest-build 34 | EXCLUDE_FROM_ALL) 35 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/cmake/helpers.cmake: -------------------------------------------------------------------------------- 1 | #set_property(GLOBAL PROPERTY USE_FOLDERS ON) 2 | set(PHMAP_IDE_FOLDER phmap) 3 | 4 | # ------------------------------------------------------------- 5 | # phmap_cc_test(NAME awesome_test 6 | # SRCS "awesome_test.cc" 7 | # DEPS phmap::awesome gmock gtest_main) 8 | # ------------------------------------------------------------- 9 | function(phmap_cc_test) 10 | cmake_parse_arguments(PHMAP_CC_TEST 11 | "" 12 | "NAME" 13 | "SRCS;COPTS;CWOPTS;CLOPTS;DEFINES;LINKOPTS;DEPS" 14 | ${ARGN} 15 | ) 16 | 17 | set(_NAME "test_${PHMAP_CC_TEST_NAME}") 18 | add_executable(${_NAME} "") 19 | target_sources(${_NAME} PRIVATE ${PHMAP_CC_TEST_SRCS}) 20 | target_include_directories(${_NAME} 21 | PUBLIC ${PHMAP_COMMON_INCLUDE_DIRS} 22 | PRIVATE ${GMOCK_INCLUDE_DIRS} ${GTEST_INCLUDE_DIRS} 23 | ) 24 | target_compile_definitions(${_NAME} 25 | PUBLIC ${PHMAP_CC_TEST_DEFINES} 26 | ) 27 | if(MSVC) 28 | target_compile_options(${_NAME} 29 | PRIVATE ${PHMAP_CC_TEST_CWOPTS} /W4 /Zc:__cplusplus /std:c++latest 30 | ) 31 | else() 32 | target_compile_options(${_NAME} 33 | PRIVATE ${PHMAP_CC_TEST_CLOPTS} 34 | ) 35 | endif() 36 | target_compile_options(${_NAME} 37 | PRIVATE ${PHMAP_CC_TEST_COPTS} 38 | ) 39 | target_link_libraries(${_NAME} 40 | PUBLIC ${PHMAP_CC_TEST_DEPS} 41 | PRIVATE ${PHMAP_CC_TEST_LINKOPTS} 42 | ) 43 | # Add all Abseil targets to a a folder in the IDE for organization. 44 | set_property(TARGET ${_NAME} PROPERTY FOLDER ${PHMAP_IDE_FOLDER}/test) 45 | 46 | set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD ${PHMAP_CXX_STANDARD}) 47 | set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD_REQUIRED ON) 48 | 49 | add_test(NAME ${_NAME} COMMAND ${_NAME}) 50 | endfunction() 51 | 52 | # ------------------------------------------------------------- 53 | function(check_target my_target) 54 | if(NOT TARGET ${my_target}) 55 | message(FATAL_ERROR " PHMAP: compiling phmap tests requires a ${my_target} CMake target in your project, 56 | see CMake/README.md for more details") 57 | endif(NOT TARGET ${my_target}) 58 | endfunction() 59 | 60 | 61 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/cmake/phmap.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | # Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) 17 | # with modifications. 18 | # 19 | # Copyright 2017 The Abseil Authors. 20 | # 21 | # Licensed under the Apache License, Version 2.0 (the "License"); 22 | # you may not use this file except in compliance with the License. 23 | # You may obtain a copy of the License at 24 | # 25 | # https://www.apache.org/licenses/LICENSE-2.0 26 | # 27 | # Unless required by applicable law or agreed to in writing, software 28 | # distributed under the License is distributed on an "AS IS" BASIS, 29 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 30 | # See the License for the specific language governing permissions and 31 | # limitations under the License. 32 | # --------------------------------------------------------------------------- 33 | 34 | 35 | include(CMakeParseArguments) 36 | 37 | function(check_target my_target) 38 | if(NOT TARGET ${my_target}) 39 | message(FATAL_ERROR " ABSL: compiling absl requires a ${my_target} CMake target in your project, 40 | see CMake/README.md for more details") 41 | endif(NOT TARGET ${my_target}) 42 | endfunction() 43 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/css/colors.css: -------------------------------------------------------------------------------- 1 | /*** 2 | 3 | colors.css v2.0.0 4 | http://clrs.cc 5 | @mrmrs 6 | MIT License 7 | 8 | ***/ 9 | /* 10 | 11 | SKINS 12 | - Backgrounds 13 | - Colors 14 | - Border colors 15 | - SVG fills 16 | - SVG Strokes 17 | 18 | */ 19 | /* Backgrounds */ 20 | .bg-navy { 21 | background-color: #001F3F; } 22 | 23 | .bg-blue { 24 | background-color: #0074D9; } 25 | 26 | .bg-aqua { 27 | background-color: #7FDBFF; } 28 | 29 | .bg-teal { 30 | background-color: #39CCCC; } 31 | 32 | .bg-olive { 33 | background-color: #3D9970; } 34 | 35 | .bg-green { 36 | background-color: #2ECC40; } 37 | 38 | .bg-lime { 39 | background-color: #01FF70; } 40 | 41 | .bg-yellow { 42 | background-color: #FFDC00; } 43 | 44 | .bg-orange { 45 | background-color: #FF851B; } 46 | 47 | .bg-red { 48 | background-color: #FF4136; } 49 | 50 | .bg-fuchsia { 51 | background-color: #F012BE; } 52 | 53 | .bg-purple { 54 | background-color: #B10DC9; } 55 | 56 | .bg-maroon { 57 | background-color: #85144B; } 58 | 59 | .bg-white { 60 | background-color: #fff; } 61 | 62 | .bg-gray { 63 | background-color: #aaa; } 64 | 65 | .bg-silver { 66 | background-color: #ddd; } 67 | 68 | .bg-black { 69 | background-color: #111; } 70 | 71 | /* Colors */ 72 | .navy { 73 | color: #001F3F; } 74 | 75 | .blue { 76 | color: #0074D9; } 77 | 78 | .aqua { 79 | color: #7FDBFF; } 80 | 81 | .teal { 82 | color: #39CCCC; } 83 | 84 | .olive { 85 | color: #3D9970; } 86 | 87 | .green { 88 | color: #2ECC40; } 89 | 90 | .lime { 91 | color: #01FF70; } 92 | 93 | .yellow { 94 | color: #FFDC00; } 95 | 96 | .orange { 97 | color: #FF851B; } 98 | 99 | .red { 100 | color: #FF4136; } 101 | 102 | .fuchsia { 103 | color: #F012BE; } 104 | 105 | .purple { 106 | color: #B10DC9; } 107 | 108 | .maroon { 109 | color: #85144B; } 110 | 111 | .white { 112 | color: #fff; } 113 | 114 | .silver { 115 | color: #ddd; } 116 | 117 | .gray { 118 | color: #aaa; } 119 | 120 | .black { 121 | color: #111; } 122 | 123 | /* Border colors 124 | 125 | Use with another border utility that sets border-width and style 126 | i.e .border { border-width: 1px; border-style: solid; } 127 | */ 128 | .border--navy { 129 | border-color: #001F3F; } 130 | 131 | .border--blue { 132 | border-color: #0074D9; } 133 | 134 | .border--aqua { 135 | border-color: #7FDBFF; } 136 | 137 | .border--teal { 138 | border-color: #39CCCC; } 139 | 140 | .border--olive { 141 | border-color: #3D9970; } 142 | 143 | .border--green { 144 | border-color: #2ECC40; } 145 | 146 | .border--lime { 147 | border-color: #01FF70; } 148 | 149 | .border--yellow { 150 | border-color: #FFDC00; } 151 | 152 | .border--orange { 153 | border-color: #FF851B; } 154 | 155 | .border--red { 156 | border-color: #FF4136; } 157 | 158 | .border--fuchsia { 159 | border-color: #F012BE; } 160 | 161 | .border--purple { 162 | border-color: #B10DC9; } 163 | 164 | .border--maroon { 165 | border-color: #85144B; } 166 | 167 | .border--white { 168 | border-color: #fff; } 169 | 170 | .border--gray { 171 | border-color: #aaa; } 172 | 173 | .border--silver { 174 | border-color: #ddd; } 175 | 176 | .border--black { 177 | border-color: #111; } 178 | 179 | /* Fills for SVG */ 180 | .fill-navy { 181 | fill: #001F3F; } 182 | 183 | .fill-blue { 184 | fill: #0074D9; } 185 | 186 | .fill-aqua { 187 | fill: #7FDBFF; } 188 | 189 | .fill-teal { 190 | fill: #39CCCC; } 191 | 192 | .fill-olive { 193 | fill: #3D9970; } 194 | 195 | .fill-green { 196 | fill: #2ECC40; } 197 | 198 | .fill-lime { 199 | fill: #01FF70; } 200 | 201 | .fill-yellow { 202 | fill: #FFDC00; } 203 | 204 | .fill-orange { 205 | fill: #FF851B; } 206 | 207 | .fill-red { 208 | fill: #FF4136; } 209 | 210 | .fill-fuchsia { 211 | fill: #F012BE; } 212 | 213 | .fill-purple { 214 | fill: #B10DC9; } 215 | 216 | .fill-maroon { 217 | fill: #85144B; } 218 | 219 | .fill-white { 220 | fill: #fff; } 221 | 222 | .fill-gray { 223 | fill: #aaa; } 224 | 225 | .fill-silver { 226 | fill: #ddd; } 227 | 228 | .fill-black { 229 | fill: #111; } 230 | 231 | /* Strokes for SVG */ 232 | .stroke-navy { 233 | stroke: #001F3F; } 234 | 235 | .stroke-blue { 236 | stroke: #0074D9; } 237 | 238 | .stroke-aqua { 239 | stroke: #7FDBFF; } 240 | 241 | .stroke-teal { 242 | stroke: #39CCCC; } 243 | 244 | .stroke-olive { 245 | stroke: #3D9970; } 246 | 247 | .stroke-green { 248 | stroke: #2ECC40; } 249 | 250 | .stroke-lime { 251 | stroke: #01FF70; } 252 | 253 | .stroke-yellow { 254 | stroke: #FFDC00; } 255 | 256 | .stroke-orange { 257 | stroke: #FF851B; } 258 | 259 | .stroke-red { 260 | stroke: #FF4136; } 261 | 262 | .stroke-fuchsia { 263 | stroke: #F012BE; } 264 | 265 | .stroke-purple { 266 | stroke: #B10DC9; } 267 | 268 | .stroke-maroon { 269 | stroke: #85144B; } 270 | 271 | .stroke-white { 272 | stroke: #fff; } 273 | 274 | .stroke-gray { 275 | stroke: #aaa; } 276 | 277 | .stroke-silver { 278 | stroke: #ddd; } 279 | 280 | .stroke-black { 281 | stroke: #111; } 282 | 283 | /* PRETTIER LINKS */ 284 | a { 285 | text-decoration: none; 286 | -webkit-transition: color .3s ease-in-out; 287 | transition: color .3s ease-in-out; } 288 | 289 | a:link { 290 | -webkit-transition: color .3s ease-in-out; 291 | transition: color .3s ease-in-out; } 292 | 293 | a:visited { } 294 | 295 | a:hover { 296 | color: #001F3F; 297 | -webkit-transition: color .3s ease-in-out; 298 | transition: color .3s ease-in-out; } 299 | 300 | a:active { 301 | -webkit-transition: color .3s ease-in-out; 302 | transition: color .3s ease-in-out; } 303 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/css/style.css: -------------------------------------------------------------------------------- 1 | /* main stylesheet */ 2 | 3 | @import url(http://fonts.googleapis.com/css?family=Signika); 4 | 5 | html { 6 | overflow-y: scroll; 7 | } 8 | 9 | body { 10 | font-size: 15px; 11 | font-family: HelveticaNeue, 'Helvetica Neue', Helvetica, Arial, sans-serif; 12 | color: #332; 13 | } 14 | 15 | h1, h2, h3, h4, h5 { 16 | color: #332; 17 | font-family: HelveticaNeue, 'Helvetica Neue', Helvetica, Arial, sans-serif; 18 | font-weight: 400; 19 | font-size: 1.4em; 20 | line-height: 1.1; 21 | margin-top: 30px; 22 | } 23 | 24 | pre code { 25 | font: 14px/19px Inconsolata, Monaco,"Lucida Console",Terminal,"Courier New",Courier; 26 | } 27 | 28 | .figure { 29 | text-align: center; 30 | } 31 | 32 | .small .figure img { 33 | height: 200px; 34 | } 35 | 36 | .pagetitle .figure { 37 | text-align: left !important; 38 | } 39 | 40 | .pagetitle .figure img { 41 | height: 36px; 42 | } 43 | 44 | table{ 45 | background:#fff; 46 | border:1px solid #ccc; 47 | border-width:2px; 48 | border-collapse:collapse; 49 | margin:5px 0 10px; 50 | 51 | margin-top: 20px; 52 | margin-bottom: 20px; 53 | } 54 | 55 | th, td{ 56 | border:1px solid #ccc; 57 | padding:3px 10px; 58 | text-align:left; 59 | vertical-align:top; 60 | } 61 | 62 | tr.even td{ 63 | background:#f7f7f7; 64 | } 65 | 66 | th{ 67 | background:#edeff0; 68 | } 69 | 70 | td code { 71 | border: 0px; 72 | } 73 | 74 | img { 75 | max-width: 100%; 76 | height: auto; 77 | } 78 | 79 | hr { 80 | border: 0px; 81 | height: 0; 82 | border-bottom: 1px solid #ccc; 83 | margin-bottom: 100px; 84 | } 85 | 86 | /* Logo */ 87 | 88 | .logo { 89 | text-align: center; 90 | } 91 | 92 | .tagline { 93 | font-family: Georgia; 94 | font-size: 18px; 95 | font-style: italic; 96 | line-height: 1.45; 97 | color: #383838; 98 | } 99 | 100 | .author { 101 | } 102 | 103 | .halfbreak { 104 | padding-bottom: 100px; 105 | } 106 | 107 | .break { 108 | padding-bottom: 200px; 109 | } 110 | 111 | /* TOC Links */ 112 | 113 | a { 114 | color: #111111; 115 | text-decoration: none; 116 | } 117 | 118 | .body li a { 119 | text-decoration: underline; 120 | } 121 | 122 | /* Math */ 123 | 124 | .MathJax_Display { 125 | padding-top: 20px; 126 | padding-bottom: 20px; 127 | } 128 | 129 | /* Body Links */ 130 | 131 | p a { 132 | text-decoration: underline; 133 | } 134 | 135 | li code, p code { 136 | font-size: 12px; 137 | border: 1px solid #ccc; 138 | margin-left: 3px; 139 | margin-right: 3px; 140 | padding-left: 2px; 141 | padding-right: 2px; 142 | } 143 | 144 | /* */ 145 | 146 | .center { 147 | text-align: center; 148 | } 149 | 150 | .bigger img { 151 | width: 120%; 152 | height: 120%; 153 | } 154 | 155 | pre { 156 | font-size: 0.9em; 157 | 158 | margin-bottom: 18px; 159 | margin-top: 18px; 160 | 161 | border-left: 1px solid #ccc; 162 | 163 | } 164 | 165 | h1 { 166 | margin-top: 0px; 167 | } 168 | 169 | .annotation { 170 | font-size: 10pt; 171 | } 172 | 173 | .annotation pre { 174 | display: block; 175 | margin: 0; 176 | padding: 7px 10px; 177 | overflow-x: auto; 178 | } 179 | 180 | .annotation.span2 { 181 | /* Override bootstrap */ 182 | margin-left: 0px !important; 183 | margin-top: 18px !important; 184 | } 185 | 186 | .annotation pre code { 187 | border: 0; 188 | padding: 0; 189 | background: transparent; 190 | } 191 | 192 | blockquote { 193 | border-left: 1px solid #ccc; 194 | font-family: Georgia, serif; 195 | font-size: 14px; 196 | font-style: italic; 197 | margin: 0.25em 0; 198 | padding-left: 10px; 199 | line-height: 1.45; 200 | color: #383838; 201 | left: 20px; 202 | } 203 | 204 | 205 | blockquote cite { 206 | color: #999999; 207 | font-size: 14px; 208 | display: block; 209 | margin-top: 5px; 210 | } 211 | 212 | ul.sections { 213 | list-style: none; 214 | padding:0 0 5px 0; 215 | margin:0; 216 | } 217 | 218 | code.sourceCode { 219 | padding: 0; 220 | background: inherit; 221 | } 222 | 223 | pre.sourceCode { 224 | padding: 10px; 225 | } 226 | 227 | ul.sections > li > div { 228 | -moz-box-sizing: border-box; /* firefox */ 229 | -ms-box-sizing: border-box; /* ie */ 230 | -webkit-box-sizing: border-box; /* webkit */ 231 | -khtml-box-sizing: border-box; /* konqueror */ 232 | box-sizing: border-box; /* css3 */ 233 | } 234 | 235 | 236 | /* Make the naviation centered and larger on small screens */ 237 | /*---------------------- (> 481px) ---------------------*/ 238 | 239 | @media only screen and (max-width: 481px) { 240 | 241 | } 242 | 243 | @media only screen and (min-width: 1025px) { 244 | body { 245 | padding: 10px; 246 | } 247 | 248 | .side { 249 | position: fixed; 250 | width: 120px !important; 251 | margin-left: 0px; 252 | z-index: 1000; 253 | } 254 | 255 | .side ul ul { 256 | display: none; 257 | } 258 | 259 | .side ul ul.active { 260 | display: block; 261 | } 262 | 263 | .side .active { 264 | font-weight: bold; 265 | } 266 | 267 | .body { 268 | margin-left: 120px !important; 269 | } 270 | 271 | } 272 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/allmaps.cc: -------------------------------------------------------------------------------- 1 | // Silly program just to test the natvis file for Visual Studio 2 | // ------------------------------------------------------------ 3 | #include 4 | #include "parallel_hashmap/phmap.h" 5 | 6 | template 7 | void test_set(const F &f) 8 | { 9 | Set s; 10 | typename Set::iterator it; 11 | for (int i=0; i<100; ++i) 12 | s.insert(f(i)); 13 | 14 | it = s.begin(); 15 | ++it; 16 | 17 | it = s.end(); 18 | it = s.begin(); 19 | while(it != s.end()) 20 | ++it; 21 | it = s.begin(); 22 | } 23 | 24 | int main(int, char **) 25 | { 26 | using namespace std; 27 | 28 | auto make_int = [](int i) { return i; }; 29 | auto make_string = [](int i) { return std::to_string(i); }; 30 | 31 | auto make_2int = [](int i) { return std::make_pair(i, i); }; 32 | auto make_2string = [](int i) { return std::make_pair(std::to_string(i), std::to_string(i)); }; 33 | 34 | 35 | test_set>(make_int); 36 | test_set>(make_string); 37 | 38 | test_set>(make_int); 39 | test_set>(make_string); 40 | 41 | test_set>(make_2int); 42 | test_set>(make_2string); 43 | 44 | test_set>(make_2int); 45 | test_set>(make_2string); 46 | 47 | test_set>(make_int); 48 | test_set>(make_string); 49 | 50 | test_set>(make_int); 51 | test_set>(make_string); 52 | 53 | test_set>(make_2int); 54 | test_set>(make_2string); 55 | 56 | test_set>(make_2int); 57 | test_set>(make_2string); 58 | 59 | // example of using default parameters in order to specify the mutex type. 60 | // 61 | // Please be aware that the iterators returned (by find for example) cannot 62 | // be safely read in a multithreaded environment. Instead use if_contains(), 63 | // which passes a reference value to the callback while holding the submap lock. 64 | // Similarly, write access can be done safely using modify_if, try_emplace_l 65 | // or lazy_emplace_l. 66 | // ---------------------------------------------------------------------------- 67 | using Map = phmap::parallel_flat_hash_map, 69 | std::equal_to, 70 | std::allocator>, 71 | 4, 72 | std::mutex>; 73 | auto make_2size_t = [](size_t i) { return std::make_pair(i, i); }; 74 | test_set(make_2size_t); 75 | } 76 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/basic.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | using phmap::flat_hash_map; 6 | 7 | int main() 8 | { 9 | // Create an unordered_map of three strings (that map to strings) 10 | flat_hash_map email; 11 | 12 | // Iterate and print keys and values 13 | for (const auto& n : email) 14 | std::cout << n.first << "'s email is: " << n.second << "\n"; 15 | 16 | // Add a new entry 17 | email["bill"] = "bg@whatever.com"; 18 | 19 | // and print it 20 | std::cout << "bill's email is: " << email["bill"] << "\n"; 21 | 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/btree.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "btree_fwd.h" 3 | #include 4 | 5 | int main() 6 | { 7 | // initialise map with some values using an initializer_list 8 | phmap::btree_map map = 9 | { { "John", 35 }, 10 | { "Jane", 32 }, 11 | { "Joe", 30 }, 12 | }; 13 | 14 | // add a couple more values using operator[]() 15 | map["lucy"] = 18; 16 | map["Andre"] = 20; 17 | 18 | auto it = map.find("Joe"); 19 | map.erase(it); 20 | 21 | map.insert(std::make_pair("Alex", 16)); 22 | map.emplace("Emily", 18); // emplace uses pair template constructor 23 | 24 | for (auto& p: map) 25 | std::cout << p.first << ", " << p.second << '\n'; 26 | 27 | IntString map2; // IntString is declared in btree_fwd.h 28 | 29 | map2.emplace(std::piecewise_construct, std::forward_as_tuple(0), std::forward_as_tuple(10, 'c')); 30 | map2.try_emplace(1, 10, 'a'); // phmap::btree_map supports c++17 API 31 | 32 | for (auto& p: map2) 33 | std::cout << p.first << ", " << p.second << '\n'; 34 | 35 | // create a btree_set of tuples 36 | using X = std::tuple; 37 | phmap::btree_set set; 38 | 39 | for (int i=0; i<10; ++i) 40 | set.insert(X((float)i, std::to_string(i))); 41 | set.emplace(15.0f, "15"); 42 | 43 | set.erase(X(1.0f, "1")); 44 | 45 | for (auto& e: set) 46 | std::cout << std::get<0>(e) << ", \"" << std::get<1>(e) << "\" \n"; 47 | 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/btree_fwd.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | using IntString = phmap::btree_map; 6 | 7 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/dump_load.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | void dump_load_uint64_uint32() { 5 | phmap::flat_hash_map mp1 = { {100, 99}, {300, 299} }; 6 | 7 | for (const auto& n : mp1) 8 | std::cout << n.first << "'s value is: " << n.second << "\n"; 9 | 10 | { 11 | phmap::BinaryOutputArchive ar_out("./dump.data"); 12 | mp1.dump(ar_out); 13 | } 14 | 15 | phmap::flat_hash_map mp2; 16 | { 17 | phmap::BinaryInputArchive ar_in("./dump.data"); 18 | mp2.load(ar_in); 19 | } 20 | 21 | for (const auto& n : mp2) 22 | std::cout << n.first << "'s value is: " << n.second << "\n"; 23 | } 24 | 25 | void dump_load_parallel_flat_hash_map() { 26 | phmap::parallel_flat_hash_map mp1 = { 27 | {100, 99}, {300, 299}, {101, 992} }; 28 | 29 | for (const auto& n : mp1) 30 | std::cout << "key: " << n.first << ", value: " << n.second << "\n"; 31 | 32 | { 33 | phmap::BinaryOutputArchive ar_out("./dump.data"); 34 | mp1.dump(ar_out); 35 | } 36 | 37 | phmap::parallel_flat_hash_map mp2; 38 | { 39 | phmap::BinaryInputArchive ar_in("./dump.data"); 40 | mp2.load(ar_in); 41 | } 42 | 43 | for (const auto& n : mp2) 44 | std::cout << "key: " << n.first << ", value: " << n.second << "\n"; 45 | } 46 | 47 | int main() 48 | { 49 | dump_load_uint64_uint32(); 50 | dump_load_parallel_flat_hash_map(); 51 | return 0; 52 | } 53 | 54 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/dump_nested.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Example of dumping a map, containing values which are phmap maps or sets 4 | * building this requires c++17 support 5 | * 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | template 12 | class MyMap : public phmap::flat_hash_map> 13 | { 14 | public: 15 | using Set = phmap::flat_hash_set; 16 | 17 | void dump(const std::string &filename) 18 | { 19 | phmap::BinaryOutputArchive ar_out (filename.c_str()); 20 | 21 | ar_out.dump(this->size()); 22 | for (auto& [k, v] : *this) 23 | { 24 | ar_out.dump(k); 25 | v.dump(ar_out); 26 | } 27 | } 28 | 29 | void load(const std::string & filename) 30 | { 31 | phmap::BinaryInputArchive ar_in(filename.c_str()); 32 | 33 | size_t size; 34 | ar_in.load(&size); 35 | this->reserve(size); 36 | 37 | while (size--) 38 | { 39 | K k; 40 | Set v; 41 | 42 | ar_in.load(&k); 43 | v.load(ar_in); 44 | 45 | this->insert_or_assign(std::move(k), std::move(v)); 46 | } 47 | } 48 | 49 | void insert(K k, V v) 50 | { 51 | Set &set = (*this)[k]; 52 | set.insert(v); 53 | } 54 | 55 | friend std::ostream& operator<<(std::ostream& os, const MyMap& map) 56 | { 57 | for (const auto& [k, m] : map) 58 | { 59 | os << k << ": ["; 60 | for (const auto& x : m) 61 | os << x << ", "; 62 | os << "]\n"; 63 | } 64 | return os; 65 | } 66 | }; 67 | 68 | int main() 69 | { 70 | MyMap m; 71 | m.insert(1, 5); 72 | m.insert(1, 8); 73 | m.insert(2, 3); 74 | m.insert(1, 15); 75 | m.insert(1, 27); 76 | m.insert(2, 10); 77 | m.insert(2, 13); 78 | 79 | std::cout << m << "\n"; 80 | 81 | m.dump("test_archive"); 82 | m.clear(); 83 | m.load("test_archive"); 84 | 85 | std::cout << m << "\n"; 86 | 87 | return 0; 88 | } 89 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/f1.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Make sure that the phmap.h header builds fine when included in two separate 3 | * source files 4 | */ 5 | #include 6 | #include 7 | 8 | using phmap::flat_hash_map; 9 | 10 | int main() 11 | { 12 | // Create an unordered_map of three strings (that map to strings) 13 | using Map = flat_hash_map; 14 | Map email = 15 | { 16 | { "tom", "tom@gmail.com"}, 17 | { "jeff", "jk@gmail.com"}, 18 | { "jim", "jimg@microsoft.com"} 19 | }; 20 | 21 | extern void f2(Map&); 22 | f2(email); 23 | 24 | return 0; 25 | } 26 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/f2.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Make sure that the phmap.h header builds fine when included in two separate 3 | * source files 4 | */ 5 | #include 6 | #include 7 | #include 8 | 9 | using phmap::flat_hash_map; 10 | using Map = flat_hash_map; 11 | 12 | void f2(Map& email) 13 | { 14 | // Iterate and print keys and values 15 | for (const auto& n : email) 16 | std::cout << n.first << "'s email is: " << n.second << "\n"; 17 | 18 | // Add a new entry 19 | email["bill"] = "bg@whatever.com"; 20 | 21 | // and print it 22 | std::cout << "bill's email is: " << email["bill"] << "\n"; 23 | } 24 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/hash_std.cc: -------------------------------------------------------------------------------- 1 | #include "hash_std.h" // defines Person with std::hash specialization 2 | 3 | #include 4 | #include 5 | 6 | int main() 7 | { 8 | // As we have defined a specialization of std::hash() for Person, 9 | // we can now create sparse_hash_set or sparse_hash_map of Persons 10 | // ---------------------------------------------------------------- 11 | phmap::flat_hash_set persons = 12 | { { "John", "Mitchell", 35 }, 13 | { "Jane", "Smith", 32 }, 14 | { "Jane", "Smith", 30 }, 15 | }; 16 | 17 | for (auto& p: persons) 18 | std::cout << p._first << ' ' << p._last << " (" << p._age << ")" << '\n'; 19 | 20 | } 21 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/hash_std.h: -------------------------------------------------------------------------------- 1 | #ifndef phmap_example_hash_std_ 2 | #define phmap_example_hash_std_ 3 | 4 | #include // minimal header providing phmap::HashState() 5 | #include 6 | using std::string; 7 | 8 | struct Person 9 | { 10 | bool operator==(const Person &o) const 11 | { 12 | return _first == o._first && _last == o._last && _age == o._age; 13 | } 14 | 15 | string _first; 16 | string _last; 17 | int _age; 18 | }; 19 | 20 | namespace std 21 | { 22 | // inject specialization of std::hash for Person into namespace std 23 | // An alternative is to provide a hash_value() friend function (see hash_value.h) 24 | // ------------------------------------------------------------------------------ 25 | template<> struct hash 26 | { 27 | std::size_t operator()(Person const &p) const 28 | { 29 | return phmap::HashState().combine(0, p._first, p._last, p._age); 30 | } 31 | }; 32 | } 33 | 34 | #endif // phmap_example_hash_std_ 35 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/hash_value.cc: -------------------------------------------------------------------------------- 1 | #include "hash_value.h" // defines Person with std::hash specialization 2 | 3 | #include 4 | #include 5 | 6 | int main() 7 | { 8 | // As we have defined a specialization of std::hash() for Person, 9 | // we can now create sparse_hash_set or sparse_hash_map of Persons 10 | // ---------------------------------------------------------------- 11 | phmap::flat_hash_set persons = 12 | { { "John", "Mitchell", 35 }, 13 | { "Jane", "Smith", 32 }, 14 | { "Jane", "Smith", 30 }, 15 | }; 16 | 17 | for (auto& p: persons) 18 | std::cout << p._first << ' ' << p._last << " (" << p._age << ")" << '\n'; 19 | 20 | } 21 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/hash_value.h: -------------------------------------------------------------------------------- 1 | #ifndef phmap_example_hash_value_ 2 | #define phmap_example_hash_value_ 3 | 4 | #include // minimal header providing phmap::HashState() 5 | #include 6 | using std::string; 7 | 8 | struct Person 9 | { 10 | bool operator==(const Person &o) const 11 | { 12 | return _first == o._first && _last == o._last && _age == o._age; 13 | } 14 | 15 | // Demonstrates how to provide the hash function as a friend member function of the class 16 | // This can be used as an alternative to providing a std::hash specialization 17 | // -------------------------------------------------------------------------------------- 18 | friend size_t hash_value(const Person &p) 19 | { 20 | return phmap::HashState().combine(0, p._first, p._last, p._age); 21 | } 22 | 23 | string _first; 24 | string _last; 25 | int _age; 26 | }; 27 | 28 | #endif // phmap_example_hash_value_ 29 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/insert_bench.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #define PHMAP_ALLOCATOR_NOTHROW 1 8 | #include 9 | 10 | // this is probably the fastest high quality 64bit random number generator that exists. 11 | // Implements Small Fast Counting v4 RNG from PractRand. 12 | class sfc64 { 13 | public: 14 | using result_type = uint64_t; 15 | 16 | // no copy ctors so we don't accidentally get the same random again 17 | sfc64(sfc64 const&) = delete; 18 | sfc64& operator=(sfc64 const&) = delete; 19 | 20 | sfc64(sfc64&&) = default; 21 | sfc64& operator=(sfc64&&) = default; 22 | 23 | sfc64(std::array const& _state) 24 | : m_a(_state[0]) 25 | , m_b(_state[1]) 26 | , m_c(_state[2]) 27 | , m_counter(_state[3]) {} 28 | 29 | static constexpr uint64_t(min)() { 30 | return (std::numeric_limits::min)(); 31 | } 32 | static constexpr uint64_t(max)() { 33 | return (std::numeric_limits::max)(); 34 | } 35 | 36 | sfc64() 37 | : sfc64(UINT64_C(0x853c49e6748fea9b)) {} 38 | 39 | sfc64(uint64_t _seed) 40 | : m_a(_seed) 41 | , m_b(_seed) 42 | , m_c(_seed) 43 | , m_counter(1) { 44 | for (int i = 0; i < 12; ++i) { 45 | operator()(); 46 | } 47 | } 48 | 49 | void seed() { 50 | *this = sfc64{std::random_device{}()}; 51 | } 52 | 53 | uint64_t operator()() noexcept { 54 | auto const tmp = m_a + m_b + m_counter++; 55 | m_a = m_b ^ (m_b >> right_shift); 56 | m_b = m_c + (m_c << left_shift); 57 | m_c = rotl(m_c, rotation) + tmp; 58 | return tmp; 59 | } 60 | 61 | // this is a bit biased, but for our use case that's not important. 62 | uint64_t operator()(uint64_t boundExcluded) noexcept { 63 | #ifdef PHMAP_HAS_UMUL128 64 | uint64_t h; 65 | (void)umul128(operator()(), boundExcluded, &h); 66 | return h; 67 | #else 68 | return 0; 69 | #endif 70 | } 71 | 72 | std::array state() const { 73 | return {{m_a, m_b, m_c, m_counter}}; 74 | } 75 | 76 | void state(std::array const& s) { 77 | m_a = s[0]; 78 | m_b = s[1]; 79 | m_c = s[2]; 80 | m_counter = s[3]; 81 | } 82 | 83 | private: 84 | template 85 | T rotl(T const x, int k) { 86 | return (x << k) | (x >> (8 * sizeof(T) - k)); 87 | } 88 | 89 | static constexpr int rotation = 24; 90 | static constexpr int right_shift = 11; 91 | static constexpr int left_shift = 3; 92 | uint64_t m_a; 93 | uint64_t m_b; 94 | uint64_t m_c; 95 | uint64_t m_counter; 96 | }; 97 | 98 | 99 | int main() 100 | { 101 | // Create an unordered_map of three strings (that map to strings) 102 | using Map = phmap::parallel_node_hash_map; 103 | static size_t const n = 50000000; 104 | sfc64 rng(123); 105 | 106 | size_t checksum = 0; 107 | 108 | if (0) 109 | { 110 | size_t const max_rng = n / 20; 111 | Map map; 112 | for (size_t i = 0; i < n; ++i) { 113 | checksum += ++map[static_cast(rng(max_rng))]; 114 | } 115 | } 116 | 117 | if (0) 118 | { 119 | size_t const max_rng = n / 4; 120 | Map map; 121 | for (size_t i = 0; i < n; ++i) { 122 | checksum += ++map[static_cast(rng(max_rng))]; 123 | } 124 | } 125 | 126 | if (1) 127 | { 128 | size_t const max_rng = n / 2; 129 | Map map; 130 | for (size_t i = 0; i < n; ++i) { 131 | checksum += ++map[static_cast(rng(max_rng))]; 132 | } 133 | } 134 | 135 | if (0) 136 | { 137 | Map map; 138 | for (size_t i = 0; i < n; ++i) { 139 | checksum += ++map[static_cast(rng())]; 140 | } 141 | } 142 | printf("%zu\n", checksum); 143 | } 144 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/matt.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | // ------------------------------------------------------------------- 13 | // ------------------------------------------------------------------- 14 | class Timer 15 | { 16 | public: 17 | Timer(std::string name) : _name(name), _start(std::chrono::high_resolution_clock::now()) {} 18 | 19 | ~Timer() 20 | { 21 | std::chrono::duration elapsed_seconds = std::chrono::high_resolution_clock::now() - _start; 22 | printf("%s: %.3fs\n", _name.c_str(), elapsed_seconds.count()); 23 | } 24 | 25 | private: 26 | std::string _name; 27 | std::chrono::high_resolution_clock::time_point _start; 28 | }; 29 | 30 | // -------------------------------------------------------------------------- 31 | // from: https://github.com/preshing/RandomSequence 32 | // -------------------------------------------------------------------------- 33 | class RSU 34 | { 35 | private: 36 | uint32_t m_index; 37 | uint32_t m_intermediateOffset; 38 | 39 | static uint32_t permuteQPR(uint32_t x) 40 | { 41 | static const uint32_t prime = 4294967291u; 42 | if (x >= prime) 43 | return x; // The 5 integers out of range are mapped to themselves. 44 | uint32_t residue = ((unsigned long long) x * x) % prime; 45 | return (x <= prime / 2) ? residue : prime - residue; 46 | } 47 | 48 | public: 49 | RSU(uint32_t seedBase, uint32_t seedOffset) 50 | { 51 | m_index = permuteQPR(permuteQPR(seedBase) + 0x682f0161); 52 | m_intermediateOffset = permuteQPR(permuteQPR(seedOffset) + 0x46790905); 53 | } 54 | 55 | uint32_t next() 56 | { 57 | return permuteQPR((permuteQPR(m_index++) + m_intermediateOffset) ^ 0x5bf03635); 58 | } 59 | }; 60 | 61 | using Perturb = std::function &)>; 62 | 63 | // -------------------------------------------------------------------------- 64 | // -------------------------------------------------------------------------- 65 | template 66 | void test(const char *name, Perturb perturb1, Perturb /* perturb2 */) 67 | { 68 | //phmap::btree_set s; 69 | Set s; 70 | 71 | unsigned int seed = 76687; 72 | RSU rsu(seed, seed + 1); 73 | 74 | for (uint32_t i=0; i order(s.begin(), s.end()); // contains sorted, randomly generated keys (when using phmap::btree_set) 78 | // or keys in the final order of a Set (when using Set). 79 | 80 | perturb1(order); // either keep them in same order, or shuffle them 81 | 82 | #if 0 83 | order.resize(N/4); 84 | perturb2(order); 85 | #endif 86 | 87 | Timer t(name); // start timer 88 | Set c; 89 | //c.reserve(order.size()); // whether this "reserve()" is present or not makes a huge difference 90 | c.insert(order.begin(), order.end()); // time for inserting the same keys into the set 91 | // should not depend on them being sorted or not. 92 | } 93 | 94 | // -------------------------------------------------------------------------- 95 | // -------------------------------------------------------------------------- 96 | template 97 | using pset = phmap::parallel_flat_hash_set, 99 | phmap::priv::hash_default_eq, 100 | phmap::priv::Allocator, // alias for std::allocator 101 | N>; 102 | 103 | // -------------------------------------------------------------------------- 104 | // -------------------------------------------------------------------------- 105 | int main() 106 | { 107 | auto shuffle = [](std::vector &order) { 108 | std::random_device rd; 109 | std::mt19937 g(rd()); 110 | std::shuffle(order.begin(), order.end(), g); 111 | }; 112 | 113 | auto noop = [](std::vector &) {}; 114 | 115 | auto perturb2 = noop; 116 | 117 | constexpr uint32_t num_keys = 10000000; 118 | using T = uint64_t; 119 | 120 | test, num_keys>("flat_hash_set ordered ", noop, perturb2); 121 | 122 | test, num_keys>("flat_hash_set shuffled", shuffle, perturb2); 123 | 124 | test, num_keys>("parallel (16) ordered ", noop, perturb2); 125 | 126 | test, num_keys>("parallel (16) shuffled", shuffle, perturb2); 127 | 128 | test, num_keys>("parallel (64) ordered ", noop, perturb2); 129 | 130 | test, num_keys>("parallel (64) shuffled", shuffle, perturb2); 131 | 132 | test, num_keys>("parallel (256) ordered ", noop, perturb2); 133 | 134 | test, num_keys>("parallel (256) shuffled", shuffle, perturb2); 135 | } 136 | 137 | 138 | 139 | 140 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/examples/serialize.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "parallel_hashmap/phmap_dump.h" 5 | 6 | #define USE_CEREAL 0 7 | 8 | #if USE_CEREAL 9 | #include "cereal/types/unordered_map.hpp" 10 | #include "cereal/types/memory.hpp" 11 | #include "cereal/types/bitset.hpp" 12 | #include "cereal/archives/binary.hpp" 13 | #include 14 | #endif 15 | #include 16 | #include 17 | #include 18 | 19 | using phmap::flat_hash_map; 20 | using namespace std; 21 | template using milliseconds = std::chrono::duration; 22 | 23 | // -------------------------------------------------------------------------- 24 | // from: https://github.com/preshing/RandomSequence 25 | // -------------------------------------------------------------------------- 26 | class RSU 27 | { 28 | private: 29 | unsigned int m_index; 30 | unsigned int m_intermediateOffset; 31 | 32 | static unsigned int permuteQPR(unsigned int x) 33 | { 34 | static const unsigned int prime = 4294967291u; 35 | if (x >= prime) 36 | return x; // The 5 integers out of range are mapped to themselves. 37 | unsigned int residue = ((unsigned long long) x * x) % prime; 38 | return (x <= prime / 2) ? residue : prime - residue; 39 | } 40 | 41 | public: 42 | RSU(unsigned int seedBase, unsigned int seedOffset) 43 | { 44 | m_index = permuteQPR(permuteQPR(seedBase) + 0x682f0161); 45 | m_intermediateOffset = permuteQPR(permuteQPR(seedOffset) + 0x46790905); 46 | } 47 | 48 | unsigned int next() 49 | { 50 | return permuteQPR((permuteQPR(m_index++) + m_intermediateOffset) ^ 0x5bf03635); 51 | } 52 | }; 53 | 54 | // -------------------------------------------------------------------------- 55 | // -------------------------------------------------------------------------- 56 | void showtime(const char *name, std::function doit) 57 | { 58 | auto t1 = std::chrono::high_resolution_clock::now(); 59 | doit(); 60 | auto t2 = std::chrono::high_resolution_clock::now(); 61 | auto elapsed = milliseconds(t2 - t1).count(); 62 | printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f); 63 | } 64 | 65 | // -------------------------------------------------------------------------- 66 | // -------------------------------------------------------------------------- 67 | int main() 68 | { 69 | using MapType = phmap::flat_hash_map; 70 | MapType table; 71 | const int num_items = 100000000; 72 | 73 | // Iterate and add keys and values 74 | // ------------------------------- 75 | showtime("build hash", [&table, num_items]() { 76 | unsigned int seed = 76687; 77 | RSU rsu(seed, seed + 1); 78 | 79 | table.reserve(num_items); 80 | for (int i=0; i < num_items; ++i) 81 | table.insert(typename MapType::value_type(rsu.next(), i)); 82 | }); 83 | 84 | // cerealize and save data 85 | // ----------------------- 86 | showtime("serialize", [&table]() { 87 | #if !USE_CEREAL 88 | phmap::BinaryOutputArchive ar_out("./dump.data"); 89 | table.dump(ar_out); 90 | #else 91 | ofstream os("out.cereal", ios::binary); 92 | cereal::BinaryOutputArchive archive(os); 93 | archive(table.size()); 94 | archive(table); 95 | #endif 96 | }); 97 | 98 | MapType table_in; 99 | 100 | // deserialize 101 | // ----------- 102 | showtime("deserialize", [&table_in]() { 103 | #if !USE_CEREAL 104 | phmap::BinaryInputArchive ar_in("./dump.data"); 105 | table_in.load(ar_in); 106 | #else 107 | ifstream is("out.cereal", ios::binary); 108 | cereal::BinaryInputArchive archive_in(is); 109 | size_t table_size; 110 | 111 | archive_in(table_size); 112 | table_in.reserve(table_size); 113 | archive_in(table_in); // deserialize from file out.cereal into table_in 114 | #endif 115 | }); 116 | 117 | 118 | if (table == table_in) 119 | printf("All checks out, table size: %zu\n", table_in.size()); 120 | else 121 | printf("FAILURE\n"); 122 | 123 | return 0; 124 | } 125 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/Makefile: -------------------------------------------------------------------------------- 1 | PANDOC = stack exec pandoc -- 2 | MATHJAX = "http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML" 3 | FLAGS = --standalone --toc --toc-depth=2 --mathjax=$(MATHJAX) --highlight-style pygments 4 | PNG_IMAGES = $(patsubst %.pdf,%.png,$(wildcard img/*.pdf)) 5 | IFORMAT = -f gfm 6 | FILTER = includes.exe 7 | FILTER_OPT = --filter=${FILTER} # includes.hs 8 | 9 | ############################### html 10 | STYLE = css/style.css 11 | TEMPLATE_HTML = template.html 12 | HTML_OPT = -c ${STYLE} --template ${TEMPLATE_HTML} -t html 13 | PGTITLE = --metadata pagetitle="The Parallel Hashmap" 14 | 15 | ############################### pdf 16 | TEMPLATE_TEX = template.latex 17 | TEX_OPT = --template $(TEMPLATE_TEX) --pdf-engine=xelatex 18 | 19 | ############################### epub 20 | EPUB_COVER = --epub-cover-image=img/cover-kindle.jpg 21 | 22 | 23 | SRC = parallel_hashmap.md 24 | 25 | OBJ = $(SRC:.md=.html) 26 | 27 | all: html 28 | 29 | includes.exe: includes.hs 30 | stack exec ghc -- -o $@ -no-keep-hi-files -no-keep-o-files includes.hs 31 | 32 | html: parallel_hashmap.md $(FILTER) ${TEMPLATE_HTML} ${STYLE} 33 | $(PANDOC) ${FILTER_OPT} ${IFORMAT} ${HTML_OPT} $(FLAGS) ${PGTITLE} -o ../index.html parallel_hashmap.md 34 | 35 | %.pdf: %.md $(FILTER) ${TEMPLATE_TEX} 36 | $(PANDOC) ${FILTER_OPT} ${IFORMAT} ${TEX_OPT} $(FLAGS) -o $@ $< 37 | 38 | pdf: $(FILTER) ${TEMPLATE_TEX} 39 | rm -f parallel_hashmap.pdf; $(PANDOC) ${FILTER_OPT} ${IFORMAT} ${TEX_OPT} $(FLAGS) -o parallel_hashmap.pdf title.md $(SRC) 40 | 41 | native: 42 | $(PANDOC) -s -t native $(SRC) 43 | 44 | native_filt: $(FILTER) 45 | $(PANDOC) ${FILTER_OPT} -s -t native $(SRC) 46 | 47 | clean: 48 | -rm -f *.html *.pdf cppi.epub 49 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/bench_results/martinus_mod/index2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |

Benchmark Results

6 | 7 | insert 100m values in map

8 | 9 | Lookup 100m ints, all present | Lookup 100m ints, few present

10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/css/colors.css: -------------------------------------------------------------------------------- 1 | /*** 2 | 3 | colors.css v2.0.0 4 | http://clrs.cc 5 | @mrmrs 6 | MIT License 7 | 8 | ***/ 9 | /* 10 | 11 | SKINS 12 | - Backgrounds 13 | - Colors 14 | - Border colors 15 | - SVG fills 16 | - SVG Strokes 17 | 18 | */ 19 | /* Backgrounds */ 20 | .bg-navy { 21 | background-color: #001F3F; } 22 | 23 | .bg-blue { 24 | background-color: #0074D9; } 25 | 26 | .bg-aqua { 27 | background-color: #7FDBFF; } 28 | 29 | .bg-teal { 30 | background-color: #39CCCC; } 31 | 32 | .bg-olive { 33 | background-color: #3D9970; } 34 | 35 | .bg-green { 36 | background-color: #2ECC40; } 37 | 38 | .bg-lime { 39 | background-color: #01FF70; } 40 | 41 | .bg-yellow { 42 | background-color: #FFDC00; } 43 | 44 | .bg-orange { 45 | background-color: #FF851B; } 46 | 47 | .bg-red { 48 | background-color: #FF4136; } 49 | 50 | .bg-fuchsia { 51 | background-color: #F012BE; } 52 | 53 | .bg-purple { 54 | background-color: #B10DC9; } 55 | 56 | .bg-maroon { 57 | background-color: #85144B; } 58 | 59 | .bg-white { 60 | background-color: #fff; } 61 | 62 | .bg-gray { 63 | background-color: #aaa; } 64 | 65 | .bg-silver { 66 | background-color: #ddd; } 67 | 68 | .bg-black { 69 | background-color: #111; } 70 | 71 | /* Colors */ 72 | .navy { 73 | color: #001F3F; } 74 | 75 | .blue { 76 | color: #0074D9; } 77 | 78 | .aqua { 79 | color: #7FDBFF; } 80 | 81 | .teal { 82 | color: #39CCCC; } 83 | 84 | .olive { 85 | color: #3D9970; } 86 | 87 | .green { 88 | color: #2ECC40; } 89 | 90 | .lime { 91 | color: #01FF70; } 92 | 93 | .yellow { 94 | color: #FFDC00; } 95 | 96 | .orange { 97 | color: #FF851B; } 98 | 99 | .red { 100 | color: #FF4136; } 101 | 102 | .fuchsia { 103 | color: #F012BE; } 104 | 105 | .purple { 106 | color: #B10DC9; } 107 | 108 | .maroon { 109 | color: #85144B; } 110 | 111 | .white { 112 | color: #fff; } 113 | 114 | .silver { 115 | color: #ddd; } 116 | 117 | .gray { 118 | color: #aaa; } 119 | 120 | .black { 121 | color: #111; } 122 | 123 | /* Border colors 124 | 125 | Use with another border utility that sets border-width and style 126 | i.e .border { border-width: 1px; border-style: solid; } 127 | */ 128 | .border--navy { 129 | border-color: #001F3F; } 130 | 131 | .border--blue { 132 | border-color: #0074D9; } 133 | 134 | .border--aqua { 135 | border-color: #7FDBFF; } 136 | 137 | .border--teal { 138 | border-color: #39CCCC; } 139 | 140 | .border--olive { 141 | border-color: #3D9970; } 142 | 143 | .border--green { 144 | border-color: #2ECC40; } 145 | 146 | .border--lime { 147 | border-color: #01FF70; } 148 | 149 | .border--yellow { 150 | border-color: #FFDC00; } 151 | 152 | .border--orange { 153 | border-color: #FF851B; } 154 | 155 | .border--red { 156 | border-color: #FF4136; } 157 | 158 | .border--fuchsia { 159 | border-color: #F012BE; } 160 | 161 | .border--purple { 162 | border-color: #B10DC9; } 163 | 164 | .border--maroon { 165 | border-color: #85144B; } 166 | 167 | .border--white { 168 | border-color: #fff; } 169 | 170 | .border--gray { 171 | border-color: #aaa; } 172 | 173 | .border--silver { 174 | border-color: #ddd; } 175 | 176 | .border--black { 177 | border-color: #111; } 178 | 179 | /* Fills for SVG */ 180 | .fill-navy { 181 | fill: #001F3F; } 182 | 183 | .fill-blue { 184 | fill: #0074D9; } 185 | 186 | .fill-aqua { 187 | fill: #7FDBFF; } 188 | 189 | .fill-teal { 190 | fill: #39CCCC; } 191 | 192 | .fill-olive { 193 | fill: #3D9970; } 194 | 195 | .fill-green { 196 | fill: #2ECC40; } 197 | 198 | .fill-lime { 199 | fill: #01FF70; } 200 | 201 | .fill-yellow { 202 | fill: #FFDC00; } 203 | 204 | .fill-orange { 205 | fill: #FF851B; } 206 | 207 | .fill-red { 208 | fill: #FF4136; } 209 | 210 | .fill-fuchsia { 211 | fill: #F012BE; } 212 | 213 | .fill-purple { 214 | fill: #B10DC9; } 215 | 216 | .fill-maroon { 217 | fill: #85144B; } 218 | 219 | .fill-white { 220 | fill: #fff; } 221 | 222 | .fill-gray { 223 | fill: #aaa; } 224 | 225 | .fill-silver { 226 | fill: #ddd; } 227 | 228 | .fill-black { 229 | fill: #111; } 230 | 231 | /* Strokes for SVG */ 232 | .stroke-navy { 233 | stroke: #001F3F; } 234 | 235 | .stroke-blue { 236 | stroke: #0074D9; } 237 | 238 | .stroke-aqua { 239 | stroke: #7FDBFF; } 240 | 241 | .stroke-teal { 242 | stroke: #39CCCC; } 243 | 244 | .stroke-olive { 245 | stroke: #3D9970; } 246 | 247 | .stroke-green { 248 | stroke: #2ECC40; } 249 | 250 | .stroke-lime { 251 | stroke: #01FF70; } 252 | 253 | .stroke-yellow { 254 | stroke: #FFDC00; } 255 | 256 | .stroke-orange { 257 | stroke: #FF851B; } 258 | 259 | .stroke-red { 260 | stroke: #FF4136; } 261 | 262 | .stroke-fuchsia { 263 | stroke: #F012BE; } 264 | 265 | .stroke-purple { 266 | stroke: #B10DC9; } 267 | 268 | .stroke-maroon { 269 | stroke: #85144B; } 270 | 271 | .stroke-white { 272 | stroke: #fff; } 273 | 274 | .stroke-gray { 275 | stroke: #aaa; } 276 | 277 | .stroke-silver { 278 | stroke: #ddd; } 279 | 280 | .stroke-black { 281 | stroke: #111; } 282 | 283 | /* PRETTIER LINKS */ 284 | a { 285 | text-decoration: none; 286 | -webkit-transition: color .3s ease-in-out; 287 | transition: color .3s ease-in-out; } 288 | 289 | a:link { 290 | -webkit-transition: color .3s ease-in-out; 291 | transition: color .3s ease-in-out; } 292 | 293 | a:visited { } 294 | 295 | a:hover { 296 | color: #001F3F; 297 | -webkit-transition: color .3s ease-in-out; 298 | transition: color .3s ease-in-out; } 299 | 300 | a:active { 301 | -webkit-transition: color .3s ease-in-out; 302 | transition: color .3s ease-in-out; } 303 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/css/style.css: -------------------------------------------------------------------------------- 1 | /* main stylesheet */ 2 | 3 | @import url(http://fonts.googleapis.com/css?family=Signika); 4 | 5 | html { 6 | overflow-y: scroll; 7 | } 8 | 9 | body { 10 | font-size: 15px; 11 | font-family: HelveticaNeue, 'Helvetica Neue', Helvetica, Arial, sans-serif; 12 | color: #332; 13 | } 14 | 15 | h1, h2, h3, h4, h5 { 16 | color: #332; 17 | font-family: "Signika"; 18 | font-weight: 400; 19 | font-size: 1.4em; 20 | line-height: 1.1; 21 | margin-top: 30px; 22 | } 23 | 24 | pre code { 25 | font: 14px/19px Inconsolata, Monaco,"Lucida Console",Terminal,"Courier New",Courier; 26 | } 27 | 28 | .figure { 29 | text-align: center; 30 | } 31 | 32 | .small .figure img { 33 | height: 200px; 34 | } 35 | 36 | .pagetitle .figure { 37 | text-align: left !important; 38 | } 39 | 40 | .pagetitle .figure img { 41 | height: 36px; 42 | } 43 | 44 | table{ 45 | background:#fff; 46 | border:1px solid #ccc; 47 | border-width:2px; 48 | border-collapse:collapse; 49 | margin:5px 0 10px; 50 | 51 | margin-top: 20px; 52 | margin-bottom: 20px; 53 | } 54 | 55 | th, td{ 56 | border:1px solid #ccc; 57 | padding:3px 10px; 58 | text-align:left; 59 | vertical-align:top; 60 | } 61 | 62 | tr.even td{ 63 | background:#f7f7f7; 64 | } 65 | 66 | th{ 67 | background:#edeff0; 68 | } 69 | 70 | td code { 71 | border: 0px; 72 | } 73 | 74 | img { 75 | max-width: 100%; 76 | height: auto; 77 | } 78 | 79 | hr { 80 | border: 0px; 81 | height: 0; 82 | border-bottom: 1px solid #ccc; 83 | margin-bottom: 100px; 84 | } 85 | 86 | /* Logo */ 87 | 88 | .logo { 89 | text-align: center; 90 | } 91 | 92 | .tagline { 93 | font-family: Georgia; 94 | font-size: 18px; 95 | font-style: italic; 96 | line-height: 1.45; 97 | color: #383838; 98 | } 99 | 100 | .author { 101 | } 102 | 103 | .halfbreak { 104 | padding-bottom: 100px; 105 | } 106 | 107 | .break { 108 | padding-bottom: 200px; 109 | } 110 | 111 | /* TOC Links */ 112 | 113 | a { 114 | color: #111111; 115 | text-decoration: none; 116 | } 117 | 118 | .body li a { 119 | text-decoration: underline; 120 | } 121 | 122 | /* Math */ 123 | 124 | .MathJax_Display { 125 | padding-top: 20px; 126 | padding-bottom: 20px; 127 | } 128 | 129 | /* Body Links */ 130 | 131 | p a { 132 | text-decoration: underline; 133 | } 134 | 135 | li code, p code { 136 | font-size: 12px; 137 | border: 1px solid #ccc; 138 | margin-left: 3px; 139 | margin-right: 3px; 140 | padding-left: 2px; 141 | padding-right: 2px; 142 | } 143 | 144 | /* */ 145 | 146 | .center { 147 | text-align: center; 148 | } 149 | 150 | .bigger img { 151 | width: 120%; 152 | height: 120%; 153 | } 154 | 155 | pre { 156 | font-size: 0.9em; 157 | 158 | margin-bottom: 18px; 159 | margin-top: 18px; 160 | 161 | border-left: 1px solid #ccc; 162 | 163 | } 164 | 165 | h1 { 166 | margin-top: 0px; 167 | } 168 | 169 | .annotation { 170 | font-size: 10pt; 171 | } 172 | 173 | .annotation pre { 174 | display: block; 175 | margin: 0; 176 | padding: 7px 10px; 177 | overflow-x: auto; 178 | } 179 | 180 | .annotation.span2 { 181 | /* Override bootstrap */ 182 | margin-left: 0px !important; 183 | margin-top: 18px !important; 184 | } 185 | 186 | .annotation pre code { 187 | border: 0; 188 | padding: 0; 189 | background: transparent; 190 | } 191 | 192 | blockquote { 193 | border-left: 1px solid #ccc; 194 | font-family: Georgia, serif; 195 | font-size: 14px; 196 | font-style: italic; 197 | margin: 0.25em 0; 198 | padding-left: 10px; 199 | line-height: 1.45; 200 | color: #383838; 201 | left: 20px; 202 | } 203 | 204 | 205 | blockquote cite { 206 | color: #999999; 207 | font-size: 14px; 208 | display: block; 209 | margin-top: 5px; 210 | } 211 | 212 | ul.sections { 213 | list-style: none; 214 | padding:0 0 5px 0; 215 | margin:0; 216 | } 217 | 218 | code.sourceCode { 219 | padding: 0; 220 | background: inherit; 221 | } 222 | 223 | pre.sourceCode { 224 | padding: 10px; 225 | } 226 | 227 | ul.sections > li > div { 228 | -moz-box-sizing: border-box; /* firefox */ 229 | -ms-box-sizing: border-box; /* ie */ 230 | -webkit-box-sizing: border-box; /* webkit */ 231 | -khtml-box-sizing: border-box; /* konqueror */ 232 | box-sizing: border-box; /* css3 */ 233 | } 234 | 235 | 236 | /* Make the naviation centered and larger on small screens */ 237 | /*---------------------- (> 481px) ---------------------*/ 238 | 239 | @media only screen and (max-width: 481px) { 240 | 241 | } 242 | 243 | @media only screen and (min-width: 1025px) { 244 | body { 245 | padding: 10px; 246 | } 247 | 248 | .side { 249 | position: fixed; 250 | width: 120px !important; 251 | margin-left: 0px; 252 | z-index: 1000; 253 | } 254 | 255 | .side ul ul { 256 | display: none; 257 | } 258 | 259 | .side ul ul.active { 260 | display: block; 261 | } 262 | 263 | .side .active { 264 | font-weight: bold; 265 | } 266 | 267 | .body { 268 | margin-left: 120px !important; 269 | } 270 | 271 | } 272 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/diagrams/closed_hashing: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | +----------------+ 5 | |"(keyi, valuei)"| "(key, value) pairs are stored directly" 6 | +----+-----------+ "into the array (no pointers)" 7 | | +--------+---------------------+ 8 | | | | | 9 | | | | | 10 | | | | | 11 | | | | | 12 | | | | | 13 | | +--------+---------------------+ 14 | +---------------> | keyi | valuei | 15 | hasher(keyi) +--------+---------------------+ 16 | | | | 17 | | | | 18 | | | | 19 | +--------+---------------------+ 20 | | | | 21 | +--------+---------------------+ 22 | | | | 23 | | | | 24 | | | | 25 | +--------+---------------------+ 26 | 27 | absl::flat_hash_map 28 | 29 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/diagrams/closed_hashing.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | (keyi, valuei) 101 | 102 | 103 | 104 | 105 | hasher(keyi) 106 | 107 | 108 | 109 | 110 | absl::flat_hash_map 111 | 112 | 113 | 114 | 115 | (key, value) pairs are stored directly 116 | 117 | 118 | 119 | 120 | into the array (no pointers) 121 | 122 | 123 | 124 | 125 | keyi 126 | 127 | 128 | 129 | 130 | valuei 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/diagrams/index_computation: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | +----------------+ 5 | |"(keyi, valuei)"| 6 | +------+---------+ 7 | | 8 | | hasher(keyi) "Parallel Hash Map" 9 | v "(with 8 submaps)" 10 | +--------+-------------+ +----------------+ 11 | | h=0x7d84ea13707f4657 | | submap0 | 12 | +---------+------------+ +----------------+ 13 | | | submap1 | 14 | | "(h ^ (h >> 3)) & 0x7" +----------------+ 15 | v | submap2 | 16 | +----+----+ +----------------+ 17 | |"idx = 5"| | submap3 | 18 | +----+----+ +----------------+ 19 | | | submap4 | 20 | | +----------------+ 21 | +------------------------------->| submap5 | 22 | +----------------+ 23 | | submap6 | 24 | +----------------+ 25 | | submap7 | 26 | +----------------+ 27 | 28 | "parallel_hash_map with 8 submaps, each submap is an absl::flat_hash_map" 29 | 30 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/closed_hashing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/closed_hashing.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/flat_mem_usage.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/flat_mem_usage.gif -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/flat_mem_usage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/flat_mem_usage.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/flat_par_mutex_4.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/flat_par_mutex_4.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/flat_par_mutex_5.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/flat_par_mutex_5.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/flat_par_mutex_5_speed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/flat_par_mutex_5_speed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/flat_par_mutex_6_speed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/flat_par_mutex_6_speed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/flat_par_speed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/flat_par_speed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/flat_peak.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/flat_peak.gif -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/flat_peak.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/flat_peak.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/hashtable_benchmarks.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/hashtable_benchmarks.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/idx_computation_cost.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/idx_computation_cost.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/index_computation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/index_computation.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/lock_various_sizes.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/lock_various_sizes.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_both.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_both.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_both_run2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_both_run2.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_mem.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_mem.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_mem_run2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_mem_run2.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_mem_run2_zoomed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_mem_run2_zoomed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_mem_zoomed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_mem_zoomed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_speed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_speed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_speed_run2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/mt_stl_flat_par_speed_run2.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/no_preselection.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/no_preselection.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/node_mem_usage.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/node_mem_usage.gif -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/node_mem_usage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/node_mem_usage.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/node_peak.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/node_peak.gif -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/node_peak.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/node_peak.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/par_align_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/par_align_test.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/par_mt_memory.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/par_mt_memory.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/par_mt_speed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/par_mt_speed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/parallel_flat_peak.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/parallel_flat_peak.gif -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/parallel_flat_peak.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/parallel_flat_peak.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/parallel_node_peak.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/parallel_node_peak.gif -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/parallel_node_peak.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/parallel_node_peak.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/phash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/phash.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/phmap_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/phmap_logo.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/spp_flat_par_both.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/spp_flat_par_both.png -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/stl_flat_both.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/stl_flat_both.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/stl_flat_mem.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/stl_flat_mem.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/stl_flat_par_both.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/stl_flat_par_both.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/stl_flat_par_mem.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/stl_flat_par_mem.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/stl_flat_par_mem_zoomed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/stl_flat_par_mem_zoomed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/stl_flat_par_speed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/stl_flat_par_speed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/img/stl_flat_speed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/img/stl_flat_speed.PNG -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/includes.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | import Text.Read 4 | import Control.Monad.State 5 | import Control.Monad 6 | import Text.Pandoc 7 | import Data.Monoid 8 | import Control.Applicative 9 | 10 | import Text.Pandoc.JSON 11 | import Text.Pandoc.Walk 12 | 13 | slice :: Int -> Int -> [a] -> [a] 14 | slice from to xs = take (to - from + 1) (drop from xs) 15 | 16 | doSlice :: Block -> IO Block 17 | doSlice cb@(CodeBlock (id, classes, namevals) contents) = do 18 | res <- return $ do 19 | upper <- readMaybe =<< lookup "upper" namevals 20 | lower <- readMaybe =<< lookup "lower" namevals 21 | file <- lookup "slice" namevals 22 | return (upper, lower, file) 23 | 24 | case res of 25 | Nothing -> return cb 26 | Just (upper, lower, f) -> do 27 | contents <- readFile f 28 | let lns = unlines $ slice lower upper (lines contents) 29 | return (CodeBlock (id, classes, namevals) lns) 30 | doSlice x = return x 31 | 32 | doInclude :: Block -> IO Block 33 | doInclude cb@(CodeBlock (id, classes, namevals) contents) = 34 | case lookup "include" namevals of 35 | Just f -> return . (CodeBlock (id, classes, namevals)) =<< readFile f 36 | Nothing -> return cb 37 | doInclude x = return x 38 | 39 | doHtml :: Block -> IO Block 40 | doHtml cb@(CodeBlock (id, classes, namevals) contents) = 41 | case lookup "literal" namevals of 42 | Just f -> return . (RawBlock "html") =<< readFile f 43 | Nothing -> return cb 44 | doHtml x = return x 45 | 46 | injectLatexMacros :: Maybe Format -> Pandoc -> IO Pandoc 47 | injectLatexMacros (Just fmt) p = do 48 | macros <- readFile "latex_macros" 49 | let block = 50 | case fmt of 51 | Format "html" -> 52 | Div ("",[],[("style","display:none")]) . (:[]) 53 | . Para . (:[]) . Math DisplayMath $ macros 54 | Format "latex" -> RawBlock "latex" macros 55 | Format "epub" -> RawBlock "latex" macros 56 | _ -> RawBlock "latex" macros 57 | return (Pandoc nullMeta [block] <> p) 58 | injectLatexMacros _ _ = return mempty 59 | 60 | main :: IO () 61 | main = toJSONFilter 62 | ((\fmt -> injectLatexMacros fmt 63 | >=> walkM doInclude 64 | >=> walkM doSlice 65 | >=> walkM doHtml) :: Maybe Format -> Pandoc -> IO Pandoc) 66 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/latex_macros: -------------------------------------------------------------------------------- 1 | \newcommand{\andalso}{\quad\quad} 2 | \newcommand{\infabbrev}[2]{\infax{#1 \quad\eqdef\quad #2}} 3 | \newcommand{\infrule}[2]{\displaystyle \dfrac{#1}{#2}} 4 | \newcommand{\ar}{\rightarrow} 5 | \newcommand{\Int}{\mathtt{Int}} 6 | \newcommand{\Bool}{\mathtt{Bool}} 7 | \newcommand{\becomes}{\Downarrow} 8 | \newcommand{\trule}[1]{(\textbf{#1})} 9 | \newcommand{\FV}[1]{\mathtt{fv}(#1)} 10 | \newcommand{\FTV}[1]{\mathtt{ftv}(#1)} 11 | \newcommand{\BV}[1]{\mathtt{bv}(#1)} 12 | \newcommand{\compiles}[1]{\text{C}\llbracket{#1}\rrbracket} 13 | \newcommand{\exec}[1]{\text{E}\llbracket{#1}\rrbracket} 14 | \renewcommand{\t}[1]{\mathtt{#1}} 15 | \newcommand{\ite}[3]{\text{if }#1\text{ then }#2\text{ else }#3} 16 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/parallel_hashmap.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_sampler/parallel-hashmap/html/parallel_hashmap.pdf -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/html/template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | The Parallel Hashmap (Gregory Popovitch) 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | $if(highlighting-css)$ 17 | 20 | $endif$ 21 | $for(css)$ 22 | 23 | $endfor$ 24 | $if(math)$ 25 | $if(html5)$ 26 | $else$ 27 | $math$ 28 | $endif$ 29 | $endif$ 30 | $for(header-includes)$ 31 | $header-includes$ 32 | $endfor$ 33 | 34 | 35 | 36 | 37 |

38 | 39 |
40 | 41 |
42 | 43 | $body$ 44 |
45 |
46 | 47 |
48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/parallel_hashmap/conanfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from conans import ConanFile, tools 5 | import os 6 | 7 | 8 | class SparseppConan(ConanFile): 9 | name = "parallel_hashmap" 10 | version = "1.27" 11 | description = "A header-only, very fast and memory-friendly hash map" 12 | 13 | # Indicates License type of the packaged library 14 | license = "https://github.com/greg7mdp/parallel-hashmap/blob/master/LICENSE" 15 | 16 | # Packages the license for the conanfile.py 17 | exports = ["LICENSE"] 18 | 19 | # Custom attributes for Bincrafters recipe conventions 20 | source_subfolder = "source_subfolder" 21 | 22 | def source(self): 23 | source_url = "https://github.com/greg7mdp/parallel-hashmap" 24 | tools.get("{0}/archive/{1}.tar.gz".format(source_url, self.version)) 25 | extracted_dir = self.name + "-" + self.version 26 | 27 | # Rename to "source_folder" is a convention to simplify later steps 28 | os.rename(extracted_dir, self.source_subfolder) 29 | 30 | def package(self): 31 | include_folder = os.path.join( 32 | self.source_subfolder, "parallel_hashmap") 33 | self.copy(pattern="LICENSE") 34 | self.copy(pattern="*", dst="include/parallel_hashmap", 35 | src=include_folder) 36 | 37 | def package_id(self): 38 | self.info.header_only() 39 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/dump_load_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "gtest/gtest.h" 4 | 5 | #include "parallel_hashmap/phmap_dump.h" 6 | 7 | namespace phmap { 8 | namespace priv { 9 | namespace { 10 | 11 | TEST(DumpLoad, FlatHashSet_uin32) { 12 | phmap::flat_hash_set st1 = { 1991, 1202 }; 13 | 14 | { 15 | phmap::BinaryOutputArchive ar_out("./dump.data"); 16 | EXPECT_TRUE(st1.dump(ar_out)); 17 | } 18 | 19 | phmap::flat_hash_set st2; 20 | { 21 | phmap::BinaryInputArchive ar_in("./dump.data"); 22 | EXPECT_TRUE(st2.load(ar_in)); 23 | } 24 | EXPECT_TRUE(st1 == st2); 25 | } 26 | 27 | TEST(DumpLoad, FlatHashMap_uint64_uint32) { 28 | phmap::flat_hash_map mp1 = { 29 | { 78731, 99}, {13141, 299}, {2651, 101} }; 30 | 31 | { 32 | phmap::BinaryOutputArchive ar_out("./dump.data"); 33 | EXPECT_TRUE(mp1.dump(ar_out)); 34 | } 35 | 36 | phmap::flat_hash_map mp2; 37 | { 38 | phmap::BinaryInputArchive ar_in("./dump.data"); 39 | EXPECT_TRUE(mp2.load(ar_in)); 40 | } 41 | 42 | EXPECT_TRUE(mp1 == mp2); 43 | } 44 | 45 | TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { 46 | phmap::parallel_flat_hash_map mp1 = { 47 | {99, 299}, {992, 2991}, {299, 1299} }; 48 | 49 | { 50 | phmap::BinaryOutputArchive ar_out("./dump.data"); 51 | EXPECT_TRUE(mp1.dump(ar_out)); 52 | } 53 | 54 | phmap::parallel_flat_hash_map mp2; 55 | { 56 | phmap::BinaryInputArchive ar_in("./dump.data"); 57 | EXPECT_TRUE(mp2.load(ar_in)); 58 | } 59 | EXPECT_TRUE(mp1 == mp2); 60 | } 61 | 62 | } 63 | } 64 | } 65 | 66 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/flat_hash_set_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef THIS_HASH_SET 16 | #define THIS_HASH_SET flat_hash_set 17 | #define THIS_TEST_NAME FlatHashSet 18 | #endif 19 | 20 | #include "parallel_hashmap/phmap.h" 21 | 22 | #include 23 | 24 | #include "hash_generator_testing.h" 25 | #include "unordered_set_constructor_test.h" 26 | #include "unordered_set_lookup_test.h" 27 | #include "unordered_set_members_test.h" 28 | #include "unordered_set_modifiers_test.h" 29 | 30 | namespace phmap { 31 | namespace priv { 32 | namespace { 33 | 34 | using ::phmap::priv::hash_internal::Enum; 35 | using ::phmap::priv::hash_internal::EnumClass; 36 | using ::testing::Pointee; 37 | using ::testing::UnorderedElementsAre; 38 | using ::testing::UnorderedElementsAreArray; 39 | 40 | template 41 | using Set = 42 | phmap::THIS_HASH_SET>; 43 | 44 | using SetTypes = 45 | ::testing::Types, Set, Set, Set>; 46 | 47 | INSTANTIATE_TYPED_TEST_SUITE_P(THIS_TEST_NAME, ConstructorTest, SetTypes); 48 | INSTANTIATE_TYPED_TEST_SUITE_P(THIS_TEST_NAME, LookupTest, SetTypes); 49 | INSTANTIATE_TYPED_TEST_SUITE_P(THIS_TEST_NAME, MembersTest, SetTypes); 50 | INSTANTIATE_TYPED_TEST_SUITE_P(THIS_TEST_NAME, ModifiersTest, SetTypes); 51 | 52 | #if PHMAP_HAVE_STD_STRING_VIEW 53 | TEST(THIS_TEST_NAME, EmplaceString) { 54 | std::vector v = {"a", "b"}; 55 | phmap::THIS_HASH_SET hs(v.begin(), v.end()); 56 | //EXPECT_THAT(hs, UnorderedElementsAreArray(v)); 57 | } 58 | #endif 59 | 60 | TEST(THIS_TEST_NAME, BitfieldArgument) { 61 | union { 62 | int n : 1; 63 | }; 64 | n = 0; 65 | phmap::THIS_HASH_SET s = {n}; 66 | s.insert(n); 67 | s.insert(s.end(), n); 68 | s.insert({n}); 69 | s.erase(n); 70 | s.count(n); 71 | s.prefetch(n); 72 | s.find(n); 73 | s.contains(n); 74 | s.equal_range(n); 75 | } 76 | 77 | TEST(THIS_TEST_NAME, MergeExtractInsert) { 78 | struct Hash { 79 | size_t operator()(const std::unique_ptr& p) const { return *p; } 80 | }; 81 | struct Eq { 82 | bool operator()(const std::unique_ptr& a, 83 | const std::unique_ptr& b) const { 84 | return *a == *b; 85 | } 86 | }; 87 | phmap::THIS_HASH_SET, Hash, Eq> set1, set2; 88 | set1.insert(phmap::make_unique(7)); 89 | set1.insert(phmap::make_unique(17)); 90 | 91 | set2.insert(phmap::make_unique(7)); 92 | set2.insert(phmap::make_unique(19)); 93 | 94 | EXPECT_THAT(set1, UnorderedElementsAre(Pointee(7), Pointee(17))); 95 | EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7), Pointee(19))); 96 | 97 | set1.merge(set2); 98 | 99 | EXPECT_THAT(set1, UnorderedElementsAre(Pointee(7), Pointee(17), Pointee(19))); 100 | EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7))); 101 | 102 | auto node = set1.extract(phmap::make_unique(7)); 103 | EXPECT_TRUE(node); 104 | EXPECT_THAT(node.value(), Pointee(7)); 105 | EXPECT_THAT(set1, UnorderedElementsAre(Pointee(17), Pointee(19))); 106 | 107 | auto insert_result = set2.insert(std::move(node)); 108 | EXPECT_FALSE(node); 109 | EXPECT_FALSE(insert_result.inserted); 110 | EXPECT_TRUE(insert_result.node); 111 | EXPECT_THAT(insert_result.node.value(), Pointee(7)); 112 | EXPECT_EQ(**insert_result.position, 7); 113 | EXPECT_NE(insert_result.position->get(), insert_result.node.value().get()); 114 | EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7))); 115 | 116 | node = set1.extract(phmap::make_unique(17)); 117 | EXPECT_TRUE(node); 118 | EXPECT_THAT(node.value(), Pointee(17)); 119 | EXPECT_THAT(set1, UnorderedElementsAre(Pointee(19))); 120 | 121 | node.value() = phmap::make_unique(23); 122 | 123 | insert_result = set2.insert(std::move(node)); 124 | EXPECT_FALSE(node); 125 | EXPECT_TRUE(insert_result.inserted); 126 | EXPECT_FALSE(insert_result.node); 127 | EXPECT_EQ(**insert_result.position, 23); 128 | EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7), Pointee(23))); 129 | } 130 | 131 | } // namespace 132 | } // namespace priv 133 | } // namespace phmap 134 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/hash_policy_testing_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "parallel_hashmap/phmap.h" 16 | #include "hash_policy_testing.h" 17 | 18 | #include "gtest/gtest.h" 19 | 20 | namespace phmap { 21 | namespace priv { 22 | namespace { 23 | 24 | TEST(_, Hash) { 25 | StatefulTestingHash h1; 26 | EXPECT_EQ(1, h1.id()); 27 | StatefulTestingHash h2; 28 | EXPECT_EQ(2, h2.id()); 29 | StatefulTestingHash h1c(h1); 30 | EXPECT_EQ(1, h1c.id()); 31 | StatefulTestingHash h2m(std::move(h2)); 32 | EXPECT_EQ(2, h2m.id()); 33 | EXPECT_EQ(0, h2.id()); 34 | StatefulTestingHash h3; 35 | EXPECT_EQ(3, h3.id()); 36 | h3 = StatefulTestingHash(); 37 | EXPECT_EQ(4, h3.id()); 38 | h3 = std::move(h1); 39 | EXPECT_EQ(1, h3.id()); 40 | } 41 | 42 | } // namespace 43 | } // namespace priv 44 | } // namespace phmap 45 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/hashtable_debug.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // This library provides APIs to debug the probing behavior of hash tables. 16 | // 17 | // In general, the probing behavior is a black box for users and only the 18 | // side effects can be measured in the form of performance differences. 19 | // These APIs give a glimpse on the actual behavior of the probing algorithms in 20 | // these hashtables given a specified hash function and a set of elements. 21 | // 22 | // The probe count distribution can be used to assess the quality of the hash 23 | // function for that particular hash table. Note that a hash function that 24 | // performs well in one hash table implementation does not necessarily performs 25 | // well in a different one. 26 | // 27 | // This library supports std::unordered_{set,map}, dense_hash_{set,map} and 28 | // phmap::{flat,node,string}_hash_{set,map}. 29 | 30 | #ifndef PHMAP_PRIV_HASHTABLE_DEBUG_H_ 31 | #define PHMAP_PRIV_HASHTABLE_DEBUG_H_ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | namespace phmap { 39 | namespace priv { 40 | 41 | // Returns the number of probes required to lookup `key`. Returns 0 for a 42 | // search with no collisions. Higher values mean more hash collisions occurred; 43 | // however, the exact meaning of this number varies according to the container 44 | // type. 45 | template 46 | size_t GetHashtableDebugNumProbes( 47 | const C& c, const typename C::key_type& key) { 48 | return phmap::priv::hashtable_debug_internal:: 49 | HashtableDebugAccess::GetNumProbes(c, key); 50 | } 51 | 52 | // Gets a histogram of the number of probes for each elements in the container. 53 | // The sum of all the values in the vector is equal to container.size(). 54 | template 55 | std::vector GetHashtableDebugNumProbesHistogram(const C& container) { 56 | std::vector v; 57 | for (auto it = container.begin(); it != container.end(); ++it) { 58 | size_t num_probes = GetHashtableDebugNumProbes( 59 | container, 60 | phmap::priv::hashtable_debug_internal::GetKey(*it, 0)); 61 | v.resize((std::max)(v.size(), num_probes + 1)); 62 | v[num_probes]++; 63 | } 64 | return v; 65 | } 66 | 67 | struct HashtableDebugProbeSummary { 68 | size_t total_elements; 69 | size_t total_num_probes; 70 | double mean; 71 | }; 72 | 73 | // Gets a summary of the probe count distribution for the elements in the 74 | // container. 75 | template 76 | HashtableDebugProbeSummary GetHashtableDebugProbeSummary(const C& container) { 77 | auto probes = GetHashtableDebugNumProbesHistogram(container); 78 | HashtableDebugProbeSummary summary = {}; 79 | for (size_t i = 0; i < probes.size(); ++i) { 80 | summary.total_elements += probes[i]; 81 | summary.total_num_probes += probes[i] * i; 82 | } 83 | summary.mean = 1.0 * summary.total_num_probes / summary.total_elements; 84 | return summary; 85 | } 86 | 87 | // Returns the number of bytes requested from the allocator by the container 88 | // and not freed. 89 | template 90 | size_t AllocatedByteSize(const C& c) { 91 | return phmap::priv::hashtable_debug_internal:: 92 | HashtableDebugAccess::AllocatedByteSize(c); 93 | } 94 | 95 | // Returns a tight lower bound for AllocatedByteSize(c) where `c` is of type `C` 96 | // and `c.size()` is equal to `num_elements`. 97 | template 98 | size_t LowerBoundAllocatedByteSize(size_t num_elements) { 99 | return phmap::priv::hashtable_debug_internal:: 100 | HashtableDebugAccess::LowerBoundAllocatedByteSize(num_elements); 101 | } 102 | 103 | } // namespace priv 104 | } // namespace phmap 105 | 106 | #endif // PHMAP_PRIV_HASHTABLE_DEBUG_H_ 107 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/node_hash_policy_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include "parallel_hashmap/phmap.h" 17 | 18 | #include "gmock/gmock.h" 19 | #include "gtest/gtest.h" 20 | 21 | namespace phmap { 22 | namespace priv { 23 | namespace { 24 | 25 | using ::testing::Pointee; 26 | 27 | struct Policy : node_hash_policy { 28 | using key_type = int; 29 | using init_type = int; 30 | 31 | template 32 | static int* new_element(Alloc*, int value) { 33 | return new int(value); 34 | } 35 | 36 | template 37 | static void delete_element(Alloc* , int* elem) { 38 | delete elem; 39 | } 40 | }; 41 | 42 | using NodePolicy = hash_policy_traits; 43 | 44 | struct NodeTest : ::testing::Test { 45 | std::allocator alloc; 46 | int n = 53; 47 | int* a = &n; 48 | }; 49 | 50 | TEST_F(NodeTest, ConstructDestroy) { 51 | NodePolicy::construct(&alloc, &a, 42); 52 | EXPECT_THAT(a, Pointee(42)); 53 | NodePolicy::destroy(&alloc, &a); 54 | } 55 | 56 | TEST_F(NodeTest, transfer) { 57 | int s = 42; 58 | int* b = &s; 59 | NodePolicy::transfer(&alloc, &a, &b); 60 | EXPECT_EQ(&s, a); 61 | } 62 | 63 | } // namespace 64 | } // namespace priv 65 | } // namespace phmap 66 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/node_hash_set_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef THIS_HASH_SET 16 | #define THIS_HASH_SET node_hash_set 17 | #define THIS_TEST_NAME NodeHashSet 18 | #endif 19 | 20 | #include "parallel_hashmap/phmap.h" 21 | 22 | #include "unordered_set_constructor_test.h" 23 | #include "unordered_set_lookup_test.h" 24 | #include "unordered_set_members_test.h" 25 | #include "unordered_set_modifiers_test.h" 26 | 27 | namespace phmap { 28 | namespace priv { 29 | namespace { 30 | using ::phmap::priv::hash_internal::Enum; 31 | using ::phmap::priv::hash_internal::EnumClass; 32 | using ::testing::Pointee; 33 | using ::testing::UnorderedElementsAre; 34 | 35 | using SetTypes = ::testing::Types< 36 | THIS_HASH_SET>, 37 | THIS_HASH_SET>, 39 | THIS_HASH_SET>, 40 | THIS_HASH_SET>>; 42 | 43 | INSTANTIATE_TYPED_TEST_SUITE_P(THIS_TEST_NAME, ConstructorTest, SetTypes); 44 | INSTANTIATE_TYPED_TEST_SUITE_P(THIS_TEST_NAME, LookupTest, SetTypes); 45 | INSTANTIATE_TYPED_TEST_SUITE_P(THIS_TEST_NAME, MembersTest, SetTypes); 46 | INSTANTIATE_TYPED_TEST_SUITE_P(THIS_TEST_NAME, ModifiersTest, SetTypes); 47 | 48 | TEST(THIS_TEST_NAME, MoveableNotCopyableCompiles) { 49 | THIS_HASH_SET> t; 50 | THIS_HASH_SET> u; 51 | u = std::move(t); 52 | } 53 | 54 | TEST(THIS_TEST_NAME, MergeExtractInsert) { 55 | struct Hash { 56 | size_t operator()(const std::unique_ptr& p) const { return *p; } 57 | }; 58 | struct Eq { 59 | bool operator()(const std::unique_ptr& a, 60 | const std::unique_ptr& b) const { 61 | return *a == *b; 62 | } 63 | }; 64 | phmap::THIS_HASH_SET, Hash, Eq> set1, set2; 65 | set1.insert(phmap::make_unique(7)); 66 | set1.insert(phmap::make_unique(17)); 67 | 68 | set2.insert(phmap::make_unique(7)); 69 | set2.insert(phmap::make_unique(19)); 70 | 71 | EXPECT_THAT(set1, UnorderedElementsAre(Pointee(7), Pointee(17))); 72 | EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7), Pointee(19))); 73 | 74 | set1.merge(set2); 75 | 76 | EXPECT_THAT(set1, UnorderedElementsAre(Pointee(7), Pointee(17), Pointee(19))); 77 | EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7))); 78 | 79 | auto node = set1.extract(phmap::make_unique(7)); 80 | EXPECT_TRUE(node); 81 | EXPECT_THAT(node.value(), Pointee(7)); 82 | EXPECT_THAT(set1, UnorderedElementsAre(Pointee(17), Pointee(19))); 83 | 84 | auto insert_result = set2.insert(std::move(node)); 85 | EXPECT_FALSE(node); 86 | EXPECT_FALSE(insert_result.inserted); 87 | EXPECT_TRUE(insert_result.node); 88 | EXPECT_THAT(insert_result.node.value(), Pointee(7)); 89 | EXPECT_EQ(**insert_result.position, 7); 90 | EXPECT_NE(insert_result.position->get(), insert_result.node.value().get()); 91 | EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7))); 92 | 93 | node = set1.extract(phmap::make_unique(17)); 94 | EXPECT_TRUE(node); 95 | EXPECT_THAT(node.value(), Pointee(17)); 96 | EXPECT_THAT(set1, UnorderedElementsAre(Pointee(19))); 97 | 98 | node.value() = phmap::make_unique(23); 99 | 100 | insert_result = set2.insert(std::move(node)); 101 | EXPECT_FALSE(node); 102 | EXPECT_TRUE(insert_result.inserted); 103 | EXPECT_FALSE(insert_result.node); 104 | EXPECT_EQ(**insert_result.position, 23); 105 | EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7), Pointee(23))); 106 | } 107 | 108 | } // namespace 109 | } // namespace priv 110 | } // namespace phmap 111 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/parallel_flat_hash_map_mutex_test.cc: -------------------------------------------------------------------------------- 1 | #define THIS_HASH_MAP parallel_flat_hash_map 2 | #define THIS_TEST_NAME ParallelFlatHashMap 3 | 4 | #if 1 5 | #define THIS_EXTRA_TPL_PARAMS , 4, std::mutex 6 | #else 7 | #include 8 | #include 9 | #define THIS_EXTRA_TPL_PARAMS , 4, boost::upgrade_mutex 10 | #endif 11 | 12 | #include "parallel_hash_map_test.cc" 13 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/parallel_flat_hash_map_test.cc: -------------------------------------------------------------------------------- 1 | #define THIS_HASH_MAP parallel_flat_hash_map 2 | #define THIS_TEST_NAME ParallelFlatHashMap 3 | 4 | #include "parallel_hash_map_test.cc" 5 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/parallel_flat_hash_set_test.cc: -------------------------------------------------------------------------------- 1 | #define THIS_HASH_SET parallel_flat_hash_set 2 | #define THIS_TEST_NAME ParallelFlatHashSet 3 | 4 | #include "flat_hash_set_test.cc" 5 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/parallel_hash_map_test.cc: -------------------------------------------------------------------------------- 1 | #ifndef THIS_HASH_MAP 2 | #define THIS_HASH_MAP parallel_flat_hash_map 3 | #define THIS_TEST_NAME ParallelFlatHashMap 4 | #endif 5 | 6 | #include "flat_hash_map_test.cc" 7 | 8 | namespace phmap { 9 | namespace priv { 10 | namespace { 11 | 12 | TEST(THIS_TEST_NAME, ThreadSafeContains) { 13 | // We can't test mutable keys, or non-copyable keys with ThisMap. 14 | // Test that the nodes have the proper API. 15 | using Map = ThisMap; 16 | 17 | { 18 | // ---------------- 19 | // test if_contains 20 | // ---------------- 21 | Map m = { {1, 7}, {2, 9} }; 22 | const Map& const_m(m); 23 | 24 | auto val = 0; 25 | auto get_value = [&val](const int& v) { val = v; }; 26 | EXPECT_TRUE(const_m.if_contains(2, get_value)); 27 | EXPECT_EQ(val, 9); 28 | 29 | EXPECT_FALSE(m.if_contains(3, get_value)); 30 | } 31 | 32 | { 33 | // -------------- 34 | // test modify_if 35 | // -------------- 36 | Map m = { {1, 7}, {2, 9} }; 37 | 38 | auto set_value = [](int& v) { v = 11; }; 39 | EXPECT_TRUE(m.modify_if(2, set_value)); 40 | EXPECT_EQ(m[2], 11); 41 | 42 | EXPECT_FALSE(m.modify_if(3, set_value)); // because m[3] does not exist 43 | } 44 | 45 | { 46 | // ------------------ 47 | // test try_emplace_l 48 | // ------------------ 49 | Map m = { {1, 7}, {2, 9} }; 50 | 51 | // overwrite an existing value 52 | m.try_emplace_l(2, [](int& v) { v = 5; }); 53 | EXPECT_EQ(m[2], 5); 54 | 55 | // insert a value that is not already present. Will be default initialised to 0 and lambda not called 56 | m.try_emplace_l(3, 57 | [](int& v) { v = 6; }, // called only when key was already present 58 | 1); // argument to construct new value is key not present 59 | EXPECT_EQ(m[3], 1); 60 | 61 | // insert a value that is not already present, provide argument to value-construct it 62 | m.try_emplace_l(4, 63 | [](int& ) {}, // called only when key was already present 64 | 999); // argument to construct new value is key not present 65 | 66 | EXPECT_EQ(m[4], 999); 67 | } 68 | 69 | { 70 | // -------------------- 71 | // test lazy__emplace_l 72 | // -------------------- 73 | Map m = { {1, 7}, {2, 9} }; 74 | 75 | // insert a value that is not already present. 76 | // right now m[5] does not exist 77 | m.lazy_emplace_l(5, 78 | [](int& v) { v = 6; }, // called only when key was already present 79 | [](const Map::constructor& ctor) { ctor(5, 13); }); // construct value_type in place when key not present 80 | 81 | EXPECT_EQ(m[5], 13); 82 | 83 | // change a value that is present. Currently m[5] == 13 84 | m.lazy_emplace_l(5, 85 | [](int& v) { v = 6; }, // called only when key was already present 86 | [](const Map::constructor& ctor) { ctor(5, 13); }); // construct value_type in place when key not present 87 | EXPECT_EQ(m[5], 6); 88 | } 89 | 90 | { 91 | // ------------- 92 | // test erase_if 93 | // ------------- 94 | Map m = { {1, 7}, {2, 9}, {5, 6} }; 95 | 96 | EXPECT_EQ(m.erase_if(9, [](int& v) { assert(0); return v==12; }), false); // m[9] not present - lambda not called 97 | EXPECT_EQ(m.erase_if(5, [](int& v) { return v==12; }), false); // m[5] == 6, so erase not performed 98 | EXPECT_EQ(m[5], 6); 99 | EXPECT_EQ(m.erase_if(5, [](int& v) { return v==6; }), true); // lambda returns true, so m[5] erased 100 | EXPECT_EQ(m[5], 0); 101 | } 102 | 103 | } 104 | 105 | } // namespace 106 | } // namespace priv 107 | } // namespace phmap 108 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/parallel_node_hash_map_test.cc: -------------------------------------------------------------------------------- 1 | #define THIS_HASH_MAP parallel_node_hash_map 2 | #define THIS_TEST_NAME ParallelNodeHashMap 3 | 4 | #include "parallel_hash_map_test.cc" 5 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/parallel_node_hash_set_test.cc: -------------------------------------------------------------------------------- 1 | #define THIS_HASH_SET parallel_node_hash_set 2 | #define THIS_TEST_NAME ParallelNodeHashSet 3 | 4 | #include "node_hash_set_test.cc" 5 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/tracked.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef PHMAP_PRIV_TRACKED_H_ 16 | #define PHMAP_PRIV_TRACKED_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | namespace phmap { 23 | namespace priv { 24 | 25 | // A class that tracks its copies and moves so that it can be queried in tests. 26 | template 27 | class Tracked { 28 | public: 29 | Tracked() {} 30 | // NOLINTNEXTLINE(runtime/explicit) 31 | Tracked(const T& val) : val_(val) {} 32 | Tracked(const Tracked& that) 33 | : val_(that.val_), 34 | num_moves_(that.num_moves_), 35 | num_copies_(that.num_copies_) { 36 | ++(*num_copies_); 37 | } 38 | Tracked(Tracked&& that) 39 | : val_(std::move(that.val_)), 40 | num_moves_(std::move(that.num_moves_)), 41 | num_copies_(std::move(that.num_copies_)) { 42 | ++(*num_moves_); 43 | } 44 | Tracked& operator=(const Tracked& that) { 45 | val_ = that.val_; 46 | num_moves_ = that.num_moves_; 47 | num_copies_ = that.num_copies_; 48 | ++(*num_copies_); 49 | } 50 | Tracked& operator=(Tracked&& that) { 51 | val_ = std::move(that.val_); 52 | num_moves_ = std::move(that.num_moves_); 53 | num_copies_ = std::move(that.num_copies_); 54 | ++(*num_moves_); 55 | } 56 | 57 | const T& val() const { return val_; } 58 | 59 | friend bool operator==(const Tracked& a, const Tracked& b) { 60 | return a.val_ == b.val_; 61 | } 62 | friend bool operator!=(const Tracked& a, const Tracked& b) { 63 | return !(a == b); 64 | } 65 | 66 | size_t num_copies() { return *num_copies_; } 67 | size_t num_moves() { return *num_moves_; } 68 | 69 | private: 70 | T val_; 71 | std::shared_ptr num_moves_ = std::make_shared(0); 72 | std::shared_ptr num_copies_ = std::make_shared(0); 73 | }; 74 | 75 | } // namespace priv 76 | } // namespace phmap 77 | 78 | #endif // PHMAP_PRIV_TRACKED_H_ 79 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/unordered_map_lookup_test.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef PHMAP_PRIV_UNORDERED_MAP_LOOKUP_TEST_H_ 16 | #define PHMAP_PRIV_UNORDERED_MAP_LOOKUP_TEST_H_ 17 | 18 | #ifdef _MSC_VER 19 | #pragma warning(push, 0) 20 | #endif 21 | 22 | #include "gmock/gmock.h" 23 | #include "gtest/gtest.h" 24 | #include "hash_generator_testing.h" 25 | #include "hash_policy_testing.h" 26 | 27 | #ifdef _MSC_VER 28 | #pragma warning(pop) 29 | #endif 30 | 31 | namespace phmap { 32 | namespace priv { 33 | 34 | template 35 | class LookupTest : public ::testing::Test {}; 36 | 37 | TYPED_TEST_SUITE_P(LookupTest); 38 | 39 | TYPED_TEST_P(LookupTest, At) { 40 | using T = hash_internal::GeneratedType; 41 | std::vector values; 42 | std::generate_n(std::back_inserter(values), 10, 43 | hash_internal::Generator()); 44 | TypeParam m(values.begin(), values.end()); 45 | for (const auto& p : values) { 46 | const auto& val = m.at(p.first); 47 | EXPECT_EQ(p.second, val) << ::testing::PrintToString(p.first); 48 | } 49 | } 50 | 51 | TYPED_TEST_P(LookupTest, OperatorBracket) { 52 | using T = hash_internal::GeneratedType; 53 | using V = typename TypeParam::mapped_type; 54 | std::vector values; 55 | std::generate_n(std::back_inserter(values), 10, 56 | hash_internal::Generator()); 57 | TypeParam m; 58 | for (const auto& p : values) { 59 | auto& val = m[p.first]; 60 | EXPECT_EQ(V(), val) << ::testing::PrintToString(p.first); 61 | val = p.second; 62 | } 63 | for (const auto& p : values) 64 | EXPECT_EQ(p.second, m[p.first]) << ::testing::PrintToString(p.first); 65 | } 66 | 67 | TYPED_TEST_P(LookupTest, Count) { 68 | using T = hash_internal::GeneratedType; 69 | std::vector values; 70 | std::generate_n(std::back_inserter(values), 10, 71 | hash_internal::Generator()); 72 | TypeParam m; 73 | for (const auto& p : values) 74 | EXPECT_EQ(0, m.count(p.first)) << ::testing::PrintToString(p.first); 75 | m.insert(values.begin(), values.end()); 76 | for (const auto& p : values) 77 | EXPECT_EQ(1, m.count(p.first)) << ::testing::PrintToString(p.first); 78 | } 79 | 80 | TYPED_TEST_P(LookupTest, Find) { 81 | using std::get; 82 | using T = hash_internal::GeneratedType; 83 | std::vector values; 84 | std::generate_n(std::back_inserter(values), 10, 85 | hash_internal::Generator()); 86 | TypeParam m; 87 | for (const auto& p : values) 88 | EXPECT_TRUE(m.end() == m.find(p.first)) 89 | << ::testing::PrintToString(p.first); 90 | m.insert(values.begin(), values.end()); 91 | for (const auto& p : values) { 92 | auto it = m.find(p.first); 93 | EXPECT_TRUE(m.end() != it) << ::testing::PrintToString(p.first); 94 | EXPECT_EQ(p.second, get<1>(*it)) << ::testing::PrintToString(p.first); 95 | } 96 | } 97 | 98 | TYPED_TEST_P(LookupTest, EqualRange) { 99 | using std::get; 100 | using T = hash_internal::GeneratedType; 101 | std::vector values; 102 | std::generate_n(std::back_inserter(values), 10, 103 | hash_internal::Generator()); 104 | TypeParam m; 105 | for (const auto& p : values) { 106 | auto r = m.equal_range(p.first); 107 | ASSERT_EQ(0, std::distance(r.first, r.second)); 108 | } 109 | m.insert(values.begin(), values.end()); 110 | for (const auto& p : values) { 111 | auto r = m.equal_range(p.first); 112 | ASSERT_EQ(1, std::distance(r.first, r.second)); 113 | EXPECT_EQ(p.second, get<1>(*r.first)) << ::testing::PrintToString(p.first); 114 | } 115 | } 116 | 117 | REGISTER_TYPED_TEST_SUITE_P(LookupTest, At, OperatorBracket, Count, Find, 118 | EqualRange); 119 | 120 | } // namespace priv 121 | } // namespace phmap 122 | 123 | #endif // PHMAP_PRIV_UNORDERED_MAP_LOOKUP_TEST_H_ 124 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/unordered_map_members_test.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef PHMAP_PRIV_UNORDERED_MAP_MEMBERS_TEST_H_ 16 | #define PHMAP_PRIV_UNORDERED_MAP_MEMBERS_TEST_H_ 17 | 18 | #include 19 | 20 | #ifdef _MSC_VER 21 | #pragma warning(push, 0) 22 | #endif 23 | 24 | #include "gmock/gmock.h" 25 | #include "gtest/gtest.h" 26 | #include "hash_generator_testing.h" 27 | #include "hash_policy_testing.h" 28 | 29 | #ifdef _MSC_VER 30 | #pragma warning(pop) 31 | #endif 32 | 33 | namespace phmap { 34 | namespace priv { 35 | 36 | template 37 | class MembersTest : public ::testing::Test {}; 38 | 39 | TYPED_TEST_SUITE_P(MembersTest); 40 | 41 | template 42 | void UseType() {} 43 | 44 | TYPED_TEST_P(MembersTest, Typedefs) { 45 | EXPECT_TRUE((std::is_same, 47 | typename TypeParam::value_type>())); 48 | EXPECT_TRUE((phmap::conjunction< 49 | phmap::negation>, 50 | std::is_integral>())); 51 | EXPECT_TRUE((phmap::conjunction< 52 | std::is_signed, 53 | std::is_integral>())); 54 | EXPECT_TRUE((std::is_convertible< 55 | decltype(std::declval()( 56 | std::declval())), 57 | size_t>())); 58 | EXPECT_TRUE((std::is_convertible< 59 | decltype(std::declval()( 60 | std::declval(), 61 | std::declval())), 62 | bool>())); 63 | EXPECT_TRUE((std::is_same())); 65 | EXPECT_TRUE((std::is_same())); 67 | EXPECT_TRUE((std::is_same())); 69 | EXPECT_TRUE((std::is_same::pointer, 71 | typename TypeParam::pointer>())); 72 | EXPECT_TRUE( 73 | (std::is_same::const_pointer, 75 | typename TypeParam::const_pointer>())); 76 | } 77 | 78 | TYPED_TEST_P(MembersTest, SimpleFunctions) { 79 | EXPECT_GT(TypeParam().max_size(), 0); 80 | } 81 | 82 | TYPED_TEST_P(MembersTest, BeginEnd) { 83 | TypeParam t = {typename TypeParam::value_type{}}; 84 | EXPECT_EQ(t.begin(), t.cbegin()); 85 | EXPECT_EQ(t.end(), t.cend()); 86 | EXPECT_NE(t.begin(), t.end()); 87 | EXPECT_NE(t.cbegin(), t.cend()); 88 | } 89 | 90 | REGISTER_TYPED_TEST_SUITE_P(MembersTest, Typedefs, SimpleFunctions, BeginEnd); 91 | 92 | } // namespace priv 93 | } // namespace phmap 94 | 95 | #endif // PHMAP_PRIV_UNORDERED_MAP_MEMBERS_TEST_H_ 96 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/unordered_set_lookup_test.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef PHMAP_PRIV_UNORDERED_SET_LOOKUP_TEST_H_ 16 | #define PHMAP_PRIV_UNORDERED_SET_LOOKUP_TEST_H_ 17 | 18 | #include "gmock/gmock.h" 19 | #include "gtest/gtest.h" 20 | #include "hash_generator_testing.h" 21 | #include "hash_policy_testing.h" 22 | 23 | namespace phmap { 24 | namespace priv { 25 | 26 | template 27 | class LookupTest : public ::testing::Test {}; 28 | 29 | TYPED_TEST_SUITE_P(LookupTest); 30 | 31 | TYPED_TEST_P(LookupTest, Count) { 32 | using T = hash_internal::GeneratedType; 33 | std::vector values; 34 | std::generate_n(std::back_inserter(values), 10, 35 | hash_internal::Generator()); 36 | TypeParam m; 37 | for (const auto& v : values) 38 | EXPECT_EQ(0, m.count(v)) << ::testing::PrintToString(v); 39 | m.insert(values.begin(), values.end()); 40 | for (const auto& v : values) 41 | EXPECT_EQ(1, m.count(v)) << ::testing::PrintToString(v); 42 | } 43 | 44 | TYPED_TEST_P(LookupTest, Find) { 45 | using T = hash_internal::GeneratedType; 46 | std::vector values; 47 | std::generate_n(std::back_inserter(values), 10, 48 | hash_internal::Generator()); 49 | TypeParam m; 50 | for (const auto& v : values) 51 | EXPECT_TRUE(m.end() == m.find(v)) << ::testing::PrintToString(v); 52 | m.insert(values.begin(), values.end()); 53 | for (const auto& v : values) { 54 | typename TypeParam::iterator it = m.find(v); 55 | static_assert(std::is_same::value, 57 | ""); 58 | static_assert(std::is_same())>::value, 60 | ""); 61 | EXPECT_TRUE(m.end() != it) << ::testing::PrintToString(v); 62 | EXPECT_EQ(v, *it) << ::testing::PrintToString(v); 63 | } 64 | } 65 | 66 | TYPED_TEST_P(LookupTest, EqualRange) { 67 | using T = hash_internal::GeneratedType; 68 | std::vector values; 69 | std::generate_n(std::back_inserter(values), 10, 70 | hash_internal::Generator()); 71 | TypeParam m; 72 | for (const auto& v : values) { 73 | auto r = m.equal_range(v); 74 | ASSERT_EQ(0, std::distance(r.first, r.second)); 75 | } 76 | m.insert(values.begin(), values.end()); 77 | for (const auto& v : values) { 78 | auto r = m.equal_range(v); 79 | ASSERT_EQ(1, std::distance(r.first, r.second)); 80 | EXPECT_EQ(v, *r.first); 81 | } 82 | } 83 | 84 | REGISTER_TYPED_TEST_SUITE_P(LookupTest, Count, Find, EqualRange); 85 | 86 | } // namespace priv 87 | } // namespace phmap 88 | 89 | #endif // PHMAP_PRIV_UNORDERED_SET_LOOKUP_TEST_H_ 90 | -------------------------------------------------------------------------------- /fast_sampler/parallel-hashmap/tests/unordered_set_members_test.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef PHMAP_PRIV_UNORDERED_SET_MEMBERS_TEST_H_ 16 | #define PHMAP_PRIV_UNORDERED_SET_MEMBERS_TEST_H_ 17 | 18 | #include 19 | #include "gmock/gmock.h" 20 | #include "gtest/gtest.h" 21 | 22 | namespace phmap { 23 | namespace priv { 24 | 25 | template 26 | class MembersTest : public ::testing::Test {}; 27 | 28 | TYPED_TEST_SUITE_P(MembersTest); 29 | 30 | template 31 | void UseType() {} 32 | 33 | TYPED_TEST_P(MembersTest, Typedefs) { 34 | EXPECT_TRUE((std::is_same())); 36 | EXPECT_TRUE((phmap::conjunction< 37 | phmap::negation>, 38 | std::is_integral>())); 39 | EXPECT_TRUE((phmap::conjunction< 40 | std::is_signed, 41 | std::is_integral>())); 42 | EXPECT_TRUE((std::is_convertible< 43 | decltype(std::declval()( 44 | std::declval())), 45 | size_t>())); 46 | EXPECT_TRUE((std::is_convertible< 47 | decltype(std::declval()( 48 | std::declval(), 49 | std::declval())), 50 | bool>())); 51 | EXPECT_TRUE((std::is_same())); 53 | EXPECT_TRUE((std::is_same())); 55 | EXPECT_TRUE((std::is_same())); 57 | EXPECT_TRUE((std::is_same::pointer, 59 | typename TypeParam::pointer>())); 60 | EXPECT_TRUE( 61 | (std::is_same::const_pointer, 63 | typename TypeParam::const_pointer>())); 64 | } 65 | 66 | TYPED_TEST_P(MembersTest, SimpleFunctions) { 67 | EXPECT_GT(TypeParam().max_size(), 0); 68 | } 69 | 70 | TYPED_TEST_P(MembersTest, BeginEnd) { 71 | TypeParam t = {typename TypeParam::value_type{}}; 72 | EXPECT_EQ(t.begin(), t.cbegin()); 73 | EXPECT_EQ(t.end(), t.cend()); 74 | EXPECT_NE(t.begin(), t.end()); 75 | EXPECT_NE(t.cbegin(), t.cend()); 76 | } 77 | 78 | REGISTER_TYPED_TEST_SUITE_P(MembersTest, Typedefs, SimpleFunctions, BeginEnd); 79 | 80 | } // namespace priv 81 | } // namespace phmap 82 | 83 | #endif // PHMAP_PRIV_UNORDERED_SET_MEMBERS_TEST_H_ 84 | -------------------------------------------------------------------------------- /fast_sampler/sample_cpu_pyg.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | #include "parallel_hashmap/phmap.h" 9 | #include "utils.hpp" 10 | 11 | using SingleSample = 12 | std::tuple; 13 | 14 | // Returns `rowptr`, `col`, `n_id`, `e_id` 15 | SingleSample sample_adj_cpu(torch::Tensor rowptr, torch::Tensor col, 16 | torch::Tensor idx, int64_t num_neighbors, 17 | bool replace, bool pin_memory = false) { 18 | CHECK_CPU(rowptr); 19 | CHECK_CPU(col); 20 | CHECK_CPU(idx); 21 | CHECK_INPUT(idx.dim() == 1); 22 | 23 | auto rowptr_data = rowptr.data_ptr(); 24 | auto col_data = col.data_ptr(); 25 | auto idx_data = idx.data_ptr(); 26 | 27 | auto out_rowptr = torch::empty(idx.numel() + 1, rowptr.options()); 28 | auto out_rowptr_data = out_rowptr.data_ptr(); 29 | out_rowptr_data[0] = 0; 30 | 31 | std::vector>> cols; // col, e_id 32 | std::vector n_ids; 33 | std::unordered_map n_id_map; 34 | 35 | int64_t i; 36 | for (int64_t n = 0; n < idx.numel(); n++) { 37 | i = idx_data[n]; 38 | cols.push_back(std::vector>()); 39 | n_id_map[i] = n; 40 | n_ids.push_back(i); 41 | } 42 | 43 | int64_t n, c, e, row_start, row_end, row_count; 44 | 45 | if (num_neighbors < 46 | 0) { // No sampling ====================================== 47 | 48 | for (int64_t i = 0; i < idx.numel(); i++) { 49 | n = idx_data[i]; 50 | row_start = rowptr_data[n], row_end = rowptr_data[n + 1]; 51 | row_count = row_end - row_start; 52 | 53 | for (int64_t j = 0; j < row_count; j++) { 54 | e = row_start + j; 55 | c = col_data[e]; 56 | 57 | if (n_id_map.count(c) == 0) { 58 | n_id_map[c] = n_ids.size(); 59 | n_ids.push_back(c); 60 | } 61 | cols[i].push_back(std::make_tuple(n_id_map[c], e)); 62 | } 63 | out_rowptr_data[i + 1] = out_rowptr_data[i] + cols[i].size(); 64 | } 65 | } 66 | 67 | else if (replace) { // Sample with replacement 68 | // =============================== 69 | 70 | for (int64_t i = 0; i < idx.numel(); i++) { 71 | n = idx_data[i]; 72 | row_start = rowptr_data[n], row_end = rowptr_data[n + 1]; 73 | row_count = row_end - row_start; 74 | 75 | for (int64_t j = 0; j < num_neighbors; j++) { 76 | e = row_start + rand() % row_count; 77 | c = col_data[e]; 78 | 79 | if (n_id_map.count(c) == 0) { 80 | n_id_map[c] = n_ids.size(); 81 | n_ids.push_back(c); 82 | } 83 | cols[i].push_back(std::make_tuple(n_id_map[c], e)); 84 | } 85 | out_rowptr_data[i + 1] = out_rowptr_data[i] + cols[i].size(); 86 | } 87 | 88 | } else { // Sample without replacement via Robert Floyd algorithm 89 | // ============ 90 | 91 | for (int64_t i = 0; i < idx.numel(); i++) { 92 | n = idx_data[i]; 93 | row_start = rowptr_data[n], row_end = rowptr_data[n + 1]; 94 | row_count = row_end - row_start; 95 | 96 | std::unordered_set perm; 97 | if (row_count <= num_neighbors) { 98 | for (int64_t j = 0; j < row_count; j++) perm.insert(j); 99 | } else { // See: https://www.nowherenearithaca.com/2013/05/ 100 | // robert-floyds-tiny-and-beautiful.html 101 | for (int64_t j = row_count - num_neighbors; j < row_count; j++) { 102 | if (!perm.insert(rand() % j).second) perm.insert(j); 103 | } 104 | } 105 | 106 | for (const int64_t &p : perm) { 107 | e = row_start + p; 108 | c = col_data[e]; 109 | 110 | if (n_id_map.count(c) == 0) { 111 | n_id_map[c] = n_ids.size(); 112 | n_ids.push_back(c); 113 | } 114 | cols[i].push_back(std::make_tuple(n_id_map[c], e)); 115 | } 116 | out_rowptr_data[i + 1] = out_rowptr_data[i] + cols[i].size(); 117 | } 118 | } 119 | 120 | int64_t N = n_ids.size(); 121 | auto out_n_id = torch::from_blob(n_ids.data(), {N}, col.options()).clone(); 122 | 123 | int64_t E = out_rowptr_data[idx.numel()]; 124 | auto out_col = torch::empty(E, col.options()); 125 | auto out_col_data = out_col.data_ptr(); 126 | auto out_e_id = torch::empty(E, col.options()); 127 | auto out_e_id_data = out_e_id.data_ptr(); 128 | 129 | i = 0; 130 | for (std::vector> &col_vec : cols) { 131 | std::sort(col_vec.begin(), col_vec.end(), 132 | [](const std::tuple &a, 133 | const std::tuple &b) -> bool { 134 | return std::get<0>(a) < std::get<0>(b); 135 | }); 136 | for (const std::tuple &value : col_vec) { 137 | out_col_data[i] = std::get<0>(value); 138 | out_e_id_data[i] = std::get<1>(value); 139 | i += 1; 140 | } 141 | } 142 | 143 | return std::make_tuple(out_rowptr, out_col, out_n_id, out_e_id); 144 | } 145 | -------------------------------------------------------------------------------- /fast_sampler/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from pathlib import Path 4 | from setuptools import setup 5 | 6 | from torch.utils.cpp_extension import BuildExtension, CppExtension 7 | from torch.__config__ import parallel_info 8 | 9 | 10 | def flags_to_list(flagstring): 11 | return list(filter(bool, flagstring.split(' '))) 12 | 13 | 14 | WITH_SYMBOLS = True if os.getenv('WITH_SYMBOLS', '0') == '1' else False 15 | CXX_FLAGS = flags_to_list(os.getenv('CXX_FLAGS', '')) 16 | ROOT_PATH = Path(__file__).resolve().parent 17 | 18 | 19 | def get_extensions(): 20 | define_macros = [] 21 | libraries = [] 22 | extra_compile_args = { 23 | 'cxx': ['-O3', '-mcpu=native', '-std=c++17', '-g'] + CXX_FLAGS} 24 | extra_link_args = [] if WITH_SYMBOLS else ['-s'] 25 | 26 | info = parallel_info() 27 | if 'backend: OpenMP' in info and 'OpenMP not found' not in info: 28 | extra_compile_args['cxx'] += ['-DAT_PARALLEL_OPENMP'] 29 | if sys.platform == 'win32': 30 | extra_compile_args['cxx'] += ['/openmp'] 31 | else: 32 | extra_compile_args['cxx'] += ['-fopenmp'] 33 | else: 34 | print('Compiling without OpenMP...') 35 | 36 | include_dirs = [ROOT_PATH, ROOT_PATH.joinpath('parallel-hashmap')] 37 | 38 | return [ 39 | CppExtension( 40 | 'fast_sampler', 41 | ['fast_sampler.cpp'], 42 | include_dirs=include_dirs, 43 | define_macros=define_macros, 44 | extra_compile_args=extra_compile_args, 45 | extra_link_args=extra_link_args, 46 | libraries=libraries, 47 | ), 48 | ] 49 | 50 | 51 | setup( 52 | name='fast_sampler', 53 | ext_modules=get_extensions(), 54 | cmdclass={ 55 | 'build_ext': BuildExtension.with_options(no_python_abi_suffix=True, use_ninja=False) 56 | }) 57 | -------------------------------------------------------------------------------- /fast_sampler/utils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #define CHECK_CPU(x) \ 6 | TORCH_INTERNAL_ASSERT(x.device().is_cpu(), #x " must be CPU tensor") 7 | #define CHECK_INPUT(x) TORCH_INTERNAL_ASSERT(x, "Input mismatch") 8 | 9 | template 10 | inline torch::Tensor vector_to_tensor(const std::vector& vec, 11 | bool pin_memory = false) { 12 | auto tensor = torch::empty( 13 | vec.size(), torch::TensorOptions() 14 | .dtype(torch::CppTypeToScalarType::value) 15 | .device(torch::kCPU) 16 | .layout(torch::kStrided) 17 | .pinned_memory(pin_memory) 18 | .requires_grad(false)); 19 | const auto tensor_data = tensor.template data_ptr(); 20 | std::copy(vec.begin(), vec.end(), tensor_data); 21 | return tensor; 22 | } 23 | -------------------------------------------------------------------------------- /fast_trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITIBMxGraph/SALIENT/51ab0e483ddfa2579930bec15a8a40ceec1ae748/fast_trainer/__init__.py -------------------------------------------------------------------------------- /fast_trainer/concepts.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional, Callable, List 2 | import torch 3 | 4 | from .samplers import PreparedBatch 5 | from .transferers import DeviceIterator 6 | 7 | TrainCore = Callable[[torch.nn.Module, PreparedBatch], Any] 8 | TrainCallback = Callable[[List[PreparedBatch], List[Any]], None] 9 | TrainImpl = Callable[[torch.nn.Module, TrainCore, DeviceIterator, 10 | torch.optim.Optimizer, Optional[TrainCallback]], None] 11 | TestCallback = Callable[[PreparedBatch], None] # should not return anything 12 | -------------------------------------------------------------------------------- /fast_trainer/distributed.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class DistributedShuffler: 5 | initial_idx: torch.Tensor 6 | current_idx: torch.Tensor 7 | world_size: int 8 | initial_seed: int 9 | generator: torch.Generator 10 | epoch: int 11 | 12 | def __init__(self, idx, world_size, initial_seed=2147483647): 13 | self.initial_idx = idx 14 | self.world_size = world_size 15 | self.initial_seed = initial_seed 16 | self.generator = torch.Generator(device='cpu') 17 | self.set_epoch(0) 18 | 19 | def set_epoch(self, epoch: int): 20 | self.epoch = epoch 21 | self.generator.manual_seed(self.initial_seed + epoch) 22 | self.current_idx = self.initial_idx[ 23 | torch.randperm(self.initial_idx.numel(), 24 | generator=self.generator, 25 | device=self.initial_idx.device)] 26 | 27 | def get_indices(self, rank): 28 | n = self.current_idx.numel() 29 | start = (n * rank) // self.world_size 30 | stop = (n * (rank + 1)) // self.world_size 31 | return self.current_idx[start: stop] 32 | -------------------------------------------------------------------------------- /fast_trainer/monkeypatch.py: -------------------------------------------------------------------------------- 1 | import torch_geometric 2 | from torch_geometric.data import NeighborSampler 3 | #from torch_geometric.loader import NeighborSampler 4 | from torch_sparse.tensor import SparseTensor 5 | 6 | if torch_geometric.__version__ < '2.0.0': 7 | from torch_geometric.data.sampler import Adj, EdgeIndex 8 | else: 9 | from torch_geometric.loader.neighbor_sampler import Adj, EdgeIndex 10 | 11 | # import faulthandler 12 | # faulthandler.enable() 13 | 14 | 15 | # NOTE: line_profiler makes __builtins__ a dict, not a module... 16 | builtins = __builtins__ if isinstance(__builtins__, dict) \ 17 | else vars(__builtins__) 18 | if 'profile' not in builtins: 19 | def profile(func): 20 | return func 21 | profile = profile 22 | 23 | 24 | # Monkey-patch torch_geometric for this 25 | def Adj__pin_memory(self, *args, **kwargs): 26 | adj = self.adj_t.pin_memory(*args, **kwargs) 27 | e_id = self.e_id.pin_memory(*args, **kwargs) if self.e_id is not None \ 28 | else None 29 | return type(self)(adj, e_id, self.size) 30 | 31 | 32 | Adj.pin_memory = Adj__pin_memory 33 | 34 | 35 | def sparse_record_stream(self: SparseTensor, stream): 36 | row = self._row 37 | if row is not None: 38 | row = row.record_stream(stream) 39 | rowptr = self._rowptr 40 | if rowptr is not None: 41 | rowptr = rowptr.record_stream(stream) 42 | col = self._col.record_stream(stream) 43 | value = self._value 44 | if value is not None: 45 | value = value.record_stream(stream) 46 | rowcount = self._rowcount 47 | if rowcount is not None: 48 | rowcount = rowcount.record_stream(stream) 49 | colptr = self._colptr 50 | if colptr is not None: 51 | colptr = colptr.record_stream(stream) 52 | colcount = self._colcount 53 | if colcount is not None: 54 | colcount = colcount.record_stream(stream) 55 | csr2csc = self._csr2csc 56 | if csr2csc is not None: 57 | csr2csc = csr2csc.record_stream(stream) 58 | csc2csr = self._csc2csr 59 | if csc2csr is not None: 60 | csc2csr = csc2csr.record_stream(stream) 61 | 62 | 63 | def Adj__record_stream(self, stream): 64 | sparse_record_stream(self.adj_t.storage, stream) 65 | if self.e_id is not None: 66 | self.e_id.record_stream(stream) 67 | 68 | 69 | Adj.record_stream = Adj__record_stream 70 | 71 | 72 | # Monkey-patch torch_geometric for this 73 | def EdgeIndex__pin_memory(self, *args, **kwargs): 74 | edge_index = self.edge_index.pin_memory(*args, **kwargs) 75 | e_id = self.e_id.pin_memory(*args, **kwargs) if self.e_id is not None \ 76 | else None 77 | return type(self)(edge_index, e_id, self.size) 78 | 79 | 80 | EdgeIndex.pin_memory = EdgeIndex__pin_memory 81 | 82 | 83 | # Monkey-patch NeighborSampler for profiling 84 | NeighborSampler.__init__ = profile(NeighborSampler.__init__) 85 | NeighborSampler.sample = profile(NeighborSampler.sample) 86 | SparseTensor.sample = profile(SparseTensor.sample) 87 | SparseTensor.sample_adj = profile(SparseTensor.sample_adj) 88 | -------------------------------------------------------------------------------- /fast_trainer/shufflers.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class Shuffler: 5 | initial_idx: torch.Tensor 6 | world_size: int 7 | initial_seed: int 8 | generator: torch.Generator 9 | epoch: int 10 | 11 | DEFAULT_INITIAL_SEED = 2147483647 12 | 13 | def __init__(self, idx, initial_seed=DEFAULT_INITIAL_SEED): 14 | assert idx.dim() == 1 15 | self.initial_idx = idx 16 | self.initial_seed = initial_seed 17 | self.generator = torch.Generator(device='cpu') 18 | self.set_epoch(0) 19 | 20 | def set_epoch(self, epoch: int): 21 | self.epoch = epoch 22 | 23 | def get_idx(self): 24 | self.generator.manual_seed(self.initial_seed + self.epoch) 25 | return self.initial_idx[torch.randperm(self.initial_idx.numel(), 26 | generator=self.generator, 27 | device=self.initial_idx.device)] 28 | 29 | 30 | class DistributedShuffler(Shuffler): 31 | world_size: int 32 | 33 | def __init__(self, idx, world_size, 34 | initial_seed=Shuffler.DEFAULT_INITIAL_SEED): 35 | super().__init__(idx, initial_seed) 36 | self.world_size = world_size 37 | 38 | def get_idx(self, rank): 39 | shuffled_idx = super().get_idx() 40 | n = shuffled_idx.numel() 41 | start = (n * rank) // self.world_size 42 | stop = (n * (rank + 1)) // self.world_size 43 | return shuffled_idx[start: stop] 44 | -------------------------------------------------------------------------------- /fast_trainer/test.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from .transferers import DeviceIterator 4 | from .concepts import TestCallback 5 | 6 | 7 | @torch.no_grad() 8 | def batchwise_test(model: torch.nn.Module, 9 | num_batches: int, 10 | devit: DeviceIterator, 11 | cb: TestCallback = None): 12 | model.eval() 13 | 14 | device, = devit.devices 15 | 16 | results = torch.empty(num_batches, dtype=torch.long, pin_memory=True) 17 | total = 0 18 | 19 | for i, inputs in enumerate(devit): 20 | inp, = inputs 21 | 22 | out = model(inp.x, inp.adjs) 23 | out = out.argmax(dim=-1, keepdim=True).reshape(-1) 24 | correct = (out == inp.y).sum() 25 | results[i].copy_(correct, non_blocking=True) 26 | total += inp.batch_size 27 | 28 | if cb is not None: 29 | cb(inp) 30 | 31 | torch.cuda.current_stream(device).synchronize() 32 | 33 | return results.sum().item(), total 34 | -------------------------------------------------------------------------------- /fast_trainer/transferers.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | #from collections.abc import Iterator 3 | from typing import Iterator 4 | import torch 5 | 6 | from .samplers import ProtoBatch, PreparedBatch 7 | from .utils import append_runtime_stats, Timer, runtime_stats_cuda 8 | import time 9 | 10 | 11 | class DeviceIterator(Iterator[List[PreparedBatch]]): 12 | ''' 13 | Abstract class that returns PreparedBatch on devices (GPUs) 14 | ''' 15 | devices: List[torch.cuda.device] 16 | 17 | def __init__(self, devices): 18 | assert len(devices) > 0 19 | self.devices = devices 20 | 21 | 22 | class DevicePrefetcher(DeviceIterator): 23 | def __init__(self, devices, it: Iterator[PreparedBatch]): 24 | super().__init__(devices) 25 | 26 | self.it = it 27 | self.streams = [torch.cuda.Stream(device) for device in devices] 28 | self.next = [] 29 | self.sampling_times = [] 30 | self.record_stream_times = [] 31 | self.start_prefetch_times = [] 32 | self.wait_stream_times = [] 33 | self.preload(False) 34 | 35 | def preload(self, timing=True): 36 | self.next = [] 37 | for device, stream in zip(self.devices, self.streams): 38 | timer_start = time.perf_counter_ns() 39 | batch = next(self.it, None) 40 | timer_end = time.perf_counter_ns() 41 | if batch is None: 42 | append_runtime_stats("total:load_batch:sampling", sum( 43 | self.sampling_times)/1000000) 44 | self.sampling_times = [] 45 | append_runtime_stats("total:load_batch:data_transfer:start_nonblocking_prefetch", sum( 46 | self.start_prefetch_times)/1000000) 47 | self.start_prefetch_times = [] 48 | break 49 | 50 | timer_start = time.perf_counter_ns() 51 | with torch.cuda.stream(stream): 52 | self.next.append(batch.to(device, non_blocking=True)) 53 | timer_end = time.perf_counter_ns() 54 | self.start_prefetch_times.append(timer_end-timer_start) 55 | 56 | def __next__(self): 57 | runtime_stats_cuda.start_region( 58 | "data_transfer", runtime_stats_cuda.get_last_event()) 59 | 60 | timer_start = time.perf_counter_ns() 61 | cur_streams = [torch.cuda.current_stream( 62 | device) for device in self.devices] 63 | 64 | for cur_stream, stream in zip(cur_streams, self.streams): 65 | cur_stream.wait_stream(stream) 66 | runtime_stats_cuda.end_region("data_transfer") 67 | 68 | runtime_stats_cuda.start_region( 69 | "sampling", runtime_stats_cuda.get_last_event()) 70 | 71 | ret = self.next 72 | timer_end = time.perf_counter_ns() 73 | self.wait_stream_times.append(timer_end-timer_start) 74 | if not ret: 75 | torch.cuda.synchronize() 76 | append_runtime_stats("total:load_batch:data_transfer:wait_stream", sum( 77 | self.wait_stream_times)/1000000) 78 | self.wait_stream_times = [] 79 | append_runtime_stats("total:load_batch:data_transfer:record_stream", sum( 80 | self.record_stream_times)/1000000) 81 | self.record_stream_times = [] 82 | raise StopIteration 83 | 84 | # TODO: this might be a bit incorrect 85 | # 86 | # in theory, we want to record this event after all the 87 | # training computation on the default stream 88 | 89 | timer_start = time.perf_counter_ns() 90 | for cur_stream, batch in zip(cur_streams, ret): 91 | batch.record_stream(cur_stream) 92 | timer_stop = time.perf_counter_ns() 93 | self.record_stream_times.append(timer_stop-timer_start) 94 | 95 | self.preload() 96 | return ret 97 | 98 | 99 | class DeviceTransferer(DeviceIterator): 100 | def __init__(self, devices, it: Iterator[PreparedBatch]): 101 | super().__init__(devices) 102 | 103 | self.it = it 104 | 105 | def __next__(self): 106 | ret = [batch.to(device, non_blocking=True) 107 | for device, batch in zip(self.devices, self.it)] 108 | if len(ret) == 0: 109 | raise StopIteration 110 | 111 | return ret 112 | 113 | 114 | class DeviceSlicerTransferer(DeviceIterator): 115 | # NOTE: This class only exists to provide functionality 116 | # that we used to have and no longer need (DATA_ON_MAIN). 117 | # You likely do not need to use this. 118 | # NOTE: x and y can be GPU tensors too! 119 | def __init__(self, devices, x: torch.Tensor, y: torch.Tensor, 120 | it: Iterator[ProtoBatch]): 121 | super().__init__(devices) 122 | 123 | self.x = x 124 | self.y = y 125 | self.it = it 126 | 127 | def __next__(self): 128 | ret = [PreparedBatch.from_proto_batch( 129 | self.x, self.y, proto_batch).to(device, non_blocking=True) 130 | for device, proto_batch in zip(self.devices, self.it)] 131 | 132 | if len(ret) == 0: 133 | raise StopIteration 134 | 135 | return ret 136 | --------------------------------------------------------------------------------