├── .github └── workflows │ ├── tests.yml │ └── wheels.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── examples ├── embedding_mammals.py └── wordnet_embedding.py ├── hyperlib ├── __init__.py ├── embedding │ ├── __init__.py │ ├── include │ │ ├── graph.h │ │ └── treerep.h │ ├── metric.py │ ├── sarkar.py │ ├── src │ │ ├── graph.cc │ │ ├── treerep.cc │ │ └── wrapper.cc │ └── treerep.py ├── loss │ ├── __init__.py │ └── constrastive_loss.py ├── manifold │ ├── __init__.py │ ├── base.py │ ├── lorentz.py │ └── poincare.py ├── models │ ├── __init__.py │ └── pehr.py ├── nn │ ├── __init__.py │ ├── layers │ │ ├── __init__.py │ │ ├── dense_attention.py │ │ └── lin_hyp.py │ └── optimizers │ │ ├── __init__.py │ │ └── rsgd.py └── utils │ ├── __init__.py │ ├── functional.py │ ├── graph.py │ ├── linalg.py │ └── multiprecision.py ├── pyproject.toml ├── setup.py └── tests ├── test_embedding.py ├── test_lorentz.py ├── test_multiprecision.py ├── test_poincare.py └── test_sarkar.py /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Lint and test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - dev 8 | - "feature/*" 9 | pull_request: 10 | branches: 11 | - main 12 | 13 | jobs: 14 | lint: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Setup python 19 | uses: actions/setup-python@v2 20 | with: 21 | python-version: "3.8" 22 | 23 | - name: Run lint 24 | run: | 25 | python -m pip install flake8 26 | # stop the build if there are Python syntax errors or undefined names 27 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 28 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 29 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 30 | 31 | test: 32 | runs-on: ${{ matrix.os }} 33 | needs: lint 34 | strategy: 35 | matrix: 36 | os: [ubuntu-latest] 37 | python-version: ["3.8","3.9"] 38 | 39 | steps: 40 | - uses: actions/checkout@v2 41 | - name: Setup python ${{ matrix.python-version }} 42 | uses: actions/setup-python@v2 43 | with: 44 | python-version: ${{ matrix.python-version }} 45 | 46 | - name: Install dependencies 47 | run: | 48 | python -m pip install --upgrade pip 49 | python -m pip install pytest pybind11 50 | 51 | - name: Install from source 52 | run: | 53 | python setup.py sdist 54 | pip install dist/*.tar.gz 55 | 56 | - name: Run tests 57 | run: | 58 | pytest -v . 59 | 60 | -------------------------------------------------------------------------------- /.github/workflows/wheels.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | jobs: 9 | build_wheels: 10 | name: Build wheels on ${{ matrix.os }} 11 | runs-on: ${{ matrix.os }} 12 | strategy: 13 | matrix: 14 | os: [ubuntu-20.04, windows-2019, macos-10.15] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | 19 | - name: Build wheels 20 | uses: pypa/cibuildwheel@v2.3.1 21 | env: 22 | CIBW_BUILD: "cp37-* cp38-* cp39-*" 23 | 24 | - uses: actions/upload-artifact@v2 25 | with: 26 | path: ./wheelhouse/*.whl 27 | 28 | build_sdist: 29 | name: Build source distribution 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v2 33 | 34 | - name: Build sdist 35 | run: pipx run build --sdist 36 | 37 | - uses: actions/upload-artifact@v2 38 | with: 39 | path: dist/*.tar.gz 40 | 41 | upload_wheels: 42 | name: Upload wheels to PyPI 43 | needs: [build_wheels, build_sdist] 44 | runs-on: ubuntu-latest 45 | if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') 46 | 47 | steps: 48 | - name: Collect sdist and wheels 49 | uses: actions/download-artifact@v2 50 | 51 | - name: Show artifacts 52 | run: ls -R 53 | 54 | - name: Publish to TestPyPI 55 | uses: pypa/gh-action-pypi-publish@v1.4.2 56 | with: 57 | user: __token__ 58 | password: ${{ secrets.TEST_PYPI_API_TOKEN }} 59 | packages_dir: artifact/ 60 | repository_url: https://test.pypi.org/legacy/ 61 | skip_existing: true 62 | 63 | - name: Publish to PyPI 64 | uses: pypa/gh-action-pypi-publish@v1.4.2 65 | with: 66 | user: __token__ 67 | password: ${{ secrets.PYPI_API_TOKEN }} 68 | packages_dir: artifact/ 69 | skip_existing: true 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # vim 132 | *.swp 133 | *.swo 134 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 nalexai 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include hyperlib/embedding/include/*.h 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HyperLib: Deep learning in the Hyperbolic space 2 | 3 | [![PyPI version](https://badge.fury.io/py/hyperlib.svg)](https://badge.fury.io/py/hyperlib) 4 | 5 | ## Background 6 | This library implements common Neural Network components in the hyperbolic space (using the Poincare model). The implementation of this library uses Tensorflow as a backend and can easily be used with Keras and is meant to help Data Scientists, Machine Learning Engineers, Researchers and others to implement hyperbolic neural networks. 7 | 8 | You can also use this library for uses other than neural networks by using the mathematical functions available in the Poincare class. In the future we may implement components that can be used in models other than neural networks. You can learn more about Hyperbolic networks [here](https://elevateailabs.com/blog/hyperlib), and in the references[^1] [^2] [^3] [^4]. 9 | 10 | ## Install 11 | The recommended way to install is with pip 12 | ``` 13 | pip install hyperlib 14 | ``` 15 | 16 | To build from source, you need to compile the pybind11 extensions. 17 | For example to build on linux: 18 | ```shell 19 | conda -n hyperlib python=3.8 gxx_linux-64 pybind11 20 | python setup.py install 21 | ``` 22 | 23 | Hyperlib works with python>=3.8 and tensorflow>=2.0. 24 | 25 | ## Example Usage 26 | 27 | Creating a hyperbolic neural network using Keras: 28 | ```python 29 | import tensorflow as tf 30 | from tensorflow import keras 31 | from hyperlib.nn.layers.lin_hyp import LinearHyperbolic 32 | from hyperlib.nn.optimizers.rsgd import RSGD 33 | from hyperlib.manifold.poincare import Poincare 34 | 35 | # Create layers 36 | hyperbolic_layer_1 = LinearHyperbolic(32, Poincare(), 1) 37 | hyperbolic_layer_2 = LinearHyperbolic(32, Poincare(), 1) 38 | output_layer = LinearHyperbolic(10, Poincare(), 1) 39 | 40 | # Create optimizer 41 | optimizer = RSGD(learning_rate=0.1) 42 | 43 | # Create model architecture 44 | model = tf.keras.models.Sequential([ 45 | hyperbolic_layer_1, 46 | hyperbolic_layer_2, 47 | output_layer 48 | ]) 49 | 50 | # Compile the model with the Riemannian optimizer 51 | model.compile( 52 | optimizer=optimizer, 53 | loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), 54 | metrics=[tf.keras.metrics.SparseCategoricalAccuracy()], 55 | ) 56 | 57 | ``` 58 | 59 | Using math functions on the Poincare ball: 60 | ```python 61 | import tensorflow as tf 62 | from hyperlib.manifold.poincare import Poincare 63 | 64 | p = Poincare() 65 | 66 | # Create two matrices 67 | a = tf.constant([[5.0,9.4,3.0],[2.0,5.2,8.9],[4.0,7.2,8.9]]) 68 | b = tf.constant([[4.8,1.0,2.3]]) 69 | 70 | # Matrix multiplication on the Poincare ball 71 | curvature = 1 72 | p.mobius_matvec(a, b, curvature) 73 | ``` 74 | 75 | ### Embeddings 76 | A big advantage of hyperbolic space is its ability to represent hierarchical data. There are several techniques for embedding data in hyperbolic space; the most common is gradient methods [^6]. 77 | 78 | If your data has a natural metric you can also use TreeRep[^5]. 79 | Input a symmetric distance matrix, or a compressed distance matrix 80 | ```python 81 | import numpy as np 82 | from hyperlib.embedding.treerep import treerep 83 | from hyperlib.embedding.sarkar import sarkar_embedding 84 | 85 | # Example: immunological distances between 8 mammals by Sarich 86 | compressed_metric = np.array([ 87 | 32., 48., 51., 50., 48., 98., 148., 88 | 26., 34., 29., 33., 84., 136., 89 | 42., 44., 44., 92., 152., 90 | 44., 38., 86., 142., 91 | 42., 89., 142., 92 | 90., 142., 93 | 148. 94 | ]) 95 | 96 | # outputs a weighted networkx Graph 97 | tree = treerep(compressed_metric, return_networkx=True) 98 | 99 | # embed the tree in 2D hyperbolic space 100 | root = 0 101 | embedding = sarkar_embedding(tree, root, tau=0.5) 102 | ``` 103 | 104 | Please see the [examples directory](https://github.com/nalexai/hyperlib/tree/main/examples) for complete examples. 105 | 106 | ## References 107 | [^1]: [Chami, I., Ying, R., Ré, C. and Leskovec, J. Hyperbolic Graph Convolutional Neural Networks. NIPS 2019.](http://web.stanford.edu/~chami/files/hgcn.pdf) 108 | 109 | [^2]: [Nickel, M. and Kiela, D. Poincaré embeddings for learning hierarchical representations. NIPS 2017.](https://papers.nips.cc/paper/2017/hash/59dfa2df42d9e3d41f5b02bfc32229dd-Abstract.html) 110 | 111 | [^3]: [Khrulkov, Mirvakhabova, Ustinova, Oseledets, Lempitsky. Hyperbolic Image Embeddings.](https://arxiv.org/pdf/1904.02239.pdf) 112 | 113 | [^4]: [Wei Peng, Varanka, Mostafa, Shi, Zhao. Hyperbolic Deep Neural Networks: A Survey.](https://arxiv.org/pdf/2101.04562.pdf) 114 | 115 | [^5]: [Rishi Sonthalia and Anna Gilbert. Tree! I am no Tree! I am a Low Dimensional Hyperbolic Embedding](https://arxiv.org/abs/2005.03847) 116 | [^6]: [De Sa et. al. Representation Tradeoffs for Hyperbolic Embeddings](https://arxiv.org/abs/1804.03329) 117 | -------------------------------------------------------------------------------- /examples/embedding_mammals.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import networkx as nx 3 | from hyperlib.embedding.treerep import treerep 4 | from hyperlib.embedding.sarkar import sarkar_embedding 5 | from hyperlib.utils.multiprecision import poincare_dist 6 | 7 | # Sarich measured "immunological distances" between 8 mammal species. 8 | # This metric should be hyperbolic because it is related to the species' 9 | # distance on the evolutionary tree. 10 | # 11 | # For more details on hyperbolicity see https://meiji163.github.io/post/combo-hyperbolic-embedding/ 12 | 13 | mammals = ["dog", "bear", "raccoon", "weasel", "seal", "sea_lion", "cat", "monkey"] 14 | labels = {i: m for i, m in enumerate(mammals)} 15 | compressed_metric = np.array([ 16 | 32., 48., 51., 50., 48., 98., 148., 17 | 26., 34., 29., 33., 84., 136., 18 | 42., 44., 44., 92., 152., 19 | 44., 38., 86., 142., 20 | 42., 89., 142., 21 | 90., 142., 22 | 148. 23 | ]) 24 | 25 | # Using TreeRep we can construct a putative evolutionary tree 26 | 27 | tree = treerep(compressed_metric, return_networkx=True) # outputs a weighted networkx Graph 28 | nx.draw(tree, labels=labels, with_labels=True) #plot tree 29 | 30 | # Using Sarkar's algorithm we can embed the tree in the Poincare ball 31 | 32 | root = 0 # label of root node 33 | tau = 0.2 # scaling factor for edges 34 | embed_2D = sarkar_embedding(tree, root, tau=tau) 35 | 36 | # calculate hyperbolic distances from the embedding 37 | poincare_dist(embed_2D[0,:], embed_2D[1,:]) 38 | -------------------------------------------------------------------------------- /examples/wordnet_embedding.py: -------------------------------------------------------------------------------- 1 | from random import choice 2 | import numpy as np 3 | import pandas as pd 4 | import tensorflow as tf 5 | from tensorflow import keras 6 | 7 | from hyperlib.manifold.lorentz import Lorentz 8 | from hyperlib.manifold.poincare import Poincare 9 | from hyperlib.models.pehr import HierarchicalEmbeddings 10 | 11 | 12 | def load_wordnet_data(file, negatives=20): 13 | noun_closure = pd.read_csv(file) 14 | noun_closure_np = noun_closure[["id1","id2"]].values 15 | 16 | edges = set() 17 | for i, j in noun_closure_np: 18 | edges.add((i,j)) 19 | 20 | unique_nouns = list(set( 21 | noun_closure["id1"].tolist()+noun_closure["id2"].tolist() 22 | )) 23 | 24 | noun_closure["neg_pairs"] = noun_closure["id1"].apply(get_neg_pairs, args=(edges, unique_nouns, 20,)) 25 | return noun_closure, unique_nouns 26 | 27 | def get_neg_pairs(noun, edges, unique_nouns, negatives=20): 28 | neg_list = [] 29 | while len(neg_list) < negatives: 30 | neg_noun = choice(unique_nouns) 31 | if neg_noun != noun \ 32 | and not neg_noun in neg_list \ 33 | and not ((noun, neg_noun) in edges or (neg_noun, noun) in edges): 34 | neg_list.append(neg_noun) 35 | return neg_list 36 | 37 | 38 | # Make training dataset 39 | noun_closure, unique_nouns = load_wordnet_data("data/mammal_closure.csv", negatives=15) 40 | noun_closure_dataset = noun_closure[["id1","id2"]].values 41 | 42 | batch_size = 16 43 | train_dataset = tf.data.Dataset.from_tensor_slices( 44 | (noun_closure_dataset, noun_closure["neg_pairs"].tolist())) 45 | train_dataset = train_dataset.shuffle(buffer_size=1024).batch(batch_size) 46 | 47 | # Create model 48 | model = HierarchicalEmbeddings(vocab=unique_nouns, embedding_dim=10) 49 | sgd = keras.optimizers.SGD(learning_rate=1e-2, momentum=0.9) 50 | 51 | # Run custom training loop 52 | model.fit(train_dataset, sgd, epochs=20) 53 | embs = model.get_embeddings() 54 | 55 | M = Poincare() 56 | mammal = M.expmap0(model(tf.constant('dog.n.01')), c=1) 57 | dists = M.dist(mammal, embs, c=1.0) 58 | top = tf.math.top_k(-dists[:,0], k=20) 59 | for i in top.indices: 60 | print(unique_nouns[i],': ',-dists[i,0].numpy()) 61 | -------------------------------------------------------------------------------- /hyperlib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nalexai/hyperlib/b7c47e121cc56b4276a387a6d8c1d05ae2445805/hyperlib/__init__.py -------------------------------------------------------------------------------- /hyperlib/embedding/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nalexai/hyperlib/b7c47e121cc56b4276a387a6d8c1d05ae2445805/hyperlib/embedding/__init__.py -------------------------------------------------------------------------------- /hyperlib/embedding/include/graph.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #ifndef GRAPH_H 13 | #define GRAPH_H 14 | typedef std::vector::iterator vitr; 15 | typedef std::vector::const_iterator const_vitr; 16 | 17 | /** 18 | * Symmetric matrix with 0's on the diagonal, 19 | * e.g. pairwise distances of N points. 20 | * 21 | * Constructors: 22 | * DistMat(unsigned N) 23 | * @param N: number of points ( > 0 ) 24 | * @param val: value to fill matrix with (default=0) 25 | * DistMat(const DistMat& D, unsigned N) 26 | * @param D: A size M <= N DistMat to copy values from 27 | * DistMat(const std::vector& dist, unsigned N) 28 | * @param dist: A size N*(N-1)/2 vector of distances d(i,j), 0<=i=0 and < N 35 | * DistMat& operator *=(double d) 36 | * multiply all entries by a scalar 37 | * int nearest(int i, const std::vector& pts) 38 | * Find the element of pts closest to point i. 39 | * @param pts: vector of non-negative ints < N 40 | * int size() 41 | * Return dimension of the matrix 42 | * double max() 43 | * Return max value in matrix 44 | * std::vector data() 45 | * Return a copy of the compressed matrix 46 | */ 47 | class DistMat{ 48 | public: 49 | DistMat(int N, double val=0); 50 | DistMat(const DistMat& D, int N); 51 | DistMat(const std::vector& dist, int N); 52 | DistMat(const double* dist, int N); 53 | double operator()(int i, int j) const; 54 | double& operator()(int i, int j); 55 | DistMat& operator*=(double d); 56 | double max() const; 57 | const_vitr nearest(int i, const std::vector& pts) const; 58 | int size() const; 59 | std::vector data(); 60 | private: 61 | int _N; 62 | double _zero; 63 | std::vector _data; 64 | }; 65 | 66 | /** 67 | * An undirected graph stored in adjacency list. Vertices are labeled with ints. 68 | * 69 | * Methods: 70 | * void add_edge(int u, int v) 71 | * Add edge (u,v). 72 | * void remove_edge(int u, int v) 73 | * Remove edge (u,v). 74 | * void remove_vertex(int v) 75 | * Remove vertex v and all its edges. 76 | * void retract(int u, int v) 77 | * Retract edge (u,v) and label the new vertex u. 78 | * bool is_adj(int u, int v) 79 | * return true if (u,v) is in the graph 80 | * std::vector neighbors(int u) 81 | * Get vector of vertices adjacent to u 82 | * void relabel(int u, int v) 83 | * relabel u as v, if u is a vertex in the graph and v is not 84 | * DistMat metric(double tol) 85 | * Calculate the shortest path distance between all vertices. 86 | * Assumes vertices are 0...N and graph is connected 87 | * @param tol: error tolerance for Floyd-Warshall 88 | * @returns: DistMat representing symmetric matrix with distances 89 | * int size() 90 | * number of vertices in the graph 91 | */ 92 | class Graph{ 93 | public: 94 | typedef std::map > vmap; //adj map 95 | typedef std::map< std::pair, double > wmap; //edge weights 96 | Graph(); 97 | void add_edge(int u, int v); 98 | void remove_edge(int u, int v); 99 | void remove_vertex(int v); 100 | void retract(int u, int v); 101 | bool is_adj(int u, int v); 102 | std::vector neighbors(int u); 103 | vmap adj_list(); 104 | DistMat metric(double tol=0.1) const; 105 | int size() const; 106 | int num_edges() const; 107 | void relabel(int u, int v); 108 | private: 109 | void _rm(int u, int v); 110 | void _insert(int u, int v); 111 | vmap _adj; 112 | }; 113 | #endif 114 | -------------------------------------------------------------------------------- /hyperlib/embedding/include/treerep.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "graph.h" 11 | 12 | #ifndef TREEREP_H 13 | #define TREEREP_H 14 | typedef std::vector< std::vector > vecvec; 15 | 16 | /** 17 | * TreeRep takes a metric and computes a weighted tree that approximates it 18 | * (R. Sonthalia & A.C. Gilbert https://arxiv.org/abs/2005.03847 ) 19 | * 20 | * @param D: DistMat holding the pairwise distances 21 | * @param tol: positive double specifying the tolerance 22 | * 23 | * Returns: weighted adjacency list, where u maps to pairs {v, w} 24 | * such that (u,v) is an edge with weight w. 25 | * 26 | */ 27 | Graph::wmap treerep(const DistMat& D, double tol=0.1); 28 | 29 | // ========== Helper functions ============= 30 | double grmv_prod(int x, int y, int z, const DistMat& W); 31 | int _treerep_recurse(Graph& G, DistMat& W, std::vector& V, std::vector& stn, 32 | int x, int y, int z); 33 | void _sort(Graph& G, DistMat& W, std::vector& V, std::vector& stn, vecvec& zns, 34 | int x, int y, int z, int r, bool rtr); 35 | void _thread_sort(Graph& G, DistMat& W, std::vector& V, vecvec& zns, std::vector& stn, 36 | int beg, int end, int x, int y, int z, int& r, bool& rtr); 37 | void _zone1(Graph& G, DistMat& W, std::vector& V, std::vector& stn,int v); 38 | void _zone2(Graph& G, DistMat& W, std::vector& V, std::vector& stn, int u, int v); 39 | std::default_random_engine& _trep_rng(int seed=1); 40 | #endif 41 | -------------------------------------------------------------------------------- /hyperlib/embedding/metric.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.spatial.distance import squareform 3 | from random import randint 4 | 5 | # there are more efficient algorithms for this 6 | # https://people.csail.mit.edu/virgi/6.890/papers/APBP.pdf 7 | def max_min(A, B): 8 | '''max-min product of two square matrices 9 | params: 10 | A, B: NxN numpy arrays ''' 11 | assert A.shape == B.shape 12 | return np.max(np.minimum(A[:, :, None], B[None, :, :]), axis=1) 13 | 14 | def mat_gromov_prod(dists, base): 15 | '''Gromov products of N-point metric space relative to base point 16 | Args: 17 | dists (ndarray): NxN matrix of pairwise distances 18 | base (int): index of the basepoint in 0...N-1 ''' 19 | assert dists.shape[0] == dists.shape[1] and 0 <= base < dists.shape[0] 20 | row = dists[base, :][None, :] 21 | col = dists[:, base][:, None] 22 | return 0.5*(row+col-dists) 23 | 24 | def delta_rel(dists, base=None): 25 | ''' Measure the delta-hyperbolicity constant of data 26 | with respect to basepoint, normalized by the diameter (max dist). 27 | Args: 28 | dists (ndarray): NxN matrix of pairwise distances 29 | base (int): index of basepoint in 0...N-1 (default = random) 30 | ''' 31 | if base is None: 32 | base = randint(0,dists.shape[0]-1) 33 | assert is_metric(dists) and 0 <= base < dists.shape[0] 34 | G = mat_gromov_prod(dists, base) 35 | delta = np.max(max_min(G,G)-G) 36 | diam = np.max(dists) 37 | return delta/diam 38 | 39 | def delta_sample(X, **kwargs): 40 | bs = kwargs.get("bs", X.shape[0]) 41 | tries = kwargs.get("tries", 10) 42 | dist = kwargs.get("dist", None) 43 | deltas = [] 44 | for i in range(tries): 45 | idx = np.random.choice(X.shape[0], bs) 46 | batch = X[idx] 47 | if dist is None: 48 | dists = np.linalg.norm( 49 | batch[None:,]-batch[:,None], 50 | axis=-1) 51 | else: 52 | dists = dist(batch,batch) 53 | deltas.append( 54 | delta_rel(dists,randint(0,bs-1)) 55 | ) 56 | return deltas 57 | 58 | def is_metric(X, tol=1e-8): 59 | return len(X.shape) == 2 and \ 60 | np.all( np.abs(X-X.T)= 0) 63 | 64 | def avg_distortion(metric1, metric2): 65 | ''' Average distortion between two metrics. 66 | Args: 67 | metric1, metric2 (ndarray): N x N distance matrices, 68 | or length N*(N-1)//2 compressed distance matrices 69 | Returns: 70 | average distortion (float) 71 | ''' 72 | assert metric1.shape == metric2.shape 73 | if len(metric1.shape) > 1: 74 | assert is_metric(metric1) 75 | X = squareform(metric1) 76 | else: 77 | X = metric1 78 | if len(metric2.shape) > 1: 79 | assert is_metric(metric2) 80 | Y = squareform(metric2) 81 | else: 82 | Y = metric2 83 | return np.mean( np.abs(X-Y)/Y ) 84 | -------------------------------------------------------------------------------- /hyperlib/embedding/sarkar.py: -------------------------------------------------------------------------------- 1 | from math import sqrt, floor, log, log2 2 | from itertools import product 3 | import networkx as nx 4 | from networkx.algorithms.shortest_paths.weighted import single_source_dijkstra_path_length 5 | import numpy as np 6 | import mpmath as mpm 7 | 8 | from ..utils.multiprecision import poincare_reflect0 9 | from ..utils.linalg import rotate_3D_mp 10 | 11 | def sarkar_embedding(tree, root, **kwargs): 12 | ''' 13 | Embed a tree in the Poincare disc using Sarkar's algorithm 14 | from "Low Distortion Delaunay Embedding of Trees in Hyperbolic Plane. 15 | Args: 16 | tree (networkx.Graph) : The tree represented with int node labels. 17 | Weighted trees should have the edge attribute "weight" 18 | root (int): The node to use as the root of the embedding 19 | Keyword Args: 20 | weighted (bool): True if the tree is weighted (default True) 21 | tau (float): the scaling factor for distances. 22 | By default it is calculated based on statistics of the tree. 23 | epsilon (float): parameter >0 controlling distortion bound (default 0.1). 24 | precision (int): number of bits of precision to use. 25 | By default it is calculated based on tau and epsilon. 26 | Returns: 27 | size N x 2 mpmath.matrix containing the coordinates of embedded nodes 28 | ''' 29 | eps = kwargs.get("epsilon",0.1) 30 | weighted = kwargs.get("weighted", True) 31 | tau = kwargs.get("tau") 32 | max_deg = max(tree.degree)[1] 33 | 34 | if tau is None: 35 | tau = (1+eps)/eps * mpm.log(2*max_deg/ mpm.pi) 36 | prc = kwargs.get("precision") 37 | if prc is None: 38 | prc = _embedding_precision(tree,root,eps) 39 | mpm.mp.dps = prc 40 | 41 | n = tree.order() 42 | emb = mpm.zeros(n,2) 43 | place = [] 44 | 45 | # place the children of root 46 | for i, v in enumerate(tree[root]): 47 | if weighted: 48 | r = mpm.tanh( tau*tree[root][v]["weight"]) 49 | else: 50 | r = mpm.tanh(tau) 51 | theta = 2*i*mpm.pi / tree.degree[root] 52 | emb[v,0] = r*mpm.cos(theta) 53 | emb[v,1] = r*mpm.sin(theta) 54 | place.append((root,v)) 55 | 56 | # TODO parallelize this 57 | while place: 58 | u, v = place.pop() # u is the parent of v 59 | p, x = emb[u,:], emb[v,:] 60 | rp = poincare_reflect0(x, p, precision=prc) 61 | arg = mpm.acos(rp[0]/mpm.norm(rp)) 62 | if rp[1] < 0: 63 | arg = 2*mpm.pi - arg 64 | 65 | theta = 2*mpm.pi / tree.degree[v] 66 | i=0 67 | for w in tree[v]: 68 | if w == u: continue 69 | i+=1 70 | if weighted: 71 | r = mpm.tanh(tau*tree[v][w]["weight"]) 72 | else: 73 | r = mpm.tanh(tau) 74 | w_emb = r * mpm.matrix([mpm.cos(arg+theta*i),mpm.sin(arg+theta*i)]).T 75 | w_emb = poincare_reflect0(x, w_emb, precision=prc) 76 | emb[w,:] = w_emb 77 | place.append((v,w)) 78 | return emb 79 | 80 | def sarkar_embedding_3D(tree, root, **kwargs): 81 | eps = kwargs.get("eps",0.1) 82 | weighted = kwargs.get("weighted", True) 83 | tau = kwargs.get("tau") 84 | max_deg = max(tree.degree)[1] 85 | 86 | if tau is None: 87 | tau = (1+eps)/eps * mpm.log(2*max_deg/ mpm.pi) 88 | prc = kwargs.get("precision") 89 | if prc is None: 90 | prc = _embedding_precision(tree,root,eps) 91 | mpm.mp.dps = prc 92 | 93 | n = tree.order() 94 | emb = mpm.zeros(n,3) 95 | place = [] 96 | 97 | # place the children of root 98 | fib = fib_2D_code(tree.degree[root]) 99 | for i, v in enumerate(tree[root]): 100 | r = mpm.tanh(tau*tree[root][v].get("weight",1.)) 101 | v_emb = r * mpm.matrix([[fib[i,0],fib[i,1],fib[i,2]]]) 102 | emb[v,:]=v_emb 103 | place.append((root,v)) 104 | 105 | while place: 106 | u, v = place.pop() # u is the parent of v 107 | u_emb, v_emb = emb[u,:], emb[v,:] 108 | 109 | # reflect and rotate so that embedding(v) is at (0,0,0) 110 | # and embedding(u) is in the direction of (0,0,1) 111 | u_emb = poincare_reflect0(v_emb, u_emb, precision=prc) 112 | R = rotate_3D_mp(mpm.matrix([[0.,0.,1.]]), u_emb) 113 | #u_emb = (R.T * u_emb).T 114 | 115 | # place children of v 116 | fib = fib_2D_code(tree.degree[v]) 117 | i=0 118 | for w in tree[v]: 119 | if w == u: # i=0 is for u (parent of v) 120 | continue 121 | i+=1 122 | r = mpm.tanh(tau*tree[w][v].get("weight",1.)) 123 | w_emb = r * mpm.matrix([[fib[i,0],fib[i,1],fib[i,2]]]) 124 | 125 | #undo reflection and rotation 126 | w_emb = (R * w_emb.T).T 127 | w_emb = poincare_reflect0(v_emb, w_emb, precision=prc) 128 | emb[w,:] = w_emb 129 | place.append((v,w)) 130 | return emb 131 | 132 | def _embedding_precision(tree, root, eps): 133 | dists = single_source_dijkstra_path_length(tree,root) 134 | max_deg = max(tree.degree)[1] 135 | l = 2*max(dists.values()) 136 | prc = floor( (log(max_deg+1)) * l/eps +1) 137 | return prc 138 | 139 | def hadamard_code(n): 140 | ''' 141 | Generate a binary Hadamard code 142 | Args: 143 | n (int): if n = 2**k + r < 2**(k+1), generates a [n, k, 2**(k-1)] Hadamard code 144 | Returns: 145 | 2**k x n np.array where code(i) = the ith row 146 | ''' 147 | k = floor( log2(n) ) 148 | r = n - 2**k 149 | gen = np.zeros((k,2**k), dtype=np.int32) 150 | for i, w in enumerate( product([0,1], repeat=k) ): 151 | gen[:,i] = w 152 | code = gen.T @ gen % 2 153 | if r > 0: 154 | code = np.concatenate( (code, code[:,:3]), axis=1 ) 155 | return code 156 | 157 | def fib_2D_code(n, eps=None): 158 | epsilons = [0.33, 1.33, 3.33, 10, 27, 75] 159 | if eps is None: 160 | if n < 24: 161 | eps = epsilons[0] 162 | elif n < 177: 163 | eps = epsilons[1] 164 | elif n < 890: 165 | eps = epsilons[2] 166 | elif n < 11000: 167 | eps = epsilons[3] 168 | elif n < 400000: 169 | eps = epsilons[4] 170 | else: 171 | eps = epsilons[5] 172 | 173 | golden = (1+sqrt(5))/2 174 | i = np.arange(0,n) 175 | theta = 2*np.pi*i / golden 176 | phi = np.arccos(1 - 2*(i+eps)/(n-1+2*eps)) 177 | M = np.stack([np.cos(theta)*np.sin(phi), np.sin(theta)*np.sin(phi), np.cos(phi)], axis=1) 178 | 179 | # rotate so that first point is (0,0,1) 180 | cos = M[0,2] 181 | sin = sqrt(1-cos**2) 182 | R = np.array([ 183 | [cos, 0., sin], 184 | [0. , 1., 0. ], 185 | [-sin,0., cos], 186 | ]) 187 | return M@R 188 | -------------------------------------------------------------------------------- /hyperlib/embedding/src/graph.cc: -------------------------------------------------------------------------------- 1 | #include "graph.h" 2 | 3 | Graph::Graph(){ 4 | } 5 | 6 | void Graph::remove_vertex(int v){ 7 | vmap::iterator i = _adj.find(v); 8 | if (i != _adj.end()){ 9 | vitr j; 10 | for (j = _adj[v].begin(); j!= _adj[v].end(); ++j){ 11 | _rm(*j,v); 12 | } 13 | _adj.erase(i); 14 | } 15 | } 16 | 17 | Graph::vmap Graph::adj_list(){ 18 | return _adj; 19 | } 20 | 21 | void Graph::add_edge(int u, int v){ 22 | _insert(u,v); 23 | _insert(v,u); 24 | } 25 | 26 | void Graph::remove_edge(int u, int v){ 27 | vmap::iterator fd = _adj.find(u); 28 | if (fd != _adj.end()){ 29 | _rm(u,v); 30 | } 31 | fd = _adj.find(v); 32 | if (fd != _adj.end()){ 33 | _rm(v,u); 34 | } 35 | } 36 | 37 | void Graph::retract(int u, int v){ 38 | vmap::iterator fd = _adj.find(v); 39 | if (fd == _adj.end() || _adj[v].empty()){ 40 | return; 41 | } 42 | for (vitr it=_adj[v].begin(); it!=_adj[v].end(); ++it){ 43 | if( u!=*it){ 44 | _rm(*it,v); 45 | add_edge(*it,u); 46 | } 47 | } 48 | _rm(u,v); 49 | _adj.erase(v); 50 | } 51 | 52 | inline void Graph::_rm(int u, int v){ 53 | vitr it = std::lower_bound(_adj[u].begin(), _adj[u].end(), v); 54 | if (it !=_adj[u].end() && *it == v){ 55 | _adj[u].erase(it); 56 | } 57 | } 58 | 59 | inline void Graph::_insert(int u, int v){ 60 | if (u == v){ //don't insert loops 61 | return; 62 | } 63 | vitr it = std::lower_bound(_adj[u].begin(), _adj[u].end(), v); 64 | if (it == _adj[u].end()){ 65 | _adj[u].push_back(v); 66 | }else if( *it != v){ 67 | _adj[u].insert(it,v); 68 | } 69 | } 70 | 71 | inline std::vector Graph::neighbors(int u){ 72 | return _adj[u]; 73 | } 74 | 75 | DistMat Graph::metric(double tol) const{ 76 | int N = _adj.size(); 77 | double infty = std::numeric_limits::infinity(); 78 | DistMat W(N, infty); 79 | vmap::const_iterator itr,jtr,ktr; 80 | for (itr=_adj.begin(); itr!=_adj.end();++itr){ 81 | for(const_vitr vit=itr->second.begin();vit!=itr->second.end(); ++vit){ 82 | W(itr->first,*vit) = 1; 83 | } 84 | } 85 | // Floyd-Warshall 86 | int i,j,k; 87 | for (k=0,ktr=_adj.begin(); ktr!=_adj.end(); ++ktr,++k){ 88 | for (i=0,itr=_adj.begin(); itr!=_adj.end(); ++itr,++i){ 89 | for (j=i+1,jtr=_adj.begin(); j W(i,k) + W(k,j) + tol ) ){ 92 | W(i,j) = W(i,k) + W(k,j); 93 | } 94 | } 95 | } 96 | } 97 | return W; 98 | } 99 | 100 | void Graph::relabel(int u, int v){ 101 | vmap::iterator fd = _adj.find(v); 102 | if( fd != _adj.end()){ 103 | return; 104 | } 105 | fd = _adj.find(u); 106 | if( fd ==_adj.end()){ 107 | return; 108 | } 109 | retract(v,u); 110 | } 111 | 112 | 113 | int Graph::size() const{ 114 | return _adj.size(); 115 | } 116 | 117 | int Graph::num_edges() const{ 118 | int len=0; 119 | for (vmap::const_iterator it=_adj.begin();it!=_adj.end(); ++it){ 120 | len += (it->second).size(); 121 | } 122 | return len/2; 123 | } 124 | 125 | DistMat::DistMat(int N, double val): _N(N), _zero(0){ 126 | _data.resize((N*(N-1))/2); 127 | for (int i=0; i<_N; ++i){ 128 | for (int j=i+1; j<_N; ++j){ 129 | (*this)(i,j) = val; 130 | } 131 | } 132 | } 133 | 134 | DistMat::DistMat(const DistMat& D, int N):_N(N), _zero(0.){ 135 | int M = D.size(); 136 | if (M > N){ 137 | throw std::invalid_argument("Incompatible size"); 138 | } 139 | _data.resize((N*(N-1))/2); 140 | for (int i = 0; i<_N; ++i){ 141 | for( int j = i+1; j<_N; ++j){ 142 | if( i& dist, int N): _N(N), _zero(0.){ 152 | int S = dist.size(); 153 | if (S != (N*(N-1))/2){ 154 | throw std::invalid_argument("Incompatible sizes "+std::to_string(S) 155 | +" and "+std::to_string(N)); 156 | } 157 | _data.resize((N*(N-1))/2); 158 | std::memcpy(_data.data(),dist.data(),S*sizeof(double)); 159 | } 160 | 161 | DistMat::DistMat(const double* dist, int N): _N(N), _zero(0.){ 162 | int S = (N*(N-1))/2; 163 | _data.reserve((N*(N-1))/2); 164 | std::memcpy(_data.data(),dist,S*sizeof(double)); 165 | } 166 | 167 | double& DistMat::operator()(int i, int j){ 168 | if(i >= _N || j >= _N || i < 0 || j < 0){ 169 | throw std::invalid_argument("index out of bounds"); 170 | }else if(i == j){ 171 | return _zero; 172 | }else if (i > j){ 173 | return _data[ _N*j + i - ((j+2)*(j+1))/2 ]; 174 | }else{ 175 | return _data[ _N*i + j - ((i+2)*(i+1))/2 ]; 176 | } 177 | } 178 | 179 | double DistMat::operator()(int i, int j) const{ 180 | if(i >= _N || j >= _N || i < 0 || j < 0){ 181 | throw std::invalid_argument("index out of bounds"); 182 | }else if(i == j){ 183 | return 0; 184 | }else if (i >j){ 185 | return _data[ _N*j + i - ((j+2)*(j+1))/2 ]; 186 | }else{ 187 | return _data[ _N*i + j - ((i+2)*(i+1))/2 ]; 188 | } 189 | } 190 | 191 | DistMat& DistMat::operator*=(double d){ 192 | for(int i=0; i< (_N*(_N-1))/2; ++i){ 193 | _data[i] *= d; 194 | } 195 | return *this; 196 | } 197 | 198 | double DistMat::max() const{ 199 | return *std::max_element(_data.begin(), _data.end()); 200 | } 201 | 202 | std::vector DistMat::data(){ 203 | return _data; 204 | } 205 | 206 | const_vitr DistMat::nearest(int i, const std::vector& pts) const{ 207 | if(pts.empty()){ 208 | throw std::invalid_argument("set of points is empty"); 209 | }else{ 210 | double min = (*this)(i,pts.front()); 211 | std::vector::const_iterator it, jt=pts.begin(); 212 | for (it = pts.begin(); it != pts.end(); ++it){ 213 | if( (*this)(i, *it) < min){ 214 | min = (*this)(i, *it); 215 | jt = it; 216 | } 217 | } 218 | return jt; 219 | } 220 | } 221 | 222 | int DistMat::size() const{ 223 | return _N; 224 | } 225 | 226 | bool Graph::is_adj(int u, int v){ 227 | vmap::iterator fd = _adj.find(u); 228 | if (fd == _adj.end() || fd->second.empty()){ 229 | return false; 230 | } 231 | vitr it = std::lower_bound(fd->second.begin(),fd->second.end(),v); 232 | return (it != fd->second.end() && *it == v); 233 | } 234 | -------------------------------------------------------------------------------- /hyperlib/embedding/src/treerep.cc: -------------------------------------------------------------------------------- 1 | #include "treerep.h" 2 | int TREP_N; 3 | double TREP_TOL; 4 | 5 | std::default_random_engine& _trep_rng(int seed){ 6 | static std::default_random_engine rng(seed); 7 | return rng; 8 | } 9 | 10 | Graph::wmap treerep(const DistMat& D, double tol){ 11 | TREP_N = D.size(); 12 | TREP_TOL = tol; 13 | DistMat W(D, 2*TREP_N); 14 | std::vector V(TREP_N); 15 | for (int i=0; i stn; 30 | stn.reserve(TREP_N); 31 | for (int i=2*TREP_N; i>=TREP_N; --i){ 32 | stn.push_back(i); 33 | } 34 | Graph G; 35 | _treerep_recurse(G,W,V,stn,x,y,z); 36 | 37 | Graph::wmap weight; 38 | for(int i=0; i& V, std::vector& stn, 52 | int x, int y, int z){ 53 | if(stn.empty()){ 54 | return 1; 55 | } 56 | 57 | // form the universal tree for x,y,z 58 | int r = stn.back(); 59 | stn.pop_back(); 60 | G.add_edge(r,x); 61 | G.add_edge(r,y); 62 | G.add_edge(r,z); 63 | 64 | bool rtr = false; 65 | W(r,x) = grmv_prod(x,y,z,W); 66 | if (std::abs(W(r,x)) < TREP_TOL && !rtr){// retract (r, x) 67 | W(r,x) = 0; 68 | G.retract(x,r); 69 | stn.push_back(r); 70 | r = x; 71 | rtr = true; 72 | } 73 | W(r,y) = grmv_prod(y,x,z,W); 74 | if (std::abs(W(r,y)) < TREP_TOL && !rtr){// retract (r, y) 75 | W(r,x) = 0; 76 | W(r,y) = 0; 77 | G.retract(y,r); 78 | stn.push_back(r); 79 | r = y; 80 | rtr = true; 81 | } 82 | W(r,z) = grmv_prod(z,x,y,W); 83 | if (std::abs(W(r,z)) < TREP_TOL && !rtr){ //retract (r, z) 84 | W(r,x) = 0; 85 | W(r,y) = 0; 86 | W(r,z) = 0; 87 | G.retract(z,r); 88 | stn.push_back(r); 89 | r = z ; 90 | rtr = true; 91 | } 92 | //sort rest of vertices into 7 zones 93 | vecvec zone(7); 94 | if( V.size() < 32){ 95 | _sort(G,W,V,stn,zone,x,y,z,r,rtr); 96 | }else{ //multithread sort 97 | std::vector tzns(4); 98 | for(int i=0; i<4; ++i){ 99 | tzns[i].resize(7); 100 | } 101 | std::thread t1(_thread_sort, std::ref(G), std::ref(W), std::ref(V), std::ref(tzns[0]),std::ref(stn), 102 | 0, V.size()/4, x,y,z,std::ref(r),std::ref(rtr)); 103 | std::thread t2(_thread_sort, std::ref(G), std::ref(W), std::ref(V), std::ref(tzns[1]), std::ref(stn), 104 | V.size()/4, V.size()/2, x,y,z,std::ref(r),std::ref(rtr)); 105 | std::thread t3(_thread_sort, std::ref(G), std::ref(W), std::ref(V), std::ref(tzns[2]), std::ref(stn), 106 | V.size()/2, 3*V.size()/4, x,y,z,std::ref(r),std::ref(rtr)); 107 | std::thread t4(_thread_sort, std::ref(G), std::ref(W), std::ref(V), std::ref(tzns[3]), std::ref(stn), 108 | 3*V.size()/4, V.size(), x,y,z,std::ref(r),std::ref(rtr)); 109 | t1.join(); 110 | t2.join(); 111 | t3.join(); 112 | t4.join(); 113 | for(int i=0; i<7; ++i){ 114 | for( int j=0; j<4; ++j){ 115 | std::move( 116 | tzns[j][i].begin(), 117 | tzns[j][i].end(), 118 | std::back_inserter(zone[i]) 119 | ); 120 | } 121 | } 122 | } 123 | _zone1(G,W,zone[0],stn,r); 124 | _zone1(G,W,zone[1],stn,z); 125 | _zone1(G,W,zone[3],stn,x); 126 | _zone1(G,W,zone[5],stn,y); 127 | _zone2(G,W,zone[2],stn,z,r); 128 | _zone2(G,W,zone[4],stn,x,r); 129 | _zone2(G,W,zone[6],stn,y,r); 130 | return 0; 131 | } 132 | 133 | void _thread_sort(Graph& G, DistMat& W, std::vector& V, vecvec& zns, std::vector& stn, 134 | int beg, int end, int x, int y, int z, int& r, bool& rtr){ 135 | for (int i = beg; i< end; ++i){ 136 | int w = V[i]; 137 | double a = grmv_prod(w,x,y,W); 138 | double b = grmv_prod(w,y,z,W); 139 | double c = grmv_prod(w,z,x,W); 140 | double max = std::max({a,b,c}); 141 | if ( std::abs(a-b)& V, std::vector& stn, vecvec& zns, 183 | int x, int y, int z, int r, bool rtr){ 184 | for (int i = 0; i< V.size(); ++i){ 185 | int w = V[i]; 186 | double a = grmv_prod(w,x,y,W); 187 | double b = grmv_prod(w,y,z,W); 188 | double c = grmv_prod(w,z,x,W); 189 | double max = std::max({a,b,c}); 190 | if ( std::abs(a-b)& V, std::vector& stn, int v){ 232 | int S = V.size(); 233 | if (S == 1){ 234 | int u = V.back(); 235 | V.pop_back(); 236 | G.add_edge(u,v); 237 | }else if (S>1){ 238 | std::shuffle(V.begin(), V.end(), _trep_rng()); 239 | int u = V.back(); 240 | V.pop_back(); 241 | int z = V.back(); 242 | V.pop_back(); 243 | _treerep_recurse(G,W,V,stn,v,u,z); 244 | } 245 | } 246 | 247 | void _zone2(Graph& G, DistMat& W, std::vector& V, std::vector& stn, 248 | int u, int v){ 249 | if (!V.empty()){ 250 | std::vector::const_iterator it = W.nearest(v, V); 251 | int z = *it; 252 | V.erase(it); 253 | G.remove_edge(u,v); 254 | _treerep_recurse(G,W,V,stn,u,v,z); 255 | } 256 | } 257 | 258 | inline double grmv_prod(int x, int y, int z, const DistMat& W){ 259 | return 0.5*(W(x,y)+W(x,z)-W(y,z)); 260 | } 261 | -------------------------------------------------------------------------------- /hyperlib/embedding/src/wrapper.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "graph.h" 6 | #include "treerep.h" 7 | 8 | namespace py = pybind11; 9 | 10 | py::array_t py_metric(const Graph& G){ 11 | DistMat D = G.metric(); 12 | std::vector metric = D.data(); 13 | auto result = py::array_t(metric.size()); 14 | auto result_buf = result.request(); 15 | int *result_ptr = (int *) result_buf.ptr; 16 | std::memcpy(result_ptr, metric.data(), metric.size()*sizeof(double)); 17 | return result; 18 | } 19 | 20 | Graph::wmap py_graph_treerep(const Graph& G, double tol=0.1){ 21 | DistMat D = G.metric(); 22 | return treerep(D,tol); 23 | } 24 | 25 | // accepts 1D numpy array of size N*(N-1)/2 as the N-point metric 26 | Graph::wmap py_treerep(py::array_t metric, int N, double tol=0.1){ 27 | if( N<2 || metric.size() != (N*(N-1))/2){ 28 | throw std::invalid_argument("array must be size N*(N-1)/2"); 29 | } 30 | DistMat M(metric.data(), N); 31 | return treerep(M,tol); 32 | } 33 | 34 | PYBIND11_MODULE(__hyperlib_embedding,m){ 35 | py::class_(m, "Graph") 36 | .def(py::init<>()) 37 | .def("add_edge", &Graph::add_edge) 38 | .def("adj_list", &Graph::adj_list) 39 | .def("remove_edge", &Graph::remove_edge) 40 | .def("retract", &Graph::retract) 41 | .def("remove_vertex", &Graph::remove_vertex); 42 | 43 | m.def("graph_metric", &py_metric); 44 | m.def("treerep_graph", 45 | &py_graph_treerep, 46 | py::arg("G"), 47 | py::arg("tol")=0.1, 48 | py::return_value_policy::move); 49 | m.def("treerep", 50 | &py_treerep, 51 | py::arg("metric"), 52 | py::arg("N"), 53 | py::arg("tol")=0.1, 54 | py::return_value_policy::move); 55 | 56 | #ifdef VERSION_INFO 57 | m.attr("__version__") = VERSION_INFO; 58 | #else 59 | m.attr("__version__") = "dev"; 60 | #endif 61 | } 62 | -------------------------------------------------------------------------------- /hyperlib/embedding/treerep.py: -------------------------------------------------------------------------------- 1 | from math import sqrt, floor 2 | from scipy.spatial.distance import squareform 3 | import numpy as np 4 | import mpmath as mpm 5 | 6 | from ..utils.graph import to_networkx 7 | from .metric import is_metric 8 | import __hyperlib_embedding 9 | 10 | def treerep(dists, **kwargs): 11 | """ 12 | TreeRep algorithm from Sonthalia & Gilbert, 'Tree! I am no Tree! I am a Low Dimensional Hyperbolic Embedding' 13 | takes a metric (distance matrix) and computes a weighted tree that approximates it. 14 | Args: 15 | metric (ndarray): size NxN distance matrix or 16 | compressed matrix of length N*(N-1)//2 ( e.g from scipy.pdist ) 17 | tol (double): tolerance for checking equalities (default=0.1) 18 | return_networkx (bool): return a networkx.Graph instead of edges (default=False) 19 | Returns: 20 | A dict mapping edges (u,v), u= N. 25 | They may or may not have an interpretation in terms of the data. 26 | - some edge weights may be 0. In that case you can either retract the edge 27 | or perturb it to a small positive number. 28 | """ 29 | tol = kwargs.get("tol",0.1) 30 | if len(dists.shape) == 1: 31 | N = floor( (1+sqrt(1+8*len(dists)))/2 ) 32 | assert N*(N-1)//2 == len(dists) 33 | W = __hyperlib_embedding.treerep(dists, N, tol) 34 | elif len(dists.shape) == 2: 35 | assert is_metric(dists) 36 | W = __hyperlib_embedding.treerep(squareform(dists), dists.shape[0], tol) 37 | else: 38 | raise ValueError("Invalid distance matrix") 39 | 40 | if kwargs.get("return_networkx", False): 41 | return to_networkx(W) 42 | return W 43 | -------------------------------------------------------------------------------- /hyperlib/loss/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nalexai/hyperlib/b7c47e121cc56b4276a387a6d8c1d05ae2445805/hyperlib/loss/__init__.py -------------------------------------------------------------------------------- /hyperlib/loss/constrastive_loss.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def contrastive_loss(pos_embs, neg_embs, M, c=1.0, clip_value=0.9): 5 | ''' 6 | The contrastive loss for embeddings used by Nickel & Kiela 7 | math:: 8 | -\log( e^{-d(x,y)} / \sum_{n \in N(x)} e^{-d(x,n)}) 9 | 10 | where (x,y) is a positive example, 11 | N(x) is the set of negative samples for x, 12 | and d(.,.) is the hyperbolic distance 13 | ''' 14 | # clip embedding norms before expmap 15 | pos_embs = tf.clip_by_norm(pos_embs, clip_value, axes=2) 16 | neg_embs = tf.clip_by_norm(neg_embs, clip_value, axes=2) 17 | 18 | x_pos = M.expmap0(pos_embs, c) 19 | x_neg = M.expmap0(neg_embs, c) 20 | 21 | batch_loss = M.dist(x_pos[:,0,:], x_pos[:,1,:], c) 22 | 23 | x = x_pos[:,0,:] 24 | x = tf.expand_dims(x, 1) 25 | x = tf.broadcast_to(x, x_neg.shape) 26 | 27 | neg_loss = tf.reduce_sum(tf.exp(-M.dist(x, x_neg, c)), axis=1) 28 | # clip to avoid log(0) 29 | neg_loss = tf.clip_by_value(neg_loss, clip_value_min=1e-15, clip_value_max=1e10) 30 | batch_loss += tf.math.log(neg_loss) 31 | return tf.reduce_sum(batch_loss, axis=0) 32 | -------------------------------------------------------------------------------- /hyperlib/manifold/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nalexai/hyperlib/b7c47e121cc56b4276a387a6d8c1d05ae2445805/hyperlib/manifold/__init__.py -------------------------------------------------------------------------------- /hyperlib/manifold/base.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | class Manifold(object): 4 | """ 5 | Abstract class to define operations on a manifold. 6 | """ 7 | 8 | def __init__(self): 9 | super().__init__() 10 | self.name = "manifold" 11 | self.min_norm = 1e-15 12 | self.eps = {tf.float32: 4e-3, tf.float64: 1e-5} 13 | 14 | 15 | def expmap(self, u, p, c): 16 | raise NotImplementedError 17 | 18 | def expmap0(self, u, c): 19 | """ 20 | Hyperbolic exponential map at zero. 21 | Args: 22 | u: tensor of size B x dimension representing tangent vectors. 23 | c: tensor of size 1 representing the hyperbolic curvature. 24 | Returns: 25 | Tensor of shape B x dimension. 26 | """ 27 | raise NotImplementedError 28 | 29 | def logmap0(self, p, c): 30 | """ 31 | Hyperbolic logarithmic map at zero. 32 | Args: 33 | p: tensor of size B x dimension representing hyperbolic points. 34 | c: tensor of size 1 representing the hyperbolic curvature. 35 | Returns: 36 | Tensor of shape B x dimension. 37 | """ 38 | raise NotImplementedError 39 | 40 | def proj(self, x, c): 41 | """ 42 | Safe projection on the manifold for numerical stability. This was mentioned in [1] 43 | 44 | Args: 45 | x : Tensor point on the Poincare ball 46 | c : Tensor of size 1 representing the hyperbolic curvature. 47 | 48 | Returns: 49 | Projected vector on the manifold 50 | 51 | References: 52 | [1] Hyperbolic Neural Networks, NIPS2018 53 | https://arxiv.org/abs/1805.09112 54 | """ 55 | 56 | raise NotImplementedError 57 | 58 | def hyp_act(self, act, c_in, c_out): 59 | """Apply an activation function to a tensor in the hyperbolic space""" 60 | raise NotImplementedError 61 | -------------------------------------------------------------------------------- /hyperlib/manifold/lorentz.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | from .base import Manifold 5 | from ..utils.functional import cosh, sinh, arcosh 6 | 7 | class Lorentz(Manifold): 8 | """ 9 | Implementation of the Lorentz/Hyperboloid manifold defined by 10 | :math: `L = \{ x \in R^d | -x_0^2 + x_1^2 + ... + x_d^2 = -K \}`, 11 | where c = 1 / K is the hyperbolic curvature and d is the manifold dimension. 12 | 13 | The point :math: `( \sqrt{K}, 0, \dots, 0 )` is referred to as "zero". 14 | """ 15 | 16 | def __init__(self): 17 | super(Lorentz).__init__() 18 | self.name = 'Lorentz' 19 | self.eps = {tf.float32: 1e-7, tf.float64: 1e-15} 20 | self.min_norm = 1e-15 21 | self.max_norm = 1e6 22 | 23 | def minkowski_dot(self, x, y, keepdim=True): 24 | res = tf.math.reduce_sum(x * y, axis=-1) - 2 * x[..., 0] * y[..., 0] 25 | if keepdim: 26 | res = tf.reshape(res, res.shape + (1,)) 27 | return res 28 | 29 | def minkowski_norm(self, u, keepdim=True): 30 | dot = self.minkowski_dot(u, u, keepdim=keepdim) 31 | t = tf.clip_by_value( 32 | dot, clip_value_min=self.eps[u.dtype], clip_value_max=self.max_norm) 33 | return tf.math.sqrt(t) 34 | 35 | def dist_squared(self, x, y, c): 36 | """Squared hyperbolic distance between x, y""" 37 | K = 1. / c 38 | theta = tf.clip_by_value( -self.minkowski_dot(x, y) / K, 39 | clip_value_min=1.0 + self.eps[x.dtype], clip_value_max=self.max_norm) 40 | return K * arcosh(theta)**2 41 | 42 | def proj(self, x, c): 43 | """Projects point (d+1)-dimensional point x to the manifold""" 44 | K = 1. / c 45 | d1 = x.shape[-1] 46 | y = x[:,1:d1] 47 | y_sqnorm = tf.math.square( 48 | tf.norm(y, ord=2, axis=1, keepdims=True)) 49 | t = tf.clip_by_value(K + y_sqnorm, 50 | clip_value_min=self.eps[x.dtype], 51 | clip_value_max=self.max_norm 52 | ) 53 | return tf.concat([tf.math.sqrt(t), y], axis=1) 54 | 55 | def proj_tan(self, u, x, c): 56 | """Projects vector u onto the tangent space at x. 57 | Note: this is not the orthogonal projection""" 58 | d1 = x.shape[-1] 59 | ud = u[:,1:d1] 60 | ux = tf.math.reduce_sum( x[:,1:d1]*ud, axis=1, keepdims=True) 61 | x0 = tf.clip_by_value(x[:,0:1], clip_value_min=self.eps[x.dtype], clip_value_max=1e5) 62 | return tf.concat( [ux/x0, ud], axis=1 ) 63 | 64 | def proj_tan0(self, u, c): 65 | """Projects vector u onto the tangent space at zero. 66 | See also: Lorentz.proj_tan""" 67 | b, d1 = u.shape 68 | z = tf.zeros((b,1), dtype=u.dtype) 69 | ud = u[:,1:d1] 70 | return tf.concat([z, ud], axis=1) 71 | 72 | def expmap(self, u, x, c): 73 | """Maps vector u in the tangent space at x onto the manifold""" 74 | K = 1. / c 75 | sqrtK = K ** 0.5 76 | normu = self.minkowski_norm(u) 77 | normu = self.clip_norm(normu) 78 | theta = normu / sqrtK 79 | theta = self.clip_norm(theta) 80 | result = cosh(theta) * x + sinh(theta) * u / theta 81 | return self.proj(result, c) 82 | 83 | def logmap(self, y, x, c): 84 | """Maps point y in the manifold to the tangent space at x""" 85 | K = 1. / c 86 | xy = tf.clip_by_value(self.minkowski_dot(x, y) + K, 87 | clip_value_min=-self.max_norm, clip_value_max=-self.eps[x.dtype]) 88 | xy -= K 89 | u = y + xy * x * c 90 | normu = self.minkowski_norm(u) 91 | normu = self.clip_norm(normu) 92 | dist = tf.math.sqrt(self.dist_squared(x, y, c)) 93 | result = dist * u / normu 94 | return self.proj_tan(result, x, c) 95 | 96 | def hyp_act(self, act, x, c_in, c_out): 97 | """Apply an activation function to a tensor in the hyperbolic space""" 98 | xt = act(self.logmap0(x, c=c_in)) 99 | return self.proj(self.expmap0(xt, c=c_out), c=c_out) 100 | 101 | def expmap0(self, u, c): 102 | """Maps vector u in the tangent space at zero onto the manifold""" 103 | K = 1. / c 104 | sqrtK = K ** 0.5 105 | d = u.shape[-1] 106 | x = tf.reshape(u[:,1:d], [-1, d-1]) 107 | x_norm = tf.norm(x, ord=2, axis=1, keepdims=True) 108 | x_norm = self.clip_norm(x_norm) 109 | theta = x_norm / sqrtK 110 | res = tf.ones_like(u) 111 | b, d = res.shape 112 | res1 = tf.ones((b,1), dtype=res.dtype) 113 | res1 *= sqrtK * cosh(theta) 114 | res2 = tf.ones((b,d-1), dtype=res.dtype) 115 | res2 *= sqrtK * sinh(theta) * (x / x_norm) 116 | res = tf.concat([res1,res2], axis=1) 117 | return self.proj(res, c) 118 | 119 | def logmap0(self, x, c): 120 | """Maps point y in the manifold to the tangent space at zero. 121 | See also: Lorentz.logmap""" 122 | K = 1. / c 123 | sqrtK = K ** 0.5 124 | b, d = x.shape 125 | y = tf.reshape(x[:,1:], [-1, d-1]) 126 | y_norm = tf.norm(y, ord=2, axis=1, keepdims=True) 127 | y_norm = self.clip_norm(y_norm) 128 | theta = tf.clip_by_value(x[:, 0:1] / sqrtK, 129 | clip_value_min=1.0+self.eps[x.dtype], clip_value_max=self.max_norm) 130 | res = sqrtK * arcosh(theta) * y / y_norm 131 | zeros = tf.zeros((b,1), dtype=res.dtype) 132 | return tf.concat([zeros, res], axis=1) 133 | 134 | def mobius_add(self, x, y, c): 135 | u = self.logmap0(y, c) 136 | v = self.ptransp0(x, u, c) 137 | return self.expmap(v, x, c) 138 | 139 | def mobius_matvec(self, m, x, c): 140 | u = self.logmap0(x, c) 141 | mu = u @ m 142 | return self.expmap0(mu, c) 143 | 144 | def ptransp(self, x, y, u, c): 145 | """Parallel transport a vector u in the tangent space at x 146 | to the tangent space at y""" 147 | log_xy = self.logmap(y, x, c) 148 | log_yx= self.logmap(x, y, c) 149 | dist_squared = self.clip_norm(self.dist_squared(x, y, c)) 150 | alpha = self.minkowski_dot(log_xy, u) / dist_squared 151 | res = u - alpha * (log_xy + log_yx) 152 | return self.proj_tan(res, y, c) 153 | 154 | def ptransp0(self, x, u, c): 155 | """Parallel transport a vector u in the tangent space at zero 156 | to the tangent space at x. 157 | See also: Lorentz.ptransp""" 158 | K = 1. / c 159 | sqrtK = K ** 0.5 160 | x0 = x[:,:1] 161 | d1 = x.shape[-1] 162 | y = x[:,1:d1] 163 | 164 | y_norm = tf.norm(y, ord=2, axis=1, keepdims=True) 165 | y_norm = self.clip_norm(y_norm) 166 | y_unit = y / y_norm 167 | v = tf.concat([y_norm, (sqrtK - x0)*y_unit], axis=1) 168 | 169 | alpha = tf.math.reduce_sum(y_unit * u[:, 1:], axis=1, keepdims=True) / sqrtK 170 | res = u - alpha * v 171 | return self.proj_tan(res, x, c) 172 | 173 | def to_poincare(self, x, c): 174 | K = 1. / c 175 | sqrtK = K ** 0.5 176 | d1 = x.shape[-1] 177 | return sqrtK * x[:,1:d1] / (x[:, 0:1] + sqrtK) 178 | 179 | def clip_norm(self, x): 180 | return tf.clip_by_value(x, clip_value_min=self.min_norm, clip_value_max=self.max_norm) 181 | -------------------------------------------------------------------------------- /hyperlib/manifold/poincare.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from ..utils.functional import tanh, atanh, asinh 3 | from .base import Manifold 4 | 5 | class Poincare(Manifold): 6 | 7 | """ 8 | Implementation of the poincare manifold,. This class can be used for mathematical functions on the poincare manifold. 9 | """ 10 | 11 | def __init__(self,): 12 | super(Poincare, self).__init__() 13 | self.name = "PoincareBall" 14 | self.min_norm = 1e-15 15 | self.eps = {tf.float32: 4e-3, tf.float64: 1e-5} 16 | 17 | def mobius_matvec(self, m, x, c): 18 | """ 19 | Generalization for matrix-vector multiplication to hyperbolic space defined as 20 | math:: 21 | M \otimes_c x = (1/\sqrt{c}) \tanh\left( 22 | \frac{\|Mx\|_2}{\|x\|_2}\tanh^{-1}(\sqrt{c}\|x\|_2) 23 | \right)\frac{Mx}{\|Mx\|_2} 24 | Args: 25 | m : Tensor for multiplication 26 | x : Tensor point on poincare ball 27 | c : Tensor of size 1 representing the hyperbolic curvature. 28 | Returns 29 | Mobius matvec result 30 | """ 31 | 32 | sqrt_c = c ** 0.5 33 | x_norm = tf.norm(x, axis=-1, keepdims=True, ord=2) 34 | max_num = tf.math.reduce_max(x_norm) 35 | x_norm = tf.clip_by_value( 36 | x_norm, clip_value_min=self.min_norm, clip_value_max=max_num 37 | ) 38 | mx = x @ m 39 | mx_norm = tf.norm(mx, axis=-1, keepdims=True, ord=2) 40 | max_num = tf.math.reduce_max(mx_norm) 41 | mx_norm = tf.clip_by_value( 42 | mx_norm, clip_value_min=self.min_norm, clip_value_max=max_num 43 | ) 44 | 45 | res_c = ( 46 | tanh(mx_norm / x_norm * atanh(sqrt_c * x_norm)) * mx / (mx_norm * sqrt_c) 47 | ) 48 | cond = tf.reduce_prod( 49 | tf.cast((mx == 0), tf.uint8, name=None), axis=-1, keepdims=True 50 | ) 51 | res_0 = tf.zeros(1, dtype=res_c.dtype) 52 | res = tf.where(tf.cast(cond, tf.bool), res_0, res_c) 53 | return res 54 | 55 | def expmap(self, u, p, c): 56 | sqrt_c = c ** 0.5 57 | u_norm = u.norm(dim=-1, p=2, keepdim=True).clamp_min(self.min_norm) 58 | second_term = ( 59 | tanh(sqrt_c / 2 * self._lambda_x(p, c) * u_norm) * u / (sqrt_c * u_norm) 60 | ) 61 | gamma_1 = self.mobius_add(p, second_term, c) 62 | return gamma_1 63 | 64 | def dist(self, x, y, c): 65 | """ 66 | Poincare distance between two points 67 | math:: 68 | 2/\sqrt{c} \artanh(\sqrt{c} ||-x \oplus_c y||) 69 | """ 70 | sqrt_c = c ** 0.5 71 | x2 = tf.reduce_sum(x * x, axis=-1, keepdims=True) 72 | y2 = tf.reduce_sum(y * y, axis=-1, keepdims=True) 73 | xy = tf.reduce_sum(x * y, axis=-1, keepdims=True) 74 | denom = 1 - 2*c * xy + c**2 * x2 * y2 75 | num = -(1 - 2*c*xy + c*y2) * x + (1 - c*x2) * y 76 | theta = tf.norm( num/denom, axis=-1, ord=2, keepdims=True) 77 | return 2/sqrt_c * atanh( sqrt_c * theta ) 78 | 79 | def expmap0(self, u, c): 80 | """ 81 | Hyperbolic exponential map at zero in the Poincare ball model. 82 | Args: 83 | u: tensor of size B x dimension representing tangent vectors. 84 | c: tensor of size 1 representing the hyperbolic curvature. 85 | Returns: 86 | Tensor of shape B x dimension. 87 | """ 88 | sqrt_c = c ** 0.5 89 | max_num = tf.math.reduce_max(u) 90 | u_norm = tf.clip_by_value( 91 | tf.norm(u, axis=-1, ord=2, keepdims=True), 92 | clip_value_min=self.min_norm, 93 | clip_value_max=max_num, 94 | ) 95 | gamma_1 = tf.math.tanh(sqrt_c * u_norm) * u / (sqrt_c * u_norm) 96 | return gamma_1 97 | 98 | def logmap0(self, p, c): 99 | """ 100 | Hyperbolic logarithmic map at zero in the Poincare ball model. 101 | Args: 102 | p: tensor of size B x dimension representing hyperbolic points. 103 | c: tensor of size 1 representing the hyperbolic curvature. 104 | Returns: 105 | Tensor of shape B x dimension. 106 | """ 107 | sqrt_c = c ** 0.5 108 | p_norm = tf.norm(p, axis=-1, ord=2, keepdims=True) 109 | max_num = tf.math.reduce_max(p_norm) 110 | p_norm = tf.clip_by_value( 111 | p_norm, clip_value_min=self.min_norm, clip_value_max=max_num 112 | ) 113 | scale = 1.0 / sqrt_c * atanh(sqrt_c * p_norm) / p_norm 114 | return scale * p 115 | 116 | def proj(self, x, c): 117 | """ 118 | Safe projection on the manifold for numerical stability. This was mentioned in [1] 119 | Args: 120 | x : Tensor point on the Poincare ball 121 | c : Tensor of size 1 representing the hyperbolic curvature. 122 | Returns: 123 | Projected vector on the manifold 124 | References: 125 | [1] Hyperbolic Neural Networks, NIPS2018 126 | https://arxiv.org/abs/1805.09112 127 | """ 128 | 129 | x_for_norm = tf.norm(x, axis=-1, keepdims=True, ord=2) 130 | max_num = tf.math.reduce_max(x_for_norm) 131 | norm = tf.clip_by_value( 132 | x_for_norm, clip_value_min=self.min_norm, clip_value_max=max_num 133 | ) 134 | maxnorm = (1 - self.eps[x.dtype]) / (c ** 0.5) # tf.math.reduce_max(x) 135 | cond = norm > maxnorm 136 | projected = x / norm * maxnorm 137 | return tf.where(cond, projected, x) 138 | 139 | def mobius_add(self, x, y, c): 140 | """Element-wise Mobius addition. 141 | Args: 142 | x: Tensor of size B x dimension representing hyperbolic points. 143 | y: Tensor of size B x dimension representing hyperbolic points. 144 | c: Tensor of size 1 representing the absolute hyperbolic curvature. 145 | Returns: 146 | Tensor of shape B x dimension representing the element-wise Mobius addition 147 | of x and y. 148 | """ 149 | cx2 = c * tf.reduce_sum(x * x, axis=-1, keepdims=True) 150 | cy2 = c * tf.reduce_sum(y * y, axis=-1, keepdims=True) 151 | cxy = c * tf.reduce_sum(x * y, axis=-1, keepdims=True) 152 | num = (1 + 2 * cxy + cy2) * x + (1 - cx2) * y 153 | denom = 1 + 2 * cxy + cx2 * cy2 154 | return self.proj(num / tf.maximum(denom, self.min_norm), c) 155 | 156 | 157 | def hyp_act(self, act, x, c_in, c_out): 158 | """Apply an activation function to a tensor in the hyperbolic space""" 159 | xt = act(self.logmap0(x, c=c_in)) 160 | return self.proj(self.expmap0(xt, c=c_out), c=c_out) 161 | 162 | def single_query_attn_scores(self, key, query, c): 163 | """ 164 | Arguments: 165 | key: Hyperbolic key with shape (batch, seq, hidden_dim) 166 | query: Hyperbolic query with shape (batch, hidden_dim) 167 | Returns: 168 | Scores as scalars in R with shape (batch,seq,1) 169 | """ 170 | euclid_key = self.logmap0(key, c) 171 | euclid_query = self.logmap0(query, c) 172 | scores = tf.matmul(euclid_query.unsqueeze(-1), euclid_key, transpose_b=True) 173 | denom = tf.norm(euclid_key, keepdims=True, axis=-1) 174 | scores = (1. / denom) * scores 175 | return scores 176 | 177 | -------------------------------------------------------------------------------- /hyperlib/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nalexai/hyperlib/b7c47e121cc56b4276a387a6d8c1d05ae2445805/hyperlib/models/__init__.py -------------------------------------------------------------------------------- /hyperlib/models/pehr.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import tensorflow as tf 4 | from tensorflow import keras 5 | 6 | from hyperlib.manifold.lorentz import Lorentz 7 | from hyperlib.manifold.poincare import Poincare 8 | from hyperlib.loss.constrastive_loss import contrastive_loss 9 | 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class HierarchicalEmbeddings(tf.keras.Model): 15 | """ 16 | Hierarchical Embeddings model from Poincaré Embeddings for 17 | Learning Hierarchical Representations by Nickel and Keila 18 | 19 | Please find an example of how to use this model in hyperlib/examples/wordnet_embedding.py 20 | """ 21 | 22 | def __init__(self, vocab, embedding_dim=2, manifold=Poincare, c=1.0, clip_value=0.9): 23 | super().__init__() 24 | 25 | initializer=keras.initializers.RandomUniform(minval=-0.001, maxval=0.001, seed=None) 26 | self.string_lookup = keras.layers.StringLookup(vocabulary=vocab, name="string_lookup") 27 | self.embedding = keras.layers.Embedding( 28 | len(vocab)+1, 29 | embedding_dim, 30 | embeddings_initializer=initializer, 31 | name="embeddings", 32 | ) 33 | self.vocab = vocab 34 | self.manifold = manifold() 35 | self.c = c 36 | self.clip_value = clip_value 37 | 38 | def call(self, inputs): 39 | indices = self.string_lookup(inputs) 40 | return self.embedding(indices) 41 | 42 | def get_embeddings(self): 43 | embeddings = self.embedding (tf.constant([i for i in range(len(self.vocab))])) 44 | embeddings_copy = tf.identity(embeddings) 45 | embeddings_hyperbolic = self.manifold.expmap0(embeddings_copy, c=self.c) 46 | return embeddings_hyperbolic 47 | 48 | def get_vocabulary(self): 49 | return self.vocab 50 | 51 | @staticmethod 52 | def get_model(vocab, embedding_dim=2): 53 | embedding_dim=2 54 | initializer=keras.initializers.RandomUniform(minval=-0.001, maxval=0.001, seed=None) 55 | string_lookup_layer = keras.layers.StringLookup(vocabulary=vocab) 56 | 57 | emb_layer = keras.layers.Embedding( 58 | len(vocab)+1, 59 | embedding_dim, 60 | embeddings_initializer=initializer, 61 | name="embeddings", 62 | ) 63 | 64 | model = keras.Sequential([string_lookup_layer, emb_layer]) 65 | return model 66 | 67 | def fit(self, train_dataset, optimizer, epochs=100): 68 | 69 | for epoch in range(epochs): 70 | log.info("Epoch %d" % (epoch,)) 71 | for step, (x_batch_train, y_batch_train) in enumerate(train_dataset): 72 | with tf.GradientTape() as tape: 73 | pos_embs = self.embedding(self.string_lookup(x_batch_train)) 74 | neg_embs = self.embedding(self.string_lookup(y_batch_train)) 75 | loss_value = contrastive_loss( 76 | pos_embs, neg_embs, self.manifold, c=self.c, clip_value=self.clip_value) 77 | 78 | grads = tape.gradient(loss_value, self.embedding.trainable_weights) 79 | optimizer.apply_gradients(zip(grads, self.embedding.trainable_weights)) 80 | 81 | if step % 100 == 0: 82 | log.info("Training loss (for one batch) at step %d: %.4f" 83 | % (step, float(loss_value))) 84 | -------------------------------------------------------------------------------- /hyperlib/nn/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nalexai/hyperlib/b7c47e121cc56b4276a387a6d8c1d05ae2445805/hyperlib/nn/__init__.py -------------------------------------------------------------------------------- /hyperlib/nn/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nalexai/hyperlib/b7c47e121cc56b4276a387a6d8c1d05ae2445805/hyperlib/nn/layers/__init__.py -------------------------------------------------------------------------------- /hyperlib/nn/layers/dense_attention.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow import keras 3 | 4 | class HypLuongAttention(keras.layers.Attention): 5 | """Dot-product attention layer, a.k.a. Luong-style attention. 6 | Inputs are `query` tensor of shape `[batch_size, Tq, dim]`, `value` tensor of 7 | shape `[batch_size, Tv, dim]` and `key` tensor of shape 8 | 9 | Call Args: 10 | inputs: List of the following tensors: 11 | * query: Query `Tensor` of shape `[batch_size, Tq, dim]`. 12 | * value: Value `Tensor` of shape `[batch_size, Tv, dim]`. 13 | Output: 14 | Attention outputs of shape `[batch_size, Tq, dim]`. 15 | # ... 16 | ``` 17 | """ 18 | 19 | def __init__(self, manifold, c=1, use_scale=False, hyperbolic_input=False, **kwargs): 20 | super(HypLuongAttention, self).__init__(**kwargs) 21 | self.use_scale = use_scale 22 | self.hyperbolc_input = hyperbolic_input 23 | self.manifold = manifold 24 | self.c = tf.Variable([c], dtype="float64") 25 | 26 | def build(self, input_shape): 27 | """Creates scale variable if use_scale==True.""" 28 | if self.use_scale: 29 | self.scale = self.add_weight( 30 | name='scale', 31 | shape=(), 32 | initializer='ones', 33 | dtype=self.dtype, 34 | trainable=True) 35 | else: 36 | self.scale = None 37 | super(HypLuongAttention, self).build(input_shape) 38 | 39 | def _calculate_scores(self, query, key): 40 | """Calculates attention scores as a query-key dot product. 41 | Args: 42 | query: Query tensor of shape `[batch_size, Tq, dim]`. 43 | key: Key tensor of shape `[batch_size, Tv, dim]`. 44 | Returns: 45 | Tensor of shape `[batch_size, Tq, Tv]`. 46 | """ 47 | if self.hyperbolc_input: 48 | scores = self.manifold.single_query_attn_scores(query, key, self.c) 49 | else: 50 | scores = tf.linalg.matmul(query, key, transpose_b=True) 51 | 52 | if self.scale is not None: 53 | scores *= self.scale 54 | 55 | return scores 56 | 57 | def get_config(self): 58 | config = {'use_scale': self.use_scale} 59 | base_config = super(keras.layers.Attention, self).get_config() 60 | return dict(list(base_config.items()) + list(config.items())) 61 | -------------------------------------------------------------------------------- /hyperlib/nn/layers/lin_hyp.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow import keras 3 | 4 | 5 | class LinearHyperbolic(keras.layers.Layer): 6 | """ 7 | Implementation of a hyperbolic linear layer for a neural network, that inherits from the keras Layer class 8 | """ 9 | 10 | def __init__(self, units, manifold, c, activation=None, use_bias=True): 11 | super().__init__() 12 | self.units = units 13 | self.c = tf.Variable([c], dtype="float64") 14 | self.manifold = manifold 15 | self.activation = keras.activations.get(activation) 16 | self.use_bias = use_bias 17 | 18 | def build(self, batch_input_shape): 19 | w_init = tf.random_normal_initializer() 20 | self.kernel = tf.Variable( 21 | initial_value=w_init(shape=(batch_input_shape[-1], self.units), dtype="float64"), dtype="float64", trainable=True, 22 | ) 23 | 24 | if self.use_bias: 25 | self.bias = self.add_weight( 26 | name="bias", 27 | shape=(self.units), 28 | initializer="zeros", 29 | dtype=tf.float64, 30 | trainable=True, 31 | ) 32 | 33 | super().build(batch_input_shape) # must be at the end 34 | 35 | def call(self, inputs): 36 | """ 37 | Called during forward pass of a neural network. Uses hyperbolic matrix multiplication 38 | """ 39 | # TODO: remove casting and instead recommend setting default tfd values to float64 40 | inputs = tf.cast(inputs, tf.float64) 41 | mv = self.manifold.mobius_matvec(self.kernel, inputs, self.c) 42 | res = self.manifold.proj(mv, c=self.c) 43 | 44 | if self.use_bias: 45 | hyp_bias = self.manifold.expmap0(self.bias, c=self.c) 46 | hyp_bias = self.manifold.proj(hyp_bias, c=self.c) 47 | res = self.manifold.mobius_add(res, hyp_bias, c=self.c) 48 | res = self.manifold.proj(res, c=self.c) 49 | 50 | return self.activation(res) 51 | 52 | def get_config(self): 53 | base_config = super().get_config() 54 | return { 55 | **base_config, 56 | "units": self.units, 57 | "activation": keras.activations.serialize(self.activation), 58 | "manifold": self.manifold, 59 | "curvature": self.c 60 | } 61 | -------------------------------------------------------------------------------- /hyperlib/nn/optimizers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nalexai/hyperlib/b7c47e121cc56b4276a387a6d8c1d05ae2445805/hyperlib/nn/optimizers/__init__.py -------------------------------------------------------------------------------- /hyperlib/nn/optimizers/rsgd.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow import keras 3 | 4 | class RSGD(keras.optimizers.Optimizer): 5 | 6 | """ 7 | Implmentation of a Riemannian Stochastic Gradient Descent. This class inherits form the keras Optimizer class. 8 | """ 9 | 10 | def __init__(self, learning_rate=0.01, name="RSGOptimizer", **kwargs): 11 | """Call super().__init__() and use _set_hyper() to store hyperparameters""" 12 | super().__init__(name, **kwargs) 13 | self._set_hyper( 14 | "learning_rate", kwargs.get("lr", tf.cast(learning_rate, tf.float64)) 15 | ) # handle lr=learning_rate 16 | self._is_first = True 17 | 18 | def _create_slots(self, var_list): 19 | """ 20 | For each model variable, create the optimizer variable associated with it 21 | """ 22 | for var in var_list: 23 | self.add_slot(var, "pv") # previous variable i.e. weight or bias 24 | for var in var_list: 25 | self.add_slot(var, "pg") # previous gradient 26 | 27 | @tf.function 28 | def _resource_apply_dense(self, grad, var): 29 | """ 30 | Update the slots and perform one optimization step for one model variable 31 | """ 32 | r_grad = self.rgrad(var, grad) 33 | r_grad = tf.math.multiply(r_grad, -self.lr) 34 | new_var_m = self.expm(var, r_grad) 35 | 36 | # slots aren't currently used - they store previous weights and gradients 37 | pv_var = self.get_slot(var, "pv") 38 | pg_var = self.get_slot(var, "pg") 39 | pv_var.assign(var) 40 | pg_var.assign(grad) 41 | # assign new weights 42 | 43 | var.assign(new_var_m) 44 | 45 | def _resource_apply_sparse(self, grad, var): 46 | raise NotImplementedError("Not implemented") 47 | 48 | def rgrad(self, var, grads): 49 | """ 50 | Transforms the gradients to Riemannian space 51 | """ 52 | vars_sqnorm = tf.math.reduce_sum(var ** 2, axis=-1, keepdims=True) 53 | grads = grads * tf.broadcast_to(((1 - vars_sqnorm) ** 2 / 4), tf.shape(grads)) 54 | return grads 55 | 56 | def expm(self, p, d_p, normalize=False, lr=None, out=None): 57 | """ 58 | Maps the variable values 59 | """ 60 | if lr is not None: 61 | d_p = tf.math.multiply(d_p, -lr) 62 | if out is None: 63 | out = p 64 | p = tf.math.add(p, d_p) 65 | if normalize: 66 | self.normalize(p) 67 | return p 68 | 69 | def normalize(self, u): 70 | d = u.shape[-1] 71 | if self.max_norm: 72 | u = tf.reshape(u, [-1, d]) 73 | u.renorm_(2, 0, self.max_norm) 74 | return u 75 | 76 | def get_config(self): 77 | base_config = super().get_config() 78 | return { 79 | **base_config, 80 | "learning_rate": self._serialize_hyperparameter("learning_rate"), 81 | } 82 | -------------------------------------------------------------------------------- /hyperlib/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nalexai/hyperlib/b7c47e121cc56b4276a387a6d8c1d05ae2445805/hyperlib/utils/__init__.py -------------------------------------------------------------------------------- /hyperlib/utils/functional.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | """ 4 | Tensorflow Math functions with clipping as required for hyperbolic functions. 5 | """ 6 | 7 | def cosh(x, clamp=15): 8 | return tf.math.cosh(tf.clip_by_value(x, clip_value_min=-clamp, clip_value_max=clamp)) 9 | 10 | def sinh(x, clamp=15): 11 | return tf.math.sinh(tf.clip_by_value(x, clip_value_min=-clamp, clip_value_max=clamp)) 12 | 13 | def tanh(x, clamp=15): 14 | return tf.math.tanh(tf.clip_by_value(x, clip_value_min=-clamp, clip_value_max=clamp)) 15 | 16 | def arcosh(x): 17 | x = tf.clip_by_value(x, clip_value_min=1+1e-15, clip_value_max=tf.reduce_max(x)) 18 | return tf.math.acosh(x) 19 | 20 | def asinh(x): 21 | return tf.math.asinh(x) 22 | 23 | def atanh(x): 24 | x = tf.clip_by_value(x, clip_value_min=-1 + 1e-15, clip_value_max=1 - 1e-15) 25 | return tf.math.atanh(x) 26 | 27 | @tf.custom_gradient 28 | def custom_artanh_cg(x): 29 | x = tf.clip_by_value(x, clip_value_min=-1 + 1e-15, clip_value_max=1 - 1e-15) 30 | z = tf.cast(x, tf.float64, name=None) 31 | temp = tf.math.subtract(tf.math.log(1 + z), (tf.math.log(1 - z))) 32 | def artanh_grad(grad): 33 | return grad/ (1 - x ** 2) 34 | 35 | return tf.cast(tf.math.multiply(temp, 0.5), x.dtype, name=None), artanh_grad 36 | -------------------------------------------------------------------------------- /hyperlib/utils/graph.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | import scipy.sparse as sp 4 | 5 | def to_networkx(edge_weights): 6 | ''' Convert edge weight dict to networkx weighted graph''' 7 | G = nx.Graph() 8 | for e in edge_weights: 9 | G.add_edge(e[0], e[1], weight=edge_weights[e]) 10 | return G 11 | 12 | def to_sparse(edge_weights): 13 | ''' Convert edge weight dict to compressed adjacency matrix, 14 | stored in a scipy.sparse.dok_matrix 15 | ''' 16 | N = max([e[1] for e in edge_weights])+1 17 | mat = sp.dok_matrix((N,N), dtype=np.float64) 18 | for e in edge_weights: 19 | mat[e[0],e[1]] = edge_weights[e] 20 | mat[e[1],e[0]] = edge_weights[e] 21 | return mat 22 | 23 | def binary_tree(depth): 24 | return n_ary_tree(2, depth) 25 | 26 | def trinary_tree(depth): 27 | return n_ary_tree(3, depth) 28 | 29 | def n_ary_tree(n, depth): 30 | assert n >= 2 31 | assert depth >= 0 32 | T = nx.Graph() 33 | for d in range(depth): 34 | for i in range(n**d,n**(d+1)): 35 | for j in range(n): 36 | T.add_edge(i, n*i+j) 37 | return nx.relabel.convert_node_labels_to_integers(T) 38 | -------------------------------------------------------------------------------- /hyperlib/utils/linalg.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This package contains linear algebra utils. 3 | Functions with suffix "_mp" operate on mpmath matrices. 4 | ''' 5 | import mpmath as mpm 6 | import numpy as np 7 | from math import sqrt 8 | 9 | 10 | def rotate_3D(x, y): 11 | '''Returns 3D rotation matrix that sends x to the direction parallel to y''' 12 | xnorm = np.linalg.norm(x) 13 | ynorm = np.linalg.norm(y) 14 | cos = np.dot(x,y)/(xnorm*ynorm) 15 | sin = sqrt(1. - cos**2) 16 | K = (np.outer(y,x) - np.outer(x,y))/(xnorm*ynorm*sin) 17 | return np.eye(3) + sin*K + (1.-cos)*(K@K) 18 | 19 | 20 | def cross_mp(x, y): 21 | return mpm.matrix([ 22 | x[1]*y[2]-x[2]*y[1], 23 | x[2]*y[0]-x[0]*y[2], 24 | x[0]*y[1]-x[1]*y[0], 25 | ]) 26 | 27 | 28 | def rotate_3D_mp(x, y): 29 | xnorm = mpm.norm(x) 30 | ynorm = mpm.norm(y) 31 | cos = mpm.fdot(x,y)/(xnorm*ynorm) 32 | sin = mpm.sqrt(1.-cos**2) 33 | K = (y.T*x-x.T*y)/(xnorm*ynorm*sin) 34 | return mpm.eye(3) + sin*K + (1.-cos)*(K*K) 35 | 36 | 37 | def rotate_mp(pts, x, y): 38 | out = mpm.zeros(pts.rows, pts.cols) 39 | v = x/mpm.norm(x) 40 | cos = mpm.fdot(x,y) / (mpm.norm(x), mpm.norm(y)) 41 | sin = mpm.sqrt(1.-cos**2) 42 | u = y - mpm.fdot(v, y) * v 43 | 44 | mat = mpm.eye(x.cols) - u.T * u - v.T * v \ 45 | + cos * u.T * u - sin * v.T * u + sin * u.T * v + cos * v.T * v 46 | return mat 47 | 48 | 49 | def from_numpy(M): 50 | '''Convert 2D numpy array to mpmath matrix''' 51 | N = mpm.matrix(*M.shape) 52 | n, m = M.shape 53 | for i in range(n): 54 | for j in range(m): 55 | N[i,j] = M[i,j] 56 | return N 57 | 58 | 59 | def to_numpy(N): 60 | '''Convert 2D mpmath matrix to numpy array''' 61 | M = np.zeros((N.rows,N.cols), dtype=np.float64) 62 | for i in range(N.rows): 63 | M[i,:] = N[i,:] 64 | return M 65 | -------------------------------------------------------------------------------- /hyperlib/utils/multiprecision.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This package contains functions for high precision calculations of hyperbolic functions. 3 | Note that a hyperbolic distance of d requires O(d) bits of precision to compute. 4 | ''' 5 | import mpmath as mpm 6 | import numpy as np 7 | 8 | def poincare_dist(x, y, c=1.0, precision=None): 9 | ''' 10 | The hyperbolic distance between points in the Poincare model with curvature -1/c 11 | Args: 12 | x, y: size 1xD mpmath matrix representing point in the D-dimensional ball |x| < 1 13 | precision (int): bits of precision to use 14 | Returns: 15 | mpmath float object. Can be converted back to regular float 16 | ''' 17 | if precision is not None: 18 | mpm.mp.dps = precision 19 | x2 = mpm.fdot(x,x) 20 | y2 = mpm.fdot(y,y) 21 | xy = mpm.fdot(x,y) 22 | sqrt_c = mpm.sqrt(c) 23 | denom = 1 - 2*c*xy + c**2 * x2*y2 24 | norm = mpm.norm( ( -(1-2*c*xy + c*y2)*x + (1.-c*x2)*y ) /denom ) 25 | return 2/sqrt_c * mpm.atanh( sqrt_c * norm ) 26 | 27 | def poincare_dist0(x, c=1.0, precision=None): 28 | ''' Distance from 0 to x in the Poincare model with curvature -1/c''' 29 | if precision is not None: 30 | mpm.mp.dps = precision 31 | x_norm = mpm.norm(x) 32 | sqrt_c = mpm.sqrt(c) 33 | return 2/sqrt_c * mpm.atanh( x_norm / sqrt_c) 34 | 35 | def poincare_metric(X, precision=None): 36 | ''' Calculate the distance matrix of points in the Poincare model with curvature -1/c 37 | Args: 38 | X (ndarray): N x D matrix representing N points in the D-dimensional ball |x|< 1 39 | precision (int): bits of precision to use 40 | Returns: 41 | distance matrix in compressed 1D form ( see scipy.squareform ) 42 | ''' 43 | if precision is not None: 44 | mpm.mp.dps = precision 45 | N = X.shape[0] 46 | out = np.zeros(shape=(N*(N-1)//2), dtype=np.float64) 47 | for i in range(N): 48 | idx = N*i-((i+1)*(i+2))//2 49 | for j in range(i+1,N): 50 | out[idx+j] = poincare_dist(X[i],X[j],precision=precision) 51 | return out 52 | 53 | def poincare_reflect(a, x, c=1.0, precision=None): 54 | ''' 55 | Spherical inversion (or "Poincare reflection") of a point x about a sphere 56 | with center at point "a", and radius = 1/c. 57 | ''' 58 | if precision is not None: 59 | mpm.mp.dps = precision 60 | a2 = mpm.fdot(a,a) 61 | x2 = mpm.fdot(x,x) 62 | xa2 = x2 + a2 - 2*mpm.fdot(x,a) 63 | r2 = a2 - 1./c 64 | scal = mpm.fdiv(r2, xa2) 65 | return scal*(x-a) + a 66 | 67 | def poincare_reflect0(z, x, c=1.0, precision=None): 68 | ''' 69 | Spherical inversion (or "Poincare reflection") of a point x 70 | such that point z is mapped to the origin. 71 | ''' 72 | if precision is not None: 73 | mpm.mp.dps = precision 74 | # the reflection is poincare_reflect(a, x) where 75 | # a = c * z / |z|**2 76 | z2 = mpm.fdot(z,z) 77 | zscal = c / z2 78 | x2 = mpm.fdot(x,x) 79 | a2 = c* zscal 80 | r2 = a2 - 1./c 81 | xa2 = x2 + a2 - 2*zscal*mpm.fdot(z,x) 82 | scal = mpm.fdiv(r2, xa2) 83 | return scal*( x - zscal * z) + zscal * z 84 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=42", 4 | "wheel", 5 | "networkx", 6 | "pybind11>=2.7.0", 7 | ] 8 | 9 | build-backend = "setuptools.build_meta" 10 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages 2 | from distutils.core import setup, Extension 3 | import pybind11 4 | from pybind11.setup_helpers import Pybind11Extension, build_ext 5 | from glob import glob 6 | import sys 7 | 8 | __version__ = "0.0.6" 9 | with open("README.md") as f: 10 | readme = f.read() 11 | 12 | ext_modules = [ 13 | Pybind11Extension("__hyperlib_embedding", sorted(glob("hyperlib/embedding/src/*.cc")), 14 | include_dirs = ["hyperlib/embedding/include", pybind11.get_include()], 15 | cxx_std=11, 16 | ) 17 | ] 18 | 19 | setup( 20 | name="hyperlib", 21 | version=__version__, 22 | packages = find_packages(), 23 | setup_requires = ["pip>=21"], 24 | install_requires = [ 25 | "numpy>=1.19.3", 26 | "tensorflow>=2.0.0", 27 | "scipy>=1.7.0", 28 | "mpmath", 29 | "networkx", 30 | "pybind11>=2.7.0", 31 | "gmpy2", 32 | ], 33 | ext_modules=ext_modules, 34 | author="Nalex.ai", 35 | author_email="info@nalexai.com", 36 | description="Library that contains implementations of machine learning components in the hyperbolic space", 37 | long_description=readme, 38 | long_description_content_type="text/markdown", 39 | project_urls = { 40 | "Source" : "https://github.com/nalexai/hyperlib", 41 | "Issues" : "https://github.com/nalexai/hyperlib/issues" 42 | }, 43 | license_files = "LICENSE", 44 | url = "https://github.com/nalexai/hyperlib", 45 | classifiers = [ 46 | "Programming Language :: Python :: 3", 47 | "License :: OSI Approved :: MIT License", 48 | "Operating System :: OS Independent", 49 | ], 50 | ) 51 | -------------------------------------------------------------------------------- /tests/test_embedding.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from random import randint 3 | from scipy.sparse.csgraph import shortest_path 4 | from scipy.spatial.distance import squareform 5 | import networkx as nx 6 | import numpy as np 7 | 8 | from hyperlib.embedding.treerep import treerep 9 | from hyperlib.embedding.metric import avg_distortion, delta_rel 10 | from hyperlib.utils.multiprecision import poincare_metric 11 | from hyperlib.utils.graph import * 12 | 13 | 14 | @pytest.fixture 15 | def sarich_data(): 16 | return np.array([ 17 | 32., 48., 51., 50., 48., 98., 148., 18 | 26., 34., 29., 33., 84., 136., 19 | 42., 44., 44., 92., 152., 20 | 44., 38., 86., 142., 21 | 42., 89., 142., 22 | 90., 142., 23 | 148. 24 | ]) 25 | 26 | def random_data(N,dim=2): 27 | # random points in the poincare ball 28 | rs = 1.-np.random.exponential(scale=1e-4, size=(N,)) 29 | pts = np.random.normal(size = (N,dim)) 30 | pts /= np.linalg.norm(pts, axis=1, keepdims=True) 31 | pts *= rs[:,None] 32 | return pts 33 | 34 | def test_treerep_sarich(sarich_data): 35 | stats = [] 36 | for _ in range(10): 37 | T = treerep(sarich_data) 38 | G = to_networkx(T) 39 | assert nx.algorithms.tree.recognition.is_tree(G) 40 | adj_mat = to_sparse(T) 41 | tree_metric = shortest_path(adj_mat, directed=False) 42 | distortion = avg_distortion(squareform(tree_metric[:8,:8]), sarich_data) 43 | stats.append(distortion) 44 | best = min(stats) 45 | print(f"Sarich Data\n\tTreerep Avg Distortion ------------ {best:.5f}") 46 | assert best < 0.1 47 | 48 | def test_treerep_rand(): 49 | pts = random_data(128) 50 | metric = poincare_metric(pts) 51 | stats = [] 52 | for _ in range(10): 53 | T = treerep(metric) 54 | G = to_networkx(T) 55 | assert nx.algorithms.tree.recognition.is_tree(G) 56 | del G 57 | adj_mat = to_sparse(T) 58 | tree_metric = shortest_path(adj_mat, directed=False) 59 | dstn = avg_distortion(squareform(tree_metric[:128,:128], checks=False), 60 | metric) 61 | stats.append(dstn) 62 | best = min(stats) 63 | print(f"Random Data\n\tTreerep Avg Distortion ------------ {best:.5f}") 64 | 65 | def test_delta_hyperbolic(): 66 | pts = random_data(128) 67 | metric = squareform(poincare_metric(pts), checks=False) 68 | delts = [] 69 | for _ in range(10): 70 | base = randint(0,127) 71 | delts.append(delta_rel(metric,base)) 72 | assert np.mean(pts) < 0.5 73 | -------------------------------------------------------------------------------- /tests/test_lorentz.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import tensorflow as tf 3 | from hyperlib.manifold.lorentz import Lorentz 4 | 5 | def is_in_tangent(v, x, tol=1e-5): 6 | dot = Lorentz().minkowski_dot(v, x) 7 | return tf.reduce_max(dot) < tol 8 | 9 | @pytest.mark.parametrize("c", [0.5, 1.0, 1.5, 2.0, 4.0]) 10 | def test_proj_on_manifold(c): 11 | tol = 1e-5 12 | x = tf.constant([ 13 | [0.5,1.0,-2.0,10.0], 14 | [1.0,-2.0,3.0,4.0], 15 | [2.4,-1.5,0.3,0.2]]) 16 | 17 | M = Lorentz() 18 | y = M.proj(x, c) 19 | dot = M.minkowski_dot(y, y) 20 | targ = -tf.ones_like(dot) 21 | assert tf.reduce_max(tf.abs(dot - targ)) 22 | 23 | @pytest.mark.parametrize("c", [0.5, 1.0, 1.5, 2.0, 4.0]) 24 | def test_proj_tan_orthogonal(c): 25 | x = tf.constant([[1.0,-3.0,0.2,-0.3],[0.0,0.3,-30.,1.]]) 26 | M = Lorentz() 27 | y = M.proj(x, c) 28 | 29 | v = tf.constant([[4.0,4.0,-1.0,0.0],[20.,20.,0.0,-1.]]) 30 | p = M.proj_tan(v, y, c) 31 | 32 | assert is_in_tangent(p, y) 33 | 34 | @pytest.mark.parametrize("c", [0.5, 1.0, 1.5, 2.0, 4.0]) 35 | def test_proj_tan_orthogonal_0(c): 36 | K = 1./c 37 | zero = tf.constant([[K**0.5,0.,0.,0.],[K**0.5,0.,0.,0.]]) 38 | v = tf.constant([[4.0,2.0,-1.0,-0.4],[20.,20.,0.0,-1.]]) 39 | v = Lorentz().proj_tan0(v, c) 40 | assert is_in_tangent(v, zero) 41 | 42 | def test_exp_log_inv(): 43 | tol = 1e-4 44 | c = 1.0 45 | x = tf.constant([[4.690416,1.,-2.,4.],[5.477226,-2.,3.,4.]]) 46 | y = tf.constant([[1.8384776,-1.5,0.3,0.2],[3.5,3.,-1.5,0.]]) 47 | 48 | M = Lorentz() 49 | v = M.logmap(y, x, c) 50 | assert is_in_tangent(v, x) 51 | 52 | expv = M.expmap(v, x, c) 53 | assert tf.reduce_max(tf.abs(expv - y)) < tol 54 | 55 | def test_exp_log_inv_0(): 56 | tol = 1e-4 57 | c = 2.0 58 | 59 | x = tf.constant([[1.6970563,-1.5,0.3,0.2],[3.4278274,3.,-1.5,0.]]) 60 | M = Lorentz() 61 | v = M.logmap0(x, c) 62 | expv = M.expmap0(v, c) 63 | assert tf.reduce_max(tf.abs(expv - x)) < tol 64 | 65 | def test_parallel_transport(): 66 | tol = 1e-4 67 | c = 1.0 68 | x = tf.constant([[4.690416,1.,-2.,4.],[5.477226,-2.,3.,4.]]) 69 | y = tf.constant([[1.8384776,-1.5,0.3,0.2],[3.5,3.,-1.5,0.]]) 70 | 71 | M = Lorentz() 72 | v = M.proj_tan( 73 | tf.constant([[-3.0,-2.0,1.0,0.0],[0.5,-3.4,4.0,1.0]]), 74 | x, c) 75 | assert is_in_tangent(v, x, tol=tol) 76 | 77 | tr_v = M.ptransp(x, y, v, c) 78 | assert is_in_tangent(tr_v, y, tol=tol) 79 | 80 | def test_parallel_transport_0(): 81 | tol = 1e-4 82 | c = 1.0 83 | zero = tf.constant([[1.,0.,0,0.],[1.,0.,0.,0.]]) 84 | x = tf.constant([[4.690416,1.,-2.,4.],[5.477226,-2.,3.,4.]]) 85 | 86 | M = Lorentz() 87 | v = M.proj_tan0(tf.constant([[-3.0,-2.0,1.0,0.0],[0.5,-3.4,4.0,1.0]]), c) 88 | M = Lorentz() 89 | assert is_in_tangent(v, zero, tol=tol) 90 | 91 | tr_v = M.ptransp0(x, v, c) 92 | assert is_in_tangent(tr_v, x, tol=tol) 93 | -------------------------------------------------------------------------------- /tests/test_multiprecision.py: -------------------------------------------------------------------------------- 1 | import mpmath as mpm 2 | 3 | from hyperlib.utils.multiprecision import * 4 | 5 | TOL = 1e-25 6 | mpm.mp.dps = 30 7 | 8 | def test_poincare_dist(): 9 | x = mpm.matrix([[0.,0.,-0.4]]) 10 | y = mpm.matrix([[0.0,0.0,0.1]]) 11 | o = mpm.zeros(1,3) 12 | 13 | assert poincare_dist(o,y)-poincare_dist0(y) < TOL 14 | assert poincare_dist(o,x)-poincare_dist0(x) < TOL 15 | assert poincare_dist(o,x)+poincare_dist(o,y)-poincare_dist(x,y) < TOL 16 | 17 | def test_poincare_reflect0(): 18 | z = mpm.matrix([[0.,0.,0.5]]) 19 | x = mpm.matrix([[0.1,-0.2,0.1]]) 20 | 21 | assert mpm.norm( poincare_reflect0(z, z) ) < TOL 22 | 23 | y = poincare_reflect0(z,x) 24 | assert mpm.norm( poincare_reflect0(z,y) - x ) < TOL 25 | 26 | def test_poincare_reflect(): 27 | x = mpm.matrix([[0.3,-0.3,0.0]]) 28 | a = mpm.matrix([[0.5,0.,0.0]]) 29 | y = poincare_reflect(a, x) 30 | 31 | R = mpm.norm(a) 32 | r1 = mpm.norm(x-a) 33 | r2 = mpm.norm(y-a) 34 | assert R**2 - r1*r2 < TOL 35 | -------------------------------------------------------------------------------- /tests/test_poincare.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from hyperlib.manifold import poincare 3 | from hyperlib.utils import functional as F 4 | from hyperlib.nn.layers import lin_hyp, dense_attention 5 | from hyperlib.nn.optimizers import rsgd 6 | import pytest 7 | 8 | class TestClass: 9 | @classmethod 10 | def setup_class(self): 11 | self.test_tensor_shape_2_2_a = tf.constant( 12 | [[1.0, 2.0], [3.0, 4.0]], dtype=tf.float64 13 | ) 14 | self.test_tensor_shape_2_2_b = tf.constant( 15 | [[0.5, 0.5], [0.5, 0.5]], dtype=tf.float64 16 | ) 17 | self.test_tensor_shape_2_1a = tf.constant([[1.0], [2.0]], dtype=tf.float64) 18 | self.test_tensor_shape_2_1b = tf.constant([[0.5], [0.5]], dtype=tf.float64) 19 | self.poincare_manifold = poincare.Poincare() 20 | self.curvature_tensor = tf.Variable([1], dtype="float64") 21 | 22 | def test_math_functions(self, x=tf.constant([1.0, 2.0, 3.0])): 23 | return F.cosh(x) 24 | 25 | def test_mobius_matvec_(self): 26 | result = self.poincare_manifold.mobius_matvec( 27 | self.test_tensor_shape_2_2_a, 28 | self.test_tensor_shape_2_2_b, c=self.curvature_tensor 29 | ) 30 | with pytest.raises(tf.errors.InvalidArgumentError): 31 | self.poincare_manifold.mobius_matvec( 32 | self.test_tensor_shape_2_2_a, 33 | self.test_tensor_shape_2_1a, c=self.curvature_tensor 34 | ) 35 | 36 | def test_expmap0(self): 37 | c = tf.Variable([1], dtype="float64") 38 | result = self.poincare_manifold.expmap0( 39 | self.test_tensor_shape_2_2_a, c=self.curvature_tensor 40 | ) 41 | 42 | @pytest.mark.skip(reason="working on a test for this") 43 | def test_logmap0(self): 44 | result = self.poincare_manifold.logmap0( 45 | self.test_tensor_shape_2_2_b, c=self.curvature_tensor 46 | ) 47 | 48 | def test_proj(self): 49 | result = self.poincare_manifold.proj(self.test_tensor_shape_2_2_a, c=self.curvature_tensor) 50 | 51 | def test_poincare_functions(self): 52 | manifold = poincare.Poincare() 53 | assert manifold.name == "PoincareBall" 54 | assert manifold.min_norm == 1e-15 55 | 56 | def test_create_layer(self, units=32): 57 | hyp_layer = lin_hyp.LinearHyperbolic( 58 | units, self.poincare_manifold, 1.0 59 | ) 60 | assert hyp_layer.units == units 61 | assert hyp_layer.manifold == self.poincare_manifold 62 | 63 | def test_attention_layer(self): 64 | sample_hidden = tf.Variable(tf.random.uniform([10, 1], 0, 100, dtype=tf.float64, seed=0)) 65 | sample_output = tf.Variable(tf.random.uniform([10, 1], 0, 100, dtype=tf.float64, seed=0)) 66 | 67 | attention_layer = dense_attention.HypLuongAttention(manifold=self.poincare_manifold, c=1, use_scale=False, 68 | hyperbolic_input=False) 69 | query_value_attention_seq = attention_layer([sample_hidden, sample_output]) 70 | 71 | assert query_value_attention_seq.shape == sample_output.shape 72 | assert attention_layer.manifold == self.poincare_manifold 73 | 74 | def test_layer_training(self, units=32): 75 | x_input = tf.zeros([units, 1]) 76 | hyp_layer = lin_hyp.LinearHyperbolic( 77 | units, self.poincare_manifold, 1.0 78 | ) 79 | output = hyp_layer(x_input) 80 | 81 | def test_layer_training_with_bias(self, units=32): 82 | x_input = tf.zeros([units, 1]) 83 | hyp_layer = lin_hyp.LinearHyperbolic( 84 | units, self.poincare_manifold, 1.0, use_bias=True 85 | ) 86 | output = hyp_layer(x_input) 87 | 88 | def test_create_optimizer(self): 89 | opt = rsgd.RSGD() 90 | -------------------------------------------------------------------------------- /tests/test_sarkar.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from random import randint 3 | import networkx as nx 4 | 5 | from hyperlib.utils.graph import binary_tree, trinary_tree 6 | from hyperlib.utils.multiprecision import poincare_metric 7 | from hyperlib.embedding.sarkar import * 8 | 9 | def test_sarkar_2D_unweighted(): 10 | T = binary_tree(4) 11 | M = sarkar_embedding(T, 0, weighted=False, precision=30) 12 | n = nx.number_of_nodes(T) 13 | assert M.rows == n 14 | assert M.cols == 2 15 | assert all([ mpm.norm(M[i,:]) < 1 for i in range(n)]) 16 | 17 | def test_sarkar_3D_unweighted(): 18 | T = trinary_tree(3) 19 | M = sarkar_embedding_3D(T, 0, weighted=False, tau=0.7, precision=50) 20 | n = nx.number_of_nodes(T) 21 | assert M.rows == n 22 | assert M.cols == 3 23 | assert all([ mpm.norm(M[i,:]) < 1 for i in range(n)]) 24 | --------------------------------------------------------------------------------