├── src ├── python │ └── ghhc │ │ ├── model │ │ ├── __init__.py │ │ └── ghhc.py │ │ ├── util │ │ ├── __init__.py │ │ ├── io.py │ │ ├── eval_dp.py │ │ ├── load.py │ │ ├── sample_triples.py │ │ ├── Config.py │ │ └── initializers.py │ │ └── inference │ │ ├── __init__.py │ │ ├── run_predict_only.py │ │ └── run_inference.py └── scala │ └── ghhc │ ├── package.scala │ └── eval │ └── EvalDendrogramPurity.scala ├── bin ├── run_inf.sh ├── sample_triples.sh ├── setup.sh ├── run_predict_only.sh ├── install_mvn.sh ├── cat_imagenet_tree.sh ├── score_tree.sh ├── launch_predict_only_imagenet.sh ├── launch_inf.sh ├── launch_samples.sh └── launch_predict_only.sh ├── config ├── glass │ ├── build_samples.json │ └── glass.json ├── aloi │ ├── build_samples.json │ └── aloi.json ├── covtype │ ├── build_samples.json │ └── covtype.json └── ilsvrc │ ├── build_samples.json │ └── ilsvrc.json ├── NOTICE.txt ├── env.yml ├── .gitignore ├── pom.xml ├── README.md ├── LICENSE.txt └── data └── covtype.evalpts5k /src/python/ghhc/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/python/ghhc/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/python/ghhc/inference/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bin/run_inf.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | input=$1 6 | 7 | python -m ghhc.inference.run_inference $input -------------------------------------------------------------------------------- /bin/sample_triples.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | config=$1 6 | 7 | python -m ghhc.util.sample_triples $config -------------------------------------------------------------------------------- /bin/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export GHHC_ROOT=`pwd` 4 | 5 | export PYTHONPATH=$GHHC_ROOT/src/python:$PYTHONPATH 6 | export PATH=$GHHC_ROOT/dep/apache-maven-3.6.3/bin:$PATH 7 | -------------------------------------------------------------------------------- /config/glass/build_samples.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset_name": "glass", 3 | "sample_dataset": "data/glass/glass.tsv", 4 | "sample_outfile": "data/glass/glass.ghhc_samples.npy", 5 | "threads": 4 6 | } -------------------------------------------------------------------------------- /config/aloi/build_samples.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset_name": "aloi", 3 | "sample_dataset": "data/aloi/aloi.tsv.1", 4 | "sample_outfile": "data/aloi/aloi.ghhc_samples.npy", 5 | "num_samples": 500000, 6 | "percent_random": 0.0, 7 | "threads": 22 8 | } -------------------------------------------------------------------------------- /bin/run_predict_only.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | input=$1 6 | filename=$2 7 | out_file=${3:-"tree.tsv"} 8 | 9 | 10 | python -m ghhc.inference.run_predict_only --config $input --data_filename $filename --output_filename $out_file -------------------------------------------------------------------------------- /config/covtype/build_samples.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset_name": "covtype", 3 | "sample_dataset": "data/covtype/covtype.50k.tsv", 4 | "sample_outfile": "data/covtype/covtype.50k.ghhc_samples.npy", 5 | "num_samples": 300000, 6 | "threads": 22, 7 | "percent_random": 0.0 8 | } -------------------------------------------------------------------------------- /config/ilsvrc/build_samples.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset_name": "ilsvrc", 3 | "sample_dataset": "data/ilsvrc/ilsvrc12.50k.tsv", 4 | "sample_outfile": "data/ilsvrc/ilsvrc12.50k.ghhc_samples.npy", 5 | "num_samples": 300000, 6 | "threads": 22, 7 | "percent_random": 0.0 8 | } -------------------------------------------------------------------------------- /config/glass/glass.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset_name": "glass", 3 | "inference_file": "data/glass/glass.tsv", 4 | "dev_file": "data/glass/glass.tsv", 5 | "dev_every": 10000, 6 | "sample_file": "data/glass/glass.ghhc_samples.npy", 7 | "num_internals": 64, 8 | "threads": 4 9 | } -------------------------------------------------------------------------------- /bin/install_mvn.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -xu 4 | 5 | mkdir $GHHC_ROOT/dep 6 | pushd $GHHC_ROOT/dep 7 | wget http://mirror.cc.columbia.edu/pub/software/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz 8 | tar -xvf apache-maven-3.6.3-bin.tar.gz 9 | popd 10 | -------------------------------------------------------------------------------- /bin/cat_imagenet_tree.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | results_dir=$1 6 | 7 | pushd $results_dir 8 | cat tree-aa.tsv.leaves tree-ab.tsv.leaves tree-ac.tsv.leaves tree-ad.tsv.leaves tree-ae.tsv.leaves tree-af.tsv.leaves tree-ag.tsv.leaves tree-ah.tsv.leaves tree-ai.tsv.leaves tree-aj.tsv.leaves tree-ak.tsv.leaves tree-al.tsv.leaves tree-am.tsv.leaves tree-aa.tsv.internals > tree.tsv 9 | popd -------------------------------------------------------------------------------- /bin/score_tree.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | tree=$1 6 | algorithm=${2:-alg} 7 | dataset=${3:-dataset} 8 | threads=${4:-24} 9 | expected_dp_point_file=${5:-"None"} 10 | 11 | java -Xmx50G -cp target/ghhc-0.1-SNAPSHOT-jar-with-dependencies.jar ghhc.eval.EvalDendrogramPurity \ 12 | --input $tree --algorithm $algorithm --dataset $dataset --threads $threads \ 13 | --print true --id-file $expected_dp_point_file -------------------------------------------------------------------------------- /bin/launch_predict_only_imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | config=$1 6 | filename=$2 7 | partition=$3 8 | mem=${4:-12000} 9 | 10 | split_names=( "aa" "ab" "ac" "ad" "ae" "af" "ag" "ah" "ai" "aj" "ak" "al" "am" ) 11 | #split_names=( "aa" "ab" "ac" ) 12 | #split_names=( "ad" "ae" "af" "ag" "ah" "ai" "aj" "ak" "al" "am" ) 13 | 14 | for i in "${split_names[@]}" 15 | do 16 | sh bin/launch_predict_only.sh $config "${filename}.split_${i}" $partition $mem "tree-$i.tsv" 17 | sleep 1 18 | done -------------------------------------------------------------------------------- /bin/launch_inf.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | config=$1 6 | partition=$2 7 | mem=${3:-12000} 8 | 9 | EMAIL=None 10 | TIME=`(date +%Y-%m-%d-%H-%M-%S)` 11 | 12 | threads=`cat $config | jq .threads` 13 | 14 | export MKL_NUM_THREADS=$threads 15 | export OPENBLAS_NUM_THREADS=$threads 16 | export OMP_NUM_THREADS=$threads 17 | 18 | log_dir=logs/inf/$TIME 19 | 20 | mkdir -p $log_dir 21 | 22 | 23 | sbatch -J inf-$TIME \ 24 | -e $log_dir/inf.err \ 25 | -o $log_dir/inf.log \ 26 | --cpus-per-task $threads \ 27 | --partition=$partition \ 28 | --ntasks=1 \ 29 | --nodes=1 \ 30 | --mem=$mem \ 31 | --time=0-08:00 \ 32 | bin/run_inf.sh $config -------------------------------------------------------------------------------- /bin/launch_samples.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | config=$1 6 | partition=$2 7 | mem=${3:-12000} 8 | 9 | threads=`cat $config | jq .threads` 10 | 11 | EMAIL=None 12 | TIME=`(date +%Y-%m-%d-%H-%M-%S)` 13 | 14 | export MKL_NUM_THREADS=$threads 15 | export OPENBLAS_NUM_THREADS=$threads 16 | export OMP_NUM_THREADS=$threads 17 | 18 | log_dir=logs/samples/$TIME 19 | 20 | mkdir -p $log_dir 21 | 22 | 23 | sbatch -J samples-$TIME \ 24 | -e $log_dir/samples.err \ 25 | -o $log_dir/samples.log \ 26 | --cpus-per-task $threads \ 27 | --partition=$partition \ 28 | --ntasks=1 \ 29 | --nodes=1 \ 30 | --mem=$mem \ 31 | --time=0-10:00 \ 32 | bin/sample_triples.sh $config -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | hyperbolic_hierarchical_clustering 2 | 3 | This software is Copyright (C) 2019 Authors of gHHC 4 | and is licensed under the 5 | terms of the Apache License, Version 2.0 (see LICENSE.txt) or (at your option) any subsequent version. 6 | 7 | The license is approved by the Open Source Initiative, and is available 8 | from their website at http://www.opensource.org. 9 | 10 | THIRD PARTY COMPONENTS 11 | 12 | factorie is Copyright (C) 2008-2019 University of Massachusetts 13 | Amherst, Department of Computer Science, and is licensed under the 14 | terms of the Apache License, Version 2.0 15 | 16 | xcluster is Copyright (C) 2017-2019 University of Massachusetts 17 | Amherst, Department of Computer Science, and is licensed under the 18 | terms of the Apache License, Version 2.0 19 | -------------------------------------------------------------------------------- /bin/launch_predict_only.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | config=$1 6 | filename=$2 7 | partition=$3 8 | mem=${4:-12000} 9 | out_file=${5:-"tree.tsv"} 10 | 11 | EMAIL=None 12 | TIME=`(date +%Y-%m-%d-%H-%M-%S)` 13 | 14 | threads=`cat $config | jq .threads` 15 | 16 | export MKL_NUM_THREADS=$threads 17 | export OPENBLAS_NUM_THREADS=$threads 18 | export OMP_NUM_THREADS=$threads 19 | 20 | log_dir=logs/inf/$TIME 21 | 22 | mkdir -p $log_dir 23 | 24 | 25 | sbatch -J inf-po-$TIME \ 26 | -e $log_dir/inf.err \ 27 | -o $log_dir/inf.log \ 28 | --cpus-per-task $threads \ 29 | --partition=$partition \ 30 | --ntasks=1 \ 31 | --nodes=1 \ 32 | --mem=$mem \ 33 | --time=0-08:00 \ 34 | bin/run_predict_only.sh $config $filename $out_file -------------------------------------------------------------------------------- /src/python/ghhc/util/io.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019 Authors of gHHC 3 | This file is part of "hyperbolic_hierarchical_clustering" 4 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | 16 | import os 17 | import errno 18 | 19 | def mkdir_p(filepath): 20 | try: 21 | os.makedirs(filepath) 22 | except OSError as exc: 23 | if exc.errno != errno.EEXIST: 24 | raise 25 | pass 26 | -------------------------------------------------------------------------------- /config/aloi/aloi.json: -------------------------------------------------------------------------------- 1 | { 2 | "alg_name": "ghhc", 3 | "batch_size": 100, 4 | "dataset_name": "aloi", 5 | "dev_every": 10000, 6 | "dev_file": "data/aloi/aloi.5k.tsv", 7 | "dev_points_file": "None", 8 | "episode_size": 5000, 9 | "exp_out_base": "exp_out", 10 | "gamma": 0.5, 11 | "inference_file": "data/aloi/aloi.tsv.1", 12 | "init_method": "randhac", 13 | "lca_type": "conditional", 14 | "loss": "sigmoid", 15 | "max_norm": 0.9, 16 | "nn_batch_size": 1000, 17 | "num_internals": 40000, 18 | "num_iterations": 5, 19 | "num_samples": 50000, 20 | "percent_random": 0.1, 21 | "random_pts_scale": 0.8 , 22 | "sample_dataset": null, 23 | "sample_file": "data/aloi/aloi.ghhc_samples.npy", 24 | "sample_outfile": null, 25 | "save_dev_pics": false, 26 | "save_every": 100000, 27 | "shuffle": true, 28 | "threads": 22, 29 | "tree_learning_rate": 0.01, 30 | "triplet_k": 5, 31 | "unit_norm": true, 32 | "zero_mean": true, 33 | "struct_prior_every": 250, 34 | "struct_prior": "pcn", 35 | "num_struct_prior_batches": 25 36 | } -------------------------------------------------------------------------------- /config/covtype/covtype.json: -------------------------------------------------------------------------------- 1 | { 2 | "alg_name": "ghhc", 3 | "batch_size": 100, 4 | "dataset_name": "covtype", 5 | "dev_every": 10000, 6 | "dev_file": "data/covtype/covtype.5k.tsv", 7 | "dev_points_file": "None", 8 | "episode_size": 5000, 9 | "exp_out_base": "exp_out", 10 | "gamma": 0.5, 11 | "inference_file": "data/covtype/covtype.50k.tsv", 12 | "init_method": "randhac", 13 | "lca_type": "conditional", 14 | "loss": "sigmoid", 15 | "max_norm": 0.9, 16 | "nn_batch_size": 1000, 17 | "num_internals": 5000, 18 | "num_iterations": 10, 19 | "num_samples": 50000, 20 | "percent_random": 0.1, 21 | "random_pts_scale": 0.8 , 22 | "random_seed": 1451, 23 | "sample_dataset": null, 24 | "sample_file": "data/covtype/covtype.50k.ghhc_samples.npy", 25 | "sample_outfile": null, 26 | "save_dev_pics": false, 27 | "save_every": 100000, 28 | "shuffle": true, 29 | "threads": 22, 30 | "tree_learning_rate": 0.01, 31 | "triplet_k": 5, 32 | "unit_norm": true, 33 | "zero_mean": true, 34 | "struct_prior_every": 1000, 35 | "struct_prior": "pcn", 36 | "num_struct_prior_batches": 100 37 | } -------------------------------------------------------------------------------- /config/ilsvrc/ilsvrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "alg_name": "ghhc", 3 | "batch_size": 100, 4 | "dataset_name": "ilsvrc", 5 | "dev_every": 10000, 6 | "dev_file": "data/ilsvrc/ilsvrc12.5k.tsv", 7 | "dev_points_file": "None", 8 | "episode_size": 5000, 9 | "exp_out_base": "exp_out", 10 | "gamma": 0.5, 11 | "inference_file": "data/ilsvrc/ilsvrc12.50k.tsv", 12 | "init_method": "randhac", 13 | "lca_type": "conditional", 14 | "loss": "sigmoid", 15 | "loss_type": "compute_soft_loss_three_sparsemax", 16 | "max_norm": 0.9, 17 | "nn_batch_size": 1000, 18 | "num_internals": 20000, 19 | "num_iterations": 10, 20 | "num_samples": 50000, 21 | "percent_random": 0.1, 22 | "random_pts_scale": 0.8 , 23 | "sample_dataset": null, 24 | "sample_file": "data/ilsvrc/ilsvrc12.50k.ghhc_samples.npy", 25 | "sample_outfile": null, 26 | "save_dev_pics": false, 27 | "save_every": 100000, 28 | "shuffle": true, 29 | "threads": 22, 30 | "tree_learning_rate": 0.01, 31 | "triplet_k": 5, 32 | "unit_norm": true, 33 | "zero_mean": true, 34 | "struct_prior_every": 5000, 35 | "struct_prior": "pcn", 36 | "num_struct_prior_batches": 100 37 | } -------------------------------------------------------------------------------- /src/python/ghhc/util/eval_dp.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019 Authors of gHHC 3 | This file is part of "hyperbolic_hierarchical_clustering" 4 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | 16 | import os 17 | 18 | 19 | def eval_dp(filename, outfile, threads, points_file, model_name='ghhc', dataset_name='dataset'): 20 | """Evaluate dendrogram purity with shell script using xcluster DP code.""" 21 | 22 | os.system("sh bin/score_tree.sh {} {} {} {} {} > {}" 23 | .format(filename, model_name, dataset_name, threads, points_file, outfile)) 24 | cost = None 25 | with open(outfile, 'r') as fin: 26 | for line in fin: 27 | splt = line.strip().split("\t") 28 | cost = float(splt[-1]) 29 | return cost 30 | -------------------------------------------------------------------------------- /env.yml: -------------------------------------------------------------------------------- 1 | name: env_ghhc 2 | channels: 3 | - defaults 4 | dependencies: 5 | - _libgcc_mutex=0.1=main 6 | - blas=1.0=mkl 7 | - c-ares=1.15.0=h7b6447c_1001 8 | - ca-certificates=2019.10.16=0 9 | - certifi=2019.9.11=py36_0 10 | - gast=0.3.2=py_0 11 | - hdf5=1.10.4=hb1b8bf9_0 12 | - intel-openmp=2019.4=243 13 | - joblib=0.14.0=py_0 14 | - keras-applications=1.0.8=py_0 15 | - keras-preprocessing=1.1.0=py_1 16 | - libedit=3.1.20181209=hc058e9b_0 17 | - libffi=3.2.1=hd88cf55_4 18 | - libgcc-ng=9.1.0=hdf63c60_0 19 | - libgfortran-ng=7.3.0=hdf63c60_0 20 | - libprotobuf=3.10.1=hd408876_0 21 | - libstdcxx-ng=9.1.0=hdf63c60_0 22 | - mkl=2019.4=243 23 | - mkl-service=2.3.0=py36he904b0f_0 24 | - mkl_fft=1.0.15=py36ha843d7b_0 25 | - mkl_random=1.1.0=py36hd6b4f25_0 26 | - mock=3.0.5=py36_0 27 | - ncurses=6.1=he6710b0_1 28 | - numpy-base=1.17.4=py36hde5b4d6_0 29 | - openssl=1.1.1d=h7b6447c_3 30 | - pip=19.3.1=py36_0 31 | - python=3.6.9=h265db76_0 32 | - readline=7.0=h7b6447c_5 33 | - scikit-learn=0.21.3=py36hd81dba3_0 34 | - scipy=1.3.1=py36h7c811a0_0 35 | - setuptools=42.0.1=py36_0 36 | - six=1.13.0=py36_0 37 | - sqlite=3.30.1=h7b6447c_0 38 | - tensorflow-base=1.13.1=mkl_py36h7ce6ba3_0 39 | - tensorflow-estimator=1.13.0=py_0 40 | - tk=8.6.8=hbc83047_0 41 | - wheel=0.33.6=py36_0 42 | - xz=5.2.4=h14c3975_4 43 | - zlib=1.2.11=h7b6447c_3 44 | - pip: 45 | - absl-py==0.8.1 46 | - astor==0.8.0 47 | - grpcio==1.25.0 48 | - h5py==2.10.0 49 | - markdown==3.1.1 50 | - numpy==1.17.4 51 | - protobuf==3.11.0 52 | - tensorboard==1.12.2 53 | - tensorflow==1.12.0 54 | - termcolor==1.1.0 55 | - werkzeug==0.16.0 56 | prefix: /path/to/anaconda3/envs/env_ghhc 57 | 58 | -------------------------------------------------------------------------------- /src/python/ghhc/util/load.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019 Authors of gHHC 3 | This file is part of "hyperbolic_hierarchical_clustering" 4 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | 16 | import numpy as np 17 | import tensorflow as tf 18 | from absl import logging 19 | 20 | tf.enable_eager_execution() 21 | 22 | def load_xcluster(filename): 23 | raw = np.loadtxt(filename,dtype=np.float32) 24 | pids = raw[:, 0].astype(np.int32) 25 | lbls = raw[:, 1].astype(np.int32) 26 | X = raw[:, 2:] 27 | return pids, lbls, X 28 | 29 | def zero_meaned(X): 30 | return X - np.mean(X, 0) 31 | 32 | def unit_normed(X, norm=1.0): 33 | un = X / np.linalg.norm(X, axis=1, keepdims=True) 34 | un = tf.clip_by_norm(un, norm, axes=[1]).numpy() 35 | return un 36 | 37 | def load(filename, config): 38 | logging.info('Loading data from filename %s' % filename) 39 | logging.info('Using xcluster format') 40 | pids, lbls, X = load_xcluster(filename) 41 | if config.zero_mean: 42 | logging.info('Zero meaning data') 43 | X = zero_meaned(X) 44 | if config.unit_norm: 45 | logging.info('Unit norming data') 46 | X = unit_normed(X, config.max_norm) 47 | return pids, lbls, X -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_STORE 2 | .idea 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # Environments 88 | .env 89 | .venv 90 | env/ 91 | venv/ 92 | ENV/ 93 | env.bak/ 94 | venv.bak/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | 109 | -------------------------------------------------------------------------------- /src/python/ghhc/inference/run_predict_only.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019 Authors of gHHC 3 | This file is part of "hyperbolic_hierarchical_clustering" 4 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | 16 | import sys 17 | import os 18 | import datetime 19 | import numpy as np 20 | 21 | from absl import logging 22 | from absl import flags 23 | from absl import app 24 | 25 | import tensorflow as tf 26 | 27 | from ghhc.util.Config import Config 28 | from ghhc.util.load import load 29 | from ghhc.util.initializers import random_pts 30 | from ghhc.model.ghhc import gHHCTree, gHHCInference 31 | from ghhc.util.io import mkdir_p 32 | 33 | tf.enable_eager_execution() 34 | 35 | FLAGS = flags.FLAGS 36 | 37 | flags.DEFINE_string('config', None, "Config file") 38 | flags.DEFINE_string('data_filename',None, 'data filename') 39 | flags.DEFINE_string('output_filename', 'tree.tsv', 'output filename') 40 | logging.set_verbosity(logging.INFO) 41 | 42 | 43 | def main(argv): 44 | config = Config(FLAGS.config) 45 | 46 | filename = FLAGS.data_filename if FLAGS.data_filename is not None else config.inference_file 47 | 48 | assert(config.exp_out_dir is not None) 49 | assert (config.last_model is not None) 50 | 51 | config.exp_out_dir = os.path.join(config.exp_out_dir, 'results') 52 | mkdir_p(config.exp_out_dir) 53 | config.save_config(config.exp_out_dir,config.to_filename() + ".json") 54 | config.save_config(config.exp_out_dir) 55 | 56 | pids, lbls, dataset = load(filename, config) 57 | 58 | dev_pids, dev_lbls, dev_dataset = load(config.dev_file, config) 59 | 60 | init_tree = random_pts(dataset, config.num_internals, config.random_pts_scale) 61 | 62 | tree = gHHCTree(init_tree.copy(), config=config) 63 | optimizer = tf.train.GradientDescentOptimizer(config.tree_learning_rate) 64 | inf = gHHCInference(tree, optimizer, config, dev_dataset, dev_lbls) 65 | inf.ckpt.restore(tf.train.latest_checkpoint(inf.config.checkpoint_dir)) 66 | 67 | tree.write_tsv(config.exp_out_dir + "/" + FLAGS.output_filename, dataset, lbls=lbls, pids=pids) 68 | 69 | if __name__ == '__main__': 70 | app.run(main) -------------------------------------------------------------------------------- /src/python/ghhc/util/sample_triples.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019 Authors of gHHC 3 | This file is part of "hyperbolic_hierarchical_clustering" 4 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | 16 | from sklearn.neighbors import NearestNeighbors 17 | 18 | import sys 19 | import numpy as np 20 | from absl import logging 21 | from ghhc.util.Config import Config 22 | from ghhc.util.load import load 23 | 24 | logging.set_verbosity(logging.INFO) 25 | 26 | def batch_find_neighbors(X,nn: NearestNeighbors,batch_size=1000): 27 | res = np.zeros((X.shape[0], nn.n_neighbors-1), dtype=np.int32) 28 | for i in range(0, X.shape[0], batch_size): 29 | logging.log_every_n_seconds(logging.INFO, 'Computed %s of %s neighbors' % (i,X.shape[0]), 5) 30 | res[i:(i + batch_size)] = nn.kneighbors(X[i:(i+batch_size), :], return_distance=False)[:,1:] 31 | logging.info('Finished batch_find_neighbors') 32 | return res 33 | 34 | def order_triple(X, i, j, k, comparison): 35 | ij = comparison(X[i, :], X[j, :]) 36 | jk = comparison(X[j, :], X[k, :]) 37 | ik = comparison(X[i, :], X[k, :]) 38 | if ij >= max(jk, ik): 39 | return np.array([i, j, k], dtype=np.int32) 40 | elif jk >= max(ij,ik): 41 | return np.array([j, k, i], dtype=np.int32) 42 | elif ik >= max(ij, jk): 43 | return np.array([i, k, j], dtype=np.int32) 44 | 45 | def sample_random(X, comparison): 46 | ijk = np.random.choice(X.shape[0],size=3,replace=False) 47 | return order_triple(X, ijk[0], ijk[1], ijk[2], comparison) 48 | 49 | def sample_nn(X, Xnn, comparison): 50 | ik = np.random.choice(Xnn.shape[0],size=2,replace=False) 51 | i = ik[0] 52 | k = ik[1] 53 | j = np.random.choice(Xnn[i,:]) 54 | while k == i or k == j: 55 | k = np.random.randint(X.shape[0],size=1)[0] 56 | return order_triple(X, i, j, k, comparison) 57 | 58 | if __name__ == "__main__": 59 | config = Config(sys.argv[1]) 60 | pids, lbls, X = load(config.sample_dataset, config) 61 | nn = NearestNeighbors(n_neighbors=config.triplet_k+1, algorithm='ball_tree').fit(X) 62 | Xnn = batch_find_neighbors(X, nn, config.nn_batch_size) 63 | samples = np.zeros((config.num_samples, 3), dtype=np.int32) 64 | def sim_fn(i, j): 65 | return 1.0 / (1.0 + np.linalg.norm(i - j)) 66 | for i in range(config.num_samples): 67 | if np.random.random() < config.percent_random: 68 | samples[i, :] = sample_random(X, sim_fn) 69 | else: 70 | samples[i, :] = sample_nn(X, Xnn, sim_fn) 71 | np.save(config.sample_outfile, samples) 72 | 73 | 74 | -------------------------------------------------------------------------------- /src/python/ghhc/inference/run_inference.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019 Authors of gHHC 3 | This file is part of "hyperbolic_hierarchical_clustering" 4 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | 16 | import sys 17 | import os 18 | import datetime 19 | import numpy as np 20 | 21 | from absl import logging 22 | import tensorflow as tf 23 | 24 | from ghhc.util.Config import Config 25 | from ghhc.util.load import load 26 | from ghhc.util.initializers import random_pts,init_from_afkmc2_and_hac,init_from_rand_and_hac 27 | from ghhc.model.ghhc import gHHCTree, gHHCInference 28 | from ghhc.util.io import mkdir_p 29 | 30 | tf.enable_eager_execution() 31 | 32 | logging.set_verbosity(logging.INFO) 33 | 34 | 35 | if __name__ == "__main__": 36 | 37 | config = Config(sys.argv[1]) 38 | 39 | now = datetime.datetime.now() 40 | ts = "{:04d}-{:02d}-{:02d}-{:02d}-{:02d}-{:02d}".format( 41 | now.year, now.month, now.day, now.hour, now.minute, now.second) 42 | config.exp_out_dir = os.path.join(config.exp_out_base, config.dataset_name, config.alg_name, "%s-%s" %(ts,config.to_filename())) 43 | config.checkpoint_dir = os.path.join(config.exp_out_dir, 'models', 'ckpt') 44 | mkdir_p(config.exp_out_dir) 45 | mkdir_p(os.path.join(config.exp_out_dir, 'models')) 46 | config.save_config(config.exp_out_dir,config.to_filename() + ".json") 47 | config.save_config(config.exp_out_dir) 48 | 49 | pids, lbls, dataset = load(config.inference_file, config) 50 | 51 | dev_pids, dev_lbls, dev_dataset = load(config.dev_file, config) 52 | 53 | if config.random_projection is not None: 54 | logging.info('Using random projection: %s', config.random_projection) 55 | _proj = np.random.randn(dataset.shape[1], config.random_projection).astype(np.float32) 56 | def p(x): 57 | projd = tf.matmul(x, _proj) 58 | projd /= tf.linalg.norm(projd,axis=1,keepdims=True) 59 | projd = tf.clip_by_norm(projd, 0.9, axes=[1]) 60 | return projd 61 | proj = lambda x: p(x) 62 | init_tree = random_pts(proj(dataset).numpy(), config.num_internals, config.random_pts_scale) 63 | else: 64 | if config.init_method == 'randompts': 65 | init_tree = random_pts(dataset, config.num_internals, config.random_pts_scale) 66 | elif config.init_method == 'afkmc2hac': 67 | init_tree = init_from_afkmc2_and_hac(dataset, config.num_internals) 68 | elif config.init_method == 'randhac': 69 | init_tree = init_from_rand_and_hac(dataset, config.num_internals, config.random_pts_scale) 70 | proj = None 71 | 72 | tree = gHHCTree(init_tree.copy(), config=config, projection=proj) 73 | optimizer = tf.train.GradientDescentOptimizer(config.tree_learning_rate) 74 | inf = gHHCInference(tree, optimizer, config, dev_dataset, dev_lbls) 75 | 76 | samples = np.load(config.sample_file) 77 | inf.inference(samples, dataset, config.batch_size) -------------------------------------------------------------------------------- /src/python/ghhc/util/Config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019 Authors of gHHC 3 | This file is part of "hyperbolic_hierarchical_clustering" 4 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | 16 | import json 17 | import random 18 | import os 19 | import numpy as np 20 | 21 | 22 | class Config(object): 23 | """Config object""" 24 | 25 | def __init__(self,filename=None): 26 | # Settings 27 | self.config_name = filename 28 | self.dataset_name = 'dataset' 29 | self.alg_name = 'ghhc' 30 | self.exp_out_base = 'exp_out' 31 | self.exp_out_dir = None 32 | 33 | self.inference_file = None 34 | 35 | # Preprocessing 36 | self.unit_norm = True 37 | self.zero_mean = True 38 | 39 | # ghhc 40 | self.max_norm = 0.9 41 | self.init_method = 'randompts' 42 | self.batch_size = 500 43 | self.tree_learning_rate = 0.01 44 | self.num_internals = 100 45 | self.random_pts_scale = 0.001 46 | self.num_iterations = 10 47 | self.shuffle = True 48 | self.dev_points_file = 'None' 49 | self.gamma = 1.0 50 | self.struct_prior_every = 100 51 | self.num_struct_prior_batches = 100 52 | self.struct_prior = 'pcn' 53 | self.last_model = None 54 | 55 | # Triplet Sampling 56 | self.num_samples = 50000 57 | self.percent_random = 0.1 58 | self.sample_file = None 59 | self.sample_dataset = None 60 | self.triplet_k = 5 61 | self.nn_batch_size = 1000 62 | self.sample_outfile = None 63 | 64 | self.dev_file = None 65 | self.dev_every = 10000 66 | self.save_dev_pics = False 67 | self.loss_type = 'threespread' 68 | self.loss = 'sigmoid' 69 | self.lca_type = 'conditional' 70 | self.threads = 1 71 | self.checkpoint_dir = None 72 | self.save_every = 100000 73 | self.random_projection = None 74 | self.random_seed = 1451 75 | 76 | self.episode_size = 5000 77 | 78 | np.random.seed(self.random_seed) 79 | 80 | if filename: 81 | self.__dict__.update(json.load(open(filename))) 82 | self.random = random.Random(self.random_seed) 83 | 84 | def to_json(self): 85 | return json.dumps(self.filter_json(self.__dict__),indent=4,sort_keys=True) 86 | 87 | def save_config(self, exp_dir, filename='config.json'): 88 | with open(os.path.join(exp_dir, filename), 'w') as fout: 89 | fout.write(self.to_json()) 90 | fout.write('\n') 91 | 92 | def to_file_name_from_fields(self, fields): 93 | return "-".join(["%s=%s" % (f,self.__dict__[f]) for f in fields]) 94 | 95 | def to_filename(self): 96 | return self.to_file_name_from_fields(['alg_name', 'init_method', 'tree_learning_rate', 97 | 'loss', 'lca_type', 'num_samples', 98 | 'batch_size', 'struct_prior']) 99 | 100 | def filter_json(self, the_dict): 101 | # print("filter_json") 102 | # print(the_dict) 103 | res = {} 104 | for k in the_dict.keys(): 105 | # print("k : {} \t {} \t {}".format(k,the_dict[k],type(the_dict[k]))) 106 | if type(the_dict[k]) is str or \ 107 | type(the_dict[k]) is float or \ 108 | type(the_dict[k]) is int or \ 109 | type(the_dict[k]) is list or \ 110 | type(the_dict[k]) is bool or \ 111 | the_dict[k] is None: 112 | # print("res[k] : {} \t {} \t {}".format(k, the_dict[k], type(the_dict[k]))) 113 | res[k] = the_dict[k] 114 | elif type(the_dict[k]) is dict: 115 | res[k] = self.filter_json(the_dict[k]) 116 | return res 117 | 118 | DefaultConfig = Config() 119 | 120 | 121 | -------------------------------------------------------------------------------- /src/scala/ghhc/package.scala: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2019 Authors of gHHC 2 | This file is part of "hyperbolic_hierarchical_clustering" 3 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. */ 13 | 14 | import java.io.{FileInputStream, InputStreamReader, BufferedReader, File} 15 | import java.io._ 16 | import java.util.UUID 17 | 18 | import cc.factorie.util.Threading 19 | 20 | import scala.collection.mutable.ArrayBuffer 21 | import cc.factorie._ 22 | 23 | package object xcluster { 24 | 25 | implicit class XClusterIndexedSeqExtras[T](seq: IndexedSeq[T]) { 26 | def pairsParallel(numThreads: Int): Iterable[(T,T)] = { 27 | val initSize = if (seq.size * (seq.size - 1) > 0) seq.size * (seq.size - 1) else Int.MaxValue-1000000 28 | val pairs = new ArrayBuffer[(T,T)](initSize) 29 | // println(s"[IndexedSeqExtras] ${seq.size * (seq.size - 1)} Pairs to find") 30 | Threading.parForeach(seq.indices,numThreads)(idx => { 31 | var i = idx + 1 32 | val p = new ArrayBuffer[(T,T)](seq.length - i) 33 | while (i < seq.length) { 34 | p += ((seq(idx),seq(i))) 35 | i += 1 36 | } 37 | synchronized { 38 | pairs ++= p 39 | // print(s"\r[IndexedSeqExtras] ${pairs.size} pairs found") 40 | } 41 | }) 42 | // println(s"\n[IndexedSeqExtras] Done finding pairs") 43 | pairs 44 | } 45 | } 46 | 47 | implicit class XClusterArrayDoubleExtras(arr: Array[Double]) { 48 | 49 | def += (arr2: Array[Double]): Unit = { 50 | val len = arr.length 51 | assert(arr2.length == len) 52 | var i = 0 53 | while (i < len) { 54 | arr(i) += arr2(i) 55 | i += 1 56 | } 57 | } 58 | 59 | def / (n: Double) = { 60 | val len = arr.length 61 | val newArr = new Array[Double](len) 62 | var i = 0 63 | while (i < len) { 64 | newArr(i) = arr(i) / n 65 | i += 1 66 | } 67 | newArr 68 | } 69 | } 70 | 71 | implicit class XClusterArrayFloatExtras(arr: Array[Float]) { 72 | 73 | def += (arr2: Array[Float]): Unit = { 74 | val len = arr.length 75 | assert(arr2.length == len) 76 | var i = 0 77 | while (i < len) { 78 | arr(i) += arr2(i) 79 | i += 1 80 | } 81 | } 82 | 83 | def / (n: Float) = { 84 | val len = arr.length 85 | val newArr = new Array[Float](len) 86 | var i = 0 87 | while (i < len) { 88 | newArr(i) = arr(i) / n 89 | i += 1 90 | } 91 | newArr 92 | } 93 | } 94 | 95 | 96 | implicit class XClusterFileExtras(file: File) { 97 | def lines(codec: String) = { 98 | new BufferedReader(new InputStreamReader(new FileInputStream(file),codec)).toIterator 99 | } 100 | 101 | def allFilesRecursively(filter: File => Boolean = _ => true): Iterable[File] = { 102 | if (file.isDirectory) 103 | file.listFiles().toIterable.filterNot(_.isDirectory).filter(filter) ++ 104 | file.listFiles().toIterable.filter(_.isDirectory).map( f => f.allFilesRecursively(filter)).flatten 105 | else 106 | if (filter(file)) 107 | Some(file) 108 | else 109 | None 110 | } 111 | 112 | 113 | } 114 | 115 | implicit class XClusterIterableDoubleExtras(i: Iterable[Double]) { 116 | 117 | def average = { 118 | var s = 0.0 119 | var c = 0 120 | i.foreach{ 121 | ii => 122 | s += ii 123 | c += 1 124 | } 125 | s / c 126 | } 127 | 128 | def fastSum(fn: Double => Double) = { 129 | var s = 0.0 130 | i.foreach{ 131 | ii => 132 | s += fn(ii) 133 | } 134 | s 135 | } 136 | 137 | def variance = { 138 | val avg = i.average 139 | var v = 0.0 140 | var c = 0 141 | i.foreach{ 142 | ii => 143 | val diff = ii - avg 144 | v += diff * diff 145 | c += 1 146 | } 147 | v / c 148 | } 149 | 150 | } 151 | 152 | def randomId: String = UUID.randomUUID().toString 153 | 154 | 155 | 156 | implicit class XClusterStringExtras(str: String) { 157 | 158 | def toFile(file: File,codec: String = "UTF-8"): Unit = { 159 | val pw = new PrintWriter(file,codec) 160 | pw.print(str) 161 | pw.close() 162 | } 163 | } 164 | 165 | } -------------------------------------------------------------------------------- /src/python/ghhc/util/initializers.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019 Authors of gHHC 3 | This file is part of "hyperbolic_hierarchical_clustering" 4 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | 16 | import numpy as np 17 | 18 | import tensorflow as tf 19 | 20 | from absl import logging 21 | 22 | from ghhc.model.ghhc import squared_euclidean_cdist 23 | from sklearn.cluster import AgglomerativeClustering 24 | 25 | tf.enable_eager_execution() 26 | 27 | 28 | def euc_dist_batched(x_i, nodes, batch_size=1000): 29 | """Batched cdist operation.""" 30 | dists = np.zeros((x_i.shape[0], nodes.shape[0]), np.float32) 31 | for i in range(0, x_i.shape[0], batch_size): 32 | logging.log_every_n_seconds(logging.INFO, 'euc_dist_batched processed %s of %s', 5, i, x_i.shape[0]) 33 | for j in range(0, nodes.shape[0], batch_size): 34 | logging.log_every_n_seconds(logging.INFO, 'euc_dist_batched processed %s of %s', 5, j, nodes.shape[0]) 35 | dists[i:(i + batch_size), j:(j + batch_size)] = squared_euclidean_cdist( 36 | x_i[i:(i + batch_size), :], nodes[j:(j + batch_size), :]).numpy() 37 | return dists 38 | 39 | 40 | def afkmc2(data, k, m=20): 41 | """Implementation of Fast and Provably Good Seedings for k-Means https://las.inf.ethz.ch/files/bachem16fast.pdf """ 42 | n = data.shape[0] 43 | c1 = np.random.randint(data.shape[0]) 44 | c1_vec = np.expand_dims(data[c1], 0) 45 | q_nom = np.squeeze(euc_dist_batched(c1_vec, data)) 46 | q_denom = np.sum(q_nom) 47 | q = 0.5 * q_nom / q_denom + 1.0 / (2.0 * n) 48 | indices = np.arange(n) 49 | c_i_minus_1 = np.zeros((k, data.shape[1]), dtype=np.float32) 50 | c_i_minus_1[0, :] = c1_vec 51 | for i in range(1, k): 52 | logging.log_every_n_seconds(logging.INFO, 'afkmc2 processed %s of %s', 5, i, k) 53 | x_ind = np.random.choice(indices, p=q) 54 | x = np.expand_dims(data[x_ind], 0) 55 | d_x = np.min(np.squeeze(euc_dist_batched(x, c_i_minus_1[:i]))) 56 | for j in range(1, m): 57 | y_ind = np.random.choice(indices, p=q) 58 | y = np.expand_dims(data[y_ind], 0) 59 | d_y = np.min(np.squeeze(euc_dist_batched(y, c_i_minus_1[:i]))) 60 | if ((d_y * q[x_ind]) / (d_x * q[y_ind])) > np.random.rand(): 61 | x = y 62 | d_x = d_y 63 | c_i_minus_1[i] = x 64 | return c_i_minus_1 65 | 66 | 67 | def init_from_rand_and_hac(data, k, scale): 68 | """Pick random points for leaves, find internals with HAC heuristic.""" 69 | centers = random_pts(data, int(k / 2), 1.0) 70 | hac_pts = init_from_hac(centers, centers.shape[0] - 1) 71 | res = np.zeros((k, data.shape[1]), dtype=np.float32) 72 | assert k % 2 == 0 73 | res[0] += scale * data[np.random.randint(data.shape[0])] 74 | res[1:centers.shape[0] + 1, :] = centers 75 | res[centers.shape[0] + 1:, :] = hac_pts 76 | res = tf.clip_by_norm(scale * res, 0.80, axes=[1]).numpy() 77 | return res 78 | 79 | 80 | def init_from_afkmc2_and_hac(data, k): 81 | """Pick leaves using afkmc2, find internals with HAC heuristic""" 82 | centers = afkmc2(data, int(k / 2)) 83 | hac_pts = init_from_hac(centers, centers.shape[0] - 1) 84 | res = np.zeros((k, data.shape[1]), dtype=np.float32) 85 | assert k % 2 == 0 86 | res[0] += 0.65 * data[np.random.randint(data.shape[0])] 87 | res[1:centers.shape[0] + 1, :] = centers 88 | res[centers.shape[0] + 1:, :] = hac_pts 89 | res = tf.clip_by_norm(0.65 * res, 0.80, axes=[1]).numpy() 90 | return res 91 | 92 | 93 | def hac_scaling_factor(n): 94 | return np.log2(n + 1 - np.arange(n)) / np.log2(n + 1) 95 | 96 | 97 | def init_from_hac(data, k): 98 | """Find internal structure using hac heuristic""" 99 | agg = AgglomerativeClustering(n_clusters=1, linkage='average') 100 | agg.fit(data) 101 | internals = np.zeros((data.shape[0] - 1, data.shape[1]), dtype=np.float32) 102 | counts = np.zeros((data.shape[0] - 1), dtype=np.float32) 103 | children = agg.children_ 104 | 105 | # find each agglomeration vector and 106 | def get_vector_for_idx(idx): 107 | if idx < data.shape[0]: 108 | return data[idx] 109 | else: 110 | return internals[idx - data.shape[0]] 111 | 112 | def get_count_for_idx(idx): 113 | if idx < data.shape[0]: 114 | return 1 115 | else: 116 | return counts[idx - data.shape[0]] 117 | 118 | for i in range(0, children.shape[0]): 119 | internals[i, :] = get_vector_for_idx(children[i, 0]) + get_vector_for_idx(children[i, 1]) 120 | counts[i] = get_count_for_idx(children[i, 0]) + get_count_for_idx(children[i, 1]) 121 | 122 | mean_internals = internals / np.expand_dims(counts, 1) 123 | normalized_internals = mean_internals / np.linalg.norm(mean_internals, axis=1, keepdims=True) 124 | selected_internals = normalized_internals[-k:, :] 125 | # print(mean_internals.shape) 126 | # print(normalized_internals.shape) 127 | # print(selected_internals.shape) 128 | # print(k) 129 | sf = hac_scaling_factor(data.shape[0])[-k:] 130 | # print(sf.shape) 131 | result = selected_internals * np.expand_dims(sf, 1) 132 | return result 133 | 134 | 135 | def random_pts(data, n, scale): 136 | """Pick random points""" 137 | x_sample = np.random.choice(data.shape[0], size=n, replace=False) 138 | return scale * data[x_sample, :] 139 | 140 | 141 | def random(data, n, scale): 142 | """Sample random points from normal(0,1)""" 143 | sample = np.random.randn(n, data.shape[1]).astype(np.float32) 144 | return scale * sample 145 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | edu.umass.cs.iesl 6 | ghhc 7 | 0.1-SNAPSHOT 8 | 2019 9 | 10 | 2.11 11 | .7 12 | 13 | 14 | 15 | 16 | scala-tools.org 17 | Scala-Tools Maven2 Repository 18 | http://scala-tools.org/repo-releases 19 | 20 | 21 | sonatype-snapshot 22 | https://oss.sonatype.org/content/repositories/snapshots 23 | 24 | 25 | 26 | 27 | 28 | scala-tools.org 29 | Scala-Tools Maven2 Repository 30 | http://scala-tools.org/repo-releases 31 | 32 | 33 | 34 | 35 | 36 | org.scala-lang 37 | scala-library 38 | ${scala.majorVersion}${scala.minorVersion} 39 | 40 | 41 | cc.factorie 42 | factorie_${scala.majorVersion} 43 | 1.2 44 | 45 | 46 | com.google.guava 47 | guava 48 | 21.0 49 | 50 | 51 | org.scala-lang 52 | scala-xml 53 | 2.11.0-M4 54 | 55 | 56 | 57 | 58 | src/scala 59 | 60 | 61 | org.scala-tools 62 | maven-scala-plugin 63 | 64 | 65 | 66 | compile 67 | testCompile 68 | 69 | 70 | 71 | 72 | ${scala.majorVersion}${scala.minorVersion} 73 | 74 | 75 | 76 | org.apache.maven.plugins 77 | maven-compiler-plugin 78 | 3.6.1 79 | 80 | 1.8 81 | 1.8 82 | 83 | 84 | 85 | org.apache.maven.plugins 86 | maven-eclipse-plugin 87 | 88 | true 89 | 90 | ch.epfl.lamp.sdt.core.scalabuilder 91 | 92 | 93 | ch.epfl.lamp.sdt.core.scalanature 94 | 95 | 96 | org.eclipse.jdt.launching.JRE_CONTAINER 97 | ch.epfl.lamp.sdt.launching.SCALA_CONTAINER 98 | 99 | 100 | 101 | 102 | maven-assembly-plugin 103 | 2.4 104 | 105 | 106 | make-assembly 107 | package 108 | 109 | attached 110 | 111 | 112 | 113 | 114 | gnu 115 | 116 | jar-with-dependencies 117 | 118 | 119 | 120 | 121 | 122 | org.apache.maven.plugins 123 | maven-surefire-plugin 124 | 2.7 125 | 126 | true 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | net.alchim31.maven 135 | scala-maven-plugin 136 | 3.2.0 137 | 138 | 139 | -deprecation 140 | -feature 141 | -optimise 142 | -Yclosure-elim 143 | -Yinline 144 | 145 | 146 | -Xms64m 147 | -Xmx1024m 148 | 149 | 150 | 160 | 161 | 162 | 163 | 164 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # gHHC 3 | 4 | Code for: [Gradient-based Hierarchical Clustering using Continuous Representations of Trees in Hyperbolic Space](https://dl.acm.org/citation.cfm?id=3330997). Nicholas Monath, Manzil Zaheer, Daniel Silva, Andrew McCallum, Amr Ahmed. KDD 2019. 5 | 6 | ## Setup 7 | 8 | In each shell session, run: 9 | 10 | ``` 11 | source bin/setup.sh 12 | ``` 13 | 14 | to set environment variables. 15 | 16 | Install jq (if not already installed): https://stedolan.github.io/jq/ 17 | 18 | Install maven (if not already installed): 19 | 20 | ``` 21 | sh bin/install_mvn.sh 22 | ``` 23 | 24 | Install python dependencies: 25 | 26 | ``` 27 | conda create -n env_ghhc pip python=3.6 28 | source activate env_ghhc 29 | # Either (linux) 30 | wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.12.0-cp36-cp36m-linux_x86_64.whl 31 | pip install tensorflow-1.12.0-cp36-cp36m-linux_x86_64.whl 32 | # or (mac) 33 | wget https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.12.0-py3-none-any.whl 34 | pip install tensorflow-1.12.0-py3-none-any.whl 35 | conda install scikit-learn 36 | conda install tensorflow-base=1.13.1 37 | ``` 38 | 39 | See [env.yml](env.yml) for a complete list of dependencies if you run into issues 40 | with the above. 41 | 42 | Build scala code: 43 | 44 | ``` 45 | mvn clean package 46 | ``` 47 | 48 | Note you may need to set `JAVA_HOME` and `JAVA_HOME_8` on your system. 49 | 50 | ALOI and Glass are downloadable from: https://github.com/iesl/xcluster 51 | 52 | Covtype is available here: https://archive.ics.uci.edu/ml/datasets/covertype 53 | 54 | Contact me regarding the ImageNet data. 55 | 56 | ## Clustering Experiments 57 | 58 | ### Step 1. Building triples for inference 59 | 60 | Sample triples of datapoints that will be used for inference: 61 | 62 | On a compute machine: 63 | 64 | ``` 65 | sh bin/sample_triples.sh config/glass/build_samples.json 66 | ``` 67 | 68 | Using slurm cluster manager: 69 | 70 | ``` 71 | sh bin/launch_samples.sh config/glass/build_samples.json 72 | ``` 73 | 74 | Note the above example is for the `glass` dataset, but the same 75 | procedure and scripts are available for all datasets. 76 | 77 | ### Step 2. Run Inference 78 | 79 | Update the representations of the internal nodes of the tree structure. 80 | 81 | On a compute machine: 82 | 83 | ``` 84 | sh bin/run_inf.sh config/glass/glass.json 85 | ``` 86 | 87 | Using slurm cluster manager: 88 | 89 | ``` 90 | sh bin/launch_inf.sh config/glass/glass.json 91 | ``` 92 | 93 | This will create a directory in `exp_out/dataset_name/ghhc/timestamp` 94 | containing the internal node parameters and configs to run the next step. 95 | For example, this would create the following: 96 | 97 | ``` 98 | exp_out/glass/ghhc/2019-11-29-20-13-29-alg_name=ghhc-init_method=randompts-tree_learning_rate=0.01-loss=sigmoid-lca_type=conditional-num_samples=50000-batch_size=500-struct_prior=pcn 99 | ``` 100 | 101 | ### Step 3. Final clustering 102 | 103 | Produce assignment of datapoints in the hierarchical clustering and 104 | produce internal structure. 105 | 106 | For datasets other than ImageNet: 107 | 108 | On a compute machine: 109 | 110 | ``` 111 | # Generally: 112 | sh bin/run_predict_only.sh exp_out/data/ghhc/timestap/config.json data/datasetname/data_to_run_on.tsv 113 | 114 | # For example: 115 | sh bin/run_predict_only.sh exp_out/glass/ghhc/2019-11-29-20-13-29-alg_name=ghhc-init_method=randompts-tree_learning_rate=0.01-loss=sigmoid-lca_type=conditional-num_samples=50000-batch_size=500-struct_prior=pcn/config.json data/glass/glass.tsv 116 | ``` 117 | 118 | Using slurm cluster manager: 119 | 120 | ``` 121 | sh bin/launch_predict_only.sh exp_out/glass/ghhc/2019-11-29-20-13-29-alg_name=ghhc-init_method=randompts-tree_learning_rate=0.01-loss=sigmoid-lca_type=conditional-num_samples=50000-batch_size=500-struct_prior=pcn/config.json data/glass/glass.tsv 122 | ``` 123 | 124 | This will create a file: `exp_out/glass/ghhc/2019-11-29-20-13-29-alg_name=ghhc-init_method=randompts-tree_learning_rate=0.01-loss=sigmoid-lca_type=conditional-num_samples=50000-batch_size=500-struct_prior=pcn/results/tree.tsv` 125 | which can be evaluated using 126 | 127 | ``` 128 | sh bin/score_tree.sh exp_out/glass/ghhc/2019-11-29-20-13-29-alg_name=ghhc-init_method=randompts-tree_learning_rate=0.01-loss=sigmoid-lca_type=conditional-num_samples=50000-batch_size=500-struct_prior=pcn/results/tree.tsv 129 | ``` 130 | 131 | When evaluating the tree for covtype, use the expected dendrogram purity point id file from the data directory: 132 | 133 | ``` 134 | sh bin/score_tree.sh /path/to/tree.tsv ghhc covtype $num_threads data/covtype.evalpts5k 135 | ``` 136 | 137 | 138 | For ImageNet: 139 | 140 | ``` 141 | sh bin/launch_predict_only_imagenet.sh exp_out/ilsvrc/ghhc/2019-11-29-08-04-23-alg_name=ghhc-init_method=randhac-tree_learning_rate=0.01-loss=sigmoid-lca_type=conditional-num_samples=50000-batch_size=100-struct_prior=pcn/config.json data/ilsvrc/ilsvrc12.tsv.1 cpu 32000 142 | ``` 143 | 144 | This assumes that the ImageNet data file has been split into 13 files: 145 | 146 | ``` 147 | data/ilsvrc/ilsvrc12.tsv.1.split_aa 148 | data/ilsvrc/ilsvrc12.tsv.1.split_ab 149 | ... 150 | data/ilsvrc/ilsvrc12.tsv.1.split_am 151 | ``` 152 | 153 | Then when all jobs finish, concatenate results: 154 | 155 | ``` 156 | sh bin/cat_imagenet_tree.sh exp_out/ilsvrc/ghhc/2019-11-29-08-04-23-alg_name=ghhc-init_method=randhac-tree_learning_rate=0.01-loss=sigmoid-lca_type=conditional-num_samples=50000-batch_size=100-struct_prior=pcn/results/ 157 | ``` 158 | 159 | This will create a file containing the entire tree: 160 | 161 | ``` 162 | exp_out/ilsvrc/ghhc/2019-11-29-08-04-23-alg_name=ghhc-init_method=randhac-tree_learning_rate=0.01-loss=sigmoid-lca_type=conditional-num_samples=50000-batch_size=100-struct_prior=pcn/results/tree.tsv 163 | ``` 164 | 165 | which can be evaluated using: 166 | 167 | ``` 168 | sh bin/score_tree.sh exp_out/ilsvrc/ghhc/2019-11-29-08-04-23-alg_name=ghhc-init_method=randhac-tree_learning_rate=0.01-loss=sigmoid-lca_type=conditional-num_samples=50000-batch_size=100-struct_prior=pcn/results/tree.tsv ghhc ilsvrc12 $num_threads data/imagenet_eval_pts.ids 169 | ``` 170 | 171 | ## Citation 172 | 173 | ``` 174 | @inproceedings{Monath:2019:GHC:3292500.3330997, 175 | author = {Monath, Nicholas and Zaheer, Manzil and Silva, Daniel and McCallum, Andrew and Ahmed, Amr}, 176 | title = {Gradient-based Hierarchical Clustering Using Continuous Representations of Trees in Hyperbolic Space}, 177 | booktitle = {Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery \& Data Mining}, 178 | series = {KDD '19}, 179 | year = {2019}, 180 | isbn = {978-1-4503-6201-6}, 181 | location = {Anchorage, AK, USA}, 182 | pages = {714--722}, 183 | numpages = {9}, 184 | url = {http://doi.acm.org/10.1145/3292500.3330997}, 185 | doi = {10.1145/3292500.3330997}, 186 | acmid = {3330997}, 187 | publisher = {ACM}, 188 | address = {New York, NY, USA}, 189 | keywords = {clustering, gradient-based clustering, hierarchical clustering}, 190 | } 191 | ``` 192 | 193 | ## License 194 | 195 | Apache License, Version 2.0 196 | 197 | ## Questions / Comments / Bugs / Issues 198 | 199 | Please contact Nicholas Monath (nmonath@cs.umass.edu). 200 | 201 | Also, please contact me for access to the data. 202 | 203 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /src/scala/ghhc/eval/EvalDendrogramPurity.scala: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2019 Authors of gHHC 2 | This file is part of "hyperbolic_hierarchical_clustering" 3 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. */ 13 | 14 | package ghhc.eval 15 | 16 | import java.io.File 17 | import java.util 18 | 19 | import cc.factorie._ 20 | import cc.factorie.util.{DefaultCmdOptions, JavaHashSet, Threading} 21 | 22 | import scala.collection.JavaConverters._ 23 | import scala.collection.{GenSet, mutable} 24 | import scala.collection.mutable.ArrayBuffer 25 | import xcluster._ 26 | 27 | 28 | /** 29 | * Tree data structure used to compute dendrogram purity 30 | * @param id The unique id for this node 31 | * @param parent The parent node or None if the node is the root 32 | * @param parentId The id of the parent node or None if the node is the root 33 | * @param children The children objects of this node 34 | * @param labels The class lables of the descendant leaves of this node. 35 | */ 36 | class EvalTreeNode( 37 | val id: String, 38 | var parent: Option[EvalTreeNode], 39 | val parentId: Option[String], 40 | var children: ArrayBuffer[EvalTreeNode], var labels: ArrayBuffer[String]) extends Comparable[EvalTreeNode]{ 41 | 42 | 43 | override def compareTo(o: EvalTreeNode): Int = id.compareTo(o.id) 44 | 45 | var best = (0,0) 46 | var local = (0,0) 47 | 48 | lazy val labelCounts = labels.groupBy(identity).mapValues(_.size).toMap[String, Int] 49 | 50 | var bestClusteringRoots = ArrayBuffer[EvalTreeNode](this) 51 | var bestClustering = new ArrayBuffer[Iterable[EvalTreeNode]]() 52 | /** 53 | * Whether or not the node is a leaf 54 | * @return True/false if this is a leaf 55 | */ 56 | def isLeaf: Boolean = this.children.isEmpty 57 | 58 | 59 | var leafCache: ArrayBuffer[EvalTreeNode] = null 60 | /** 61 | * Return all of the descendant leaves of this node 62 | * @return Iterable of nodes 63 | */ 64 | def leaves(): Iterable[EvalTreeNode] = { 65 | if (leafCache == null) { 66 | leafCache = { 67 | if (this.isLeaf) 68 | ArrayBuffer(this) 69 | else { 70 | var curr_node = this 71 | val q = new mutable.Queue[EvalTreeNode]() 72 | q.enqueue(curr_node) 73 | val leaves = new ArrayBuffer[EvalTreeNode]() 74 | while (q.nonEmpty) { 75 | curr_node = q.dequeue() 76 | curr_node.children.foreach { 77 | c => 78 | if (c.isLeaf) 79 | leaves += c 80 | else 81 | q.enqueue(c) 82 | } 83 | } 84 | leaves 85 | } 86 | } 87 | } 88 | leafCache 89 | } 90 | 91 | 92 | /** 93 | * Compute the purity of this node from the cached labels 94 | * @param wrt Class label about which purity should be computed 95 | * @return Purity value for the given class 96 | */ 97 | def purity(wrt: String): Double = { 98 | // TODO: We could cache these scores 99 | // labels.count(_ == wrt).toDouble / labels.size 100 | labelCounts(wrt).toDouble / labels.size 101 | } 102 | 103 | /** 104 | * Some leaves may have missing labels. Purity strict 105 | * differs from purity by penalizing the inclusion of these 106 | * leaves with missing labels in the cluster. 107 | * @param wrt Class label about which purity should be computed 108 | * @return Purity value for the given class 109 | */ 110 | def purityStrict(wrt: String): Double = { 111 | // TODO: We could cache these scores 112 | labels.count(_ == wrt).toDouble / this.leaves().size 113 | } 114 | 115 | 116 | 117 | /** 118 | * The purity of the most frequently appearing class in this node 119 | * @return Purity value for the given class 120 | */ 121 | def purity(): Double = { 122 | val mostFreq = labels.groupBy(identity).mapValues(_.size).maxBy(_._2)._1 123 | purity(mostFreq) 124 | } 125 | 126 | 127 | def depth() = ancestors().length 128 | 129 | /** 130 | * The ancestors of this node (does not include this node). 131 | * @return An iterable of nodes 132 | */ 133 | def ancestors(): IndexedSeq[EvalTreeNode] = { 134 | val ancestors = new ArrayBuffer[EvalTreeNode]() 135 | var currNode = this 136 | while (currNode.parent.isDefined) { 137 | currNode = currNode.parent.get 138 | ancestors += currNode 139 | } 140 | ancestors 141 | } 142 | 143 | /** 144 | * The descendants of this node (does include this node) 145 | * @return The descendants of the node and this node itself 146 | */ 147 | def descendants(): IndexedSeq[EvalTreeNode] = { 148 | val ds = new ArrayBuffer[EvalTreeNode]() 149 | val q = new mutable.Queue[EvalTreeNode]() 150 | q.enqueue(this) 151 | while (q.nonEmpty) { 152 | val target = q.dequeue() 153 | ds.append(target) 154 | target.children.foreach(q.enqueue(_)) 155 | } 156 | ds.toIndexedSeq 157 | } 158 | 159 | /** 160 | * The root of the tree. 161 | * @return root 162 | */ 163 | def root():EvalTreeNode = { 164 | var currNode = this 165 | while (currNode.parent.isDefined) { 166 | currNode = currNode.parent.get 167 | } 168 | currNode 169 | } 170 | 171 | 172 | /** 173 | * The least common ancestor of this node and the given other node 174 | * @param other A node in the tree 175 | * @return lca(this,other) 176 | */ 177 | def lca(other: EvalTreeNode): EvalTreeNode = { 178 | val thisAncestors = this.ancestors() 179 | val otherAncestors = other.ancestors() 180 | var i = 0 181 | var found = false 182 | var lca = this.root() 183 | while (i < thisAncestors.length && !found) { 184 | var j = 0 185 | while(j < otherAncestors.length && !found) { 186 | if (thisAncestors(i) == otherAncestors(j)) { 187 | found = true 188 | lca = thisAncestors(i) 189 | } 190 | j += 1 191 | } 192 | i += 1 193 | } 194 | i-=1 195 | lca 196 | } 197 | 198 | } 199 | 200 | /** 201 | * Loaders for evaluation tree file format: 202 | * NodeId \t ParentId \t Label 203 | * 204 | * Parent and or label can be none or null if empty. 205 | */ 206 | object LoadEvalTree { 207 | 208 | /** 209 | * Load a serialized evaluation tree into memory and return the root of the tree 210 | * @param filename The filename 211 | */ 212 | def load(filename: String): EvalTreeNode = { 213 | val forest = loadForest(filename) 214 | assert(forest.size == 1, s"File: $filename contained a forest of trees, not one tree.") 215 | forest.head 216 | } 217 | 218 | /** 219 | * Load a forest of evaluation trees into memory returning the roots of all the trees. 220 | * @param fn 221 | * @return 222 | */ 223 | def loadForest(fn: String) = { 224 | 225 | val nodes = new java.util.HashMap[String,EvalTreeNode]().asScala 226 | 227 | new File(fn).lines("UTF-8").zipWithIndex.foreach { 228 | case (line,idx) => 229 | // if (idx % 100 == 0) 230 | // println(s"Read $idx lines of $fn") 231 | val Array(id,parent,label) = line.split("\t") 232 | val parentId = if (parent.equalsIgnoreCase("none") || parent.equalsIgnoreCase("null")) None else Some(parent) 233 | val labels = new ArrayBuffer[String]() 234 | if (!(label.equalsIgnoreCase("none") || label.equalsIgnoreCase("null"))) 235 | labels += label 236 | val node = new EvalTreeNode(id,None,parentId,new ArrayBuffer[EvalTreeNode](),labels) 237 | nodes.put(node.id,node) 238 | } 239 | nodes.values.foreach{ 240 | n => 241 | if (n.parentId.isDefined) { 242 | n.parent = nodes.get(n.parentId.get) 243 | if (n.parent.isEmpty) 244 | System.err.println(s"Missing ${n.parentId.get}") 245 | n.parent.get.children += n 246 | 247 | } 248 | } 249 | 250 | def propagateLabel(leaf: EvalTreeNode) = { 251 | val label = leaf.labels.head 252 | var currNode = leaf 253 | while (currNode.parent.isDefined) { 254 | currNode = currNode.parent.get 255 | currNode.labels += label 256 | } 257 | } 258 | val leaves = nodes.values.filter(_.isLeaf) 259 | leaves.filter(_.labels.nonEmpty).foreach(propagateLabel) 260 | nodes.values.zipWithIndex.foreach{ 261 | case (n,idx) => 262 | n.leaves() 263 | if (idx % 10000 == 0) 264 | System.err.print(s"\r[eval dp] pre-processed ${idx} nodes") 265 | } 266 | System.err.println() 267 | nodes.values.filter(_.parent.isEmpty) 268 | } 269 | 270 | 271 | } 272 | 273 | trait DendrogramPurityOpts extends DefaultCmdOptions { 274 | val input = new CmdOption[String]("input","the tree file to score") 275 | val algorithm = new CmdOption[String]("algorithm","Algorithm","STRING","the algorithm name") 276 | val dataset = new CmdOption[String]("dataset","dataset","STRING","the dataset name") 277 | val threads = new CmdOption[Int]("threads",4,"INT","number of threads to use") 278 | val print = new CmdOption[Boolean]("print",false,"BOOLEAN","print status updates for computation defaults to be false") 279 | val idFile = new CmdOption[String]("id-file","The file containing the point ids on which to evaluate dendrogram purity. Leave blank or as None to do exact dendrogram purity") 280 | val strictDP = new CmdOption[Boolean]("strict",false,"BOOLEAN","Whether to use version of purity which penalizes missing labels. ") 281 | } 282 | 283 | /** 284 | * Executable for dendrogram purity and expected 285 | * dendrogram purity. By default, exact dendrogram purity is computed 286 | * if an id-file is passed in then expected dendrogram 287 | * purity is computed on only those ids. 288 | */ 289 | object EvalDendrogramPurity { 290 | def main(args: Array[String]): Unit = { 291 | val opts = new DendrogramPurityOpts {} 292 | opts.parse(args) 293 | if (opts.idFile.wasInvoked && opts.idFile.value.toLowerCase != "none") 294 | ExpectedDendrogramPurity.run(opts) 295 | else 296 | DendrogramPurity.run(opts) 297 | } 298 | } 299 | 300 | 301 | object DendrogramPurity { 302 | 303 | /** 304 | * Find all pairs of points which have the same class label 305 | * @param root The root of the tree to evaluate 306 | * @param numThreads The number of threads to use 307 | * @return All pairs of points in dendrogram purity calculations 308 | */ 309 | def allPairsForEval(root: EvalTreeNode, numThreads: Int): ArrayBuffer[(EvalTreeNode,EvalTreeNode)] = { 310 | val leaves = root.leaves() 311 | val byClass = new util.HashMap[String,ArrayBuffer[EvalTreeNode]]().asScala 312 | leaves.filter(_.labels.nonEmpty).zipWithIndex.foreach{ 313 | case (l,idx) => 314 | // if (idx % 100 == 0) 315 | // println(s"Processed $idx leaves") 316 | // assert(l.labels.size == 1) 317 | // if (!byClass.contains(l.labels.head)) 318 | // byClass.put(l.labels.head, new ArrayBuffer[EvalTreeNode]()) 319 | // byClass(l.labels.head) += l 320 | // only include laves that have a label. 321 | if (l.labels.size == 1) { 322 | byClass.getOrElseUpdate(l.labels.head, new ArrayBuffer[EvalTreeNode]()) += l 323 | } 324 | } 325 | // println("Finding pairs.") 326 | val allPairs = new ArrayBuffer[(EvalTreeNode,EvalTreeNode)](byClass.values.map(f => f.size * f.size).sum) 327 | byClass.values.foreach{ 328 | classOnlyPoints => 329 | if (classOnlyPoints.length > 100) 330 | allPairs ++= classOnlyPoints.pairsParallel(numThreads) 331 | else 332 | allPairs ++= classOnlyPoints.pairs 333 | } 334 | allPairs 335 | } 336 | 337 | /** 338 | * Compute the dendrogram purity for the given pairs 339 | * @param pairs Pairs of points to evaluate 340 | * @param threads Number of threads to use 341 | * @param print Whether or not to print status updates 342 | * @return Dendrogram purity 343 | */ 344 | def evalPar(pairs: IndexedSeq[(EvalTreeNode,EvalTreeNode)],threads:Int,print:Boolean, strict: Boolean =false): Double = { 345 | val bufferSize = 10000 346 | val startIndexes = 0 until pairs.size by bufferSize 347 | @volatile var sum_purities = 0.0 348 | @volatile var N = 0.0 349 | Threading.parForeach(startIndexes,threads)({ 350 | start => 351 | var local_n = 0.0 352 | var local_purity = 0.0 353 | var i = start 354 | val end = math.min(start+ bufferSize,pairs.length) 355 | while (i < end) { 356 | val pair = pairs(i) 357 | if (strict) 358 | local_purity += pair._1.lca(pair._2).purityStrict(pair._1.labels.head) 359 | else 360 | local_purity += pair._1.lca(pair._2).purity(pair._1.labels.head) 361 | local_n += 1.0 362 | i += 1 363 | } 364 | synchronized{ 365 | if (print) 366 | System.err.print(s"\rThread ${Thread.currentThread().getId} Computing purities for pairs ${start} to ${end} of ${pairs.size} = ${100*end/pairs.size.toFloat}% done") 367 | N += local_n 368 | sum_purities += local_purity 369 | } 370 | }) 371 | sum_purities / N 372 | } 373 | 374 | /** 375 | * Run the main executable. 376 | * @param opts 377 | */ 378 | def run(opts: DendrogramPurityOpts): Unit = { 379 | val root = LoadEvalTree.load(opts.input.value) 380 | val pairs = allPairsForEval(root,opts.threads.value) 381 | val score = evalPar(pairs,opts.threads.value,opts.print.value,opts.strictDP.value) 382 | println(s"${opts.algorithm.value}\t${opts.dataset.value}\t${score}") 383 | } 384 | 385 | def main(args: Array[String]): Unit = { 386 | val opts = new DendrogramPurityOpts {} 387 | opts.parse(args) 388 | run(opts) 389 | } 390 | 391 | 392 | } 393 | 394 | /** 395 | * Expected dendrogram computations 396 | */ 397 | object ExpectedDendrogramPurity { 398 | 399 | def main(args: Array[String]): Unit = { 400 | val opts = new DendrogramPurityOpts {} 401 | opts.parse(args) 402 | run(opts) 403 | } 404 | 405 | 406 | 407 | 408 | def run(opts: DendrogramPurityOpts) = { 409 | val ids = io.Source.fromFile(opts.idFile.value)("UTF-8").getLines().toSet[String] 410 | val root = LoadEvalTree.load(opts.input.value) 411 | val pairs = all_pairs_for_eval(root,ids,opts.threads.value) 412 | val score = evalPar(pairs,opts.threads.value,opts.print.value,opts.strictDP.value) 413 | System.err.println() 414 | println(s"${opts.algorithm.value}\t${opts.dataset.value}\t${score}") 415 | } 416 | 417 | /** 418 | * Compute all pairs for evaluation with respect to a given set of point ids 419 | * @param root the root node 420 | * @param idSet the ids 421 | * @param numThreads number of threads 422 | * @return 423 | */ 424 | def all_pairs_for_eval(root: EvalTreeNode, idSet: Set[String],numThreads: Int): ArrayBuffer[(EvalTreeNode,EvalTreeNode)] = { 425 | val leaves = root.leaves() 426 | val byClass = new util.HashMap[String,ArrayBuffer[EvalTreeNode]]().asScala 427 | leaves.filter(l => idSet.contains(l.id)).zipWithIndex.foreach{ 428 | case (l,idx) => 429 | // if (idx % 100 == 0) 430 | // println(s"Processed $idx leaves") 431 | // if labels are empty do not include in evaluation. 432 | if (l.labels.size == 1) { 433 | byClass.getOrElseUpdate(l.labels.head, new ArrayBuffer[EvalTreeNode]()) += l 434 | } 435 | } 436 | // println("Finding pairs.") 437 | val allPairs = new ArrayBuffer[(EvalTreeNode,EvalTreeNode)](byClass.values.map(f => f.size * f.size).sum) 438 | byClass.values.foreach{ 439 | classOnlyPoints => 440 | if (classOnlyPoints.length > 100) 441 | allPairs ++= classOnlyPoints.pairsParallel(numThreads) 442 | else 443 | allPairs ++= classOnlyPoints.pairs 444 | } 445 | allPairs 446 | } 447 | 448 | /** 449 | * Evaluate expected dendrogram purity 450 | * @param pairs the pairs to evaluate on 451 | * @param threads the number of threads 452 | * @param print whether or not to print status updates 453 | * @return 454 | */ 455 | def evalPar(pairs: IndexedSeq[(EvalTreeNode,EvalTreeNode)],threads:Int,print:Boolean,strict: Boolean = false) = { 456 | val bufferSize = 1000 457 | val startIndexes = 0 until pairs.size by bufferSize 458 | @volatile var sum_purities = 0.0 459 | @volatile var N = 0.0 460 | Threading.parForeach(startIndexes,threads)({ 461 | start => 462 | var local_n = 0.0 463 | var local_purity = 0.0 464 | var i = start 465 | val end = math.min(start+ bufferSize,pairs.length) 466 | while (i < end) { 467 | val pair = pairs(i) 468 | // if the code below breaks you tried to evaluate on a pair that has no label. 469 | assert(pair._1.labels.size == 1) 470 | assert(pair._2.labels.size == 1) 471 | if (strict) 472 | local_purity += pair._1.lca(pair._2).purityStrict(pair._1.labels.head) 473 | else 474 | local_purity += pair._1.lca(pair._2).purity(pair._1.labels.head) 475 | local_n += 1.0 476 | i += 1 477 | } 478 | synchronized{ 479 | if (print) 480 | System.err.println(s"Thread ${Thread.currentThread().getId} Computing purities for pairs ${start} to ${end} of ${pairs.size} = ${100*end/pairs.size.toFloat}% done") 481 | N += local_n 482 | sum_purities += local_purity 483 | } 484 | }) 485 | sum_purities / N 486 | } 487 | 488 | 489 | } 490 | -------------------------------------------------------------------------------- /src/python/ghhc/model/ghhc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019 Authors of gHHC 3 | This file is part of "hyperbolic_hierarchical_clustering" 4 | http://github.com/nmonath/hyperbolic_hierarchical_clustering 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | 16 | import os 17 | import time 18 | 19 | import numpy as np 20 | import tensorflow as tf 21 | 22 | from ghhc.util.eval_dp import eval_dp 23 | from ghhc.util.io import mkdir_p 24 | 25 | from absl import logging 26 | 27 | tf.enable_eager_execution() 28 | 29 | def squared_norm(x, axis=1, keepdims=True): 30 | """Squared L2 Norm of x.""" 31 | return tf.reduce_sum(tf.pow(x, 2), axis=axis, keepdims=keepdims) 32 | 33 | 34 | def squared_euclidean_cdist(x, y): 35 | """Squared euclidean distance 36 | 37 | Computed as: ||x||^2 + ||y||^2 - 2 x^T y. 38 | 39 | Args: 40 | x: N by D matrix 41 | y: M by D matrix 42 | :returns matrix (N by M) such that result[i,j] = || x[i,:] - y[j,;] ||^2 43 | """ 44 | norms = squared_norm(x, axis=1, keepdims=True) + tf.transpose(squared_norm(y, axis=1, keepdims=True)) 45 | dot = 2.0*tf.matmul(x, y, transpose_b=True) 46 | return norms - dot 47 | 48 | def poincare_cdist(x, y): 49 | """Poincare distance 50 | 51 | Args: 52 | x: N by D matrix 53 | y: M by D matrix 54 | :returns matrix (N by M) such that result[i,j] = ppoincare dist(x[i,:], y[j,:]) 55 | """ 56 | numerator = squared_euclidean_cdist(x, y) 57 | denom = (1.0 - squared_norm(x)) * (1.0 - tf.transpose(squared_norm(y, axis=1, keepdims=True))) 58 | arccosh_arg = 1.0 + 2.0 * numerator / denom 59 | res = tf.math.acosh(1e-8 + arccosh_arg) 60 | return res 61 | 62 | 63 | def squared_euclidean_dist(x, y): 64 | """Squared euclidean distance 65 | 66 | Computed as: ||x||^2 + ||y||^2 - 2 x^T y. 67 | 68 | Args: 69 | x: N by D matrix 70 | y: N by D matrix 71 | :returns vector (N by 1) such that the ith element is || x[i,:] - y[i,;] ||^2 72 | """ 73 | norms = squared_norm(x, axis=1, keepdims=True) + squared_norm(y, axis=1, keepdims=True) 74 | dot = 2*tf.reduce_sum(tf.multiply(x, y), axis=1, keepdims=True) 75 | return norms - dot 76 | 77 | 78 | def poincare_dist(x, y): 79 | """Poincare distance between x and y. 80 | 81 | Args: 82 | x: N by D matrix 83 | y: N by D matrix 84 | :returns vector (N by 1) such that the ith element is poincare dist(x[i,:], y[i,:]) 85 | """ 86 | numerator = squared_euclidean_dist(x, y) 87 | denom = (1.0 - squared_norm(x)) * (1.0 - squared_norm(y)) 88 | arccosh_arg = 1.0 + 2.0 * numerator / denom 89 | res = tf.math.acosh(arccosh_arg) 90 | return res 91 | 92 | 93 | def poincare_norm(x, axis=1, keepdims=True): 94 | """Squared poincare norm of x.""" 95 | return 2.0*tf.math.atanh(tf.linalg.norm(x, axis=axis, keepdims=keepdims)) 96 | 97 | 98 | def parent_order_penalty(p, c, marg): 99 | """Penalty for parents to have smaller norm than children.""" 100 | return tf.maximum(0.0, poincare_norm(p) - poincare_norm(c) + marg) + 1.0 101 | 102 | 103 | def parent_order_penalty_cdist(p, c, marg): 104 | """Penalty for parents to have smaller norm than children.""" 105 | return tf.maximum(0.0, tf.transpose(poincare_norm(p)) - poincare_norm(c) + marg) + 1.0 106 | 107 | 108 | class gHHCTree(tf.keras.Model): 109 | """Object for a ghhc tree.""" 110 | 111 | def __init__(self, init_tree=None, gamma=0.25, config=None, projection=None): 112 | super(gHHCTree, self).__init__() 113 | self.internals = tf.get_variable('internals', initializer=init_tree) 114 | self.max_norm = 0.8 115 | self.internals_so_far = 0 116 | self.gamma = gamma 117 | self.config = config 118 | self.projection = None 119 | self.cached_pairs = None 120 | if projection is not None: 121 | self.projection = projection 122 | else: 123 | self.projection = lambda x: x 124 | 125 | def project(self, x_i, x_j, x_k): 126 | return self.projection(x_i), self.projection(x_j), self.projection(x_k) 127 | 128 | def clip(self): 129 | tf.assign(self.internals, tf.clip_by_norm(self.internals, self.max_norm, axes=[1])) 130 | 131 | def p_par_broadcast(self, x_i): 132 | return self.p_par_to_broadcast(x_i, self.internals) 133 | 134 | def p_par_to_broadcast(self, x_i, nodes): 135 | dists = poincare_cdist(x_i, nodes) 136 | res = tf.multiply(dists, parent_order_penalty_cdist(nodes, x_i, self.gamma)) 137 | return res 138 | 139 | def p_par_to(self, x_i, nodes): 140 | dists = poincare_dist(x_i, nodes) 141 | res = tf.multiply(dists, parent_order_penalty(nodes, x_i, self.gamma)) 142 | return res 143 | 144 | def p_par_to_batched_np(self, x_i, nodes, batch_size=1000): 145 | dists = np.zeros((x_i.shape[0], nodes.shape[0]), np.float32) 146 | for i in range(0, x_i.shape[0], batch_size): 147 | logging.log_every_n_seconds(logging.INFO,'p_par_to_batched_np processed %s of %s', 5, i, x_i.shape[0]) 148 | for j in range(0, nodes.shape[0], batch_size): 149 | dists[i:(i+batch_size), j:(j+batch_size)] = self.p_par_to_broadcast(x_i[i:(i + batch_size), :], nodes[j:(j + batch_size), :]).numpy() 150 | return dists 151 | 152 | def compute_loss(self, x_i, x_j, x_k): 153 | x_i, x_j, x_k = self.project(x_i, x_j, x_k) 154 | 155 | x_i_dists = self.p_par_to_broadcast(x_i, self.internals) 156 | x_j_dists = self.p_par_to_broadcast(x_j, self.internals) 157 | x_k_dists = self.p_par_to_broadcast(x_k, self.internals) 158 | 159 | max_dists_ij = tf.maximum(x_i_dists, x_j_dists) 160 | gumbel_ij_noise = tf.log(-tf.log(tf.random_uniform(tf.shape(max_dists_ij)))) 161 | gumbel_ijk_noise = tf.log(-tf.log(tf.random_uniform(tf.shape(max_dists_ij)))) 162 | max_dists_ijk = tf.maximum(x_k_dists, max_dists_ij) 163 | lca_ij_softmax = tf.nn.softmax(-max_dists_ij+gumbel_ij_noise, axis=1) 164 | lca_ij_idx = tf.argmin(max_dists_ij, axis=1) 165 | offset = np.zeros_like(max_dists_ij) 166 | offset[np.arange(offset.shape[0]), lca_ij_idx] = 1000 167 | 168 | max_dists_ijk += offset 169 | lca_ijk_softmax = tf.nn.softmax(-max_dists_ijk + gumbel_ijk_noise, axis=1) 170 | 171 | logits1 = lca_ij_softmax * x_i_dists - lca_ijk_softmax * x_i_dists 172 | logits2 = lca_ij_softmax * x_j_dists - lca_ijk_softmax * x_j_dists 173 | logits3 = lca_ijk_softmax * x_k_dists - lca_ij_softmax * x_k_dists 174 | 175 | per_ex_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.zeros_like(logits1), logits=logits1) \ 176 | + tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.zeros_like(logits2), logits=logits2) \ 177 | + tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.zeros_like(logits3), logits=logits3) 178 | loss = tf.reduce_sum(per_ex_loss) 179 | return loss 180 | 181 | def p_par_assign_to_internal(self, children, parents, proj_child=True): 182 | if proj_child: 183 | children = self.projection(children) 184 | 185 | internal_norm = tf.norm(parents, axis=1, keepdims=True) 186 | internal_ordering = tf.argsort(-tf.squeeze(internal_norm)).numpy() 187 | back_to_orig = tf.argsort(internal_ordering) 188 | parents = parents[internal_ordering,:] 189 | children = children[internal_ordering,:] 190 | 191 | dists = self.p_par_to_batched_np(children, parents) 192 | dists[np.tril_indices_from(dists)] = np.Inf 193 | np.fill_diagonal(dists, np.Inf) 194 | dists = dists[back_to_orig,:][:,back_to_orig] 195 | assignments = np.argmin(dists, axis=1) 196 | mindists = np.min(dists, axis=1) 197 | assignments[mindists == np.Inf] = -1 198 | return assignments 199 | 200 | def p_par_assign_to(self, children, parents, exclude_diag=False, proj_child=True): 201 | if proj_child: 202 | children = self.projection(children) 203 | dists = self.p_par_to_batched_np(children, parents) 204 | children_norm = tf.norm(children, axis=1, keepdims=True) 205 | internal_norm = tf.norm(parents, axis=1, keepdims=True) 206 | eligible = tf.less(-children_norm + tf.transpose(internal_norm), 0).numpy() 207 | dists[eligible == False] = np.Inf 208 | if exclude_diag: 209 | np.fill_diagonal(dists, np.Inf) 210 | assignments = np.argmin(dists, axis=1) 211 | mindists = np.min(dists, axis=1) 212 | assignments[mindists == np.Inf] = -1 213 | return assignments 214 | 215 | def write_tsv(self, filename, leaves, pids=None, lbls=None, update_cache=True): 216 | logging.info('Writing tree tsv to %s' % filename) 217 | logging.info('num leaves %s' % leaves.shape[0]) 218 | logging.info('pids is None? %s' % (pids is None)) 219 | logging.info('lbls is None? %s' % (lbls is None)) 220 | internals = self.internals.numpy() 221 | leaf_to_par_assign = self.p_par_assign_to(leaves, internals) 222 | internal_to_par_assign = self.p_par_assign_to_internal(internals, internals, proj_child=False) 223 | self.cached_pairs = np.concatenate([ np.expand_dims(np.arange(internal_to_par_assign.shape[0]),1), np.expand_dims(internal_to_par_assign,1)],axis=1) 224 | self.cached_pairs = self.cached_pairs[self.cached_pairs[:,1]!=-1] 225 | with open(filename + '.internals', 'w') as fouti: 226 | with open(filename + '.leaves', 'w') as foutl: 227 | with open(filename, 'w') as fout: 228 | i = -1 229 | pid = 'int_%s' % i 230 | best_pid = 'best_int_%s' % i 231 | par_id = 'None' 232 | fout.write('%s\t%s\tNone\n' % (pid, par_id)) 233 | fout.write('%s\t%s\tNone\n' % (best_pid, pid)) 234 | 235 | fouti.write('%s\t%s\tNone\n' % (pid, par_id)) 236 | fouti.write('%s\t%s\tNone\n' % (best_pid, pid)) 237 | 238 | for i in range(leaf_to_par_assign.shape[0]): 239 | logging.log_every_n_seconds(logging.INFO,'Wrote %s leaves' % i,5) 240 | pid = 'pt_%s' % i if pids is None else pids[i] 241 | lbl = pid if lbls is None else lbls[i] 242 | par_id = 'best_int_%s' % leaf_to_par_assign[i] 243 | fout.write('%s\t%s\t%s\n' % (pid, par_id, lbl)) 244 | foutl.write('%s\t%s\t%s\n' % (pid, par_id, lbl)) 245 | 246 | for i in range(internal_to_par_assign.shape[0]): 247 | logging.log_every_n_seconds(logging.INFO,'Wrote %s internals' % i,5) 248 | pid = 'int_%s' % i 249 | par_id = 'int_%s' % internal_to_par_assign[i] 250 | best_pid = 'best_int_%s' % i 251 | fout.write('%s\t%s\tNone\n' % (pid, par_id)) 252 | fout.write('%s\t%s\tNone\n' % (best_pid, par_id)) 253 | fouti.write('%s\t%s\tNone\n' % (pid, par_id)) 254 | fouti.write('%s\t%s\tNone\n' % (best_pid, par_id)) 255 | 256 | def plot_tree(self, leaves, filename): 257 | internals = self.internals.numpy() 258 | leaf_to_par_assign = self.p_par_assign_to(leaves, internals) 259 | internal_to_par_assign = self.p_par_assign_to_internal(internals, internals, proj_child=False) 260 | import matplotlib 261 | matplotlib.use('Agg') 262 | import matplotlib.pyplot as plt 263 | fig, ax = plt.subplots(figsize=(10, 10)) 264 | ax.spines['top'].set_visible(False) 265 | ax.spines['right'].set_visible(False) 266 | ax.spines['bottom'].set_visible(False) 267 | ax.spines['left'].set_visible(False) 268 | ax.tick_params(axis='x', which='both', bottom='off', top='off', 269 | color='white') 270 | ax.tick_params(axis='y', which='both', left='off', right='off', 271 | color='white') 272 | # plt.scatter(0, 0, label='root', marker='^', zorder=2) 273 | # plt.annotate('root', xy=(0,0), size=3) 274 | 275 | for idx in range(internals.shape[0]): 276 | plt.scatter(internals[idx, 0], internals[idx, 1], label='int_%s' % idx, s=100, marker='^', zorder=2) 277 | # plt.annotate('int_%s' % idx, xy=(internals[idx,0], internals[idx,1]), size=5) 278 | for idx in range(internals.shape[0]): 279 | if internal_to_par_assign[idx] != -1: 280 | plt.plot([internals[idx,0], internals[internal_to_par_assign[idx],0]], 281 | [internals[idx,1], internals[internal_to_par_assign[idx],1]], linewidth=2, 282 | c='k', zorder=1) 283 | # else: 284 | # plt.plot([internals[idx, 0], 0], 285 | # [internals[idx, 1], 0], linewidth=1, 286 | # c='k', zorder=1) 287 | for idx in range(leaves.shape[0]): 288 | plt.scatter(leaves[idx, 0], leaves[idx, 1], s=100, label='%s' % idx, marker='o', zorder=2) 289 | 290 | # plt.annotate('pt_%s' % idx, xy=(leaves[idx, 0], leaves[idx, 1]), size=5) 291 | for idx in range(leaves.shape[0]): 292 | if leaf_to_par_assign[idx] != -1: 293 | # print('gpid %s lpid %s' % (grinch_par_id, leaf_to_par_assign[idx])) 294 | plt.plot([leaves[idx, 0], internals[leaf_to_par_assign[idx], 0]], 295 | [leaves[idx, 1], internals[leaf_to_par_assign[idx], 1]], linewidth=2, 296 | c='k', zorder=1) 297 | # else: 298 | # plt.plot([leaves[idx, 0], 0], 299 | # [leaves[idx, 1], 0], linewidth=1, 300 | # c='k', zorder=1) 301 | plt.xlim([-1.1, 1.1]) 302 | plt.ylim([-1.1, 1.1]) 303 | circle = plt.Circle((0, 0), 1, color='r',linewidth=5, fill=False) 304 | ax.add_artist(circle) 305 | plt.axis('off') 306 | plt.savefig(filename) 307 | 308 | def structure_loss(self): 309 | res = tf.reduce_sum(self.child_parent_norm_loss(self.cached_pairs)) 310 | # logging.log_every_n(logging.INFO,'cp res: %s', 10,res ) 311 | return res 312 | 313 | def child_parent_norm_loss(self, pairs): 314 | internal_norms = poincare_norm(self.internals) 315 | children = tf.gather(internal_norms, pairs[:,0]) 316 | parents = tf.gather(internal_norms, pairs[:,1]) 317 | logits1 = tf.nn.relu(parents - children + self.gamma) 318 | min_norm = tf.argmin(internal_norms).numpy()[0] 319 | logging.log_every_n(logging.INFO,'min_norm %s %s',500,min_norm,internal_norms[min_norm]) 320 | max_norm = tf.argmax(internal_norms).numpy()[0] 321 | logging.log_every_n(logging.INFO, 'max_norm %s %s', 500, max_norm, internal_norms[max_norm]) 322 | return tf.reduce_sum(logits1) 323 | 324 | 325 | def rsgd_or_sgd(grads_and_vars, rsgd=True): 326 | if rsgd: 327 | res = [] 328 | for g,v in grads_and_vars: 329 | scale = ((1.0 - tf.reduce_sum(tf.multiply(v,v),axis=1,keepdims=True)) ** 2) / 4.0 330 | res.append((scale*g, v)) 331 | return res 332 | else: 333 | return grads_and_vars 334 | 335 | class gHHCInference(object): 336 | def __init__(self, ghhcTree, optimizer, config, dev_set, dev_lbls): 337 | self.ghhcTree = ghhcTree 338 | self.optimizer = optimizer 339 | self.config = config 340 | self.dev_set = dev_set 341 | self.dev_lbls = dev_lbls 342 | self.best_dev_dp_score = 0.0 343 | self.best_dev_iter = 0.0 344 | self.last_dev_dp_score = 0.0 345 | self.last_dev_iter = 0.0 346 | self.checkpoint_prefix = self.config.checkpoint_dir + "/ckpt" 347 | self.ckpt = tf.train.Checkpoint(optimizer=optimizer, 348 | model=ghhcTree, 349 | optimizer_step=tf.train.get_or_create_global_step()) 350 | 351 | def update(self, c1, c2, par_id, gp_id, steps=100): 352 | for i in range(steps): 353 | with tf.GradientTape() as tape: 354 | loss = self.ghhcTree.pull_close_par_gp(c1, c2, par_id, gp_id) 355 | grads = tape.gradient(loss, self.ghhcTree.trainable_variables) 356 | self.optimizer.apply_gradients(rsgd_or_sgd(zip(grads, self.ghhcTree.trainable_variables)), 357 | global_step=tf.train.get_or_create_global_step()) 358 | self.ghhcTree.clip() 359 | 360 | def episode_inference(self, x_i, x_j, x_k, dataset, batch_size=1000, examples_so_far=0): 361 | time_so_far = 0.0 362 | loss_so_far = 0.0 363 | struct_loss_so_far = 0.0 364 | 365 | for idx in range(0, x_i.shape[0], batch_size): 366 | 367 | if self.config.struct_prior is not None and idx+examples_so_far > 0: 368 | if self.ghhcTree.cached_pairs is None: 369 | self.dev_eval(idx + examples_so_far) 370 | 371 | if (idx + examples_so_far) % self.config.struct_prior_every == 0: 372 | for idx2 in range(self.config.num_struct_prior_batches): 373 | start_time = time.time() 374 | logging.log_every_n(logging.INFO, 375 | '[STRUCTURE] Processed %s of %s batches || Avg. Loss %s || Avg Time %s' % (idx2, 100, struct_loss_so_far / max(idx2, 1), time_so_far / max(idx2, 1)),100) 376 | with tf.GradientTape() as tape: 377 | sloss = self.ghhcTree.structure_loss() 378 | struct_loss_so_far += sloss.numpy() 379 | grads = tape.gradient(sloss, self.ghhcTree.trainable_variables) 380 | self.optimizer.apply_gradients(rsgd_or_sgd(zip(grads, self.ghhcTree.trainable_variables)), 381 | global_step=tf.train.get_or_create_global_step()) 382 | self.ghhcTree.clip() 383 | end_time = time.time() 384 | time_so_far += end_time - start_time 385 | logging.log(logging.INFO, '[STRUCTURE] Processed %s of %s batches || Avg. Loss %s || Avg Time %s' % (self.config.num_struct_prior_batches, 100, struct_loss_so_far / max(self.config.num_struct_prior_batches, 1), time_so_far / max(self.config.num_struct_prior_batches, 1))) 386 | 387 | if (idx + examples_so_far) % self.config.dev_every == 0: 388 | self.dev_eval(idx + examples_so_far) 389 | elif (idx + examples_so_far ) % self.config.save_every == 0: 390 | self.ckpt.save(self.checkpoint_prefix) 391 | self.config.last_model = tf.train.latest_checkpoint(self.config.checkpoint_dir) 392 | self.config.save_config(self.config.exp_out_dir, filename='config.json') 393 | 394 | start_time = time.time() 395 | if idx % 100 == 0 and idx > 0: 396 | logging.info('Processed %s of %s batches || Avg. Loss %s || Avg Time %s' % (idx, x_i.shape[0], loss_so_far/idx, time_so_far / max(idx,1))) 397 | with tf.GradientTape() as tape: 398 | bx_i = dataset[x_i[idx:(idx + batch_size)], :] 399 | bx_j = dataset[x_j[idx:(idx + batch_size)], :] 400 | bx_k = dataset[x_k[idx:(idx + batch_size)], :] 401 | loss = self.ghhcTree.compute_loss(bx_i, bx_j, bx_k) 402 | loss_so_far += loss.numpy() 403 | grads = tape.gradient(loss, self.ghhcTree.trainable_variables) 404 | self.optimizer.apply_gradients(rsgd_or_sgd(zip(grads, self.ghhcTree.trainable_variables)), 405 | global_step=tf.train.get_or_create_global_step()) 406 | self.ghhcTree.clip() 407 | end_time = time.time() 408 | time_so_far += end_time - start_time 409 | 410 | logging.info('Processed %s of %s batches || Avg. Loss %s || Avg Time %s' % (x_i.shape[0], x_i.shape[0], loss_so_far / x_i.shape[0], time_so_far / max(x_i.shape[0], 1))) 411 | 412 | # save model at the end of training 413 | self.ckpt.save(self.checkpoint_prefix) 414 | self.config.last_model = tf.train.latest_checkpoint(self.config.checkpoint_dir) 415 | # record the last model in the config. 416 | self.config.save_config(self.config.exp_out_dir, filename='config.json') 417 | return x_i.shape[0] 418 | 419 | def dev_eval(self, steps): 420 | if self.dev_set is not None: 421 | start_dev = time.time() 422 | mkdir_p(os.path.join(self.config.exp_out_dir, 'dev')) 423 | filename = os.path.join(self.config.exp_out_dir, 'dev', 'dev_tree_%s.tsv' % steps) 424 | self.ghhcTree.write_tsv(filename,self.dev_set,lbls=self.dev_lbls) 425 | dp = eval_dp(filename, os.path.join(self.config.exp_out_dir, 'dev', 'dev_score_%s.tsv' % steps), 426 | self.config.threads, self.config.dev_points_file) 427 | logging.info('DEV EVAL @ %s minibatches || %s DP' % (steps,dp)) 428 | end_dev = time.time() 429 | logging.info('Finished Dev Eval in %s seconds' % (end_dev-start_dev)) 430 | if self.config.save_dev_pics: 431 | filename = os.path.join(self.config.exp_out_dir, 'dev', 'dev_tree_%s.png' % steps) 432 | self.ghhcTree.plot_tree(self.dev_set, filename) 433 | 434 | # record the best dev score to try to understand if we end up doing worse, not used at inference time 435 | # last model is used at inference. 436 | self.best_dev_dp_score = max(self.best_dev_dp_score,dp) 437 | self.best_dev_iter = steps if self.best_dev_dp_score == dp else self.best_dev_iter 438 | self.last_dev_dp_score = dp 439 | self.last_dev_iter = steps 440 | # save every time we run this eval 441 | self.ckpt.save(self.checkpoint_prefix) 442 | self.config.last_model = tf.train.latest_checkpoint(self.config.checkpoint_dir) 443 | if self.best_dev_dp_score == dp: 444 | self.config.best_model = tf.train.latest_checkpoint(self.config.checkpoint_dir) 445 | self.config.save_config(self.config.exp_out_dir, filename='config.json') 446 | return dp 447 | else: 448 | return 0.0 449 | 450 | def inference(self, indexes, dataset, batch_size=1000, episode_size=5000): 451 | batches_so_far = 0 452 | curr_idx = 0 453 | episode_size = self.config.episode_size 454 | if self.config.shuffle: 455 | indexes = indexes[np.random.permutation(indexes.shape[0]), :] 456 | for i in range(self.config.num_iterations): 457 | if curr_idx > indexes.shape[0]: 458 | logging.info('Restarting....') 459 | curr_idx = 0 460 | if self.config.shuffle: 461 | indexes = indexes[np.random.permutation(indexes.shape[0]), :] 462 | logging.info('Starting iteration %s of %s' % (i, self.config.num_iterations)) 463 | batches_so_far += self.episode_inference(indexes[curr_idx:(curr_idx+episode_size), 0], 464 | indexes[curr_idx:(curr_idx+episode_size), 1], 465 | indexes[curr_idx:(curr_idx+episode_size), 2], 466 | dataset, batch_size, examples_so_far=batches_so_far) 467 | 468 | 469 | -------------------------------------------------------------------------------- /data/covtype.evalpts5k: -------------------------------------------------------------------------------- 1 | 13993 2 | 104991 3 | 89769 4 | 232503 5 | 318623 6 | 361776 7 | 257285 8 | 507206 9 | 262982 10 | 461493 11 | 3152 12 | 323632 13 | 59203 14 | 138675 15 | 187478 16 | 299388 17 | 137665 18 | 242625 19 | 353839 20 | 423959 21 | 9891 22 | 444187 23 | 17334 24 | 150662 25 | 423867 26 | 319346 27 | 527354 28 | 390147 29 | 287552 30 | 204990 31 | 285914 32 | 65171 33 | 1094 34 | 508459 35 | 161722 36 | 46799 37 | 498771 38 | 330456 39 | 419492 40 | 62598 41 | 18414 42 | 520603 43 | 368022 44 | 369989 45 | 474302 46 | 87955 47 | 418224 48 | 342932 49 | 157850 50 | 163820 51 | 549090 52 | 494486 53 | 417438 54 | 71247 55 | 124274 56 | 259612 57 | 48076 58 | 572160 59 | 550677 60 | 307567 61 | 104083 62 | 394470 63 | 54661 64 | 164806 65 | 50949 66 | 336968 67 | 552700 68 | 12672 69 | 324451 70 | 489363 71 | 529725 72 | 417900 73 | 13518 74 | 466051 75 | 280427 76 | 449118 77 | 290543 78 | 297050 79 | 415250 80 | 425092 81 | 351474 82 | 239283 83 | 6563 84 | 25416 85 | 259516 86 | 410754 87 | 567437 88 | 289200 89 | 90578 90 | 452127 91 | 222144 92 | 159672 93 | 235539 94 | 552057 95 | 221480 96 | 255772 97 | 303373 98 | 406871 99 | 498140 100 | 12661 101 | 343842 102 | 185591 103 | 319422 104 | 140376 105 | 265242 106 | 74381 107 | 272825 108 | 360245 109 | 485185 110 | 73173 111 | 414292 112 | 273792 113 | 419536 114 | 36978 115 | 75237 116 | 188839 117 | 281645 118 | 417584 119 | 441454 120 | 91411 121 | 338334 122 | 538409 123 | 13417 124 | 149734 125 | 137482 126 | 135076 127 | 479475 128 | 446565 129 | 308191 130 | 154845 131 | 372791 132 | 65844 133 | 139204 134 | 524839 135 | 445304 136 | 292010 137 | 579658 138 | 76362 139 | 12423 140 | 282277 141 | 539832 142 | 204890 143 | 227526 144 | 291807 145 | 108398 146 | 216317 147 | 110832 148 | 339779 149 | 89658 150 | 189962 151 | 25985 152 | 220412 153 | 294575 154 | 318596 155 | 549912 156 | 87174 157 | 545421 158 | 447054 159 | 378753 160 | 509243 161 | 44012 162 | 351707 163 | 163834 164 | 305816 165 | 89875 166 | 194729 167 | 28695 168 | 539953 169 | 219935 170 | 193040 171 | 251974 172 | 302035 173 | 423376 174 | 310718 175 | 523156 176 | 58497 177 | 53787 178 | 189849 179 | 538691 180 | 186206 181 | 287717 182 | 356605 183 | 434031 184 | 346394 185 | 9268 186 | 239837 187 | 478531 188 | 256604 189 | 113744 190 | 448708 191 | 324133 192 | 546717 193 | 98353 194 | 151890 195 | 141589 196 | 219761 197 | 542281 198 | 434780 199 | 432174 200 | 545185 201 | 555701 202 | 54553 203 | 374595 204 | 158250 205 | 572436 206 | 576640 207 | 90970 208 | 436821 209 | 183728 210 | 106114 211 | 404903 212 | 243733 213 | 369940 214 | 257680 215 | 163101 216 | 243321 217 | 302758 218 | 40359 219 | 308436 220 | 227212 221 | 16819 222 | 439342 223 | 424494 224 | 334718 225 | 457153 226 | 579190 227 | 183836 228 | 461612 229 | 211841 230 | 378092 231 | 139706 232 | 417142 233 | 474665 234 | 242151 235 | 235873 236 | 560733 237 | 368136 238 | 244660 239 | 99337 240 | 106155 241 | 112964 242 | 79909 243 | 570872 244 | 215306 245 | 558553 246 | 556070 247 | 28351 248 | 5165 249 | 240585 250 | 409321 251 | 563346 252 | 176383 253 | 456235 254 | 99719 255 | 378983 256 | 68863 257 | 1227 258 | 429989 259 | 380738 260 | 556076 261 | 170687 262 | 496314 263 | 149308 264 | 190487 265 | 303750 266 | 276169 267 | 257909 268 | 343493 269 | 396975 270 | 66078 271 | 63795 272 | 564395 273 | 192965 274 | 558954 275 | 411023 276 | 15373 277 | 542366 278 | 470387 279 | 490612 280 | 430897 281 | 577257 282 | 253968 283 | 359604 284 | 468252 285 | 372436 286 | 544906 287 | 328631 288 | 311500 289 | 426229 290 | 189866 291 | 502699 292 | 381349 293 | 356045 294 | 501507 295 | 206631 296 | 400462 297 | 437314 298 | 316729 299 | 186407 300 | 532840 301 | 175553 302 | 354904 303 | 127058 304 | 425625 305 | 112215 306 | 96956 307 | 323485 308 | 426156 309 | 514743 310 | 478087 311 | 17394 312 | 296283 313 | 551165 314 | 341282 315 | 159829 316 | 211099 317 | 467458 318 | 24869 319 | 534854 320 | 66910 321 | 221134 322 | 310392 323 | 395726 324 | 16030 325 | 138695 326 | 110104 327 | 483220 328 | 344229 329 | 499495 330 | 228326 331 | 189379 332 | 468866 333 | 141850 334 | 203665 335 | 471466 336 | 413862 337 | 143259 338 | 87804 339 | 100105 340 | 274176 341 | 181277 342 | 370000 343 | 451151 344 | 92801 345 | 552918 346 | 132664 347 | 88070 348 | 315631 349 | 410864 350 | 198896 351 | 367707 352 | 452281 353 | 10419 354 | 477703 355 | 286864 356 | 166904 357 | 497478 358 | 369421 359 | 192913 360 | 236972 361 | 503726 362 | 236786 363 | 162657 364 | 535932 365 | 19415 366 | 360944 367 | 305133 368 | 46589 369 | 385946 370 | 121885 371 | 312616 372 | 386987 373 | 108968 374 | 396839 375 | 425387 376 | 81021 377 | 192120 378 | 301420 379 | 210067 380 | 363413 381 | 359989 382 | 254338 383 | 260107 384 | 146605 385 | 172475 386 | 84104 387 | 501377 388 | 21241 389 | 268265 390 | 515127 391 | 25466 392 | 122510 393 | 178871 394 | 434519 395 | 184593 396 | 454407 397 | 77621 398 | 299112 399 | 309618 400 | 100544 401 | 43556 402 | 502291 403 | 449891 404 | 195860 405 | 196955 406 | 76592 407 | 566329 408 | 396943 409 | 208793 410 | 53030 411 | 242905 412 | 103801 413 | 253783 414 | 119916 415 | 501799 416 | 414432 417 | 249116 418 | 60703 419 | 325083 420 | 414061 421 | 240147 422 | 247962 423 | 239112 424 | 323583 425 | 362845 426 | 273680 427 | 442606 428 | 409407 429 | 178021 430 | 153065 431 | 427703 432 | 388043 433 | 206931 434 | 484309 435 | 415381 436 | 315239 437 | 577247 438 | 335810 439 | 575621 440 | 348108 441 | 4517 442 | 521312 443 | 166920 444 | 73694 445 | 129567 446 | 482918 447 | 177341 448 | 495998 449 | 409569 450 | 288732 451 | 75517 452 | 138431 453 | 479139 454 | 160811 455 | 286170 456 | 342830 457 | 567135 458 | 62750 459 | 10146 460 | 301587 461 | 233027 462 | 302046 463 | 570827 464 | 439626 465 | 3309 466 | 504247 467 | 55567 468 | 571967 469 | 552828 470 | 172815 471 | 551412 472 | 354620 473 | 255068 474 | 174089 475 | 213216 476 | 354455 477 | 41365 478 | 2809 479 | 492291 480 | 103138 481 | 411854 482 | 200188 483 | 499028 484 | 489844 485 | 522631 486 | 160285 487 | 388737 488 | 394150 489 | 443814 490 | 358047 491 | 189662 492 | 464635 493 | 3738 494 | 253254 495 | 506554 496 | 454606 497 | 145322 498 | 561188 499 | 247653 500 | 567896 501 | 310784 502 | 445369 503 | 161556 504 | 281988 505 | 253419 506 | 365829 507 | 152127 508 | 381132 509 | 440121 510 | 414707 511 | 569338 512 | 38371 513 | 477002 514 | 574239 515 | 210513 516 | 503459 517 | 100256 518 | 384119 519 | 433484 520 | 474458 521 | 553894 522 | 278758 523 | 148853 524 | 249795 525 | 517682 526 | 378804 527 | 198311 528 | 386466 529 | 296070 530 | 51968 531 | 206416 532 | 423975 533 | 390421 534 | 551280 535 | 20646 536 | 525695 537 | 467467 538 | 347393 539 | 519739 540 | 160283 541 | 202616 542 | 452794 543 | 516422 544 | 117780 545 | 354594 546 | 192251 547 | 519257 548 | 274652 549 | 378822 550 | 136521 551 | 250458 552 | 289731 553 | 229571 554 | 280047 555 | 539072 556 | 172411 557 | 201407 558 | 214674 559 | 35906 560 | 540360 561 | 92800 562 | 34662 563 | 400703 564 | 103688 565 | 516889 566 | 66074 567 | 208538 568 | 538744 569 | 230649 570 | 305222 571 | 546583 572 | 220189 573 | 306653 574 | 395364 575 | 13988 576 | 103065 577 | 207410 578 | 546664 579 | 198287 580 | 338724 581 | 234980 582 | 218296 583 | 548951 584 | 312862 585 | 225245 586 | 552195 587 | 456783 588 | 158466 589 | 573859 590 | 573874 591 | 68177 592 | 540808 593 | 543376 594 | 332108 595 | 387730 596 | 537972 597 | 140472 598 | 317118 599 | 329846 600 | 206753 601 | 383074 602 | 188917 603 | 415764 604 | 80235 605 | 496197 606 | 307114 607 | 439396 608 | 478803 609 | 260340 610 | 116766 611 | 549297 612 | 217147 613 | 93357 614 | 54272 615 | 521963 616 | 6876 617 | 487382 618 | 198056 619 | 313525 620 | 286314 621 | 476810 622 | 554255 623 | 538569 624 | 327938 625 | 457336 626 | 143934 627 | 128302 628 | 506919 629 | 341676 630 | 463182 631 | 61125 632 | 462257 633 | 414484 634 | 217266 635 | 475064 636 | 250413 637 | 498193 638 | 403434 639 | 234714 640 | 338164 641 | 533204 642 | 148990 643 | 213887 644 | 573267 645 | 229131 646 | 88254 647 | 430563 648 | 273091 649 | 15076 650 | 320500 651 | 134088 652 | 243572 653 | 72391 654 | 340089 655 | 513704 656 | 496703 657 | 552174 658 | 576937 659 | 512683 660 | 456387 661 | 480379 662 | 179638 663 | 497177 664 | 124367 665 | 559757 666 | 288902 667 | 226752 668 | 251521 669 | 78857 670 | 452550 671 | 158261 672 | 274589 673 | 67743 674 | 341735 675 | 95058 676 | 183880 677 | 579745 678 | 42420 679 | 562281 680 | 396608 681 | 156377 682 | 111605 683 | 149594 684 | 404685 685 | 421775 686 | 143450 687 | 554582 688 | 484522 689 | 298306 690 | 257197 691 | 89521 692 | 269851 693 | 335688 694 | 375852 695 | 406375 696 | 331623 697 | 160523 698 | 415410 699 | 473136 700 | 185755 701 | 75752 702 | 501887 703 | 381573 704 | 533685 705 | 469322 706 | 344030 707 | 126852 708 | 466248 709 | 26629 710 | 151042 711 | 115515 712 | 104341 713 | 236722 714 | 398740 715 | 367834 716 | 340941 717 | 208735 718 | 542736 719 | 110303 720 | 574926 721 | 517730 722 | 165050 723 | 145880 724 | 393702 725 | 197437 726 | 304196 727 | 574258 728 | 379777 729 | 450188 730 | 335287 731 | 320937 732 | 297350 733 | 5158 734 | 343915 735 | 3036 736 | 44492 737 | 476662 738 | 438712 739 | 483006 740 | 368774 741 | 422027 742 | 196562 743 | 527292 744 | 203459 745 | 306094 746 | 61014 747 | 30149 748 | 191009 749 | 134022 750 | 140845 751 | 190297 752 | 387783 753 | 218989 754 | 278806 755 | 183481 756 | 4445 757 | 553285 758 | 534914 759 | 71643 760 | 572493 761 | 112106 762 | 453984 763 | 133264 764 | 184311 765 | 499523 766 | 516346 767 | 374266 768 | 319773 769 | 542834 770 | 42458 771 | 484825 772 | 338362 773 | 316117 774 | 132536 775 | 362528 776 | 489453 777 | 226305 778 | 140822 779 | 253591 780 | 500969 781 | 329231 782 | 334755 783 | 150149 784 | 248693 785 | 225782 786 | 318685 787 | 436494 788 | 173042 789 | 392328 790 | 165094 791 | 351111 792 | 125571 793 | 443150 794 | 82080 795 | 465841 796 | 265927 797 | 163816 798 | 13337 799 | 344145 800 | 76633 801 | 310288 802 | 255432 803 | 499681 804 | 557872 805 | 237656 806 | 137773 807 | 42437 808 | 523124 809 | 61059 810 | 52505 811 | 214007 812 | 200924 813 | 249724 814 | 512473 815 | 336576 816 | 17455 817 | 308727 818 | 284233 819 | 102364 820 | 399605 821 | 473883 822 | 532210 823 | 251292 824 | 322940 825 | 132525 826 | 502728 827 | 75883 828 | 86566 829 | 555740 830 | 210094 831 | 471726 832 | 91154 833 | 179323 834 | 223907 835 | 542491 836 | 342344 837 | 215634 838 | 465781 839 | 523538 840 | 570866 841 | 128234 842 | 550229 843 | 576801 844 | 310811 845 | 376272 846 | 166056 847 | 458349 848 | 409325 849 | 537329 850 | 348065 851 | 277568 852 | 459415 853 | 576883 854 | 111387 855 | 549375 856 | 572911 857 | 159017 858 | 443022 859 | 172504 860 | 116554 861 | 135690 862 | 361775 863 | 426911 864 | 70500 865 | 412330 866 | 437834 867 | 424390 868 | 61072 869 | 500273 870 | 444127 871 | 198133 872 | 191177 873 | 60710 874 | 262305 875 | 144128 876 | 447874 877 | 134905 878 | 572461 879 | 63332 880 | 140339 881 | 437363 882 | 433545 883 | 268428 884 | 380576 885 | 374864 886 | 23027 887 | 569995 888 | 87865 889 | 455240 890 | 53133 891 | 113655 892 | 230685 893 | 48907 894 | 264319 895 | 321817 896 | 148306 897 | 427383 898 | 413324 899 | 532060 900 | 86104 901 | 540707 902 | 402462 903 | 231836 904 | 138287 905 | 169885 906 | 490468 907 | 573922 908 | 289968 909 | 240459 910 | 110078 911 | 141788 912 | 18590 913 | 560844 914 | 256054 915 | 369461 916 | 248689 917 | 420575 918 | 374641 919 | 383438 920 | 183874 921 | 252037 922 | 314029 923 | 330151 924 | 234971 925 | 129918 926 | 10936 927 | 349621 928 | 124338 929 | 444039 930 | 492486 931 | 20979 932 | 237167 933 | 539059 934 | 443263 935 | 492424 936 | 543938 937 | 448816 938 | 528084 939 | 191193 940 | 53441 941 | 388163 942 | 176160 943 | 84511 944 | 255403 945 | 174206 946 | 134954 947 | 303956 948 | 544815 949 | 241353 950 | 2003 951 | 550641 952 | 115528 953 | 523119 954 | 91794 955 | 396931 956 | 286690 957 | 442211 958 | 100405 959 | 100386 960 | 296969 961 | 450532 962 | 65487 963 | 85149 964 | 48737 965 | 213600 966 | 356699 967 | 161254 968 | 120902 969 | 51669 970 | 251723 971 | 76937 972 | 546455 973 | 405027 974 | 470603 975 | 182822 976 | 267729 977 | 184491 978 | 95644 979 | 351226 980 | 319709 981 | 250607 982 | 180845 983 | 459861 984 | 165521 985 | 168088 986 | 556765 987 | 171263 988 | 551153 989 | 94521 990 | 441229 991 | 510496 992 | 521014 993 | 266666 994 | 458097 995 | 52542 996 | 413897 997 | 469607 998 | 194689 999 | 502546 1000 | 342635 1001 | 278393 1002 | 73291 1003 | 273856 1004 | 21160 1005 | 57626 1006 | 225785 1007 | 153498 1008 | 401492 1009 | 148286 1010 | 352545 1011 | 227797 1012 | 275105 1013 | 9811 1014 | 29691 1015 | 153228 1016 | 450423 1017 | 16546 1018 | 401880 1019 | 541885 1020 | 332316 1021 | 388327 1022 | 330032 1023 | 580408 1024 | 311448 1025 | 359282 1026 | 84604 1027 | 351185 1028 | 97529 1029 | 50961 1030 | 30752 1031 | 138658 1032 | 245341 1033 | 94710 1034 | 567654 1035 | 80952 1036 | 20799 1037 | 538216 1038 | 288197 1039 | 288884 1040 | 94731 1041 | 4631 1042 | 546438 1043 | 253913 1044 | 134758 1045 | 79120 1046 | 176036 1047 | 286704 1048 | 401191 1049 | 569186 1050 | 458129 1051 | 142178 1052 | 258743 1053 | 297352 1054 | 200669 1055 | 512029 1056 | 285982 1057 | 177733 1058 | 295515 1059 | 59687 1060 | 230466 1061 | 483199 1062 | 248597 1063 | 289726 1064 | 571108 1065 | 102751 1066 | 232696 1067 | 196971 1068 | 277123 1069 | 150780 1070 | 566169 1071 | 508667 1072 | 524806 1073 | 367451 1074 | 68168 1075 | 395468 1076 | 259283 1077 | 496333 1078 | 197186 1079 | 91738 1080 | 149844 1081 | 489228 1082 | 244260 1083 | 21106 1084 | 283315 1085 | 243899 1086 | 125148 1087 | 100728 1088 | 353519 1089 | 385815 1090 | 446709 1091 | 192017 1092 | 302004 1093 | 471555 1094 | 171046 1095 | 123724 1096 | 552865 1097 | 154629 1098 | 339184 1099 | 519529 1100 | 387900 1101 | 296931 1102 | 574607 1103 | 407805 1104 | 61624 1105 | 103427 1106 | 215726 1107 | 500353 1108 | 223758 1109 | 47232 1110 | 96508 1111 | 296610 1112 | 345414 1113 | 89260 1114 | 441444 1115 | 476518 1116 | 384880 1117 | 506313 1118 | 355306 1119 | 60885 1120 | 506884 1121 | 196031 1122 | 240600 1123 | 377967 1124 | 249783 1125 | 340233 1126 | 169950 1127 | 460079 1128 | 10536 1129 | 70708 1130 | 534450 1131 | 463952 1132 | 39505 1133 | 226338 1134 | 394740 1135 | 130144 1136 | 288878 1137 | 249966 1138 | 331117 1139 | 281690 1140 | 68542 1141 | 2355 1142 | 160268 1143 | 525949 1144 | 97977 1145 | 340888 1146 | 446049 1147 | 192564 1148 | 386584 1149 | 475035 1150 | 454012 1151 | 508892 1152 | 255134 1153 | 88338 1154 | 140293 1155 | 309294 1156 | 182083 1157 | 179883 1158 | 147385 1159 | 577776 1160 | 156511 1161 | 37028 1162 | 7747 1163 | 170433 1164 | 170623 1165 | 393356 1166 | 144411 1167 | 361278 1168 | 382307 1169 | 421474 1170 | 18388 1171 | 534048 1172 | 245577 1173 | 118424 1174 | 152008 1175 | 184428 1176 | 488487 1177 | 388096 1178 | 368293 1179 | 109381 1180 | 220312 1181 | 498795 1182 | 307288 1183 | 49408 1184 | 225311 1185 | 572768 1186 | 17153 1187 | 426676 1188 | 562517 1189 | 379646 1190 | 14733 1191 | 164775 1192 | 242249 1193 | 499313 1194 | 448457 1195 | 127004 1196 | 40912 1197 | 562110 1198 | 442769 1199 | 49963 1200 | 81549 1201 | 246799 1202 | 277235 1203 | 366373 1204 | 47638 1205 | 423971 1206 | 469451 1207 | 222840 1208 | 100725 1209 | 332667 1210 | 470016 1211 | 224301 1212 | 76142 1213 | 572070 1214 | 69826 1215 | 420216 1216 | 190324 1217 | 522190 1218 | 393841 1219 | 478337 1220 | 411787 1221 | 160443 1222 | 514093 1223 | 438497 1224 | 267195 1225 | 558088 1226 | 426635 1227 | 91335 1228 | 132501 1229 | 62196 1230 | 274182 1231 | 394487 1232 | 539090 1233 | 302188 1234 | 62115 1235 | 384837 1236 | 266010 1237 | 446783 1238 | 357392 1239 | 535965 1240 | 557550 1241 | 572944 1242 | 425398 1243 | 334547 1244 | 57303 1245 | 137281 1246 | 78797 1247 | 514380 1248 | 242902 1249 | 540178 1250 | 22317 1251 | 175599 1252 | 239917 1253 | 489928 1254 | 578798 1255 | 100968 1256 | 396695 1257 | 217882 1258 | 449168 1259 | 74275 1260 | 469997 1261 | 206764 1262 | 504105 1263 | 360866 1264 | 381332 1265 | 559818 1266 | 237250 1267 | 251065 1268 | 180896 1269 | 580016 1270 | 167421 1271 | 442103 1272 | 344144 1273 | 54081 1274 | 393112 1275 | 190168 1276 | 129377 1277 | 185377 1278 | 188089 1279 | 218186 1280 | 495348 1281 | 535422 1282 | 42569 1283 | 248862 1284 | 345018 1285 | 210200 1286 | 375445 1287 | 251000 1288 | 505392 1289 | 310454 1290 | 566985 1291 | 346522 1292 | 129927 1293 | 546890 1294 | 147287 1295 | 447866 1296 | 472683 1297 | 34245 1298 | 331483 1299 | 410653 1300 | 529579 1301 | 410014 1302 | 218004 1303 | 441507 1304 | 107398 1305 | 18804 1306 | 575714 1307 | 208235 1308 | 364118 1309 | 227840 1310 | 552070 1311 | 539115 1312 | 472609 1313 | 575527 1314 | 50369 1315 | 142864 1316 | 171035 1317 | 260334 1318 | 562373 1319 | 541439 1320 | 254687 1321 | 159096 1322 | 139590 1323 | 373341 1324 | 330620 1325 | 518273 1326 | 498596 1327 | 381074 1328 | 337202 1329 | 439098 1330 | 467022 1331 | 455599 1332 | 384089 1333 | 173378 1334 | 481643 1335 | 60602 1336 | 206455 1337 | 94387 1338 | 449418 1339 | 541884 1340 | 513879 1341 | 470708 1342 | 403401 1343 | 330348 1344 | 296196 1345 | 235080 1346 | 112895 1347 | 458748 1348 | 180380 1349 | 251119 1350 | 449234 1351 | 533188 1352 | 226129 1353 | 389177 1354 | 499860 1355 | 41884 1356 | 454957 1357 | 8491 1358 | 244345 1359 | 406099 1360 | 173832 1361 | 185921 1362 | 276289 1363 | 557925 1364 | 164939 1365 | 48044 1366 | 560222 1367 | 347904 1368 | 442860 1369 | 540927 1370 | 541773 1371 | 459870 1372 | 54765 1373 | 338519 1374 | 253977 1375 | 286680 1376 | 273851 1377 | 303745 1378 | 257654 1379 | 459051 1380 | 365996 1381 | 480875 1382 | 531847 1383 | 65645 1384 | 207029 1385 | 430811 1386 | 264111 1387 | 268322 1388 | 333503 1389 | 469583 1390 | 528409 1391 | 111143 1392 | 539408 1393 | 157945 1394 | 466488 1395 | 345838 1396 | 357580 1397 | 126100 1398 | 426299 1399 | 552770 1400 | 406447 1401 | 521930 1402 | 306316 1403 | 204340 1404 | 569603 1405 | 315196 1406 | 563090 1407 | 508536 1408 | 359862 1409 | 431913 1410 | 18107 1411 | 534860 1412 | 226883 1413 | 244989 1414 | 526919 1415 | 182455 1416 | 402730 1417 | 46234 1418 | 86117 1419 | 193673 1420 | 396328 1421 | 27628 1422 | 262045 1423 | 40717 1424 | 337985 1425 | 446307 1426 | 369132 1427 | 246148 1428 | 250135 1429 | 509255 1430 | 176491 1431 | 19242 1432 | 111460 1433 | 329315 1434 | 339738 1435 | 94298 1436 | 232733 1437 | 94674 1438 | 189331 1439 | 232332 1440 | 221368 1441 | 220909 1442 | 344094 1443 | 73646 1444 | 315073 1445 | 552025 1446 | 9046 1447 | 147910 1448 | 122912 1449 | 191494 1450 | 129846 1451 | 133199 1452 | 492041 1453 | 560476 1454 | 184523 1455 | 291235 1456 | 191626 1457 | 429897 1458 | 191499 1459 | 487673 1460 | 391220 1461 | 171109 1462 | 28244 1463 | 281866 1464 | 377996 1465 | 401057 1466 | 517698 1467 | 347224 1468 | 235760 1469 | 120560 1470 | 196223 1471 | 462009 1472 | 233460 1473 | 353196 1474 | 336802 1475 | 381760 1476 | 462146 1477 | 352175 1478 | 450165 1479 | 247949 1480 | 67512 1481 | 540053 1482 | 535804 1483 | 401428 1484 | 172645 1485 | 4705 1486 | 412139 1487 | 239419 1488 | 64642 1489 | 438590 1490 | 408813 1491 | 208557 1492 | 446399 1493 | 267315 1494 | 499831 1495 | 41970 1496 | 362983 1497 | 452836 1498 | 417673 1499 | 118139 1500 | 161773 1501 | 202675 1502 | 210791 1503 | 23965 1504 | 562260 1505 | 121126 1506 | 281993 1507 | 216695 1508 | 566862 1509 | 453175 1510 | 33945 1511 | 149025 1512 | 304653 1513 | 22433 1514 | 56231 1515 | 481904 1516 | 145519 1517 | 422332 1518 | 274257 1519 | 321341 1520 | 398309 1521 | 272613 1522 | 333689 1523 | 448972 1524 | 554201 1525 | 54287 1526 | 22010 1527 | 536064 1528 | 154094 1529 | 100915 1530 | 26225 1531 | 111515 1532 | 414500 1533 | 185579 1534 | 194144 1535 | 296390 1536 | 93171 1537 | 573229 1538 | 504322 1539 | 376734 1540 | 269832 1541 | 12354 1542 | 343864 1543 | 289412 1544 | 178196 1545 | 34496 1546 | 432449 1547 | 20516 1548 | 210803 1549 | 18955 1550 | 455722 1551 | 240026 1552 | 131035 1553 | 261607 1554 | 218427 1555 | 146934 1556 | 434570 1557 | 551196 1558 | 133840 1559 | 491742 1560 | 17777 1561 | 518508 1562 | 411058 1563 | 47428 1564 | 225808 1565 | 310357 1566 | 225263 1567 | 148594 1568 | 320532 1569 | 245506 1570 | 318525 1571 | 414039 1572 | 25845 1573 | 341346 1574 | 452940 1575 | 235201 1576 | 85693 1577 | 422255 1578 | 336033 1579 | 432394 1580 | 543729 1581 | 498420 1582 | 478091 1583 | 387484 1584 | 116009 1585 | 483842 1586 | 171272 1587 | 9462 1588 | 175537 1589 | 572989 1590 | 323735 1591 | 90149 1592 | 219893 1593 | 204512 1594 | 394820 1595 | 149082 1596 | 41328 1597 | 119491 1598 | 169201 1599 | 366046 1600 | 316133 1601 | 231200 1602 | 237883 1603 | 65755 1604 | 97604 1605 | 282330 1606 | 539519 1607 | 172225 1608 | 391617 1609 | 497275 1610 | 439351 1611 | 211934 1612 | 140959 1613 | 251128 1614 | 355613 1615 | 505516 1616 | 9809 1617 | 190255 1618 | 413779 1619 | 168290 1620 | 329336 1621 | 64565 1622 | 116645 1623 | 208490 1624 | 68208 1625 | 289370 1626 | 371912 1627 | 436812 1628 | 497361 1629 | 64044 1630 | 470794 1631 | 245877 1632 | 501328 1633 | 213063 1634 | 192297 1635 | 208448 1636 | 545155 1637 | 353344 1638 | 125096 1639 | 529638 1640 | 458179 1641 | 442012 1642 | 146483 1643 | 112251 1644 | 471505 1645 | 311911 1646 | 494442 1647 | 574961 1648 | 357778 1649 | 537500 1650 | 126477 1651 | 285650 1652 | 62902 1653 | 47954 1654 | 359254 1655 | 148489 1656 | 527014 1657 | 354049 1658 | 406043 1659 | 404545 1660 | 135275 1661 | 267707 1662 | 464445 1663 | 414963 1664 | 49584 1665 | 34942 1666 | 124768 1667 | 354852 1668 | 54721 1669 | 149372 1670 | 167817 1671 | 161841 1672 | 168485 1673 | 568731 1674 | 397219 1675 | 531461 1676 | 391419 1677 | 85871 1678 | 242141 1679 | 64191 1680 | 110733 1681 | 512630 1682 | 284722 1683 | 68844 1684 | 369324 1685 | 145636 1686 | 532055 1687 | 28058 1688 | 205328 1689 | 311018 1690 | 48622 1691 | 56014 1692 | 340046 1693 | 525170 1694 | 168960 1695 | 384957 1696 | 49961 1697 | 347147 1698 | 502041 1699 | 117328 1700 | 372175 1701 | 129701 1702 | 118381 1703 | 383863 1704 | 418665 1705 | 224205 1706 | 571001 1707 | 130690 1708 | 88621 1709 | 436890 1710 | 103617 1711 | 553563 1712 | 565930 1713 | 295593 1714 | 421258 1715 | 67047 1716 | 494757 1717 | 262259 1718 | 119474 1719 | 548354 1720 | 405154 1721 | 239520 1722 | 147962 1723 | 3233 1724 | 332295 1725 | 437822 1726 | 154372 1727 | 55027 1728 | 488424 1729 | 261105 1730 | 321489 1731 | 182594 1732 | 138321 1733 | 556469 1734 | 568333 1735 | 377080 1736 | 514831 1737 | 552966 1738 | 243012 1739 | 220698 1740 | 282936 1741 | 84654 1742 | 434842 1743 | 559300 1744 | 557649 1745 | 195413 1746 | 16537 1747 | 76349 1748 | 372404 1749 | 389712 1750 | 52571 1751 | 489294 1752 | 547977 1753 | 265007 1754 | 399977 1755 | 541159 1756 | 204240 1757 | 441939 1758 | 488900 1759 | 95492 1760 | 289474 1761 | 131618 1762 | 569674 1763 | 556837 1764 | 302071 1765 | 144287 1766 | 436427 1767 | 130787 1768 | 560874 1769 | 132506 1770 | 34502 1771 | 516799 1772 | 477513 1773 | 6465 1774 | 287184 1775 | 119913 1776 | 549497 1777 | 108928 1778 | 85822 1779 | 189175 1780 | 242713 1781 | 415065 1782 | 291566 1783 | 171834 1784 | 102214 1785 | 12323 1786 | 204351 1787 | 434820 1788 | 484631 1789 | 224384 1790 | 365386 1791 | 570478 1792 | 563736 1793 | 142300 1794 | 123703 1795 | 363697 1796 | 125557 1797 | 331402 1798 | 130184 1799 | 576337 1800 | 309631 1801 | 499295 1802 | 468690 1803 | 240802 1804 | 236152 1805 | 329763 1806 | 361421 1807 | 103327 1808 | 249051 1809 | 243600 1810 | 264075 1811 | 425336 1812 | 53002 1813 | 252084 1814 | 368993 1815 | 149979 1816 | 135087 1817 | 385562 1818 | 320249 1819 | 340040 1820 | 396647 1821 | 424948 1822 | 153980 1823 | 107826 1824 | 203624 1825 | 237528 1826 | 470727 1827 | 445644 1828 | 31238 1829 | 2040 1830 | 396717 1831 | 183935 1832 | 512085 1833 | 282550 1834 | 136081 1835 | 261895 1836 | 121949 1837 | 551772 1838 | 113245 1839 | 182061 1840 | 171928 1841 | 234936 1842 | 580940 1843 | 295333 1844 | 345936 1845 | 75545 1846 | 138678 1847 | 17350 1848 | 315633 1849 | 437056 1850 | 266627 1851 | 530430 1852 | 47339 1853 | 436956 1854 | 530967 1855 | 40629 1856 | 497304 1857 | 352016 1858 | 327670 1859 | 382742 1860 | 310830 1861 | 579878 1862 | 442794 1863 | 335552 1864 | 269058 1865 | 422940 1866 | 54142 1867 | 102656 1868 | 112927 1869 | 146271 1870 | 356120 1871 | 440400 1872 | 285733 1873 | 218448 1874 | 329991 1875 | 395227 1876 | 176538 1877 | 338695 1878 | 65494 1879 | 392768 1880 | 27632 1881 | 294823 1882 | 563112 1883 | 232151 1884 | 101336 1885 | 156955 1886 | 168954 1887 | 269988 1888 | 335576 1889 | 282089 1890 | 467752 1891 | 575356 1892 | 104832 1893 | 274068 1894 | 574826 1895 | 153758 1896 | 519890 1897 | 482883 1898 | 401155 1899 | 429274 1900 | 192514 1901 | 249722 1902 | 247397 1903 | 164704 1904 | 134125 1905 | 5561 1906 | 426222 1907 | 505290 1908 | 220789 1909 | 237084 1910 | 455971 1911 | 150526 1912 | 551475 1913 | 20268 1914 | 261563 1915 | 495417 1916 | 107551 1917 | 121172 1918 | 150091 1919 | 17365 1920 | 214553 1921 | 297467 1922 | 477107 1923 | 529323 1924 | 532357 1925 | 340251 1926 | 303307 1927 | 490804 1928 | 43984 1929 | 180728 1930 | 518492 1931 | 572503 1932 | 507896 1933 | 472469 1934 | 523155 1935 | 493572 1936 | 425320 1937 | 535101 1938 | 22050 1939 | 138205 1940 | 519157 1941 | 191623 1942 | 472402 1943 | 407777 1944 | 572080 1945 | 3198 1946 | 388868 1947 | 386923 1948 | 113155 1949 | 297228 1950 | 428726 1951 | 241770 1952 | 68545 1953 | 52883 1954 | 68294 1955 | 110215 1956 | 124047 1957 | 64902 1958 | 150041 1959 | 431433 1960 | 495168 1961 | 229176 1962 | 36052 1963 | 324725 1964 | 534714 1965 | 130812 1966 | 366031 1967 | 178363 1968 | 471522 1969 | 229778 1970 | 380973 1971 | 57902 1972 | 364542 1973 | 578759 1974 | 339740 1975 | 39695 1976 | 324529 1977 | 268958 1978 | 578422 1979 | 56292 1980 | 62442 1981 | 434999 1982 | 144545 1983 | 471406 1984 | 51709 1985 | 53839 1986 | 216925 1987 | 98373 1988 | 429220 1989 | 309307 1990 | 165947 1991 | 85429 1992 | 451272 1993 | 53101 1994 | 205451 1995 | 346801 1996 | 56252 1997 | 126877 1998 | 190744 1999 | 324314 2000 | 498472 2001 | 356720 2002 | 540457 2003 | 250080 2004 | 65815 2005 | 449096 2006 | 111230 2007 | 342205 2008 | 262216 2009 | 523290 2010 | 480376 2011 | 511287 2012 | 351407 2013 | 66259 2014 | 197485 2015 | 462949 2016 | 4610 2017 | 75542 2018 | 85231 2019 | 205063 2020 | 399612 2021 | 490539 2022 | 372360 2023 | 281800 2024 | 313564 2025 | 237937 2026 | 86879 2027 | 478174 2028 | 32700 2029 | 91725 2030 | 239288 2031 | 504980 2032 | 542335 2033 | 136513 2034 | 75922 2035 | 95809 2036 | 358681 2037 | 367634 2038 | 312336 2039 | 339449 2040 | 274259 2041 | 391978 2042 | 70305 2043 | 225586 2044 | 421209 2045 | 306336 2046 | 2589 2047 | 343346 2048 | 552282 2049 | 537541 2050 | 322531 2051 | 139909 2052 | 245977 2053 | 435518 2054 | 110249 2055 | 419479 2056 | 208641 2057 | 495622 2058 | 359353 2059 | 49006 2060 | 375637 2061 | 196400 2062 | 362304 2063 | 509888 2064 | 102041 2065 | 180352 2066 | 155055 2067 | 202077 2068 | 235417 2069 | 466115 2070 | 334009 2071 | 571055 2072 | 288973 2073 | 376783 2074 | 440670 2075 | 442501 2076 | 312519 2077 | 20145 2078 | 46493 2079 | 172547 2080 | 287799 2081 | 457045 2082 | 29834 2083 | 182409 2084 | 476180 2085 | 46 2086 | 248130 2087 | 35110 2088 | 538380 2089 | 305442 2090 | 539678 2091 | 150403 2092 | 277716 2093 | 67005 2094 | 9412 2095 | 541571 2096 | 256828 2097 | 18843 2098 | 91006 2099 | 520323 2100 | 47064 2101 | 428356 2102 | 367165 2103 | 42771 2104 | 134278 2105 | 509136 2106 | 230546 2107 | 338545 2108 | 352873 2109 | 433140 2110 | 97947 2111 | 393465 2112 | 110450 2113 | 259714 2114 | 100879 2115 | 257434 2116 | 573418 2117 | 438035 2118 | 466719 2119 | 214026 2120 | 385598 2121 | 106618 2122 | 180054 2123 | 522553 2124 | 275303 2125 | 357160 2126 | 270162 2127 | 283183 2128 | 556896 2129 | 54326 2130 | 169704 2131 | 228284 2132 | 236043 2133 | 440989 2134 | 391241 2135 | 127571 2136 | 305481 2137 | 111618 2138 | 209626 2139 | 561034 2140 | 507930 2141 | 291408 2142 | 233721 2143 | 181887 2144 | 279448 2145 | 277821 2146 | 358262 2147 | 470300 2148 | 340874 2149 | 322770 2150 | 112886 2151 | 160276 2152 | 524031 2153 | 15833 2154 | 547025 2155 | 318395 2156 | 328070 2157 | 512041 2158 | 438751 2159 | 398403 2160 | 10863 2161 | 268921 2162 | 414960 2163 | 353733 2164 | 451185 2165 | 489434 2166 | 256348 2167 | 301596 2168 | 366529 2169 | 437636 2170 | 431791 2171 | 537344 2172 | 26013 2173 | 535941 2174 | 538592 2175 | 119107 2176 | 289260 2177 | 561496 2178 | 261307 2179 | 123417 2180 | 170026 2181 | 315782 2182 | 236398 2183 | 500068 2184 | 283484 2185 | 327151 2186 | 377717 2187 | 418591 2188 | 37518 2189 | 568653 2190 | 230474 2191 | 118276 2192 | 531498 2193 | 348525 2194 | 343688 2195 | 53130 2196 | 349173 2197 | 100906 2198 | 288023 2199 | 185372 2200 | 292837 2201 | 84819 2202 | 227005 2203 | 201809 2204 | 523094 2205 | 517813 2206 | 531286 2207 | 409474 2208 | 384482 2209 | 256549 2210 | 429987 2211 | 265438 2212 | 321175 2213 | 555888 2214 | 355652 2215 | 55996 2216 | 433068 2217 | 45205 2218 | 107062 2219 | 534319 2220 | 109708 2221 | 197517 2222 | 473065 2223 | 116010 2224 | 122926 2225 | 479526 2226 | 171144 2227 | 132145 2228 | 486063 2229 | 432553 2230 | 51692 2231 | 495363 2232 | 428853 2233 | 187379 2234 | 463531 2235 | 257056 2236 | 429809 2237 | 126599 2238 | 168514 2239 | 402344 2240 | 382697 2241 | 251375 2242 | 30631 2243 | 477739 2244 | 411970 2245 | 105596 2246 | 572888 2247 | 482547 2248 | 271601 2249 | 65795 2250 | 229012 2251 | 156781 2252 | 457348 2253 | 395380 2254 | 240790 2255 | 480536 2256 | 72869 2257 | 290917 2258 | 396549 2259 | 181857 2260 | 415997 2261 | 420078 2262 | 237729 2263 | 377558 2264 | 127909 2265 | 270627 2266 | 229402 2267 | 373800 2268 | 468494 2269 | 106775 2270 | 393907 2271 | 83240 2272 | 499819 2273 | 37020 2274 | 337606 2275 | 289684 2276 | 517986 2277 | 252604 2278 | 318631 2279 | 119588 2280 | 107823 2281 | 46044 2282 | 18443 2283 | 495877 2284 | 118074 2285 | 170203 2286 | 244946 2287 | 454662 2288 | 458 2289 | 531697 2290 | 268476 2291 | 393947 2292 | 513409 2293 | 93143 2294 | 510488 2295 | 457987 2296 | 293908 2297 | 568984 2298 | 333028 2299 | 261832 2300 | 99748 2301 | 575049 2302 | 407007 2303 | 313232 2304 | 346483 2305 | 93789 2306 | 242325 2307 | 235591 2308 | 563258 2309 | 263246 2310 | 200896 2311 | 61304 2312 | 6607 2313 | 363487 2314 | 297125 2315 | 217517 2316 | 239586 2317 | 500155 2318 | 185942 2319 | 377802 2320 | 461917 2321 | 3145 2322 | 166546 2323 | 300447 2324 | 145909 2325 | 429957 2326 | 385778 2327 | 440524 2328 | 62254 2329 | 331216 2330 | 261679 2331 | 506561 2332 | 423661 2333 | 259947 2334 | 141611 2335 | 52618 2336 | 77222 2337 | 426512 2338 | 175778 2339 | 397130 2340 | 259864 2341 | 371593 2342 | 170860 2343 | 557345 2344 | 51084 2345 | 434334 2346 | 556699 2347 | 177744 2348 | 54560 2349 | 117546 2350 | 371033 2351 | 279119 2352 | 312148 2353 | 221014 2354 | 263688 2355 | 61534 2356 | 70654 2357 | 536120 2358 | 132788 2359 | 491124 2360 | 171328 2361 | 177870 2362 | 355572 2363 | 235324 2364 | 181736 2365 | 373078 2366 | 424386 2367 | 482709 2368 | 350044 2369 | 335836 2370 | 478829 2371 | 442354 2372 | 60466 2373 | 236862 2374 | 579587 2375 | 384377 2376 | 126729 2377 | 340809 2378 | 261545 2379 | 87629 2380 | 505809 2381 | 170917 2382 | 489596 2383 | 297098 2384 | 293472 2385 | 162907 2386 | 281260 2387 | 308533 2388 | 206744 2389 | 88576 2390 | 481318 2391 | 198607 2392 | 517051 2393 | 163589 2394 | 283973 2395 | 333119 2396 | 29626 2397 | 87352 2398 | 61246 2399 | 515521 2400 | 286100 2401 | 487424 2402 | 235128 2403 | 95696 2404 | 227718 2405 | 227550 2406 | 98317 2407 | 190070 2408 | 392708 2409 | 214670 2410 | 485800 2411 | 313926 2412 | 420654 2413 | 435555 2414 | 566292 2415 | 443879 2416 | 280545 2417 | 389352 2418 | 368642 2419 | 66619 2420 | 522543 2421 | 39537 2422 | 558778 2423 | 337583 2424 | 110298 2425 | 37415 2426 | 148069 2427 | 495073 2428 | 2688 2429 | 446651 2430 | 187321 2431 | 190932 2432 | 25502 2433 | 346247 2434 | 285889 2435 | 102036 2436 | 293220 2437 | 120428 2438 | 86311 2439 | 560339 2440 | 173051 2441 | 185918 2442 | 289056 2443 | 323001 2444 | 191844 2445 | 259033 2446 | 209014 2447 | 501405 2448 | 486055 2449 | 78088 2450 | 457971 2451 | 554020 2452 | 509842 2453 | 464051 2454 | 431220 2455 | 224010 2456 | 578801 2457 | 566427 2458 | 465736 2459 | 76273 2460 | 577073 2461 | 416088 2462 | 424212 2463 | 532584 2464 | 181830 2465 | 477462 2466 | 514100 2467 | 501961 2468 | 19168 2469 | 358186 2470 | 330959 2471 | 322880 2472 | 332276 2473 | 77072 2474 | 249589 2475 | 69589 2476 | 45622 2477 | 390173 2478 | 468127 2479 | 49177 2480 | 440197 2481 | 257100 2482 | 532430 2483 | 459094 2484 | 282767 2485 | 239980 2486 | 9385 2487 | 246619 2488 | 402859 2489 | 524387 2490 | 131284 2491 | 308116 2492 | 411659 2493 | 220132 2494 | 87039 2495 | 141470 2496 | 188856 2497 | 57138 2498 | 559982 2499 | 455873 2500 | 286767 2501 | 282708 2502 | 182913 2503 | 73388 2504 | 569063 2505 | 198291 2506 | 11201 2507 | 423540 2508 | 475794 2509 | 401029 2510 | 414696 2511 | 324340 2512 | 288560 2513 | 300306 2514 | 554976 2515 | 453488 2516 | 416741 2517 | 479636 2518 | 194300 2519 | 503803 2520 | 74187 2521 | 394100 2522 | 149460 2523 | 320080 2524 | 30154 2525 | 299241 2526 | 150273 2527 | 51680 2528 | 115587 2529 | 486083 2530 | 145026 2531 | 239343 2532 | 253413 2533 | 53337 2534 | 38366 2535 | 432239 2536 | 35929 2537 | 140260 2538 | 395695 2539 | 25314 2540 | 82911 2541 | 339988 2542 | 163928 2543 | 445425 2544 | 120675 2545 | 555979 2546 | 66723 2547 | 446272 2548 | 120912 2549 | 88660 2550 | 397691 2551 | 63670 2552 | 319870 2553 | 85443 2554 | 257535 2555 | 129300 2556 | 324424 2557 | 365086 2558 | 367459 2559 | 529689 2560 | 120321 2561 | 65541 2562 | 89172 2563 | 509835 2564 | 169827 2565 | 81588 2566 | 123371 2567 | 114086 2568 | 542163 2569 | 44718 2570 | 509884 2571 | 165524 2572 | 344424 2573 | 172647 2574 | 134161 2575 | 468265 2576 | 468977 2577 | 31910 2578 | 415082 2579 | 366422 2580 | 109980 2581 | 296723 2582 | 549100 2583 | 507271 2584 | 417953 2585 | 55766 2586 | 93043 2587 | 572086 2588 | 136312 2589 | 72454 2590 | 212397 2591 | 561810 2592 | 447662 2593 | 350694 2594 | 163099 2595 | 276994 2596 | 211632 2597 | 508125 2598 | 238595 2599 | 300659 2600 | 112057 2601 | 513688 2602 | 27000 2603 | 402078 2604 | 327141 2605 | 269455 2606 | 142519 2607 | 183602 2608 | 333165 2609 | 559715 2610 | 33063 2611 | 311677 2612 | 451772 2613 | 543517 2614 | 97884 2615 | 201726 2616 | 533028 2617 | 1702 2618 | 280534 2619 | 501566 2620 | 564588 2621 | 328827 2622 | 49477 2623 | 225761 2624 | 153504 2625 | 142755 2626 | 573772 2627 | 363759 2628 | 333047 2629 | 179116 2630 | 217590 2631 | 82284 2632 | 26761 2633 | 44889 2634 | 301713 2635 | 326216 2636 | 326832 2637 | 62331 2638 | 438514 2639 | 361109 2640 | 575749 2641 | 447318 2642 | 309139 2643 | 386643 2644 | 446314 2645 | 410391 2646 | 567725 2647 | 446238 2648 | 128352 2649 | 157950 2650 | 316430 2651 | 485731 2652 | 111304 2653 | 174875 2654 | 538121 2655 | 57443 2656 | 350758 2657 | 112950 2658 | 310595 2659 | 307442 2660 | 359946 2661 | 466730 2662 | 566471 2663 | 389682 2664 | 5832 2665 | 147730 2666 | 576268 2667 | 551584 2668 | 369235 2669 | 6395 2670 | 401004 2671 | 216061 2672 | 571780 2673 | 41575 2674 | 449978 2675 | 300098 2676 | 67359 2677 | 249006 2678 | 472526 2679 | 111786 2680 | 335114 2681 | 158011 2682 | 35858 2683 | 92515 2684 | 529141 2685 | 109586 2686 | 506572 2687 | 523960 2688 | 536220 2689 | 235010 2690 | 324573 2691 | 467966 2692 | 93258 2693 | 221043 2694 | 346661 2695 | 225882 2696 | 532145 2697 | 172667 2698 | 527736 2699 | 385109 2700 | 312191 2701 | 245580 2702 | 385725 2703 | 465747 2704 | 273493 2705 | 71550 2706 | 370999 2707 | 18455 2708 | 249077 2709 | 35078 2710 | 295881 2711 | 392956 2712 | 345377 2713 | 55538 2714 | 578728 2715 | 203213 2716 | 225102 2717 | 30849 2718 | 525883 2719 | 206990 2720 | 411372 2721 | 65987 2722 | 79515 2723 | 65993 2724 | 290825 2725 | 339203 2726 | 121697 2727 | 374610 2728 | 317216 2729 | 69890 2730 | 545116 2731 | 374248 2732 | 76658 2733 | 525345 2734 | 503850 2735 | 391814 2736 | 369 2737 | 79078 2738 | 546257 2739 | 161047 2740 | 559231 2741 | 161813 2742 | 223446 2743 | 500734 2744 | 119860 2745 | 526739 2746 | 127037 2747 | 107710 2748 | 151632 2749 | 258324 2750 | 539555 2751 | 173241 2752 | 355000 2753 | 548873 2754 | 193549 2755 | 41215 2756 | 62342 2757 | 534354 2758 | 118649 2759 | 387679 2760 | 38623 2761 | 272941 2762 | 246264 2763 | 433347 2764 | 529607 2765 | 310217 2766 | 181711 2767 | 250557 2768 | 155528 2769 | 41339 2770 | 385551 2771 | 344976 2772 | 524425 2773 | 402899 2774 | 497532 2775 | 562721 2776 | 368968 2777 | 509892 2778 | 24081 2779 | 165116 2780 | 219008 2781 | 368401 2782 | 170004 2783 | 136260 2784 | 458620 2785 | 513571 2786 | 535333 2787 | 243749 2788 | 478498 2789 | 347572 2790 | 454094 2791 | 577042 2792 | 224052 2793 | 379641 2794 | 423619 2795 | 295041 2796 | 242213 2797 | 384607 2798 | 528639 2799 | 543129 2800 | 547415 2801 | 256738 2802 | 478638 2803 | 159066 2804 | 380144 2805 | 403212 2806 | 119878 2807 | 209472 2808 | 34339 2809 | 156314 2810 | 238819 2811 | 515293 2812 | 557520 2813 | 157925 2814 | 118529 2815 | 54497 2816 | 84733 2817 | 90459 2818 | 481683 2819 | 508605 2820 | 357849 2821 | 201229 2822 | 163090 2823 | 456753 2824 | 367280 2825 | 304646 2826 | 192009 2827 | 227278 2828 | 406185 2829 | 323101 2830 | 67402 2831 | 290225 2832 | 313456 2833 | 284395 2834 | 444951 2835 | 544529 2836 | 307179 2837 | 297318 2838 | 64659 2839 | 213314 2840 | 175234 2841 | 514496 2842 | 252505 2843 | 121386 2844 | 51432 2845 | 210788 2846 | 233200 2847 | 167823 2848 | 550065 2849 | 498081 2850 | 130320 2851 | 328562 2852 | 173415 2853 | 87507 2854 | 537013 2855 | 54962 2856 | 86414 2857 | 120261 2858 | 89170 2859 | 407395 2860 | 506123 2861 | 80678 2862 | 174832 2863 | 398107 2864 | 121962 2865 | 481996 2866 | 196146 2867 | 443155 2868 | 72634 2869 | 466297 2870 | 193329 2871 | 574939 2872 | 324382 2873 | 216591 2874 | 257258 2875 | 507867 2876 | 263638 2877 | 375399 2878 | 425385 2879 | 7260 2880 | 342374 2881 | 1145 2882 | 92324 2883 | 424902 2884 | 398609 2885 | 398173 2886 | 49839 2887 | 265379 2888 | 127351 2889 | 33097 2890 | 124662 2891 | 374653 2892 | 465351 2893 | 40703 2894 | 300463 2895 | 104802 2896 | 556183 2897 | 110032 2898 | 305282 2899 | 262647 2900 | 300368 2901 | 264054 2902 | 305144 2903 | 133707 2904 | 499726 2905 | 212183 2906 | 277919 2907 | 444256 2908 | 39948 2909 | 122521 2910 | 69899 2911 | 111974 2912 | 274920 2913 | 505564 2914 | 402628 2915 | 55687 2916 | 234391 2917 | 241642 2918 | 492747 2919 | 492649 2920 | 65446 2921 | 85411 2922 | 530523 2923 | 455679 2924 | 259149 2925 | 333968 2926 | 532543 2927 | 259038 2928 | 513468 2929 | 560209 2930 | 28675 2931 | 277263 2932 | 304324 2933 | 541002 2934 | 397678 2935 | 258475 2936 | 123562 2937 | 75167 2938 | 203991 2939 | 129759 2940 | 29355 2941 | 450453 2942 | 229377 2943 | 482487 2944 | 316163 2945 | 578402 2946 | 257317 2947 | 221034 2948 | 268395 2949 | 280208 2950 | 32747 2951 | 164481 2952 | 425495 2953 | 257799 2954 | 37721 2955 | 285879 2956 | 506094 2957 | 209398 2958 | 207746 2959 | 300848 2960 | 260168 2961 | 450700 2962 | 150193 2963 | 146851 2964 | 69771 2965 | 104667 2966 | 441677 2967 | 218906 2968 | 294960 2969 | 229805 2970 | 60525 2971 | 116628 2972 | 140144 2973 | 350098 2974 | 533455 2975 | 386848 2976 | 3513 2977 | 267376 2978 | 565399 2979 | 63961 2980 | 475960 2981 | 317346 2982 | 436534 2983 | 12874 2984 | 264046 2985 | 303772 2986 | 86032 2987 | 314092 2988 | 246473 2989 | 342887 2990 | 97617 2991 | 209075 2992 | 477526 2993 | 141279 2994 | 267962 2995 | 244967 2996 | 321981 2997 | 35118 2998 | 51903 2999 | 244245 3000 | 520773 3001 | 238613 3002 | 461292 3003 | 71110 3004 | 57472 3005 | 477012 3006 | 404778 3007 | 252947 3008 | 68521 3009 | 380743 3010 | 401897 3011 | 322443 3012 | 474480 3013 | 37877 3014 | 83351 3015 | 69912 3016 | 261630 3017 | 362225 3018 | 101776 3019 | 380240 3020 | 10845 3021 | 276111 3022 | 457637 3023 | 385179 3024 | 153538 3025 | 358007 3026 | 397479 3027 | 87056 3028 | 143820 3029 | 118602 3030 | 309586 3031 | 326333 3032 | 552615 3033 | 289317 3034 | 17392 3035 | 564542 3036 | 108427 3037 | 361272 3038 | 498110 3039 | 437327 3040 | 166043 3041 | 234944 3042 | 380191 3043 | 124901 3044 | 415946 3045 | 339298 3046 | 488209 3047 | 269165 3048 | 291531 3049 | 69054 3050 | 14007 3051 | 426795 3052 | 455030 3053 | 32474 3054 | 311662 3055 | 170454 3056 | 314521 3057 | 371011 3058 | 425749 3059 | 251254 3060 | 235652 3061 | 4556 3062 | 526338 3063 | 64474 3064 | 522068 3065 | 313989 3066 | 532743 3067 | 129806 3068 | 512394 3069 | 266088 3070 | 285025 3071 | 73294 3072 | 333555 3073 | 527288 3074 | 555872 3075 | 239641 3076 | 203854 3077 | 361114 3078 | 312447 3079 | 248887 3080 | 213089 3081 | 25488 3082 | 145732 3083 | 229934 3084 | 288683 3085 | 432783 3086 | 549201 3087 | 184487 3088 | 84183 3089 | 398061 3090 | 127437 3091 | 383621 3092 | 305294 3093 | 73743 3094 | 205075 3095 | 120131 3096 | 378396 3097 | 266642 3098 | 539458 3099 | 40695 3100 | 506865 3101 | 312952 3102 | 318511 3103 | 466499 3104 | 84087 3105 | 456884 3106 | 218488 3107 | 470878 3108 | 458884 3109 | 379002 3110 | 114648 3111 | 277208 3112 | 10244 3113 | 95947 3114 | 242826 3115 | 529281 3116 | 528819 3117 | 70866 3118 | 361530 3119 | 555086 3120 | 555125 3121 | 84140 3122 | 352068 3123 | 13678 3124 | 480910 3125 | 324577 3126 | 168210 3127 | 271111 3128 | 501243 3129 | 237235 3130 | 188504 3131 | 85766 3132 | 347619 3133 | 313673 3134 | 567233 3135 | 547294 3136 | 202162 3137 | 222355 3138 | 470598 3139 | 184885 3140 | 436742 3141 | 206210 3142 | 13670 3143 | 412454 3144 | 95381 3145 | 419032 3146 | 339705 3147 | 105277 3148 | 449948 3149 | 71072 3150 | 235904 3151 | 207795 3152 | 312107 3153 | 103994 3154 | 551215 3155 | 403234 3156 | 544300 3157 | 542345 3158 | 149931 3159 | 77399 3160 | 285081 3161 | 436115 3162 | 411957 3163 | 200467 3164 | 263636 3165 | 491206 3166 | 441312 3167 | 61409 3168 | 384317 3169 | 393456 3170 | 428380 3171 | 151922 3172 | 401915 3173 | 135085 3174 | 45126 3175 | 579792 3176 | 51848 3177 | 246739 3178 | 341778 3179 | 80779 3180 | 193643 3181 | 307911 3182 | 400792 3183 | 572468 3184 | 82552 3185 | 85130 3186 | 476858 3187 | 58565 3188 | 495099 3189 | 28282 3190 | 288156 3191 | 297834 3192 | 72186 3193 | 211054 3194 | 508061 3195 | 387552 3196 | 435487 3197 | 329200 3198 | 150586 3199 | 414579 3200 | 564435 3201 | 469937 3202 | 264372 3203 | 28078 3204 | 252249 3205 | 277693 3206 | 161552 3207 | 537879 3208 | 427029 3209 | 254960 3210 | 132019 3211 | 498514 3212 | 463734 3213 | 186223 3214 | 165234 3215 | 169439 3216 | 440500 3217 | 434054 3218 | 110338 3219 | 120592 3220 | 45056 3221 | 441867 3222 | 295772 3223 | 384695 3224 | 572668 3225 | 64560 3226 | 60034 3227 | 118755 3228 | 44508 3229 | 477213 3230 | 563504 3231 | 212911 3232 | 130332 3233 | 568613 3234 | 53416 3235 | 151352 3236 | 312609 3237 | 491424 3238 | 27158 3239 | 258207 3240 | 301041 3241 | 324956 3242 | 307634 3243 | 72240 3244 | 165400 3245 | 538431 3246 | 103294 3247 | 6315 3248 | 529009 3249 | 568884 3250 | 423820 3251 | 211052 3252 | 487278 3253 | 545710 3254 | 41460 3255 | 266607 3256 | 566380 3257 | 449431 3258 | 10779 3259 | 447920 3260 | 156925 3261 | 154558 3262 | 280957 3263 | 480166 3264 | 3891 3265 | 17214 3266 | 372591 3267 | 473697 3268 | 303719 3269 | 37642 3270 | 562325 3271 | 533396 3272 | 84345 3273 | 272808 3274 | 575520 3275 | 460624 3276 | 305110 3277 | 424791 3278 | 181721 3279 | 366852 3280 | 308975 3281 | 478357 3282 | 332412 3283 | 105453 3284 | 129901 3285 | 469023 3286 | 461807 3287 | 102274 3288 | 413679 3289 | 370148 3290 | 13491 3291 | 17460 3292 | 144950 3293 | 347583 3294 | 179359 3295 | 124537 3296 | 575731 3297 | 41267 3298 | 360639 3299 | 470272 3300 | 256660 3301 | 539999 3302 | 57587 3303 | 141638 3304 | 461714 3305 | 124109 3306 | 382073 3307 | 505337 3308 | 92793 3309 | 480083 3310 | 21121 3311 | 330544 3312 | 146290 3313 | 423990 3314 | 215407 3315 | 224513 3316 | 257001 3317 | 339508 3318 | 13177 3319 | 390775 3320 | 328112 3321 | 376667 3322 | 274169 3323 | 490912 3324 | 322221 3325 | 50526 3326 | 508598 3327 | 269940 3328 | 519451 3329 | 79165 3330 | 476664 3331 | 517807 3332 | 151051 3333 | 154760 3334 | 488206 3335 | 155600 3336 | 380356 3337 | 493271 3338 | 352443 3339 | 66303 3340 | 128662 3341 | 504288 3342 | 426283 3343 | 331599 3344 | 301178 3345 | 561636 3346 | 245319 3347 | 29983 3348 | 309719 3349 | 378956 3350 | 136431 3351 | 8770 3352 | 95344 3353 | 381576 3354 | 371062 3355 | 541803 3356 | 38422 3357 | 261184 3358 | 340245 3359 | 75646 3360 | 323500 3361 | 104536 3362 | 472304 3363 | 368721 3364 | 194525 3365 | 142459 3366 | 540686 3367 | 485134 3368 | 24759 3369 | 326277 3370 | 230791 3371 | 110272 3372 | 97143 3373 | 195177 3374 | 473573 3375 | 463095 3376 | 379389 3377 | 6853 3378 | 263907 3379 | 134647 3380 | 246780 3381 | 500431 3382 | 423474 3383 | 218762 3384 | 560516 3385 | 189381 3386 | 327705 3387 | 279482 3388 | 260897 3389 | 208507 3390 | 417485 3391 | 481024 3392 | 499651 3393 | 122165 3394 | 270688 3395 | 118008 3396 | 421838 3397 | 486418 3398 | 109676 3399 | 287514 3400 | 239089 3401 | 569983 3402 | 348389 3403 | 34113 3404 | 85627 3405 | 284205 3406 | 364041 3407 | 121750 3408 | 176543 3409 | 264784 3410 | 48930 3411 | 8665 3412 | 297355 3413 | 92809 3414 | 153367 3415 | 473246 3416 | 423124 3417 | 580360 3418 | 337768 3419 | 6965 3420 | 205686 3421 | 576569 3422 | 550709 3423 | 182902 3424 | 479697 3425 | 304864 3426 | 568829 3427 | 162428 3428 | 14065 3429 | 351134 3430 | 277420 3431 | 186280 3432 | 333611 3433 | 161240 3434 | 4738 3435 | 253201 3436 | 213203 3437 | 16896 3438 | 422494 3439 | 218640 3440 | 288094 3441 | 483803 3442 | 221005 3443 | 73620 3444 | 16024 3445 | 489635 3446 | 469496 3447 | 381494 3448 | 3447 3449 | 139643 3450 | 163270 3451 | 51282 3452 | 162607 3453 | 236209 3454 | 339495 3455 | 513362 3456 | 114267 3457 | 14037 3458 | 455969 3459 | 373579 3460 | 157737 3461 | 162371 3462 | 410053 3463 | 349764 3464 | 483752 3465 | 468054 3466 | 2387 3467 | 353888 3468 | 258885 3469 | 243346 3470 | 8466 3471 | 242760 3472 | 467809 3473 | 381900 3474 | 218198 3475 | 368561 3476 | 460472 3477 | 500627 3478 | 173020 3479 | 108516 3480 | 526048 3481 | 334754 3482 | 497747 3483 | 575692 3484 | 387311 3485 | 432617 3486 | 535641 3487 | 353512 3488 | 319995 3489 | 267930 3490 | 193105 3491 | 571680 3492 | 363950 3493 | 460875 3494 | 234432 3495 | 257624 3496 | 354842 3497 | 555764 3498 | 461410 3499 | 272549 3500 | 180066 3501 | 427565 3502 | 343434 3503 | 70814 3504 | 205934 3505 | 342055 3506 | 273666 3507 | 383268 3508 | 210586 3509 | 267534 3510 | 568398 3511 | 241725 3512 | 291899 3513 | 325904 3514 | 129583 3515 | 575095 3516 | 271355 3517 | 303808 3518 | 377345 3519 | 341304 3520 | 427267 3521 | 572561 3522 | 23311 3523 | 339497 3524 | 157105 3525 | 341279 3526 | 224367 3527 | 504481 3528 | 113006 3529 | 87218 3530 | 547830 3531 | 238150 3532 | 449033 3533 | 287368 3534 | 168409 3535 | 314600 3536 | 352937 3537 | 319829 3538 | 335573 3539 | 120061 3540 | 25633 3541 | 23415 3542 | 33257 3543 | 544921 3544 | 524791 3545 | 228317 3546 | 530857 3547 | 94618 3548 | 41848 3549 | 386852 3550 | 223590 3551 | 122060 3552 | 563712 3553 | 221382 3554 | 127568 3555 | 469751 3556 | 432688 3557 | 158962 3558 | 132940 3559 | 464424 3560 | 15070 3561 | 509681 3562 | 324836 3563 | 114411 3564 | 119236 3565 | 181390 3566 | 1373 3567 | 46397 3568 | 336865 3569 | 238439 3570 | 207440 3571 | 273987 3572 | 183134 3573 | 305029 3574 | 534930 3575 | 550751 3576 | 396584 3577 | 537148 3578 | 272021 3579 | 462079 3580 | 325211 3581 | 146547 3582 | 293465 3583 | 371809 3584 | 530310 3585 | 580531 3586 | 275694 3587 | 84223 3588 | 257893 3589 | 474965 3590 | 453700 3591 | 265076 3592 | 318106 3593 | 339021 3594 | 351715 3595 | 79141 3596 | 229378 3597 | 225039 3598 | 224257 3599 | 538735 3600 | 404261 3601 | 229382 3602 | 136222 3603 | 364692 3604 | 180929 3605 | 250474 3606 | 86231 3607 | 521525 3608 | 232249 3609 | 544374 3610 | 375841 3611 | 268335 3612 | 545756 3613 | 85111 3614 | 379688 3615 | 57425 3616 | 110460 3617 | 83422 3618 | 446588 3619 | 57977 3620 | 209240 3621 | 358518 3622 | 382691 3623 | 286149 3624 | 303431 3625 | 444839 3626 | 474495 3627 | 208089 3628 | 383449 3629 | 305934 3630 | 412626 3631 | 434942 3632 | 171170 3633 | 236536 3634 | 208624 3635 | 19922 3636 | 67874 3637 | 62936 3638 | 116264 3639 | 499258 3640 | 532616 3641 | 390621 3642 | 222846 3643 | 35385 3644 | 26915 3645 | 342017 3646 | 309759 3647 | 213872 3648 | 177558 3649 | 565014 3650 | 1759 3651 | 538558 3652 | 403929 3653 | 204892 3654 | 191579 3655 | 484819 3656 | 423212 3657 | 309562 3658 | 136811 3659 | 267881 3660 | 182680 3661 | 193524 3662 | 166279 3663 | 149870 3664 | 402863 3665 | 284013 3666 | 560871 3667 | 6492 3668 | 56421 3669 | 291695 3670 | 96044 3671 | 525689 3672 | 575986 3673 | 513472 3674 | 283884 3675 | 455837 3676 | 19828 3677 | 144414 3678 | 499670 3679 | 320339 3680 | 318061 3681 | 573913 3682 | 204056 3683 | 404499 3684 | 452829 3685 | 550788 3686 | 198987 3687 | 62480 3688 | 63706 3689 | 449321 3690 | 503967 3691 | 260514 3692 | 344974 3693 | 141440 3694 | 346045 3695 | 576863 3696 | 491868 3697 | 542381 3698 | 181339 3699 | 318290 3700 | 240981 3701 | 459695 3702 | 446992 3703 | 107751 3704 | 270843 3705 | 126275 3706 | 224962 3707 | 237044 3708 | 112794 3709 | 186124 3710 | 378399 3711 | 498979 3712 | 382510 3713 | 205319 3714 | 577461 3715 | 288797 3716 | 226275 3717 | 377015 3718 | 334194 3719 | 216156 3720 | 389719 3721 | 321042 3722 | 346375 3723 | 193835 3724 | 161269 3725 | 477872 3726 | 321998 3727 | 361788 3728 | 109403 3729 | 129053 3730 | 317798 3731 | 381102 3732 | 7080 3733 | 293069 3734 | 420693 3735 | 272127 3736 | 489478 3737 | 58452 3738 | 150429 3739 | 580889 3740 | 277116 3741 | 247431 3742 | 277939 3743 | 520497 3744 | 118571 3745 | 386165 3746 | 1346 3747 | 309685 3748 | 376788 3749 | 487244 3750 | 375033 3751 | 305347 3752 | 466593 3753 | 358984 3754 | 47600 3755 | 202125 3756 | 490645 3757 | 291727 3758 | 95795 3759 | 210450 3760 | 561442 3761 | 34835 3762 | 98194 3763 | 34893 3764 | 79613 3765 | 92004 3766 | 82126 3767 | 269065 3768 | 515942 3769 | 448572 3770 | 140832 3771 | 5784 3772 | 33416 3773 | 481704 3774 | 305120 3775 | 495626 3776 | 423076 3777 | 150817 3778 | 253253 3779 | 484170 3780 | 215279 3781 | 378488 3782 | 442520 3783 | 181472 3784 | 153589 3785 | 172429 3786 | 275810 3787 | 18077 3788 | 468051 3789 | 575303 3790 | 541437 3791 | 12307 3792 | 85599 3793 | 437693 3794 | 580044 3795 | 5897 3796 | 275571 3797 | 537308 3798 | 236052 3799 | 18727 3800 | 531963 3801 | 157050 3802 | 457375 3803 | 449919 3804 | 350874 3805 | 201986 3806 | 83646 3807 | 386798 3808 | 432099 3809 | 175558 3810 | 370479 3811 | 475239 3812 | 505318 3813 | 449737 3814 | 481747 3815 | 574449 3816 | 15785 3817 | 549150 3818 | 132926 3819 | 577739 3820 | 347365 3821 | 277938 3822 | 118150 3823 | 538848 3824 | 481103 3825 | 32956 3826 | 239085 3827 | 232043 3828 | 402290 3829 | 333229 3830 | 572917 3831 | 319159 3832 | 205878 3833 | 267023 3834 | 442784 3835 | 58589 3836 | 574282 3837 | 18441 3838 | 220086 3839 | 248610 3840 | 378863 3841 | 332825 3842 | 490458 3843 | 489681 3844 | 251484 3845 | 247205 3846 | 114780 3847 | 556882 3848 | 340672 3849 | 553436 3850 | 457875 3851 | 197050 3852 | 424164 3853 | 268223 3854 | 319576 3855 | 264623 3856 | 296 3857 | 82174 3858 | 156977 3859 | 260975 3860 | 288001 3861 | 377804 3862 | 421819 3863 | 389336 3864 | 248633 3865 | 52830 3866 | 155702 3867 | 496138 3868 | 194607 3869 | 390835 3870 | 27361 3871 | 123200 3872 | 509219 3873 | 52220 3874 | 240331 3875 | 381274 3876 | 324056 3877 | 495077 3878 | 472737 3879 | 53744 3880 | 332398 3881 | 33721 3882 | 545760 3883 | 189044 3884 | 348627 3885 | 202861 3886 | 312714 3887 | 157367 3888 | 278224 3889 | 389255 3890 | 464706 3891 | 559601 3892 | 329201 3893 | 175704 3894 | 111103 3895 | 218196 3896 | 504019 3897 | 128642 3898 | 243726 3899 | 265556 3900 | 520866 3901 | 235141 3902 | 580939 3903 | 440157 3904 | 157696 3905 | 360159 3906 | 198552 3907 | 68088 3908 | 521715 3909 | 318783 3910 | 489502 3911 | 478269 3912 | 45124 3913 | 162286 3914 | 293359 3915 | 563829 3916 | 553835 3917 | 570641 3918 | 444433 3919 | 544109 3920 | 255682 3921 | 575410 3922 | 560806 3923 | 233980 3924 | 422947 3925 | 176980 3926 | 233824 3927 | 198286 3928 | 195084 3929 | 250176 3930 | 78134 3931 | 482581 3932 | 455935 3933 | 141430 3934 | 332565 3935 | 126501 3936 | 439427 3937 | 366923 3938 | 366068 3939 | 419397 3940 | 465901 3941 | 576102 3942 | 293149 3943 | 197060 3944 | 110727 3945 | 280052 3946 | 386751 3947 | 154734 3948 | 454884 3949 | 269053 3950 | 531868 3951 | 477681 3952 | 150844 3953 | 77762 3954 | 337988 3955 | 327519 3956 | 334475 3957 | 29964 3958 | 293819 3959 | 272805 3960 | 337089 3961 | 18025 3962 | 230519 3963 | 80466 3964 | 357886 3965 | 518723 3966 | 526980 3967 | 150194 3968 | 386554 3969 | 559208 3970 | 333524 3971 | 441015 3972 | 31167 3973 | 52974 3974 | 110285 3975 | 395670 3976 | 197305 3977 | 62610 3978 | 225044 3979 | 422506 3980 | 578868 3981 | 351898 3982 | 378307 3983 | 153981 3984 | 224731 3985 | 478652 3986 | 260778 3987 | 150753 3988 | 580769 3989 | 435025 3990 | 443654 3991 | 511863 3992 | 482836 3993 | 255655 3994 | 176341 3995 | 404201 3996 | 179158 3997 | 47087 3998 | 552648 3999 | 509181 4000 | 242523 4001 | 87762 4002 | 55807 4003 | 548091 4004 | 22032 4005 | 180329 4006 | 352484 4007 | 115859 4008 | 561396 4009 | 369844 4010 | 155855 4011 | 421974 4012 | 555070 4013 | 99750 4014 | 170788 4015 | 309671 4016 | 111878 4017 | 74637 4018 | 567513 4019 | 19531 4020 | 365310 4021 | 579533 4022 | 40599 4023 | 505934 4024 | 82298 4025 | 155944 4026 | 440950 4027 | 171005 4028 | 180586 4029 | 58662 4030 | 225279 4031 | 22802 4032 | 165297 4033 | 161537 4034 | 418590 4035 | 475699 4036 | 26931 4037 | 535130 4038 | 89122 4039 | 461305 4040 | 499336 4041 | 171718 4042 | 443666 4043 | 294246 4044 | 409205 4045 | 525565 4046 | 559913 4047 | 537141 4048 | 159978 4049 | 264769 4050 | 344260 4051 | 432373 4052 | 221945 4053 | 520357 4054 | 92092 4055 | 332061 4056 | 373269 4057 | 288497 4058 | 143810 4059 | 456347 4060 | 418722 4061 | 520732 4062 | 196248 4063 | 152435 4064 | 471869 4065 | 298920 4066 | 129868 4067 | 153970 4068 | 373446 4069 | 107114 4070 | 143442 4071 | 320697 4072 | 501769 4073 | 63190 4074 | 347935 4075 | 62445 4076 | 48662 4077 | 114742 4078 | 399260 4079 | 566771 4080 | 378209 4081 | 78151 4082 | 528674 4083 | 205663 4084 | 131978 4085 | 432405 4086 | 88563 4087 | 9016 4088 | 347922 4089 | 425207 4090 | 474875 4091 | 537195 4092 | 479220 4093 | 59480 4094 | 318369 4095 | 536168 4096 | 487390 4097 | 161732 4098 | 90385 4099 | 389791 4100 | 42792 4101 | 92485 4102 | 417123 4103 | 277917 4104 | 205482 4105 | 516867 4106 | 53592 4107 | 101893 4108 | 441494 4109 | 188243 4110 | 242447 4111 | 318029 4112 | 220091 4113 | 253578 4114 | 114656 4115 | 24753 4116 | 225674 4117 | 263522 4118 | 18914 4119 | 59156 4120 | 570136 4121 | 344724 4122 | 331962 4123 | 300013 4124 | 326925 4125 | 542263 4126 | 530112 4127 | 413580 4128 | 10613 4129 | 245185 4130 | 308976 4131 | 92498 4132 | 387135 4133 | 179190 4134 | 234104 4135 | 244985 4136 | 497562 4137 | 190248 4138 | 512419 4139 | 173541 4140 | 556172 4141 | 264911 4142 | 516234 4143 | 356199 4144 | 125611 4145 | 563514 4146 | 333698 4147 | 207480 4148 | 93172 4149 | 87545 4150 | 354148 4151 | 215797 4152 | 139438 4153 | 389797 4154 | 152988 4155 | 151839 4156 | 42116 4157 | 131695 4158 | 371677 4159 | 290343 4160 | 270658 4161 | 228785 4162 | 189532 4163 | 222002 4164 | 550452 4165 | 53430 4166 | 362411 4167 | 483802 4168 | 131700 4169 | 221336 4170 | 410086 4171 | 273806 4172 | 556228 4173 | 390757 4174 | 452020 4175 | 323565 4176 | 405712 4177 | 381330 4178 | 429469 4179 | 534779 4180 | 48287 4181 | 492780 4182 | 493334 4183 | 88819 4184 | 233627 4185 | 400693 4186 | 422187 4187 | 196781 4188 | 371466 4189 | 196931 4190 | 95768 4191 | 135507 4192 | 515586 4193 | 156784 4194 | 342032 4195 | 136189 4196 | 580635 4197 | 83204 4198 | 569350 4199 | 423785 4200 | 561795 4201 | 74756 4202 | 337584 4203 | 533147 4204 | 295438 4205 | 499924 4206 | 291844 4207 | 57319 4208 | 293001 4209 | 416839 4210 | 76928 4211 | 407332 4212 | 126213 4213 | 527790 4214 | 561926 4215 | 417865 4216 | 258729 4217 | 524628 4218 | 498987 4219 | 433046 4220 | 437160 4221 | 341134 4222 | 146159 4223 | 85418 4224 | 254831 4225 | 69246 4226 | 169770 4227 | 178221 4228 | 574833 4229 | 147424 4230 | 303698 4231 | 237206 4232 | 441498 4233 | 487905 4234 | 278793 4235 | 454760 4236 | 461748 4237 | 215304 4238 | 246207 4239 | 563340 4240 | 81633 4241 | 443448 4242 | 553713 4243 | 486199 4244 | 261339 4245 | 352075 4246 | 275166 4247 | 300361 4248 | 286504 4249 | 461092 4250 | 571471 4251 | 187249 4252 | 162004 4253 | 578456 4254 | 540120 4255 | 492932 4256 | 480578 4257 | 205585 4258 | 476081 4259 | 515940 4260 | 82794 4261 | 343007 4262 | 498589 4263 | 467811 4264 | 117397 4265 | 430063 4266 | 219434 4267 | 125114 4268 | 359637 4269 | 228265 4270 | 30166 4271 | 195585 4272 | 352653 4273 | 219850 4274 | 521121 4275 | 430992 4276 | 105288 4277 | 9731 4278 | 571877 4279 | 302108 4280 | 347567 4281 | 529164 4282 | 223011 4283 | 385976 4284 | 314162 4285 | 328215 4286 | 273132 4287 | 363424 4288 | 512702 4289 | 172952 4290 | 252725 4291 | 342813 4292 | 375824 4293 | 493114 4294 | 200437 4295 | 458175 4296 | 377200 4297 | 405039 4298 | 261539 4299 | 241498 4300 | 338514 4301 | 274022 4302 | 254245 4303 | 509191 4304 | 151305 4305 | 376634 4306 | 522173 4307 | 187921 4308 | 85010 4309 | 298139 4310 | 327759 4311 | 171906 4312 | 330793 4313 | 299869 4314 | 30484 4315 | 426551 4316 | 343569 4317 | 566066 4318 | 461746 4319 | 95905 4320 | 211310 4321 | 184259 4322 | 59962 4323 | 217529 4324 | 470331 4325 | 493396 4326 | 51822 4327 | 52180 4328 | 335707 4329 | 422831 4330 | 329577 4331 | 367693 4332 | 312459 4333 | 563231 4334 | 231134 4335 | 3527 4336 | 256879 4337 | 411129 4338 | 282880 4339 | 321430 4340 | 76232 4341 | 83104 4342 | 487615 4343 | 169070 4344 | 370770 4345 | 498229 4346 | 401854 4347 | 428255 4348 | 554337 4349 | 78306 4350 | 260782 4351 | 569229 4352 | 205825 4353 | 364252 4354 | 339924 4355 | 360519 4356 | 491592 4357 | 88617 4358 | 425285 4359 | 534712 4360 | 136503 4361 | 177190 4362 | 396611 4363 | 216301 4364 | 167150 4365 | 352635 4366 | 225307 4367 | 222511 4368 | 134570 4369 | 252680 4370 | 203385 4371 | 209432 4372 | 493288 4373 | 402422 4374 | 486504 4375 | 113835 4376 | 335928 4377 | 394055 4378 | 391257 4379 | 525305 4380 | 368855 4381 | 431941 4382 | 33071 4383 | 191101 4384 | 263359 4385 | 247858 4386 | 56146 4387 | 1720 4388 | 88639 4389 | 428914 4390 | 348718 4391 | 229488 4392 | 18784 4393 | 453442 4394 | 204238 4395 | 42519 4396 | 428525 4397 | 383603 4398 | 54391 4399 | 414548 4400 | 160095 4401 | 112376 4402 | 526220 4403 | 315166 4404 | 529289 4405 | 197260 4406 | 189926 4407 | 542543 4408 | 253296 4409 | 172913 4410 | 469894 4411 | 412167 4412 | 497018 4413 | 542528 4414 | 1540 4415 | 173860 4416 | 11243 4417 | 425993 4418 | 83390 4419 | 251988 4420 | 80021 4421 | 353929 4422 | 469687 4423 | 333191 4424 | 541396 4425 | 36863 4426 | 321628 4427 | 335796 4428 | 210480 4429 | 526617 4430 | 401126 4431 | 360947 4432 | 246291 4433 | 136032 4434 | 376454 4435 | 444480 4436 | 276217 4437 | 212031 4438 | 533957 4439 | 280556 4440 | 440705 4441 | 204370 4442 | 275579 4443 | 56656 4444 | 398767 4445 | 216240 4446 | 98934 4447 | 478457 4448 | 392433 4449 | 187014 4450 | 513179 4451 | 395200 4452 | 186166 4453 | 539292 4454 | 86504 4455 | 515355 4456 | 285361 4457 | 516167 4458 | 198620 4459 | 192013 4460 | 41311 4461 | 441445 4462 | 178431 4463 | 523396 4464 | 267349 4465 | 360477 4466 | 542450 4467 | 238167 4468 | 525794 4469 | 249805 4470 | 535050 4471 | 86602 4472 | 524779 4473 | 292699 4474 | 400485 4475 | 295397 4476 | 350659 4477 | 499848 4478 | 70722 4479 | 460593 4480 | 444483 4481 | 398880 4482 | 137596 4483 | 321398 4484 | 348554 4485 | 395569 4486 | 558823 4487 | 390916 4488 | 449994 4489 | 97431 4490 | 531612 4491 | 213998 4492 | 368510 4493 | 53539 4494 | 220135 4495 | 152514 4496 | 479777 4497 | 358574 4498 | 185454 4499 | 196204 4500 | 64293 4501 | 230945 4502 | 557631 4503 | 417735 4504 | 507054 4505 | 209512 4506 | 201412 4507 | 228980 4508 | 279641 4509 | 105133 4510 | 243921 4511 | 386997 4512 | 95155 4513 | 301871 4514 | 39052 4515 | 555643 4516 | 342415 4517 | 162074 4518 | 58265 4519 | 114459 4520 | 154881 4521 | 468909 4522 | 87519 4523 | 106477 4524 | 462635 4525 | 288849 4526 | 260476 4527 | 4381 4528 | 274960 4529 | 499541 4530 | 79566 4531 | 186594 4532 | 133422 4533 | 283846 4534 | 306461 4535 | 403919 4536 | 133805 4537 | 280825 4538 | 114584 4539 | 261027 4540 | 29525 4541 | 580966 4542 | 233163 4543 | 117920 4544 | 44770 4545 | 85587 4546 | 238745 4547 | 465755 4548 | 178283 4549 | 104161 4550 | 424163 4551 | 432232 4552 | 357070 4553 | 40821 4554 | 362750 4555 | 195882 4556 | 162132 4557 | 99122 4558 | 239870 4559 | 136320 4560 | 471311 4561 | 6012 4562 | 411940 4563 | 118173 4564 | 147855 4565 | 86773 4566 | 293552 4567 | 3282 4568 | 306412 4569 | 23875 4570 | 473039 4571 | 553061 4572 | 134785 4573 | 346393 4574 | 417161 4575 | 547012 4576 | 304989 4577 | 383186 4578 | 348265 4579 | 57136 4580 | 115197 4581 | 421372 4582 | 441035 4583 | 473453 4584 | 42497 4585 | 250221 4586 | 158918 4587 | 289686 4588 | 261505 4589 | 407681 4590 | 108393 4591 | 73142 4592 | 243846 4593 | 183556 4594 | 443071 4595 | 211117 4596 | 215759 4597 | 579700 4598 | 321099 4599 | 512901 4600 | 504707 4601 | 551882 4602 | 469616 4603 | 478988 4604 | 495329 4605 | 95557 4606 | 548222 4607 | 391204 4608 | 89136 4609 | 84797 4610 | 162056 4611 | 254343 4612 | 316183 4613 | 350707 4614 | 542161 4615 | 34421 4616 | 559620 4617 | 549322 4618 | 219789 4619 | 165160 4620 | 74582 4621 | 195092 4622 | 572257 4623 | 408812 4624 | 168143 4625 | 115242 4626 | 468824 4627 | 358566 4628 | 318094 4629 | 256512 4630 | 321153 4631 | 576657 4632 | 237779 4633 | 127121 4634 | 70773 4635 | 139726 4636 | 278143 4637 | 412877 4638 | 108458 4639 | 271218 4640 | 502570 4641 | 239337 4642 | 349923 4643 | 11025 4644 | 379371 4645 | 491718 4646 | 359619 4647 | 281454 4648 | 501512 4649 | 478200 4650 | 294262 4651 | 546772 4652 | 473418 4653 | 541826 4654 | 571246 4655 | 534867 4656 | 245550 4657 | 352041 4658 | 425628 4659 | 472493 4660 | 122965 4661 | 547652 4662 | 539952 4663 | 205985 4664 | 570145 4665 | 454597 4666 | 32671 4667 | 95192 4668 | 9019 4669 | 432301 4670 | 267044 4671 | 50506 4672 | 213213 4673 | 546006 4674 | 455454 4675 | 509304 4676 | 123421 4677 | 466143 4678 | 435257 4679 | 142179 4680 | 53691 4681 | 170555 4682 | 77482 4683 | 348186 4684 | 430657 4685 | 25071 4686 | 200619 4687 | 185421 4688 | 87255 4689 | 183644 4690 | 220352 4691 | 434417 4692 | 262943 4693 | 574119 4694 | 552352 4695 | 185900 4696 | 207416 4697 | 298002 4698 | 242776 4699 | 161199 4700 | 88588 4701 | 503658 4702 | 287027 4703 | 357081 4704 | 574989 4705 | 426665 4706 | 360313 4707 | 546708 4708 | 295602 4709 | 295163 4710 | 329047 4711 | 105863 4712 | 95106 4713 | 475808 4714 | 169970 4715 | 512296 4716 | 254455 4717 | 141782 4718 | 327390 4719 | 491337 4720 | 181069 4721 | 148992 4722 | 423842 4723 | 279600 4724 | 242038 4725 | 183652 4726 | 497888 4727 | 520863 4728 | 212135 4729 | 315840 4730 | 360260 4731 | 34724 4732 | 83998 4733 | 547317 4734 | 13674 4735 | 457029 4736 | 288048 4737 | 381796 4738 | 455560 4739 | 301456 4740 | 580052 4741 | 472240 4742 | 168542 4743 | 251280 4744 | 364561 4745 | 579963 4746 | 180484 4747 | 106842 4748 | 45422 4749 | 448527 4750 | 108937 4751 | 353147 4752 | 200542 4753 | 305129 4754 | 61836 4755 | 117649 4756 | 232765 4757 | 8044 4758 | 55667 4759 | 317404 4760 | 225452 4761 | 86950 4762 | 23446 4763 | 295930 4764 | 457636 4765 | 133569 4766 | 183097 4767 | 484700 4768 | 307041 4769 | 428523 4770 | 261116 4771 | 151623 4772 | 406279 4773 | 356776 4774 | 452559 4775 | 100910 4776 | 117634 4777 | 505994 4778 | 60 4779 | 508539 4780 | 6831 4781 | 492979 4782 | 402553 4783 | 317075 4784 | 46535 4785 | 1797 4786 | 206069 4787 | 8035 4788 | 448603 4789 | 461635 4790 | 271093 4791 | 288508 4792 | 93438 4793 | 407283 4794 | 543058 4795 | 282958 4796 | 489564 4797 | 335508 4798 | 227043 4799 | 174916 4800 | 98241 4801 | 300781 4802 | 52169 4803 | 175647 4804 | 16963 4805 | 189113 4806 | 76771 4807 | 396206 4808 | 38136 4809 | 347085 4810 | 88336 4811 | 120274 4812 | 465128 4813 | 526172 4814 | 326542 4815 | 531759 4816 | 201369 4817 | 85719 4818 | 253701 4819 | 476379 4820 | 283565 4821 | 326665 4822 | 333645 4823 | 289202 4824 | 273885 4825 | 342928 4826 | 324321 4827 | 91524 4828 | 280040 4829 | 129669 4830 | 177727 4831 | 551953 4832 | 345256 4833 | 325934 4834 | 527698 4835 | 185463 4836 | 395672 4837 | 167742 4838 | 47408 4839 | 476508 4840 | 235798 4841 | 547827 4842 | 210446 4843 | 202101 4844 | 536596 4845 | 139511 4846 | 486271 4847 | 324 4848 | 128775 4849 | 83759 4850 | 577250 4851 | 536019 4852 | 23458 4853 | 87108 4854 | 454428 4855 | 557669 4856 | 106411 4857 | 260624 4858 | 230295 4859 | 317458 4860 | 338460 4861 | 374845 4862 | 282424 4863 | 322976 4864 | 288175 4865 | 212603 4866 | 13877 4867 | 425481 4868 | 310360 4869 | 318256 4870 | 209370 4871 | 210152 4872 | 272912 4873 | 519885 4874 | 694 4875 | 167271 4876 | 158588 4877 | 116439 4878 | 185716 4879 | 332922 4880 | 421935 4881 | 580714 4882 | 32274 4883 | 311583 4884 | 78741 4885 | 271183 4886 | 201491 4887 | 65819 4888 | 237520 4889 | 327494 4890 | 380914 4891 | 507120 4892 | 248135 4893 | 349475 4894 | 51322 4895 | 525844 4896 | 304063 4897 | 346434 4898 | 387958 4899 | 177476 4900 | 522431 4901 | 64382 4902 | 503277 4903 | 317479 4904 | 189670 4905 | 546550 4906 | 289244 4907 | 39741 4908 | 266979 4909 | 260631 4910 | 211932 4911 | 548506 4912 | 571313 4913 | 369424 4914 | 418167 4915 | 156481 4916 | 447951 4917 | 461264 4918 | 309213 4919 | 200998 4920 | 271948 4921 | 54445 4922 | 205377 4923 | 248869 4924 | 570656 4925 | 399662 4926 | 346772 4927 | 22061 4928 | 168128 4929 | 161046 4930 | 236256 4931 | 414323 4932 | 415047 4933 | 100191 4934 | 508213 4935 | 84680 4936 | 439514 4937 | 141515 4938 | 577616 4939 | 569838 4940 | 59567 4941 | 329717 4942 | 370083 4943 | 230267 4944 | 491371 4945 | 167582 4946 | 410470 4947 | 107293 4948 | 474399 4949 | 499536 4950 | 519947 4951 | 74347 4952 | 252883 4953 | 505551 4954 | 162059 4955 | 567387 4956 | 129468 4957 | 195207 4958 | 239298 4959 | 298221 4960 | 568005 4961 | 435543 4962 | 531189 4963 | 417635 4964 | 484028 4965 | 381336 4966 | 520797 4967 | 485140 4968 | 531560 4969 | 356076 4970 | 571019 4971 | 69903 4972 | 231735 4973 | 239793 4974 | 489347 4975 | 277504 4976 | 572205 4977 | 314923 4978 | 554181 4979 | 528927 4980 | 550792 4981 | 387189 4982 | 98755 4983 | 88223 4984 | 208546 4985 | 241116 4986 | 421677 4987 | 72014 4988 | 555925 4989 | 250262 4990 | 370893 4991 | 731 4992 | 144311 4993 | 65026 4994 | 579123 4995 | 217136 4996 | 53442 4997 | 536070 4998 | 557943 4999 | 505791 5000 | 544933 5001 | --------------------------------------------------------------------------------