├── .gitignore ├── python ├── SpringRank │ ├── __init__.py │ ├── _spring_rank.py │ └── _core.py ├── README.md ├── test_calculate_SpringRank.py ├── SprinRank_tools.py ├── tools.py └── test_spring_rank_dense_vs_sparse.py ├── matlab ├── eigenvectorCentrality.m ├── globalAccuracy_BTL.m ├── localAccuracy_BTL.m ├── katzCentrality.m ├── davidScore.m ├── pageRank.m ├── syncRank.m ├── generativeModel.m ├── betaGlobal.m ├── betaLocal.m ├── localAccuracy.m ├── globalAccuracy.m ├── colleyMatrix.m ├── springRankHamiltonian.m ├── btl.m ├── rankCentrality.m ├── serialRank.m ├── springRank.m ├── pvalueNullModel.m ├── shuffle.m ├── networkComponents.m ├── demo.m ├── ranks2svg.m ├── mvr.m └── crossValidation.m ├── r ├── test_springrank.R └── springrank.R ├── data ├── README.md ├── US_CS_SpringRank_a0.0_l0_1.0_l1_1.0.dat ├── US_CS_nodes.dat └── US_CS_adjacency.dat ├── iml_sas └── springrank.mod ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.DS_store 3 | *__pycache__ 4 | *~ -------------------------------------------------------------------------------- /python/SpringRank/__init__.py: -------------------------------------------------------------------------------- 1 | from ._spring_rank import SpringRank -------------------------------------------------------------------------------- /matlab/eigenvectorCentrality.m: -------------------------------------------------------------------------------- 1 | function [V] = eigenvectorCentrality(A,regularization) 2 | % Eigenvector Centrality implemented below 3 | 4 | % Let's get the Perron Frobenius eigenvector of A 5 | [V,~] = eigs(A+regularization,1); 6 | -------------------------------------------------------------------------------- /matlab/globalAccuracy_BTL.m: -------------------------------------------------------------------------------- 1 | % Code by Daniel Larremore 2 | % Santa Fe Institute 3 | % larremore@santafe.edu 4 | % http://danlarremore.com 5 | 6 | % evaluate the local accuracy of edge direction prediction 7 | 8 | function y = globalAccuracy_BTL(A,g) 9 | n = length(g); 10 | y = 0; 11 | for i=1:n 12 | for j=1:n 13 | p = g(i)/(g(i)+g(j)); % BTL probability 14 | if p==0 || p==1 || isnan(p) 15 | % do nothing 16 | else 17 | y = y + A(i,j)*log(p)+A(j,i)*log(1-p); 18 | end 19 | end 20 | end 21 | 22 | -------------------------------------------------------------------------------- /matlab/localAccuracy_BTL.m: -------------------------------------------------------------------------------- 1 | % Code by Daniel Larremore 2 | % Santa Fe Institute 3 | % larremore@santafe.edu 4 | % http://danlarremore.com 5 | 6 | % evaluate the local accuracy of edge direction prediction 7 | 8 | function a = localAccuracy_BTL(A,g) 9 | m = sum(sum(A)); 10 | n = length(g); 11 | y = 0; 12 | for i=1:n 13 | for j=1:n 14 | p = g(i)/(g(i)+g(j)); % BTL probability 15 | if isnan(p) 16 | % do nothing 17 | else 18 | y = y + abs( A(i,j)-( A(i,j) + A(j,i) )*p ); 19 | end 20 | end 21 | end 22 | a = 1-0.5*y/m; 23 | -------------------------------------------------------------------------------- /matlab/katzCentrality.m: -------------------------------------------------------------------------------- 1 | function [c] = katzCentrality(A,regularization) 2 | % Katz Centrality implemented below 3 | 4 | % number of nodes 5 | N = max(size(A)); 6 | 7 | % Get the eigenvalues of A+regularization 8 | % Take their absolute value. 9 | v = abs(eig(A+regularization)); 10 | 11 | % Set the damping factor as half the inverse largest eigenvalue 12 | alph = 0.5/max(v); 13 | % Set the scaling parameter b to 1 14 | b = 1; 15 | 16 | % x = a Ac + b 17 | % x - a Ac = b1 18 | % Ix - a Ac = b1 19 | % (I-aA)c = b1 20 | % c = (I-aA) \ b1 21 | 22 | c = (eye(N)-alph*A) \ (b*ones(N,1)) ; 23 | 24 | 25 | -------------------------------------------------------------------------------- /r/test_springrank.R: -------------------------------------------------------------------------------- 1 | # This just runs the test script, same as the Python version, but a 2 | # bit more barebones. 3 | 4 | library(readr) 5 | library(igraph) 6 | library(Matrix) 7 | 8 | source("springrank.R") 9 | 10 | df <- read_delim("../data/US_CS_adjacency.dat", 11 | delim = " ", 12 | col_names = c("i", "j", "weight")) 13 | 14 | G <- graph.data.frame(df) 15 | A <- as.matrix(as_adjacency_matrix(G, attr = "weight")) 16 | A <- Matrix(A) # sparsify A 17 | 18 | alpha <- 0.0 19 | l0 <- 1.0 20 | l1 <- 1.0 21 | 22 | r <- spring_rank(A, alpha, l0, l1) 23 | print(round(rev(r[order(r)]), 3)) -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | The sample dataset `US_CS_adjacency.dat` represents the North American faculty hiring network in Computer Science. It can be found at http://tuvalu.santafe.edu/~aaronc/facultyhiring/. 2 | An edge `node1 node2 weight` means that a number of `weight` faculties with a PhD obtained at institution `node1` have been hired at institution `node2`. 3 | 4 | The file `US_CS_nodes.dat` contains information about the nodes, specifically it maps the integer node ids to the names of the institutions. 5 | 6 | The SpringRank scores are shown in `US_CS_SpringRank_a0.0_l0_1.0_l1_1.0.dat` and they refer to a choice of parameters `alpha=0` and `l_0=l_1=1`. 7 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | - `SpringRank/` folder containing the main function to calculate SpringRank scores. To use it in your code, add it by writing on top: 2 | - `from SpringRank import SpringRank` 3 | see `test_calculate_SpringRank.py` for an example. 4 | 5 | - `SpringRank_tools.py` containes the function to generate a network according to the SpringRank generative model. 6 | 7 | - `tools.py` containes auxiliary functions needed to process the input and output. 8 | 9 | - `test_calculate_SpringRank.py` is a sample script for testing the code. It gives an example of how to use the code: it calculates the SpringRank scores for the sample network contained in `../data`. 10 | 11 | - `test_spring_rank_dense_vs_sparse` is a sample script for testing the efficency using sparse vs dense matrices. Use always sparse matrices when you can! 12 | -------------------------------------------------------------------------------- /matlab/davidScore.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % s = davidScore(A) 11 | % INPUTS: 12 | % A is a NxN matrix representing a directed network 13 | % A can be weighted (integer or non-integer) 14 | % A(i,j) = # of dominance interactions by i toward j. 15 | % A(i,j) = # of times that j endorsed i. 16 | % OUTPUTS: 17 | % s is the Nx1 vector of Davids Score 18 | 19 | function [s] = davidScore(A) 20 | 21 | P = A./(A+A'); % Pij = Aij / (Aij + Aji) 22 | P(isnan(P)) = 0; 23 | P(1:size(P,1)+1:end) = 0; % ensure there are no entries on the diagonal 24 | w = sum(P,2); 25 | l = sum(transpose(P),2); 26 | w2 = P*w; 27 | l2 = transpose(P)*l; 28 | s = w+w2-l-l2; 29 | -------------------------------------------------------------------------------- /matlab/pageRank.m: -------------------------------------------------------------------------------- 1 | % Parameter M adjacency matrix where M_i,j represents the link from 'j' to 'i', such that for all 'j' 2 | % sum(i, M_i,j) = 1 3 | % Parameter d damping factor 4 | % Parameter v_quadratic_error quadratic error for v 5 | % Return v, a vector of ranks such that v_i is the i-th rank from [0, 1] 6 | 7 | function v = pageRank(A, d, v_quadratic_error) 8 | N = max(size(A)); % N is equal to either dimension of M and the number of documents 9 | M = zeros(N,N); 10 | for j=1:N 11 | if sum(A(:,j)) > 0 12 | M(:,j)=A(:,j)/sum(A(:,j)); 13 | else 14 | M(:,j) = 1/N; 15 | end 16 | end 17 | % v = rand(N, 1); 18 | v = ones(N,1); 19 | v = v ./ norm(v, 1); % This is now L1, not L2 20 | last_v = ones(N, 1) * inf; 21 | M_hat = (d .* M) + (((1 - d) / N) .* ones(N, N)); 22 | 23 | while(norm(v - last_v, 2) > v_quadratic_error) 24 | last_v = v; 25 | v = M_hat * v; 26 | v = v/norm(v,1); 27 | % removed the L2 norm of the iterated PR 28 | end -------------------------------------------------------------------------------- /matlab/syncRank.m: -------------------------------------------------------------------------------- 1 | function [sy] = syncRank(A) 2 | 3 | N = size(A,2); 4 | 5 | % 1. Form C 6 | % Whenever Aij > Aji we set Cij = 1 7 | % Whenever Aij < Aji we set Cij = -1 8 | % Else, Cij = 0; 9 | % This means that Cij = sign(Aij - Aji) 10 | C = sign(A-transpose(A)); 11 | 12 | % 2. Form Theta 13 | T = pi*C/(N-1); 14 | 15 | % 3. Form H 16 | H = spalloc(N,N,nnz(T)); 17 | H(T~=0) = exp(1i*T(T~=0)); 18 | 19 | % 4. Form Dinv 20 | Dinv = diag(1./sum(abs(H))); 21 | 22 | % 5. Form fancyH 23 | fancyH = Dinv*H; 24 | 25 | % 6. Leading eigenvector of fancyH 26 | [V,~] = eigs(fancyH,1); 27 | 28 | % 7. Get angles in complex plane. 29 | angles = angle(V); 30 | 31 | % 8. Get order from angles 32 | [~,idx] = sort(angles,'ascend'); 33 | sy(idx) = 1:N; 34 | 35 | % 10. Choose the rank permutation that minimizes violations. 36 | viols = zeros(N,1); 37 | for ii=1:N 38 | sy_perm = mod(sy + ii - 2,N) + 1; 39 | idx_perm(sy_perm) = 1:N; 40 | viols(ii) = sum(sum(triu(A(idx_perm,idx_perm)))); 41 | end 42 | best = find(viols==min(viols)); 43 | 44 | sy = (mod(sy + best(1) -2,N)+1)'; 45 | -------------------------------------------------------------------------------- /matlab/generativeModel.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % [A,P] = generativeModel(c,b,s) 11 | % INPUTS: 12 | % c is the overall sparsity constant 13 | % b is the inverse temperature (called beta in the paper) 14 | % s is the Nx1 vector of planted node positions 15 | % OUTPUTS: 16 | % A is a directed network adjacency matrix; A(i,j)=1 if i dominates j, e.g. 17 | % P is a full NxN matrix; P(i,j) = expected number of edges from i to j 18 | 19 | function [A,P] = generativeModel(c,b,s) 20 | 21 | % number of vertices 22 | N = length(s); 23 | % preallocate P 24 | P = zeros(N,N); 25 | for i=1:N 26 | for j=1:N 27 | % compute expected number of edges from i to j 28 | P(i,j) = c*exp(-b/2*(s(i)-s(j)-1)^2); 29 | end 30 | end 31 | % draw poisson random numbers from P and return 32 | A = sparse(poissrnd(P)); -------------------------------------------------------------------------------- /iml_sas/springrank.mod: -------------------------------------------------------------------------------- 1 | /* this module implements the SpringRank routine of Larremore, based on his 2 | github matlab code. Note that as implemented it assumes a connected 3 | component. 4 | 5 | Author: Moody 6 | Source: https://github.com/cdebacco/SpringRank 7 | Date: 4.10.2018 8 | 9 | Input: An adjacency matrix for a connected component. 10 | output: a 1-dimensional score of position in the hierarchy, relative to 11 | node n which has value 0. 12 | 13 | */ 14 | 15 | start springrank(a); 16 | n=nrow(a); 17 | dout=a[,+]; 18 | din=a[+,]; 19 | din=din`; 20 | 21 | *print dout din; 22 | dNout=dout[n]; 23 | dNin=din[n]; 24 | 25 | dout=diag(dout[1:N-1]); 26 | din=diag(din[1:n-1]); 27 | 28 | *out-degree and in-degree for vertex N; 29 | dNout = dout[N]; 30 | dNin = din[N]; 31 | at=a`; 32 | 33 | B=Dout+Din - a[1:n-1,1:n-1] - a[1:n-1,1:n-1]` - repeat(a[n,1:n-1],n-1,1) 34 | -repeat(at[n,1:n-1] ,n-1,1); 35 | 36 | lb = vecdiag(Dout)-vecdiag(Din)+dNout-dNin; 37 | t = solve(B, lb); 38 | *print b lb t; 39 | 40 | s=t//0; 41 | return(s); 42 | 43 | finish; 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 cdebacco 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /matlab/betaGlobal.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % b = betaGlobal(A,s,varargin) 11 | % 12 | % INPUTS: 13 | % A is a NxN matrix representing a directed network 14 | % A can be weighted (integer or non-integer) 15 | % A(i,j) = # of dominance interactions by i toward j. 16 | % A(i,j) = # of times that j endorsed i. 17 | % s is the Nx1 vector of node positions (ranks) 18 | % 19 | % OUTPUTS: 20 | % b is the optimal inverse temperature (beta) under the GLOBAL accuracy, 21 | % which we call \sigma_\ell in the paper. 22 | 23 | function [b] = betaGlobal(A,s) 24 | global M r 25 | M = A; 26 | r = s; 27 | b = fzero(@f,0.1); 28 | end 29 | 30 | function [y] = f(b) 31 | global M r 32 | n = length(r); 33 | y = 0; 34 | for i=1:n 35 | for j=1:n 36 | d = r(i) - r(j); 37 | pij = (1+exp(-2*b*d))^(-1); 38 | y = y + d*(M(i,j) - (M(i,j)+M(j,i))*pij); 39 | end 40 | end 41 | end -------------------------------------------------------------------------------- /matlab/betaLocal.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % b = betaLocal(A,s) 11 | % INPUTS: 12 | % A is a NxN matrix representing a directed network 13 | % A can be weighted (integer or non-integer) 14 | % A(i,j) = # of dominance interactions by i toward j. 15 | % A(i,j) = # of times that j endorsed i. 16 | % s is the Nx1 vector of node positions (ranks) 17 | % OUTPUTS: 18 | % b is the optimal inverse temperature (beta) under the LOCAL accuracy, 19 | % which we call \sigma_a in the paper. 20 | 21 | function [b] = betaLocal(A,s) 22 | 23 | global M r 24 | M = A; 25 | r = s; 26 | b = fminbnd(@negacc,1e-6,1000); 27 | end 28 | 29 | function [a] = negacc(b) 30 | global M r 31 | m = sum(sum(M)); 32 | n = length(r); 33 | y = 0; 34 | for i=1:n 35 | for j=1:n 36 | d = r(i) - r(j); 37 | y = y + abs( M(i,j)- (M(i,j) + M(j,i))*((1+exp(-2*b*d))^(-1)) ); 38 | end 39 | end 40 | a = y/m-1; 41 | end -------------------------------------------------------------------------------- /matlab/localAccuracy.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % a = localAccuracy(A,s,b) 11 | % 12 | % INPUTS: 13 | % A is a NxN matrix representing a directed network 14 | % A can be weighted (integer or non-integer) 15 | % A(i,j) = # of dominance interactions by i toward j. 16 | % A(i,j) = # of times that j endorsed i. 17 | % s is the Nx1 vector of node positions (ranks) 18 | % b is the inverse temperature (called beta in the paper) 19 | % 20 | % OUTPUTS: 21 | % a is the local accuracy (called \sigma_a in the paper) 22 | 23 | function a = localAccuracy(A,s,b) 24 | % total edges 25 | m = sum(sum(A)); 26 | 27 | % number of vertices 28 | n = length(s); 29 | 30 | % accumulate accuracy of predictions 31 | y = 0; 32 | for i=1:n 33 | for j=1:n 34 | d = s(i) - s(j); 35 | p = (1+exp(-2*b*d))^(-1); 36 | y = y + abs( A(i,j)-( A(i,j) + A(j,i) )*p ); 37 | end 38 | end 39 | 40 | % cleanup 41 | a = 1-0.5*y/m; -------------------------------------------------------------------------------- /matlab/globalAccuracy.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % y = globalAccuracy(A,s,b) 11 | % 12 | % INPUTS: 13 | % A is a NxN matrix representing a directed network 14 | % A can be weighted (integer or non-integer) 15 | % A(i,j) = # of dominance interactions by i toward j. 16 | % A(i,j) = # of times that j endorsed i. 17 | % s is the Nx1 vector of node positions (ranks) 18 | % b is the inverse temperature (called beta in the paper) 19 | % 20 | % OUTPUTS: 21 | % y is the global accuracy (called \sigma_L in the paper) 22 | 23 | function y = globalAccuracy(A,s,b) 24 | 25 | % number of nodes 26 | n = length(s); 27 | % accumulate the log likelihood score elementwise 28 | y = 0; 29 | for i=1:n 30 | for j=1:n 31 | d = s(i) - s(j); 32 | p = (1+exp(-2*b*d))^(-1); 33 | if p==0 || p==1 34 | % do nothing 35 | else 36 | y = y + A(i,j)*log(p)+A(j,i)*log(1-p); 37 | end 38 | end 39 | end 40 | end -------------------------------------------------------------------------------- /matlab/colleyMatrix.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % r = colleyMatrix(A) 11 | % INPUTS: 12 | % A is a NxN matrix representing a directed network 13 | % A can be weighted (integer or non-integer) 14 | % A(i,j) = # of dominance interactions by i toward j. 15 | % A(i,j) = # of times that j endorsed i. 16 | % OUTPUTS: 17 | % r is the Nx1 vector of the Colley Matrix ranks 18 | 19 | function [r] = colleyMatrix(A) 20 | %Aij = i beats j 21 | %therefore out-degree = sum over j = wins 22 | %therefore in-degree = sum over i = losses 23 | A(1:max(size(A))+1:end) = 0; 24 | wins = sum(A,2); 25 | losses = sum(A,1)'; 26 | total = wins+losses; 27 | matches = A+transpose(A); 28 | 29 | n = size(A,1); 30 | C = zeros(n,n); 31 | b = zeros(n,1); 32 | for i=1:n 33 | b(i) = 1 + (wins(i)-losses(i))/2; 34 | for j=1:n 35 | if i==j 36 | C(i,j) = 2+total(i); 37 | else 38 | C(i,j) = -matches(i,j); 39 | end 40 | end 41 | end 42 | r = C\b; -------------------------------------------------------------------------------- /matlab/springRankHamiltonian.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % H = springRankHamiltonian(s,A,mu) 11 | % INPUTS: 12 | % s is a N-vector of node positions 13 | % A is a NxN matrix representing a directed network 14 | % A can be weighted (integer or non-integer) 15 | % A(i,j) = # of dominance interactions by i toward j. 16 | % A(i,j) = # of times that j endorsed i. 17 | % mu can be a scalar or a NxN matrix 18 | % OUTPUTS: 19 | % H is the scalar spring energy of the system; 20 | % NOTE: assumes spring rest length of 1 21 | 22 | function [H] = springRankHamiltonian(s,A,mu) 23 | % assuming A is sparse, faster to go through entries of A. 24 | [r,c,v] = find(A); 25 | % preallocate container of hamiltonian values. 26 | h = zeros(size(v)); 27 | 28 | % NOTE: i = r(n) and j = c(n) 29 | 30 | % Probably could be faster 31 | if length(mu)==1 % SCALAR spring constant 32 | for n = 1:length(v) 33 | h(n) = v(n) * (s(r(n))-s(c(n))-1)^2; 34 | end 35 | h = h*mu; 36 | else % MATRIX of spring constants 37 | for n = 1:length(v) 38 | h(n) = v(n) * mu(r(n),c(n)) * (s(r(n))-s(c(n))-1)^2; 39 | end 40 | end 41 | 42 | % sum up, divide by 2, and return 43 | H = sum(h)/2; -------------------------------------------------------------------------------- /python/test_calculate_SpringRank.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Example of SpringRank usage 3 | 4 | From a given network, it extracts the SpringRank scores 5 | 6 | ''' 7 | 8 | import networkx as nx 9 | import numpy as np 10 | from SpringRank import SpringRank 11 | import tools as tl 12 | 13 | network='US_CS' 14 | 15 | alpha=0. 16 | l0=1. 17 | l1=1. 18 | 19 | ''' 20 | Builds graph and Adjacency matrix from input network 21 | ''' 22 | inadjacency='../data/'+network+'_adjacency.dat' 23 | 24 | G=tl.build_graph_from_adjacency(inadjacency) 25 | 26 | nodes=list(G.nodes()) # determines the order of the entries of matrix A 27 | 28 | A = nx.to_scipy_sparse_matrix(G, dtype=float,nodelist=nodes) 29 | 30 | 31 | ''' 32 | Extracts SpringRank 33 | ''' 34 | rank=SpringRank(A,alpha=alpha,l0=l0,l1=l1) 35 | 36 | rank=tl.shift_rank(rank) # (optional) shifts so that the min is in zero and the others are positive 37 | 38 | ''' 39 | Order results so that the first node is the highest-ranked one 40 | ''' 41 | X=[(nodes[i],rank[i]) for i in range(G.number_of_nodes())] 42 | X= sorted(X, key=lambda tup: tup[1],reverse=True) 43 | ''' 44 | Prints results 45 | ''' 46 | print('SpringRank scores:') 47 | outfile='../data/'+network+'_SpringRank_'+'a'+str(alpha)+'_l0_'+str(l0)+'_l1_'+str(l1)+'.dat' 48 | outf=open(outfile,'w') 49 | 50 | for i in range(G.number_of_nodes()): 51 | outf.write("{} {}\n".format(X[i][0],X[i][1])) 52 | # print nodes[i],rank[i] 53 | print(X[i][0],X[i][1]) 54 | print('Results saved in:', outfile) 55 | outf.close() 56 | -------------------------------------------------------------------------------- /matlab/btl.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % g = btl(A,tol) 11 | % INPUTS: 12 | % A is a NxN matrix representing a directed network 13 | % A can be weighted (integer or non-integer) 14 | % A(i,j) = # of dominance interactions by i toward j. 15 | % A(i,j) = # of times that j endorsed i. 16 | % tol is the accuracy tolerance desired for successive iterations 17 | % OUTPUTS: 18 | % s is the Nx1 vector of Davids Score 19 | % NOTE: implementation of a regularized version (for dangling node) 20 | % version of the algorithm presented in 21 | % Hunter DR (2004) MM algorithms for generalized Bradley-Terry models. 22 | % Annals of Statistics pp. 384?406 23 | 24 | function [g] = btl(A,tol) 25 | 26 | A(1:max(size(A))+1:end) = 0; 27 | N = size(A,1); 28 | g = rand(1,N); % random initial guesss 29 | wins = sum(A,2); 30 | matches = A+transpose(A); 31 | totalMatches = sum(matches); 32 | g_prev = rand(1,N); 33 | eps=1e-6; 34 | while norm(g-g_prev) > tol 35 | g_prev = g; 36 | for i=1:N 37 | if totalMatches(i)>0 38 | q = matches(i,:)./(g_prev(i)+g_prev); 39 | q(i) = []; 40 | g(i) = (wins(i)+eps)/sum(q); 41 | else 42 | g(i) = 0; 43 | end 44 | end 45 | g = g/sum(g); 46 | end 47 | g = transpose(g); 48 | end -------------------------------------------------------------------------------- /matlab/rankCentrality.m: -------------------------------------------------------------------------------- 1 | % Rank Centrality 2 | % Implemented by Dan Larremore, University of Colorado Boulder 3 | % April 8, 2018 4 | % 5 | % Based on the manuscript 6 | % Rank Centrality: Ranking from Pairwise Comparisons 7 | % Sahand Negahban, Sewoong Oh, Devavrat Shah 8 | % 2017 9 | % 10 | function [rc] = rankCentrality(A) 11 | % In their text, a_ij = # of times j is preferred over i. 12 | % In the SpringRank paper, we usually assume the opposite. 13 | % Here, we'll use the authors' direction, but note that whenever we call 14 | % this code, we'll have to pass the transpose of A. 15 | 16 | % Note that there are no self-loops in this model, so we will check, 17 | % discard, and warn 18 | N = size(A,1); 19 | if sum(diag(A)) > 0 20 | % fprintf('Warning: self-loops detected (and ignored)\n') 21 | A(1:N+1:end) = 0; 22 | end 23 | 24 | 25 | % see Eq 5 of https://arxiv.org/pdf/1209.1688.pdf 26 | % We're going to regularize. 27 | % They suggest epsilon = 1. 28 | % This seems extreme? 29 | 30 | % Not listed in the paper, but this is important. We have to regularize the 31 | % matrix A before we compute dmax. 32 | regularization = 1; 33 | A = A+regularization; 34 | 35 | % Find dmax 36 | dout = sum(A,2); 37 | dmax = max(dout); 38 | 39 | % Eq 5 40 | 41 | P = (1/dmax) * A./(A+transpose(A)); 42 | 43 | % But we need to make sure that the matrix remains stochastic by making the 44 | % rows sum to 1. Without regularization, Eq 1 says P(i,i) = 1 - dout(i)/dmax; 45 | % Instead, we're going to just do this "manually" 46 | 47 | P(1:N+1:end) = 0; 48 | for i=1:N 49 | P(i,i) = 1-sum(P(i,:)); 50 | end 51 | 52 | [V,~] = eigs(transpose(P),1); 53 | 54 | rc = V / sum(V); 55 | 56 | end -------------------------------------------------------------------------------- /matlab/serialRank.m: -------------------------------------------------------------------------------- 1 | function [serr] = serialRank(A) 2 | 3 | % In serialRank, C(i,j) = 1 if j was preferred over i. 4 | % The way that we construct the adj matrices in SpringRank, it's the opp. 5 | % Therefore, pass in A transpose and implement serialRank faithfullly from 6 | % the formulas. 7 | 8 | 9 | % According to the SerialRank tutorial, we have 10 | % If Aij = Aji = 0, then Qij = Qji = 1/2 11 | % Else, Qij = 1/mij * sum (cij + 1)/2 where the sum is over the mij outcomes 12 | % Noting that mij from SerialRank is Aij + Aji, 13 | % And noting that the sum is (Aji - Aji + Aji + Aij) 14 | % Else, Qij = ( 1 / 2(Aij+Aji) ) * (A_ij - A_ji + A_ij + A_ji) 15 | % i.e. Qij = ( 1 / 2(Aij+Aji) ) * 2A_ij 16 | % i.e. Qij = ( 1 / (Aij+Aji) ) * A_ij 17 | % i.e. Qij = A_ij / (Aij+Aji) 18 | 19 | N = size(A,1); 20 | Q = A; 21 | M = A+A'; 22 | Q(A~=0) = A(A~=0)./M(A~=0); 23 | Q(M==0) = 1/2; 24 | 25 | % Sij = sum over k, where B(i,k) and B(j,k) are nonzero, the quantity 26 | % (1 - abs(Q(i,k)-Q(j,k)/2) 27 | % and (sum) the quantity 1/2 otherwise. 28 | % Hmmm. I'm not sure how to do this without a for loop... Probably there's 29 | % a way, but I don't feel like optimizing someone else's alg. :/ 30 | % Also, S is dense and symmetric, I think. 31 | 32 | S = zeros(N); 33 | for i=1:N 34 | for j=i+1:N 35 | % Form the indicator Bik * Bjk == 0 36 | indicator = M(i,:).*M(j,:); 37 | S(i,j) = S(i,j) + sum(indicator==0)/2; 38 | nonzero = find(indicator>0); 39 | if ~isempty(nonzero) 40 | S(i,j) = S(i,j) + length(nonzero) - sum(abs(Q(i,nonzero) - Q(j,nonzero)))/2; 41 | end 42 | end 43 | end 44 | S = S+S'; 45 | L = diag(sum(S)) - S; 46 | [V,~] = eigs(L,2,'smallestabs'); 47 | serr = V(:,2); 48 | 49 | % [~,idx] = sort(W,'descend'); 50 | % serr(idx) = 1:N; 51 | % serr = serr'; -------------------------------------------------------------------------------- /matlab/springRank.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % s = springRank(A) 11 | % INPUTS: 12 | % A is a NxN matrix representing a directed network 13 | % A can be weighted (integer or non-integer) 14 | % A(i,j) = # of dominance interactions by i toward j. 15 | % A(i,j) = # of times that j endorsed i. 16 | % OUTPUTS: 17 | % s is a N-vector of node positions according to SpringRank 18 | 19 | function [s] = springRank(A) 20 | 21 | % Input check for sparsity 22 | if issparse(A) 23 | % Great. We like sparse. 24 | else 25 | A = sparse(A); 26 | % Still works, but slower 27 | warning('Input matrix not sparse; much faster if A is a sparse matrix.') 28 | end 29 | 30 | % Number of vertices 31 | N = size(A,2); 32 | % out-degree and in-degree VECTORS 33 | dout = sum(A,2); 34 | din = sum(A,1); 35 | % out-degree and in-degree DIAGONAL MATRICES 36 | Dout = diag(dout(1:N-1)); 37 | Din = diag(din(1:N-1)); 38 | % out-degree and in-degree for vertex N 39 | dNout = dout(N); 40 | dNin = din(N); 41 | 42 | B = Dout + Din - A(1:N-1,1:N-1) - transpose(A(1:N-1,1:N-1))... 43 | - repmat(A(N,1:N-1),N-1,1) - repmat(transpose(A(1:N-1,N)),N-1,1); 44 | b = diag(Dout)-diag(Din)+dNout-dNin; 45 | 46 | % Sparse solve. Use [t,~] to suppress warnings. 47 | [t,~] = bicgstab(B,b,1e-12,200); 48 | 49 | % ranks 50 | s = [t;0]; 51 | 52 | % adjust mean of each component to be 0 53 | [nComponents,~,members] = networkComponents(A); 54 | for n = 1:nComponents 55 | s(members{n}) = s(members{n})-mean(s(members{n})); 56 | end 57 | -------------------------------------------------------------------------------- /python/SprinRank_tools.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | 4 | def SpringRank_planted_network(N,beta,alpha,K,prng,l0=0.5,l1=1.): 5 | ''' 6 | 7 | Uses the SpringRank generative model to build a directed, possibly weigthed and having self-loops, network. 8 | Can be used to generate benchmarks for hierarchical networks 9 | 10 | Steps: 11 | 1. Generates the scores (default is factorized Gaussian) 12 | 2. Extracts A_ij entries (network edges) from Poisson distribution with average related to SpringRank energy 13 | 14 | INPUT: 15 | 16 | N=# of nodes 17 | beta= inverse temperature, controls noise 18 | alpha=controls prior's variance 19 | K=E/N --> average degree, controls sparsity 20 | l0=prior spring's rest length 21 | l1=interaction spring's rest lenght 22 | 23 | OUTPUT: 24 | G: nx.DiGraph() Directed (possibly weighted graph, there can be self-loops) 25 | 26 | ''' 27 | G=nx.DiGraph() 28 | 29 | scores=prng.normal(l0,1./np.sqrt(alpha*beta),N) # planted scores ---> uses factorized Gaussian 30 | for i in range(N):G.add_node(i,score=scores[i]) 31 | 32 | # ---- Fixing sparsity i.e. the average degree ---- 33 | Z=0. 34 | for i in range(N): 35 | for j in range(N): 36 | Z+=np.exp(-0.5*beta*np.power(scores[i]-scores[j]-l1,2)) 37 | c=float(K*N)/Z 38 | # -------------------------------------------------- 39 | 40 | # ---- Building the graph ------------------------ 41 | for i in range(N): 42 | for j in range(N): 43 | 44 | H_ij=0.5*np.power((scores[i]-scores[j]-l1),2) 45 | lambda_ij=c*np.exp(-beta*H_ij) 46 | 47 | A_ij=prng.poisson(lambda_ij,1)[0] 48 | 49 | if A_ij>0:G.add_edge(i,j,weight=A_ij) 50 | 51 | return G 52 | -------------------------------------------------------------------------------- /python/tools.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | from scipy.optimize import brentq 4 | 5 | def build_graph_from_adjacency(inadjacency): 6 | """ 7 | Takes an adjacency_list like: "23 41 18" or 18 times "23 41 1" (edge from 23 --> 41) 8 | possibly having multiple edges and build a graph with no multiple edges but weigths representing how many of them there are 9 | Necessary in case of using algorithms that do not accept MultiGraphs. E.g. eigenvector centrality. 10 | """ 11 | 12 | adjacency_list=open(inadjacency,'r') 13 | edges={} 14 | for row in adjacency_list: 15 | a=row.split() 16 | e=(a[0],a[1]) 17 | w=int(a[2]) 18 | if(e not in edges):edges[e]=w 19 | else:edges[e]+=w 20 | G=nx.DiGraph() 21 | for e in edges: G.add_edge(e[0],e[1],weight=edges[e]) 22 | adjacency_list.close() 23 | 24 | return G 25 | 26 | 27 | def shift_rank(ranks): 28 | ''' 29 | Shifts all scores so that the minimum is in zero and the others are all positive 30 | ''' 31 | min_r=min(ranks) 32 | N=len(ranks) 33 | for i in range(N): ranks[i]=ranks[i]-min_r 34 | return ranks 35 | 36 | def btl(A,tol): 37 | N = np.shape(A)[0] 38 | g = np.random.rand(N) 39 | wins = np.array(np.sum(A,axis=1)).flatten(); 40 | matches = np.array(A+np.transpose(A)); 41 | totalMatches = np.array(np.sum(matches,axis=0)).flatten() 42 | g_prev = np.random.rand(N) 43 | eps = 1e-6 44 | while np.linalg.norm(g-g_prev) > tol: 45 | g_prev = g 46 | for i in range(N): 47 | if totalMatches[i]>0: 48 | q = np.divide(matches[i,:],g_prev[i]+g_prev) 49 | q[i] = 0 50 | g[i] = (wins[i]+eps)/np.sum(q) 51 | else: 52 | g[i] = 0 53 | g = g/np.sum(g) 54 | return np.log(g) 55 | 56 | def eqs39(beta,s,A): 57 | N = np.shape(A)[0] 58 | x = 0 59 | for i in range(N): 60 | for j in range(i+1,N): 61 | if (A[i,j] == 0) & (A[j,i] == 0): 62 | continue 63 | else: 64 | x += (s[i]-s[j]) * ( A[i,j] - (A[i,j]+A[j,i]) / (1+np.exp(-2*beta*(s[i]-s[j]))) ) 65 | return x 66 | 67 | def get_optimal_temperature(ranks,A): 68 | return brentq(eqs39,0.01,20,args=(ranks,A)) 69 | -------------------------------------------------------------------------------- /python/SpringRank/_spring_rank.py: -------------------------------------------------------------------------------- 1 | """ 2 | New version developed by Nicolò Ruggeri, Max Planck Institute for Intelligent Systems, Tuebingen, Germany, March-2020 3 | It forces to use sparse matrices when possible, results in much more efficent implementation, especially for large matrices 4 | """ 5 | 6 | import warnings 7 | 8 | import scipy.sparse 9 | 10 | from ._core import build_from_dense, build_from_sparse, solve_linear_system 11 | 12 | 13 | def SpringRank(A, alpha=0., l0=1., l1=1., solver='bicgstab', verbose=False, force_dense=False): 14 | """ 15 | Main routine to calculate SpringRank by a solving linear system. 16 | 17 | Parameters 18 | ---------- 19 | A : numpy.ndarray or scipy.sparse.spmatrix 20 | Has tobe 2 dimensional and with same dimensions. 21 | alpha, l0, l1: float 22 | Defined as in the SpringRank paper 23 | https://arxiv.org/abs/1709.09002 24 | solver: str 25 | One between 'spsolve' (direct, slower) and 'bicgstab' (iterative, faster). 26 | The solver to be used for the linear system returning the ranks. 27 | verbose: bool 28 | force_dense: bool 29 | By default A is converted to a sparse matrix scipy.sparse.csr, if it is not already sparse. 30 | If force_dense is set to True and a dense ndarray A is input, then it is not converted to sparse. 31 | 32 | Returns 33 | ------- 34 | rank 35 | numpy.ndarray of ranks. Indices represent the nodes' indices used in the matrix A. 36 | 37 | """ 38 | 39 | # check if input is sparse or can be converted to sparse. 40 | use_sparse = True 41 | if force_dense and not scipy.sparse.issparse(A): 42 | try: 43 | A = scipy.sparse.csr_matrix(A) 44 | except: 45 | warnings.warn('The input parameter A could not be converted to scipy.sparse.csr_matrix. ' 46 | 'Using a dense representation.') 47 | use_sparse = False 48 | elif force_dense: 49 | use_sparse = False 50 | 51 | # build array to feed linear system solver 52 | if use_sparse: 53 | A, B = build_from_sparse(A, alpha, l0, l1) 54 | else: 55 | A, B = build_from_dense(A, alpha, l0, l1) 56 | 57 | rank = solve_linear_system(A, B, solver, verbose) 58 | 59 | return rank 60 | 61 | 62 | -------------------------------------------------------------------------------- /python/test_spring_rank_dense_vs_sparse.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test developed by Nicolò Ruggeri, Max Planck Institute for Intelligent Systems, Tuebingen, Germany, March-2020 3 | It shows that working with sparse matrices it's much faster, this can be noticed for Adjacency matrices of size ~10^4 4 | """ 5 | import scipy.sparse 6 | import numpy as np 7 | import networkx as nx 8 | import timeit 9 | 10 | from SpringRank import SpringRank 11 | 12 | 13 | # test correctness (outputs are the same for old and new function) 14 | def test_correctness(alpha, rtol=1.e-10): 15 | print('\nTest correctness with alpha = ', alpha) 16 | res_dense = SpringRank(B, alpha=alpha, verbose=verbose, force_dense=True) 17 | res_sparse = SpringRank(A, alpha=alpha, verbose=verbose) 18 | assert np.allclose(res_dense, res_sparse, rtol=rtol) 19 | print('Passed.') 20 | 21 | 22 | # test speed (having the adjacency matrix already instanciated) 23 | def test_speed(alpha, rep): 24 | # print(f'\nTest speed with alpha = {alpha} and {rep} repetitions') 25 | print("\nTest speed with alpha={alpha} and {rep} repetitions".format(alpha=alpha, rep=rep)) 26 | sr_dense = lambda: SpringRank(B, alpha=alpha, verbose=False, force_dense=True) 27 | sr_sparse = lambda: SpringRank(A, alpha=alpha, verbose=False) 28 | 29 | print('Time for SpringRank with dense arrays: ', timeit.timeit(sr_dense, number=rep)) 30 | print('Time for SpringRank with sparse arrays:', timeit.timeit(sr_sparse, number=rep)) 31 | 32 | 33 | if __name__ == '__main__': 34 | # define experiment instance 35 | experiment = 4 36 | 37 | if experiment == 1: 38 | A = scipy.sparse.csr_matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=float) 39 | B = A.toarray() 40 | elif experiment == 2: 41 | A = scipy.sparse.csr_matrix(np.random.rand(100, 100)) 42 | B = A.toarray() 43 | elif experiment == 3: 44 | G = nx.scale_free_graph(1000) 45 | A = nx.to_scipy_sparse_matrix(G, dtype=float) 46 | B = A.toarray() 47 | elif experiment == 4: 48 | print('Generating synsthetic scale free graph...') 49 | G = nx.scale_free_graph(10000) 50 | print('Done.') 51 | A = nx.to_scipy_sparse_matrix(G, dtype=float) 52 | B = A.toarray() 53 | 54 | verbose = False 55 | solver = 'spsolve' # 'bicgstab' 56 | rep = 10 57 | 58 | # test correctness 59 | test_correctness(alpha=0.) 60 | test_correctness(alpha=1.) 61 | 62 | # test speed 63 | test_speed(alpha=0., rep=rep) 64 | test_speed(alpha=1., rep=rep) 65 | -------------------------------------------------------------------------------- /matlab/pvalueNullModel.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % [p,H0,H] = pvalueNullModel(A,n_repetitions) 11 | % 12 | % INPUTS: 13 | % A is a NxN matrix representing a directed network 14 | % A can be weighted (integer or non-integer) 15 | % A(i,j) = # of dominance interactions by i toward j. 16 | % A(i,j) = # of times that j endorsed i. 17 | % n_repetitions is the number of randomizations that you would like to use 18 | % to calculate the p-value. Higher numbers mean a better estimate of 19 | % probability that the p-value is meant to represent. Lower numbers may 20 | % be required for very large networks whose randomizations are expensive 21 | % or slow. 22 | % 23 | % OUTPUTS: 24 | % p is the p-value described in the paper for the probability that a 25 | % network A whose edge directions are randomized would have a lower 26 | % ground-state energy than the original network A 27 | % H0 is the ground state energy of A 28 | % H is the vector of ground state energies associated with randomizations 29 | % of the directions of A. 30 | 31 | function [p,H0,H] = pvalueNullModel(A,n_repetitions) 32 | 33 | % First determine what kind of matrix we have. Integer or non-integer? 34 | [~,~,v] = find(A); 35 | if sum(mod(v,1)==0)==length(v) 36 | isInteger = 1; 37 | Abar = A+A'; 38 | else 39 | isInteger = 0; 40 | end 41 | 42 | % Preallocate 43 | H = zeros(n_repetitions,1); 44 | H0 = springRankHamiltonian(springRank(A),A,1); 45 | 46 | % Iterate over repetitions 47 | for n = 1:n_repetitions 48 | % Two different randomization schemes, depending on whether or not the 49 | % network is integer or scalar. 50 | if isInteger 51 | B = randomEdgeDirectionsInt(Abar); 52 | else 53 | B = randomEdgeDirectionsScalar(A); 54 | end 55 | H(n) = springRankHamiltonian(springRank(B),B,1); 56 | end 57 | 58 | p = sum(H - a vector of component sizes, sorted, 20 | % descending. 21 | % members cell> a cell array of vectors, each 22 | % entry of which is a membership list for that component, sorted, 23 | % descending by component size. 24 | % 25 | % Example: (uncomment and copy and paste into MATLAB command window) 26 | % % Generate a 1000 node network adjacency matrix, A 27 | % A = floor(1.0015*rand(1000,1000)); A=A+A'; A(A==2)=1; A(1:1001:end) = 0; 28 | % % Call networkComponents function 29 | % [nComponents,sizes,members] = networkComponents(A); 30 | % % get the size of the largest component 31 | % sizeLC = sizes(1); 32 | % % get a network adjacency matrix for ONLY the largest component 33 | % LC = A(members{1},members{1}); 34 | 35 | function [nComponents,sizes,members] = networkComponents(A) 36 | % Number of nodes 37 | N = size(A,1); 38 | % Remove diagonals 39 | A(1:N+1:end) = 0; 40 | % make symmetric, just in case it isn't 41 | A=A+A'; 42 | % Have we visited a particular node yet? 43 | isDiscovered = zeros(N,1); 44 | % Empty members cell 45 | members = {}; 46 | % check every node 47 | for n=1:N 48 | if ~isDiscovered(n) 49 | % started a new group so add it to members 50 | members{end+1} = n; 51 | % account for discovering n 52 | isDiscovered(n) = 1; 53 | % set the ptr to 1 54 | ptr = 1; 55 | while (ptr <= length(members{end})) 56 | % find neighbors 57 | nbrs = find(A(:,members{end}(ptr))); 58 | % here are the neighbors that are undiscovered 59 | newNbrs = nbrs(isDiscovered(nbrs)==0); 60 | % we can now mark them as discovered 61 | isDiscovered(newNbrs) = 1; 62 | % add them to member list 63 | members{end}(end+1:end+length(newNbrs)) = newNbrs; 64 | % increment ptr so we check the next member of this component 65 | ptr = ptr+1; 66 | end 67 | end 68 | end 69 | % number of components 70 | nComponents = length(members); 71 | for n=1:nComponents 72 | % compute sizes of components 73 | sizes(n) = length(members{n}); 74 | end 75 | 76 | [sizes,idx] = sort(sizes,'descend'); 77 | members = members(idx); 78 | 79 | end -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SpringRank 2 | 3 | SpringRank is a method for ranking the nodes in a directed network or from a set of pairwise comparisons. 4 | 5 | In this repo, we provide three implementations of the SpringRank model described in: 6 | 7 | * [1] C. De Bacco, D. B. Larremore and C. Moore, *A physical model for efficient ranking in networks*, Science Advances, Vol 4, **7**, eaar8260, 2018. [Science Advances PDF](http://advances.sciencemag.org/content/4/7/eaar8260). [arXiv PDF](https://arxiv.org/abs/1709.09002) and [here](http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf). 8 | 9 | If you use this code please cite [1]. 10 | 11 | 12 | ## What's included: 13 | - `python` : Python code and a test script. 14 | - `matlab` : MATLAB code and a test script. 15 | - `r` : R code and a test script. 16 | - `iml_sas:` SAS/IML code. 17 | - `data` : Contains sample adjacency files to test the code and a sample result. 18 | 19 | ## Other implementations: 20 | - `Haskell` : by Jon Zingale [github.com/jonzingale/Haskell/](https://github.com/jonzingale/Haskell/tree/master/SpringRank). 21 | - `Python` : Fast Sparse alternative code by Larremore Lab on PyPI [github.com/LarremoreLab/SpringRank](https://github.com/LarremoreLab/SpringRank) 22 | 23 | ## Python Notes: 24 | Need to make a directory called `data` at the same level of the `python` folder. 25 | To make one, just type from the command line, inside that folder: 26 | * `mkdir data` 27 | 28 | #### Input format. 29 | The directed adjacency matrix should be formatted as an edge list with 3 columns: 30 | 31 | `node1 node2 3 ` 32 | 33 | The first and second columns are the source and target nodes of that edge, respectively; the third is the edge weigth (must be integer). In this example the edge node1 --> node2 exists with weight 3. 34 | 35 | #### Output. 36 | One file will be generated inside the `data` folder containg the SpringRank scores ordered from highest to lowest. The output file will be inside `data` folder with names: 37 | - `networkname_SpringRank_a0.0_l0_1.0_l1_1.0.dat` for the case `alpha=0` and `l_0=l_1=1.0`. 38 | 39 | The first column is the node id, the second column is the SpringRank score. 40 | 41 | 42 | 43 | 44 | Copyright (c) 2017 [Caterina De Bacco](http://cdebacco.com) and [Daniel B Larremore](https://larremorelab.github.io) 45 | 46 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 47 | 48 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 49 | 50 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 51 | -------------------------------------------------------------------------------- /matlab/demo.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % This code will: 11 | % 1. Generate a network via generativeModel.m 12 | % 2. Use methods to find rankings from the resulting network 13 | % springRank.m 14 | % mvr.m 15 | % colleyMatrix.m 16 | % davidScore.m 17 | % btl.m 18 | % eigenvectorCentrality.m 19 | % 3. Compute the optimal inv. temperatures beta_local and beta_global 20 | 21 | %% 1. Create a network 22 | 23 | % choose preferred degree 24 | deg = 5; 25 | % choose inverse temperature beta 26 | beta0 = 0.4; 27 | % choose number of nodes N 28 | N = 102; 29 | 30 | % create planted ranks 31 | s0 = [normrnd(4,sqrt(1),N/3,1);... 32 | normrnd(0,sqrt(0.5),N/3,1);... 33 | normrnd(-4,sqrt(2),N/3,1)]; 34 | 35 | % precompute spring energy matrix to keep degree fixed below 36 | spr = zeros(N,N); 37 | for i=1:N 38 | for j=1:N 39 | spr(i,j) = (s0(i) - s0(j) -1)^2; 40 | end 41 | end 42 | spr(1:N+1:end) = 0; 43 | % expected degrees at inverse temperature B(i) 44 | M = sum(sum(exp(-beta0/2*spr))); 45 | % adjust c to give expected degrees = k*n; 46 | c = deg*N/M; 47 | % create network using generative model 48 | A = generativeModel(c,beta0,s0); 49 | 50 | % find number of components and analyze only the largest component 51 | % THIS CODE CALLS networkComponents.m by Dan Larremore 52 | % Included in the SpringRank github repo 53 | % or mathworks.com/matlabcentral/fileexchange/42040-find-network-components 54 | [nComponents,sizes,members] = networkComponents(A); 55 | if nComponents > 1 56 | % extract largest component 57 | A = A(members{1},members{1}); 58 | s0 = s0(members{1}); 59 | end 60 | 61 | %remove self loops 62 | A(1:N+1:end) = 0; 63 | 64 | fprintf('Network created.\n'); 65 | %% 2. Analyze the network 66 | fprintf('Running SpringRank...\t\t') 67 | tic 68 | s_spr = springRank(A); 69 | fprintf('%f seconds.\n',toc) 70 | 71 | fprintf('Running BTL...\t\t\t') 72 | tic 73 | s_btl = btl(A,1e-3); 74 | fprintf('%f seconds.\n',toc) 75 | 76 | fprintf('Running Colley Matrix...\t') 77 | tic 78 | s_col = colleyMatrix(A); 79 | fprintf('%f seconds.\n',toc) 80 | 81 | fprintf('Running David Score...\t\t') 82 | tic 83 | s_dav = davidScore(A); 84 | fprintf('%f seconds.\n',toc) 85 | 86 | fprintf('Running Katz centrality...\t') 87 | tic 88 | s_eig = eigenvectorCentrality(A,1e-6); 89 | fprintf('%f seconds.\n',toc) 90 | 91 | fprintf('Running MVR...\t\t\t') 92 | tic 93 | s_mvr = mvr(A,5); 94 | fprintf('%f seconds.\n',toc) 95 | 96 | fprintf('Running PageRank...\t\t') 97 | tic 98 | s_pag = pageRank(A,0.8,1e-12); 99 | fprintf('%f seconds.\n',toc) 100 | 101 | %% 3. Compute beta values (local and global) 102 | fprintf('Computing beta values.') 103 | beta_local = betaLocal(A,s_spr); 104 | beta_global = betaGlobal(A,s_spr); 105 | 106 | %% 4. Computer accuracy via cross-validation 107 | [sig_a,sig_L] = crossValidation(A,5,1); 108 | 109 | %% 5. Test whether the network's hierarchy is statistically significant 110 | [p,H0,H] = pvalueNullModel(A,100); -------------------------------------------------------------------------------- /matlab/ranks2svg.m: -------------------------------------------------------------------------------- 1 | % Code by Daniel Larremore 2 | % Santa Fe Institute 3 | % larremore@santafe.edu 4 | % http://danlarremore.com 5 | % v3 6 | function [energy] = ranks2svg(A,s,filename) 7 | [r,c,v] = find(A); 8 | energy = (s(r)-s(c)-1).^2; 9 | energy = lin(energy/max(energy),0.05,0.3); 10 | 11 | % wid = 800; % Must be at least 400 12 | hei = 800; % Must be at least 400 13 | aspectRatio = 0.4; 14 | wid = hei*aspectRatio; 15 | s = max(s)-s; %flip orientation so top ranked shows up highest 16 | y = (s-min(s))/(max(s)-min(s))*(7/8)*hei+hei/16; 17 | 18 | % colors 19 | c_fill = [10,10,10]; %color - circle fill 20 | % c_fill = 'none'; 21 | c_stroke = [10,10,10]; %color - circle stroke 22 | c_down = [41,143,170]; %color - edges down 23 | c_up = [170,48,41]; %color - edges up 24 | s_min = 1; %edge stroke min 25 | s_max = 4; %edge stroke max 26 | r_min = 1; %circle min radius 27 | r_max = 3; %circle max radius 28 | swirl = 0.75; 29 | 30 | % bind sizes of circles to k. 31 | Q = A; 32 | Q(1:size(Q,1)+1:end)=0; % kill diagonal 33 | k = sum(Q,2); % bind to out degree. 34 | k = lin(k,r_min,r_max); 35 | 36 | % transform edge weights in v to range [s_min,s_max] 37 | w = lin(v,s_min,s_max); 38 | 39 | fid = fopen(filename,'w'); 40 | fprintf(fid,'',wid,hei); 41 | fprintf(fid,'\n'); 42 | 43 | % Paths 44 | for i=1:length(v) 45 | fr = y(r(i)); 46 | to = y(c(i)); 47 | anchor_y = swirl*fr+(1-swirl)*to; 48 | anchor_x = (wid/2)-sign(fr-to)*abs(fr-to)*aspectRatio; 49 | if sign(fr-to) < 0 50 | rgb = getColor_identity(c_down,energy); 51 | opacity = 0.15; 52 | fprintf(fid,'',... 53 | wid/2,fr,... 54 | anchor_x,anchor_y,wid/2,to,... 55 | w(i),... 56 | rgb(1),... 57 | rgb(2),... 58 | rgb(3),... 59 | opacity); 60 | else 61 | rgb = getColor_identity(c_up,energy); 62 | opacity = 0.15; 63 | fprintf(fid,'',... 64 | wid/2,fr,... 65 | anchor_x,anchor_y,wid/2,to,... 66 | w(i),... 67 | rgb(1),... 68 | rgb(2),... 69 | rgb(3),... 70 | opacity); 71 | end 72 | fprintf(fid,'\n'); 73 | end 74 | 75 | % Circles 76 | n = size(A,1); 77 | for i=1:n 78 | if strcmp(c_fill,'none')==1 79 | fprintf(fid,'',... 80 | k(i),c_stroke(1),c_stroke(2),c_stroke(3),wid/2,y(i) ); 81 | else 82 | fprintf(fid,'',... 83 | k(i),c_fill(1),c_fill(2),c_fill(3),c_stroke(1),c_stroke(2),c_stroke(3),wid/2,y(i) ); 84 | end 85 | fprintf(fid,'\n'); 86 | end 87 | 88 | fprintf(fid,''); 89 | 90 | fclose(fid); 91 | 92 | end 93 | 94 | function [y] = lin(x,m,M) 95 | y = (x - min(x))/(max(x)-min(x))*(M-m)+m; 96 | end 97 | function [rgb] = getColor_linearToWhite(rgb_base,scalar) 98 | M = max(rgb_base); 99 | rgb = round(rgb_base+(M-rgb_base)*scalar); 100 | end 101 | function [rgb] = getColor_identity(rgb_base,scalar) 102 | rgb = rgb_base; 103 | end 104 | -------------------------------------------------------------------------------- /python/SpringRank/_core.py: -------------------------------------------------------------------------------- 1 | """ 2 | New version developed by Nicolò Ruggeri, Max Planck Institute for Intelligent Systems, Tuebingen, Germany, March-2020 3 | It forces to use sparse matrices when possible, results in much more efficent implementation, especially for large matrices 4 | """ 5 | 6 | import warnings 7 | 8 | import numpy as np 9 | import scipy.sparse 10 | import scipy.sparse.linalg 11 | import sparse 12 | 13 | 14 | def build_from_dense(A, alpha, l0, l1): 15 | """ 16 | Given as input a 2d numpy array, build the matrices A and B to feed to the linear system solver for SpringRank. 17 | """ 18 | n = A.shape[0] 19 | k_in = np.sum(A, 0) 20 | k_out = np.sum(A, 1) 21 | 22 | D1 = k_in + k_out # to be seen as diagonal matrix, stored as 1d array 23 | D2 = l1 * (k_out - k_in) # to be seen as diagonal matrix, stored as 1d array 24 | 25 | if alpha != 0.: 26 | B = np.ones(n) * (alpha * l0) + D2 27 | A = - (A + A.T) 28 | A[np.arange(n), np.arange(n)] = alpha + D1 + np.diagonal(A) 29 | else: 30 | last_row_plus_col = (A[n - 1, :] + A[:, n - 1]).reshape((1, n)) 31 | A = A + A.T 32 | A += last_row_plus_col 33 | 34 | A[np.arange(n), np.arange(n)] = A.diagonal() + D1 35 | D3 = np.ones(n) * (l1 * (k_out[n - 1] - k_in[n - 1])) # to be seen as diagonal matrix, stored as 1d array 36 | B = D2 + D3 37 | 38 | return scipy.sparse.csr_matrix(A), B 39 | 40 | 41 | def build_from_sparse(A, alpha, l0, l1): 42 | """ 43 | Given as input a sparse 2d scipy array, build the matrices A and B to feed to the linear system solver for 44 | SpringRank. 45 | """ 46 | n = A.shape[0] 47 | k_in = np.sum(A, 0).A1 # convert matrix of shape (1, n) into 1-dimensional array 48 | k_out = np.sum(A, 1).A1 # same with (n, 1) matrix 49 | 50 | D1 = k_in + k_out # to be seen as diagonal matrix, stored as 1d array 51 | D2 = l1 * (k_out - k_in) # to be seen as diagonal matrix, stored as 1d array 52 | 53 | if alpha != 0.: 54 | B = np.ones(n) * (alpha * l0) + D2 55 | A = - (A + A.T) 56 | # convert to lil matrix for more efficient computations 57 | A = A.tolil(copy=False) 58 | A.setdiag(alpha + D1 + A.diagonal()) 59 | else: 60 | last_row_plus_col = sparse.COO.from_scipy_sparse(A[n - 1, :] + A[:, n - 1].T) # create sparse 1d COO array 61 | A = A + A.T 62 | A += last_row_plus_col # broadcast on rows 63 | A = -A.tocsr() # reconvert to csr scipy matrix 64 | 65 | # Notice that a scipy.sparse.SparseEfficiencyWarning will be raised by calling A.setdiag(). 66 | # However converting to lil matrix with 67 | # A.tolil(copy=False) 68 | # is not computationally convenient. Just suppress the warning during the call of A.setdiag(...) 69 | with warnings.catch_warnings(): 70 | warnings.simplefilter("ignore", scipy.sparse.SparseEfficiencyWarning) 71 | A.setdiag(A.diagonal() + D1) 72 | 73 | D3 = np.ones(n) * (l1 * (k_out[n-1] - k_in[n-1])) # to be seen as diagonal matrix, stored as 1d array 74 | B = D2 + D3 75 | 76 | return A, B 77 | 78 | 79 | def solve_linear_system(A, B, solver, verbose): 80 | if solver not in ['spsolve', 'bicgstab']: 81 | warnings.warn('Unknown parameter {solver} for argument solver. Setting solver = "bicgstab"'.format(solver=solver)) 82 | solver = 'bicgstab' 83 | 84 | if verbose: 85 | print('Using scipy.sparse.linalg.{solver}(A,B)'.format(solver=solver)) 86 | 87 | if solver == 'spsolve': 88 | sol = scipy.sparse.linalg.spsolve(A, B) 89 | elif solver == 'bicgstab': 90 | sol = scipy.sparse.linalg.bicgstab(A, B)[0] 91 | 92 | return sol.reshape((-1,)) 93 | -------------------------------------------------------------------------------- /matlab/mvr.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % [order,violations,A] = mvr(A) 11 | % INPUTS: 12 | % A is a NxN matrix representing a directed network 13 | % A can be weighted (integer or non-integer) 14 | % A(i,j) = # of dominance interactions by i toward j. 15 | % A(i,j) = # of times that j endorsed i. 16 | % n_samples is an integer number of independent replicates of the MVR MCMC 17 | % search procedure. 18 | % OUTPUTS: 19 | % best_ranks is a vector of ranks. ONE IS BEST. N IS WORST 20 | % best_violations is the number of violations 21 | % best_A is the reordered matrix whose lower triangle contains min. viols. 22 | 23 | function [best_ranks,best_violations,best_A] = mvr(A,n_samples) 24 | 25 | best_violations = size(A,1)^2; 26 | 27 | for n = 1:n_samples 28 | [ranks,violations,A] = mvr_single(A); 29 | if violations < best_violations 30 | best_violations = violations; 31 | best_ranks = ranks; 32 | best_A = A; 33 | end 34 | end 35 | 36 | end 37 | 38 | function [ranks,violations,A] = mvr_single(A) 39 | violations = compute_violations(A); 40 | 41 | N = size(A,1); 42 | %order = shuffle(1:N); 43 | order =1:N; 44 | A(:,:) = A(order,:); 45 | A(:,:) = A(:,order); 46 | 47 | step = 1; 48 | fails = 0; 49 | % fprintf('Initial\t\t\t\tviolations\t%i\n',violations); 50 | hist_viols(step) = violations; 51 | hist_viols_backup(step) = violations; 52 | hist_fails(step) = fails; 53 | 54 | % RANDOM STEPS - Randomly swap till N^2 failures in a row. 55 | while 1 56 | i = randi(N); % choose random node 57 | j = randi(N); % choose second random node. 58 | while j==i % make sure different 59 | i = randi(N); 60 | j = randi(N); 61 | end 62 | dx = compute_violations_change(A,i,j); 63 | if dx < 0 64 | order([i,j]) = order([j,i]); 65 | A([i,j],:) = A([j,i],:); 66 | A(:,[i,j]) = A(:,[j,i]); 67 | step = step+1; 68 | hist_swaps(step,:) = [i,j]; 69 | hist_fails(step) = fails; 70 | hist_viols(step) = hist_viols(step-1)+dx; 71 | violations = compute_violations(A); 72 | hist_viols_backup(step) = violations; 73 | % fprintf('swap %i ~ %i \t --> %i\tviolations\t%i\t%i\n',i,j,dx,violations,fails) 74 | fails = 0; 75 | else 76 | fails = fails+1; 77 | end 78 | if fails == N^2 79 | A(1,:); 80 | % fprintf('----- Too much fails -----\n'); 81 | break 82 | end 83 | end 84 | 85 | % DETERMINISTIC STEPS - Find any local steps deterministically by search. 86 | while 1 87 | dxbest = 0; 88 | for i=1:N-1 89 | for j=i+1:N 90 | dx = compute_violations_change(A,i,j); 91 | if dx < dxbest 92 | bestSwap = [i,j]; 93 | dxbest = dx; 94 | end 95 | end 96 | end 97 | if dxbest==0 98 | % fprintf('---- no improvement, exiting ----\n'); 99 | [~,ranks] = sort(order); 100 | return; 101 | end 102 | i = bestSwap(1); 103 | j = bestSwap(2); 104 | order([i,j]) = order([j,i]); 105 | % before = compute_violations(A); 106 | A([i,j],:) = A([j,i],:); 107 | A(:,[i,j]) = A(:,[j,i]); 108 | % after = compute_violations(A); 109 | step = step+1; 110 | hist_swaps(step,:) = [i,j]; 111 | hist_viols(step) = hist_viols(step-1)+dxbest; 112 | violations = compute_violations(A); 113 | hist_viols_backup(step) = violations; 114 | % fprintf('swap %i ~ %i \t --> %i\tviolations\t%i\n',i,j,dxbest,violations) 115 | end 116 | 117 | end 118 | 119 | function dx = compute_violations_change(A,ii,jj) 120 | % Let's arbitrarily choose i to fall (larger rank number) and j to rise 121 | % (smaller rank number). 122 | i = min(ii,jj); 123 | j = max(ii,jj); 124 | dx= full(-sum(A(j,i:j-1)) ... 125 | +sum(A(i,i+1:j)) ... 126 | -sum(A(i+1:j-1,i)) ... 127 | +sum(A(i+1:j-1,j))); 128 | end 129 | 130 | function x = compute_violations(B) 131 | x = full(sum(sum(tril(B,-1)))); 132 | end 133 | -------------------------------------------------------------------------------- /r/springrank.R: -------------------------------------------------------------------------------- 1 | library(igraph) 2 | library(Matrix) 3 | library(Rlinsolve) 4 | 5 | spring_rank <- function(A, alpha = 0, l0 = 1.0, l1 = 1.0, 6 | shift = TRUE, solver = Rlinsolve::lsolve.bicgstab) { 7 | #' Core function for calculating SpringRank. 8 | #' Default parameters follow stanadard model. 9 | #' 10 | #' @param A The adjacency matrix of the graph. This should be a sparse dgCMatrix; 11 | #' if it isn't it will be coerced to a dgCMatrix. 12 | #' @param alpha Controls the impact regularization term. 13 | #' @param l0 Regularization spring's rest length. 14 | #' @param l1 Interaction springs' rest length. 15 | #' @param shift (Optional, default TRUE) normalize such that the lowest-ranked 16 | #' node has a SpringRank value of zero. 17 | #' @param solver (Optional, default Rlinsolve::bicgstab) your preferred 18 | #' solver for Ax=B. Should be able to handle dgCMatrix sparse matrices. 19 | #' if the solve is not from Rlinsolve may throw a spurious warning for 20 | #' unused parameters. 21 | #' 22 | #' @return A vector of SpringRank scores for each node. Sort or order the 23 | #' vector for ordinal rankings of each node. 24 | 25 | if (class(A) == "matrix") { 26 | # coerce dense matrix to sparse matrice so the user doesn't have to. 27 | A <- as(Matrix(A, sparse = TRUE, doDiag = FALSE), "dgCMatrix") 28 | } else { 29 | # confirm it's the right kind of sparse matrix. might throw an error 30 | # if it's one of the less common sparse matrix types. 31 | A <- as(A, "dgCMatrix") 32 | } 33 | 34 | N <- dim(A)[1] 35 | k_in <- colSums(A) 36 | k_out <- rowSums(A) 37 | One <- as(Matrix(rep(1, N)), "dgCMatrix") 38 | C <- A + t(A) 39 | 40 | D1 <- as(Matrix(0, ncol = N, nrow = N), "dgCMatrix") 41 | D2 <- as(Matrix(0, ncol = N, nrow = N), "dgCMatrix") 42 | 43 | for (i in 1:N) { 44 | D1[i, i] <- k_out[i] + k_in[i] 45 | D2[i, i] <- l1 * (k_out[i] - k_in[i]) 46 | } 47 | 48 | if (alpha != 0) { 49 | print("assuming invertible matrix") 50 | 51 | B = One * l0 + D2 %*% One 52 | A_ = alpha * diag(nrow = N, ncol = N) + D1 - C 53 | } else { 54 | print("fixing a rank degree of freedom") 55 | 56 | C <- C + 57 | matrix(rep(A[N,] , times = N), 58 | ncol = N, 59 | nrow = N, 60 | byrow = T) + 61 | matrix(rep(A[, N], times = N), 62 | ncol = N, 63 | nrow = N, 64 | byrow = T) 65 | 66 | D3 <- as(Matrix(0, ncol = N, nrow = N), "dgCMatrix") 67 | for (i in 1:N) { 68 | D3[i, i] <- l1 * (k_out[N] - k_in[N]) 69 | } 70 | 71 | B <- D2 %*% One + D3 %*% One 72 | A_ <- D1 - C 73 | } 74 | 75 | rank <- solver(A_, B, verbose = F) 76 | if (class(rank) == "list") { 77 | rank <- rank$x # accomodates both Rlinsolve and Matrix solves 78 | } 79 | 80 | if (shift) { 81 | rank <- rank - min(rank) 82 | } 83 | 84 | # coerce matrix to vector, so we can use names() 85 | rank <- rank[,1] 86 | names(rank) <- colnames(A) 87 | return(rank) 88 | } 89 | 90 | 91 | spring_rank_network <- function(N, beta, alpha, K, l0 = 0.5, l1 = 1.0) { 92 | #' Generative model for SpringRank. Builds a weighted graph with self-loops. 93 | #' The model first generates node scores using a normal distribution, then 94 | #' generates edges (and edge weights) from a Poisson distribution with mean 95 | #' equal to the energy of the system. 96 | #' 97 | #' @param N The number of nodes. 98 | #' @param beta Inverse temperature, a noise parameter. 99 | #' @param alpha Variance of the Normal prior. 100 | #' @param K The average degree of the network. 101 | #' @param l0 Prior spring's rest length. 102 | #' @param l1 Interaction spring's rest length. 103 | #' 104 | #' @return A directed graph (potentially weighted, potentially containing 105 | #' self-loops) 106 | 107 | scores <- rnorm(N, l0, 1/sqrt(alpha*beta)) 108 | Z <- 0 109 | for (i in 1:N) { 110 | for (j in 1:N) { 111 | Z <- Z + exp(-0.5 * beta * (scores[i] - scores[j] - l1)^2) 112 | } 113 | } 114 | C <- (K*N)/Z 115 | 116 | # for loops are slow in R so make a matrix of element-wise subtractions, 117 | # each element i, j being scores[i]-scores[j] 118 | # basically, trading off increased memory usage (dense matrix) for speed. 119 | scores_mat <- matrix(1, length(scores), 1) %*% t(scores) 120 | scores_mat <- scores_mat - scores - l1 121 | H <- .5 * scores_mat^2 122 | lambda <- C * exp(-1*beta*H) 123 | A <- rpois(length(scores)^2, lambda) %>% matrix(nrow = dim(lambda)[1]) 124 | 125 | return(graph_from_adjacency_matrix(A, mode = "directed", weighted = "weight")) 126 | } 127 | -------------------------------------------------------------------------------- /matlab/crossValidation.m: -------------------------------------------------------------------------------- 1 | % SpringRank 2 | % CODE -> https://github.com/cdebacco/SpringRank 3 | % PAPER -> http://danlarremore.com/pdf/SpringRank_2017_PrePrint.pdf 4 | % Code by Daniel Larremore 5 | % University of Colorado at Boulder 6 | % BioFrontiers Institute & Dept of Computer Science 7 | % daniel.larremore@colorado.edu 8 | % http://danlarremore.com 9 | % 10 | % [sig_a,sig_L] = crossValidation(A,folds,reps) 11 | % 12 | % INPUTS: 13 | % A is a NxN matrix representing a directed network 14 | % A can be weighted (integer or non-integer) 15 | % A(i,j) = # of dominance interactions by i toward j. 16 | % A(i,j) = # of times that j endorsed i. 17 | % folds is the number of folks k in a k-fold cross validation 18 | % reps is the number of random repetitions desired over the k-folds. For 19 | % example, folds=5 and reps=7 would divide the data into 5 folds, 20 | % performing tests, and repeating those tests 7 times independently. 21 | % 22 | % OUTPUTS: 23 | % sig_a is the local accuracy (\sigma_a in the paper) 24 | % sig_L is the global accuracy (\sigma_L in the paper) 25 | 26 | function [sig_a,sig_L] = crossValidation(A,folds,reps) 27 | 28 | % convert to sparse for improved performance 29 | if ~issparse(A) 30 | A = sparse(A); 31 | end 32 | 33 | % NOTE: We perform cross validation over *interacting pairs* so we're not 34 | % dividing the edges of the network into k groups, but dividing the 35 | % interactions into k groups. Here is an example that clarifies the 36 | % difference. If there is a pair of nodes (i,j) with A(i,j) = 1 and 37 | % A(j,i)=3, this would count as ONE interacting pair, not four. There are 38 | % other interpretations of how to split edges into a training and a test 39 | % set, and these may be application dependent. All this writing here is 40 | % just to clarify *exactly* how this code works. 41 | 42 | % Find interacting pairs 43 | [r,c,v] = find(triu(A+transpose(A))); 44 | % Number of interacting pairs 45 | M = length(v); 46 | % Size of each fold 47 | foldSize = floor(M/folds); 48 | 49 | % preallocate; note that DIM2 should be increased if you are testing more 50 | % than one method. 51 | sig_a = zeros(reps*folds,1); 52 | sig_L = zeros(reps*folds,1); 53 | 54 | % iterate over reps 55 | for rep = 1:reps 56 | 57 | % shuffle interactions 58 | idx = shuffle(1:M); 59 | % build K-1 folds of equal size 60 | for f = 1:folds-1 61 | fold{f} = idx( (f-1)*foldSize+1 : f*foldSize); 62 | end 63 | % then put the remainder in the final Kth fold 64 | fold{folds} = idx( (folds-1)*foldSize+1 : end); 65 | 66 | % iterate over folds 67 | for f = 1:folds 68 | % Print 69 | fprintf('Cross validation progress: Rep %i/%i, Fold %i/%i.\n',... 70 | rep,reps,f,folds); 71 | % bookkeeping 72 | foldrep = f+(rep-1)*folds; 73 | % build the testset of indices 74 | test_i = r(fold{f}); 75 | test_j = c(fold{f}); 76 | test = sub2ind(size(A),test_i,test_j); 77 | testpose = sub2ind(size(A),test_j,test_i); 78 | % build the training set by setting testset interactions to zero. 79 | TRAIN = A; 80 | TRAIN(test) = 0; 81 | TRAIN(testpose) = 0; 82 | % Build the TEST set. 83 | TEST = A-TRAIN; 84 | numTestEdges = sum(sum(TEST)); 85 | 86 | % train springRank on the TRAINset 87 | s0 = springRank(TRAIN); 88 | bloc0 = betaLocal(TRAIN,s0); 89 | bglob0 = betaGlobal(TRAIN,s0); 90 | % springRank accuracies on TEST set 91 | sig_a(foldrep,1) = localAccuracy(TEST,s0,bloc0); 92 | sig_L(foldrep,1) = -globalAccuracy(TEST,s0,bglob0)/numTestEdges; 93 | 94 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 95 | % NOTE THAT IF YOU WANT TO COMPARE OTHER METHODS TO SPRINGRANK, 96 | % THIS IS THE PLACE THAT THEY SHOULD BE INCLUDED. 97 | % Commented out, below, you can see the call to springRank with the 98 | % regularization, as well as the call to BTL and its separate 99 | % accuracy. 100 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 101 | 102 | % % train regularized springRank on the TRAINset 103 | % s2 = springRankFull(TRAIN,2); 104 | % bloc2 = betaLocal2(TRAIN,s2); 105 | % bglob2 = betaGlobal(TRAIN,s2); 106 | % % regularized springRank accuracies on TEST set 107 | % sig_a(foldrep,2) = localAccuracy(TEST,s2,bloc2); 108 | % sig_L(foldrep,2) = -globalAccuracy(TEST,s2,bglob2)/numTestEdges; 109 | 110 | % % train BTL on the TRAINset 111 | % bt = btl(TRAIN,1e-3); 112 | % % BTL accuracies on TEST set 113 | % sig_a(foldrep,3) = localAccuracy_BTL(TEST,bt); 114 | % sig_L(foldrep,3) = -globalAccuracy_BTL(TEST,bt)/numTestEdges; 115 | end 116 | end -------------------------------------------------------------------------------- /data/US_CS_SpringRank_a0.0_l0_1.0_l1_1.0.dat: -------------------------------------------------------------------------------- 1 | 2 2.4226395524688336 2 | 1 2.4159544580786303 3 | 3 2.3898244144620295 4 | 4 2.290468716060098 5 | 5 2.220954376089337 6 | 8 2.0923129533538525 7 | 6 2.0427917574204755 8 | 9 1.9992673532610523 9 | 7 1.955643923000736 10 | 12 1.9460508043398215 11 | 10 1.9425079078489735 12 | 14 1.9180905048489676 13 | 11 1.9174836990878514 14 | 13 1.915442916586008 15 | 15 1.8816379837492883 16 | 21 1.8561689136110406 17 | 19 1.838389447305988 18 | 18 1.8333021029651098 19 | 16 1.7943542203773655 20 | 20 1.781586141906679 21 | 17 1.7556438265383987 22 | 25 1.7293378829344797 23 | 23 1.7167292349975285 24 | 27 1.7088957693089923 25 | 22 1.7027840538299506 26 | 31 1.683475048488325 27 | 28 1.681346937891377 28 | 24 1.6787202493105327 29 | 26 1.646210564700957 30 | 35 1.615016240852257 31 | 41 1.6136333113511194 32 | 29 1.6068159055302382 33 | 44 1.601401656259858 34 | 36 1.585352475498902 35 | 34 1.580799027974649 36 | 30 1.5677075758607903 37 | 32 1.5443642951313394 38 | 39 1.5374024109594886 39 | 38 1.5003041810387212 40 | 46 1.4973003647588141 41 | 40 1.491459278955829 42 | 37 1.487338074811235 43 | 48 1.4830187326940427 44 | 33 1.4757560096533708 45 | 45 1.4526303983198634 46 | 42 1.4115071702075466 47 | 50 1.3847385368786618 48 | 43 1.3481585567478305 49 | 56 1.316523404215118 50 | 49 1.3006313053887304 51 | 68 1.2888483344799317 52 | 59 1.2869768129635706 53 | 52 1.2867208556994867 54 | 55 1.2791840091489282 55 | 58 1.2742868417893225 56 | 51 1.2716120702771156 57 | 69 1.2341303684435283 58 | 47 1.2027363645930178 59 | 122 1.193348480182236 60 | 71 1.1899851450790286 61 | 57 1.157213781344293 62 | 63 1.1544886733960515 63 | 76 1.145620083554946 64 | 54 1.1413133778321052 65 | 83 1.1406832868028962 66 | 70 1.140589401162388 67 | 67 1.1328483267919196 68 | 65 1.1128282745513 69 | 61 1.0916937315169153 70 | 75 1.075030802185001 71 | 82 1.0565888509376513 72 | 74 1.0533858030017118 73 | 66 1.0487833970689937 74 | 60 1.0482251808763192 75 | 80 1.0456272710774899 76 | 111 1.0373362738552894 77 | 72 1.0352058993630184 78 | 101 1.0280865938771186 79 | 73 1.0248352985145215 80 | 97 1.0240314136783861 81 | 79 1.0225199518451877 82 | 78 1.0206657553110865 83 | 81 1.0160322782908842 84 | 85 1.0157803415009115 85 | 86 1.0110508177357167 86 | 53 1.00594551978043 87 | 84 1.0046083546873001 88 | 90 0.9998740661913751 89 | 62 0.9879085522210397 90 | 64 0.9804176832599036 91 | 77 0.9738708108442564 92 | 103 0.9706346245766725 93 | 93 0.9597823747003857 94 | 96 0.943452683604567 95 | 154 0.9424228613119161 96 | 92 0.9411040071718573 97 | 89 0.936651305636945 98 | 121 0.9232661604575396 99 | 87 0.9177125711917503 100 | 88 0.9135678015452544 101 | 102 0.9048818119275754 102 | 112 0.9038330291939597 103 | 175 0.8891609187718665 104 | 113 0.8651860475035721 105 | 95 0.8644141303825011 106 | 124 0.8590674272096227 107 | 105 0.8562308839188757 108 | 133 0.8462759344496075 109 | 100 0.8378351504323203 110 | 107 0.8291969834607334 111 | 91 0.8231539774707435 112 | 104 0.8193210717035699 113 | 98 0.8015071920172973 114 | 115 0.7917734540896127 115 | 114 0.7887604862760151 116 | 106 0.7843606952548952 117 | 147 0.7843176170202061 118 | 150 0.7774738561074689 119 | 142 0.7756246607790684 120 | 128 0.7754882778153482 121 | 120 0.7723122368297564 122 | 135 0.7701258297762773 123 | 145 0.7548539858939127 124 | 94 0.7531297795206247 125 | 117 0.7504035409293139 126 | 123 0.7503100392871709 127 | 116 0.7490342788956013 128 | 99 0.7474482239812225 129 | 110 0.7393823536953144 130 | 125 0.7361306493798307 131 | 166 0.7336203489778327 132 | 139 0.732783140639014 133 | 164 0.7275777115059809 134 | 118 0.7270737696299221 135 | 161 0.7239406413455253 136 | 109 0.714291591490622 137 | 151 0.7034431692622395 138 | 119 0.6966103873678284 139 | 148 0.6963737540307889 140 | 137 0.6857858789484859 141 | 155 0.6819357167839643 142 | 108 0.6731985883521726 143 | 130 0.6596928790636526 144 | 131 0.6464425415943673 145 | 126 0.6383373700108058 146 | 132 0.6358664330228669 147 | 138 0.6350699034081638 148 | 170 0.616658464353957 149 | 160 0.6124674573146425 150 | 152 0.6073256967999283 151 | 181 0.6031624882618123 152 | 127 0.600700954123478 153 | 141 0.5990609330465375 154 | 167 0.594071190874031 155 | 149 0.5655401288928243 156 | 146 0.5617635032488146 157 | 136 0.5614614151666686 158 | 140 0.5553472929912455 159 | 129 0.5535239159051117 160 | 153 0.5457247851116838 161 | 191 0.5443709614979411 162 | 156 0.5437856220897981 163 | 163 0.5323475994219757 164 | 157 0.5303356998270012 165 | 134 0.5040485057059758 166 | 188 0.4875178747982014 167 | 169 0.460736631855698 168 | 144 0.45594740450817506 169 | 173 0.4510310687988399 170 | 174 0.4488129407181638 171 | 165 0.44643516409738093 172 | 171 0.4405592312046671 173 | 168 0.4291962697505849 174 | 200 0.3887551505501796 175 | 162 0.38149983269521437 176 | 184 0.34777075237481925 177 | 179 0.31791024192024486 178 | 201 0.3174870859833512 179 | 194 0.31590967280414284 180 | 143 0.30750218540375074 181 | 187 0.30572806553625 182 | 159 0.28455294090477334 183 | 178 0.27872040144359245 184 | 190 0.27847984270514003 185 | 193 0.2593250043126931 186 | 204 0.24464381811097713 187 | 192 0.2365413270893737 188 | 186 0.22889306483381033 189 | 189 0.21713577865886724 190 | 158 0.2052971707091642 191 | 172 0.1974143507756838 192 | 199 0.1802543269613175 193 | 182 0.1702307604353277 194 | 203 0.1632922802386867 195 | 196 0.15878544214213686 196 | 177 0.14969625010043197 197 | 202 0.1424458607422081 198 | 176 0.12507361195587197 199 | 180 0.11631147240675954 200 | 185 0.10851458916847756 201 | 198 0.10669528573128595 202 | 183 0.08894897375884114 203 | 195 0.03910328256237117 204 | 197 0.003436086038484776 205 | 205 0.0 206 | -------------------------------------------------------------------------------- /data/US_CS_nodes.dat: -------------------------------------------------------------------------------- 1 | u pi USN2010 NRC95 Region institution 2 | 1 2.23 1 1 West Stanford University 3 | 2 2.31 1 3 West UC Berkeley 4 | 3 3.52 1 2 Northeast MIT 5 | 4 5.24 11 12 West California Institute of Technology 6 | 5 6.12 17 11 Northeast Harvard University 7 | 6 8.29 5 5 Northeast Cornell University 8 | 7 9.28 1 4 Northeast Carnegie Mellon University 9 | 8 9.32 8 6 Northeast Princeton University 10 | 9 9.98 20 14 Northeast Yale University 11 | 10 11.06 7 9 West University of Washington 12 | 11 13.43 5 8 Midwest University of Illinois, Urbana Champaign 13 | 12 14.12 11 10 Midwest University of Wisconsin, Madison 14 | 13 15.31 17 22 Northeast University of Pennsylvania 15 | 14 16.44 20 18 South Rice University 16 | 15 17.82 14 14 West UCLA 17 | 16 18.23 28 16 Northeast New York University 18 | 17 18.82 35 22 Midwest University of Chicago 19 | 18 19.44 8 7 South University of Texas, Austin 20 | 19 21.92 20 13 Northeast Brown University 21 | 20 22.53 17 21 Northeast Columbia University 22 | 21 24.79 . 92 Canada University of Toronto 23 | 22 25.64 47 27 Northeast University of Rochester 24 | 23 26.17 20 19 West University of Southern California 25 | 24 26.30 28 34 South Johns Hopkins University 26 | 25 26.95 20 17 Northeast University of Massachusetts, Amherst 27 | 26 27.93 14 21 West UC San Diego 28 | 27 28.65 14 15 South University of Maryland, College Park 29 | 28 29.37 13 20 Midwest University of Michigan 30 | 29 29.55 20 26 South University of North Carolina, Chapel Hill 31 | 30 30.50 27 25 South Duke University 32 | 31 36.01 44 28 Northeast State University of New York, Stony Brook 33 | 32 37.36 28 31 West UC Irvine 34 | 33 37.90 44 51 Northeast Dartmouth College 35 | 34 38.11 28 32 South University of Virginia 36 | 35 38.27 20 23 Midwest Purdue University 37 | 36 38.94 35 43 Midwest University of Minnesota, Minneapolis 38 | 37 39.69 10 29 South Georgia Tech 39 | 38 40.46 28 24 Northeast Rutgers University 40 | 39 40.57 47 30 West University of Arizona 41 | 40 41.22 28 50 Northeast Pennsylvania State University 42 | 41 42.17 28 36 Midwest Ohio State University 43 | 42 44.47 35 35 Midwest Northwestern University 44 | 43 46.40 39 48 Midwest Washington University, St. Louis 45 | 44 46.81 53 39 Northeast University of Pittsburgh 46 | 45 51.73 47 52 Northeast Boston University 47 | 46 51.75 . 92 Canada University of British Columbia 48 | 47 54.86 63 57 West University of Oregon 49 | 48 56.74 63 40 Northeast Syracuse University 50 | 49 56.76 35 44 West UC Santa Barbara 51 | 50 59.10 39 37 West University of Utah 52 | 51 61.39 39 52 West UC Davis 53 | 52 63.14 47 56 South Texas A&M 54 | 53 64.97 110 66 South University of Houston 55 | 54 66.24 58 49 Midwest Michigan State University 56 | 55 67.81 . 92 Canada University of Waterloo 57 | 56 68.17 39 37 West University of Colorado, Boulder 58 | 57 69.84 . 92 Canada McGill University 59 | 58 70.04 79 92 Northeast CUNY Graduate Center 60 | 59 70.82 63 60 Midwest Case Western Reserve University 61 | 60 71.14 53 92 West UC Riverside 62 | 61 74.40 79 75 Midwest University of Kansas, Lawrence 63 | 62 74.98 79 82 South Florida State University 64 | 63 75.20 79 92 South College of William and Mary 65 | 64 76.47 99 83 Northeast Lehigh University 66 | 65 78.54 . 92 Canada University of Montreal 67 | 66 78.97 . 92 Canada Simon Fraser University 68 | 67 79.33 58 47 Midwest University of Illinois, Chicago 69 | 68 79.33 99 92 Midwest University of Cincinnati 70 | 69 79.49 53 46 West UC Santa Cruz 71 | 70 79.68 47 45 Northeast Rensselaer Polytechnic Institute 72 | 71 80.22 110 78 South Southern Methodist University 73 | 72 81.51 91 72 South University of Central Florida 74 | 73 81.65 79 92 West University of New Mexico 75 | 74 81.90 47 53 South North Carolina State University 76 | 75 82.16 61 52 Northeast State University of New York, Buffalo 77 | 76 82.40 99 89 Northeast Stevens Institute of Technology 78 | 77 83.35 79 70 West Washington State University, Pullman 79 | 78 84.01 . 92 Canada McMaster University 80 | 79 84.55 72 92 Northeast Brandeis University 81 | 80 86.24 . 92 Canada University of Alberta 82 | 81 87.19 79 60 South George Washington University 83 | 82 87.42 72 92 Northeast Polytechnic Institute of NYU 84 | 83 87.49 63 55 Midwest University of Iowa 85 | 84 87.66 72 92 South University of Delaware 86 | 85 92.45 63 68 Midwest Iowa State University 87 | 86 92.50 58 64 South Vanderbilt University 88 | 87 94.94 39 42 South University of Florida 89 | 88 97.34 63 92 South George Mason University 90 | 89 98.80 63 92 Midwest University of Notre Dame 91 | 90 99.58 110 69 Midwest Wayne State University 92 | 91 100.27 . 92 Canada University of Calgary 93 | 92 100.62 127 63 South University of Louisiana, Lafayette 94 | 93 100.90 . 92 Canada University of Western Ontario 95 | 94 101.73 . 92 Canada Carleton University 96 | 95 101.86 . 92 Canada Queens University 97 | 96 102.07 110 92 Midwest University of Wisconsin, Milwaukee 98 | 97 102.17 99 68 South Louisiana State University 99 | 98 106.62 99 74 South University of Texas, Arlington 100 | 99 107.42 . 92 Canada University of Ottawa 101 | 100 108.32 72 80 Northeast University of Connecticut 102 | 101 108.71 79 38 West Oregon Health and Science University 103 | 102 108.72 53 54 West Arizona State University 104 | 103 109.31 99 71 Midwest Kansas State University 105 | 104 110.18 91 92 Northeast State University of New York, Albany 106 | 105 111.94 99 61 South University of South Florida 107 | 106 112.40 44 59 South Virginia Tech 108 | 107 113.61 . 92 West Portland State University 109 | 108 113.75 99 81 South University of South Carolina 110 | 109 113.87 121 92 Midwest Missouri University of Science and Technology 111 | 110 116.24 110 87 South University of Oklahoma 112 | 111 116.75 53 33 Midwest Indiana University 113 | 112 117.72 110 79 Midwest Illinois Institute of Technology 114 | 113 119.03 63 92 South University of Tennessee, Knoxville 115 | 114 120.24 . 92 South University of Miami 116 | 115 120.85 91 92 Northeast New Jersey Institute of Technology 117 | 116 123.11 91 58 South University of Kentucky 118 | 117 124.35 110 90 Northeast State University of New York, Binghamton 119 | 118 124.60 79 92 West Colorado State University 120 | 119 125.51 . 92 West University of Colorado, Denver 121 | 120 125.87 79 67 South University of Texas, Dallas 122 | 121 126.00 . 92 Northeast University of Rhode Island 123 | 122 126.16 . 92 Midwest Toyota Technological Institute at Chicago 124 | 123 126.69 91 92 South Auburn University 125 | 124 127.37 99 92 Northeast Drexel University 126 | 125 127.72 63 62 West Oregon State University 127 | 126 129.12 . 92 Canada University of Manitoba 128 | 127 130.03 . 92 Canada University of Regina 129 | 128 130.66 79 65 Midwest University of Nebraska, Lincoln 130 | 129 130.86 . 92 Canada University of Saskatchewan 131 | 130 131.20 127 92 South Florida Atlantic University 132 | 131 133.22 . 92 South University of Texas, San Antonio 133 | 132 133.81 . 92 West University of Denver 134 | 133 134.13 72 78 South University of Maryland, Baltimore County 135 | 134 134.48 127 92 Midwest Oakland University (Michigan) 136 | 135 135.62 127 92 Northeast University of Maine 137 | 136 137.31 . 92 Northeast Clarkson University 138 | 137 137.49 121 69 West New Mexico State University 139 | 138 137.66 127 92 South University of Memphis 140 | 139 137.87 . 92 Canada University of Victoria 141 | 140 138.05 . 92 Canada Concordia University, Montreal 142 | 141 138.72 127 92 Midwest University of Missouri, Kansas City 143 | 142 139.45 110 92 Midwest University of Missouri, Columbia 144 | 143 139.97 . 92 Canada Dalhousie University 145 | 144 139.99 . 92 South Virginia Commonwealth University 146 | 145 140.75 91 89 Northeast Worcester Polytechnic Institute 147 | 146 142.05 127 77 South Old Dominion University 148 | 147 142.20 . 92 South University of Texas, El Paso 149 | 148 142.23 72 73 West Naval Postgraduate School 150 | 149 143.20 . 92 Northeast University of Massachusetts, Boston 151 | 150 144.80 . 92 West University of Wyoming 152 | 151 145.24 . 92 Canada York University 153 | 152 145.66 91 92 South University of Georgia 154 | 153 146.44 121 85 South Mississippi State University 155 | 154 146.50 72 92 Northeast Tufts University 156 | 155 146.92 127 92 South Florida International University 157 | 156 148.23 . 92 Midwest Southern Illinois University, Carbondale 158 | 157 150.08 . 92 Midwest Ohio University 159 | 158 150.95 121 92 West Claremont Graduate University 160 | 159 151.04 . 92 South Catholic University of America 161 | 160 151.45 79 92 South Clemson University 162 | 161 151.46 127 84 Northeast University of Massachusetts, Lowell 163 | 162 151.76 127 92 South Oklahoma State University 164 | 163 152.67 99 92 South University of North Carolina, Charlotte 165 | 164 153.44 127 84 South University of Alabama, Birmingham 166 | 165 153.65 . 92 West Santa Clara University 167 | 166 154.88 127 92 West University of Hawaii, Manoa 168 | 167 156.16 110 62 Northeast Temple University 169 | 168 156.35 . 92 Midwest University of Toledo 170 | 169 156.45 127 92 Midwest Michigan Technological University 171 | 170 156.98 . 92 Northeast University of New Hampshire 172 | 171 157.66 . 92 West Montana State University 173 | 172 158.12 . 92 Canada University of New Brunswick 174 | 173 159.20 121 92 Midwest Wright State University 175 | 174 163.13 . 92 Midwest Western Michigan University 176 | 175 163.79 61 92 Northeast Northeastern University 177 | 176 163.89 127 76 South University of Alabama, Huntsville 178 | 177 164.22 127 92 West University of Idaho, Moscow 179 | 178 164.46 121 92 South Texas Tech University 180 | 179 164.68 127 92 South University of Tulsa 181 | 180 165.50 127 92 South University of Mississippi 182 | 181 166.53 91 92 West Brigham Young University 183 | 182 168.42 . 92 Northeast Long Island University 184 | 183 169.02 127 92 South University of Southern Mississippi 185 | 184 169.16 . 92 West University of Colorado, Colorado Springs 186 | 185 169.65 110 92 West Colorado School of Mines 187 | 186 169.71 99 92 South Georgia State University 188 | 187 171.62 . 92 West University of Nevada, Las Vegas 189 | 188 171.68 127 86 Midwest Kent State University 190 | 189 173.29 . 92 Canada Memorial University of Newfoundland 191 | 190 173.75 . 92 West Utah State University 192 | 191 174.47 127 92 Midwest DePaul University 193 | 192 176.03 127 92 Midwest North Dakota State University 194 | 193 178.55 127 92 South Nova Southeastern University 195 | 194 178.70 127 92 South University of Arkansas, Fayetteville 196 | 195 178.73 . 92 Northeast University of Bridgeport 197 | 196 179.12 127 92 South University of Louisville 198 | 197 179.97 . 92 Midwest University of Nebraska, Omaha 199 | 198 180.75 127 92 South Florida Institute of Technology 200 | 199 181.14 . 92 South University of Arkansas, Little Rock 201 | 200 181.68 . 92 Northeast Rochester Institute of Technology 202 | 201 182.16 . 92 Northeast Pace University 203 | 202 182.28 127 91 West New Mexico Institute of Mining and Technology 204 | 203 182.47 . 92 West University of Nevada, Reno 205 | 204 186.81 127 92 South University of Alabama, Tuscaloosa 206 | 205 186.84 127 82 South University of North Texas, Denton 207 | 206 206.00 . . Earth All others 208 | -------------------------------------------------------------------------------- /data/US_CS_adjacency.dat: -------------------------------------------------------------------------------- 1 | 133 133 1 2 | 133 167 1 3 | 133 191 1 4 | 164 164 1 5 | 164 199 1 6 | 164 156 1 7 | 131 90 1 8 | 131 131 1 9 | 130 153 1 10 | 130 130 4 11 | 130 167 1 12 | 130 198 1 13 | 137 147 1 14 | 137 197 1 15 | 137 202 1 16 | 137 137 2 17 | 137 192 1 18 | 136 195 1 19 | 136 152 1 20 | 135 135 1 21 | 139 198 1 22 | 139 57 1 23 | 139 172 3 24 | 139 91 1 25 | 139 139 1 26 | 139 189 3 27 | 139 80 1 28 | 138 89 1 29 | 93 143 4 30 | 93 55 1 31 | 93 168 1 32 | 93 178 1 33 | 93 128 1 34 | 93 129 2 35 | 93 93 3 36 | 93 172 1 37 | 24 39 1 38 | 24 27 2 39 | 24 14 1 40 | 24 169 1 41 | 24 49 1 42 | 24 18 1 43 | 24 117 1 44 | 24 37 1 45 | 24 43 1 46 | 24 35 2 47 | 24 115 1 48 | 24 75 1 49 | 24 148 1 50 | 24 7 1 51 | 24 154 1 52 | 24 171 1 53 | 24 96 1 54 | 25 151 1 55 | 25 153 1 56 | 25 154 1 57 | 25 156 1 58 | 25 194 1 59 | 25 50 2 60 | 25 133 2 61 | 25 61 1 62 | 25 117 1 63 | 25 202 1 64 | 25 110 1 65 | 25 161 2 66 | 25 67 1 67 | 25 68 1 68 | 25 80 1 69 | 25 81 2 70 | 25 87 1 71 | 25 84 3 72 | 25 25 2 73 | 25 20 3 74 | 25 22 1 75 | 25 160 1 76 | 25 44 1 77 | 25 28 3 78 | 25 40 3 79 | 25 118 4 80 | 25 179 1 81 | 25 5 1 82 | 25 7 2 83 | 25 6 2 84 | 25 166 2 85 | 25 145 2 86 | 25 184 1 87 | 25 98 3 88 | 25 74 1 89 | 25 73 2 90 | 25 72 1 91 | 25 128 1 92 | 25 100 1 93 | 25 95 1 94 | 25 107 3 95 | 25 79 1 96 | 25 38 1 97 | 25 59 1 98 | 25 54 1 99 | 25 57 1 100 | 25 56 1 101 | 25 37 1 102 | 25 36 2 103 | 25 35 2 104 | 25 34 1 105 | 25 175 1 106 | 25 169 1 107 | 26 152 1 108 | 26 155 1 109 | 26 60 2 110 | 26 131 1 111 | 26 64 1 112 | 26 110 1 113 | 26 67 1 114 | 26 69 2 115 | 26 139 1 116 | 26 81 1 117 | 26 118 1 118 | 26 26 2 119 | 26 49 3 120 | 26 28 1 121 | 26 2 1 122 | 26 148 2 123 | 26 108 1 124 | 26 72 1 125 | 26 70 1 126 | 26 165 2 127 | 26 11 1 128 | 26 10 1 129 | 26 12 1 130 | 26 15 1 131 | 26 19 1 132 | 26 32 1 133 | 26 56 1 134 | 26 37 1 135 | 26 35 2 136 | 26 111 1 137 | 27 138 1 138 | 27 151 3 139 | 27 198 1 140 | 27 197 1 141 | 27 60 1 142 | 27 88 4 143 | 27 116 1 144 | 27 64 2 145 | 27 66 1 146 | 27 112 1 147 | 27 83 1 148 | 27 80 1 149 | 27 81 1 150 | 27 175 1 151 | 27 85 1 152 | 27 25 1 153 | 27 27 2 154 | 27 21 3 155 | 27 48 1 156 | 27 49 2 157 | 27 46 1 158 | 27 44 2 159 | 27 45 1 160 | 27 28 1 161 | 27 43 1 162 | 27 41 1 163 | 27 167 2 164 | 27 146 1 165 | 27 200 1 166 | 27 145 2 167 | 27 204 1 168 | 27 76 1 169 | 27 38 2 170 | 27 124 1 171 | 27 71 1 172 | 27 70 2 173 | 27 102 4 174 | 27 93 1 175 | 27 101 1 176 | 27 106 2 177 | 27 107 1 178 | 27 133 2 179 | 27 39 1 180 | 27 12 2 181 | 27 22 1 182 | 27 121 1 183 | 27 98 2 184 | 27 19 1 185 | 27 18 1 186 | 27 30 1 187 | 27 37 1 188 | 27 36 1 189 | 27 35 3 190 | 27 34 1 191 | 27 55 1 192 | 27 32 1 193 | 20 198 1 194 | 20 51 2 195 | 20 193 1 196 | 20 60 1 197 | 20 102 1 198 | 20 88 2 199 | 20 130 1 200 | 20 175 1 201 | 20 25 1 202 | 20 27 1 203 | 20 20 1 204 | 20 49 1 205 | 20 28 1 206 | 20 29 1 207 | 20 40 1 208 | 20 41 2 209 | 20 3 1 210 | 20 7 3 211 | 20 201 1 212 | 20 145 1 213 | 20 184 1 214 | 20 120 2 215 | 20 98 1 216 | 20 124 1 217 | 20 70 1 218 | 20 91 1 219 | 20 96 1 220 | 20 13 2 221 | 20 38 1 222 | 20 15 1 223 | 20 17 1 224 | 20 16 1 225 | 20 18 1 226 | 20 31 1 227 | 20 37 2 228 | 20 36 1 229 | 20 52 1 230 | 20 32 2 231 | 21 151 15 232 | 21 153 1 233 | 21 189 3 234 | 21 60 1 235 | 21 102 1 236 | 21 111 1 237 | 21 65 3 238 | 21 66 6 239 | 21 67 1 240 | 21 69 1 241 | 21 80 8 242 | 21 139 3 243 | 21 170 1 244 | 21 27 1 245 | 21 21 17 246 | 21 22 2 247 | 21 46 5 248 | 21 47 1 249 | 21 175 1 250 | 21 45 1 251 | 21 3 1 252 | 21 7 3 253 | 21 6 1 254 | 21 8 1 255 | 21 96 1 256 | 21 143 1 257 | 21 141 1 258 | 21 120 1 259 | 21 77 1 260 | 21 75 1 261 | 21 124 1 262 | 21 72 1 263 | 21 129 4 264 | 21 91 3 265 | 21 59 1 266 | 21 93 2 267 | 21 95 6 268 | 21 94 3 269 | 21 78 5 270 | 21 10 4 271 | 21 13 1 272 | 21 38 2 273 | 21 15 1 274 | 21 55 14 275 | 21 18 1 276 | 21 31 1 277 | 21 51 1 278 | 21 99 5 279 | 21 32 1 280 | 21 57 3 281 | 22 155 1 282 | 22 200 2 283 | 22 193 1 284 | 22 63 1 285 | 22 64 1 286 | 22 66 1 287 | 22 80 1 288 | 22 87 1 289 | 22 25 1 290 | 22 26 1 291 | 22 27 2 292 | 22 22 2 293 | 22 23 1 294 | 22 44 1 295 | 22 45 1 296 | 22 41 2 297 | 22 181 1 298 | 22 147 1 299 | 22 140 1 300 | 22 70 1 301 | 22 161 1 302 | 22 104 1 303 | 22 105 1 304 | 22 10 1 305 | 22 38 1 306 | 22 14 2 307 | 22 37 1 308 | 22 50 1 309 | 23 56 1 310 | 23 50 1 311 | 23 60 1 312 | 23 61 1 313 | 23 88 1 314 | 23 110 2 315 | 23 81 2 316 | 23 173 2 317 | 23 87 3 318 | 23 26 1 319 | 23 27 1 320 | 23 23 3 321 | 23 47 1 322 | 23 28 1 323 | 23 41 2 324 | 23 181 1 325 | 23 6 1 326 | 23 203 2 327 | 23 120 2 328 | 23 76 1 329 | 23 108 1 330 | 23 123 1 331 | 23 72 1 332 | 23 71 1 333 | 23 102 2 334 | 23 115 1 335 | 23 19 1 336 | 23 32 1 337 | 23 49 1 338 | 23 36 2 339 | 23 53 2 340 | 23 52 2 341 | 23 131 1 342 | 23 54 2 343 | 95 200 1 344 | 95 140 1 345 | 95 55 1 346 | 95 99 1 347 | 95 111 1 348 | 95 126 2 349 | 95 129 2 350 | 95 95 2 351 | 95 172 1 352 | 28 30 1 353 | 28 42 1 354 | 28 196 1 355 | 28 191 2 356 | 28 62 1 357 | 28 63 1 358 | 28 111 1 359 | 28 179 1 360 | 28 67 2 361 | 28 82 1 362 | 28 175 2 363 | 28 174 1 364 | 28 171 1 365 | 28 24 1 366 | 28 25 3 367 | 28 21 1 368 | 28 47 1 369 | 28 28 3 370 | 28 40 2 371 | 28 1 1 372 | 28 7 2 373 | 28 6 1 374 | 28 8 1 375 | 28 147 1 376 | 28 108 1 377 | 28 73 1 378 | 28 72 1 379 | 28 102 1 380 | 28 90 3 381 | 28 100 1 382 | 28 101 1 383 | 28 106 2 384 | 28 107 4 385 | 28 163 1 386 | 28 11 4 387 | 28 18 1 388 | 28 31 1 389 | 28 56 1 390 | 28 37 4 391 | 28 88 1 392 | 28 52 1 393 | 28 54 1 394 | 28 169 1 395 | 29 152 1 396 | 29 194 2 397 | 29 191 2 398 | 29 50 2 399 | 29 133 2 400 | 29 61 1 401 | 29 116 2 402 | 29 111 1 403 | 29 87 1 404 | 29 27 1 405 | 29 20 1 406 | 29 23 1 407 | 29 160 3 408 | 29 29 2 409 | 29 1 1 410 | 29 146 1 411 | 29 120 1 412 | 29 108 1 413 | 29 74 2 414 | 29 128 1 415 | 29 90 1 416 | 29 166 1 417 | 29 106 1 418 | 29 59 1 419 | 29 32 2 420 | 29 37 1 421 | 29 36 1 422 | 29 35 2 423 | 29 52 1 424 | 94 99 1 425 | 94 189 1 426 | 94 95 1 427 | 94 94 7 428 | 94 143 4 429 | 4 54 2 430 | 4 111 1 431 | 4 90 1 432 | 4 86 1 433 | 4 85 1 434 | 4 21 1 435 | 4 23 1 436 | 4 41 1 437 | 4 1 3 438 | 4 3 2 439 | 4 2 1 440 | 4 4 2 441 | 4 7 2 442 | 4 6 3 443 | 4 140 1 444 | 4 70 1 445 | 4 164 1 446 | 4 59 1 447 | 4 101 1 448 | 4 106 1 449 | 4 11 1 450 | 4 10 1 451 | 4 15 1 452 | 4 16 1 453 | 4 19 1 454 | 4 18 1 455 | 4 30 1 456 | 4 37 1 457 | 4 52 1 458 | 4 32 3 459 | 8 151 1 460 | 8 56 1 461 | 8 155 1 462 | 8 154 2 463 | 8 42 1 464 | 8 193 1 465 | 8 60 1 466 | 8 61 1 467 | 8 88 1 468 | 8 63 1 469 | 8 64 4 470 | 8 113 2 471 | 8 139 1 472 | 8 81 1 473 | 8 86 1 474 | 8 118 1 475 | 8 24 1 476 | 8 25 1 477 | 8 26 2 478 | 8 27 1 479 | 8 20 2 480 | 8 21 1 481 | 8 23 1 482 | 8 46 1 483 | 8 175 1 484 | 8 45 1 485 | 8 28 4 486 | 8 40 1 487 | 8 1 1 488 | 8 3 1 489 | 8 2 1 490 | 8 5 3 491 | 8 4 1 492 | 8 7 1 493 | 8 9 1 494 | 8 8 2 495 | 8 105 1 496 | 8 142 1 497 | 8 204 1 498 | 8 120 1 499 | 8 122 1 500 | 8 103 1 501 | 8 100 1 502 | 8 107 2 503 | 8 79 1 504 | 8 78 1 505 | 8 11 1 506 | 8 10 1 507 | 8 13 2 508 | 8 38 2 509 | 8 14 1 510 | 8 17 1 511 | 8 16 1 512 | 8 33 1 513 | 8 18 2 514 | 8 49 1 515 | 8 51 1 516 | 8 36 1 517 | 8 35 1 518 | 8 34 2 519 | 8 55 1 520 | 8 32 1 521 | 8 114 1 522 | 8 133 1 523 | 163 150 1 524 | 163 202 1 525 | 163 160 1 526 | 163 154 1 527 | 163 184 1 528 | 119 83 1 529 | 120 203 1 530 | 120 173 1 531 | 120 195 1 532 | 120 18 1 533 | 120 192 1 534 | 120 98 1 535 | 120 187 1 536 | 120 177 1 537 | 120 160 1 538 | 121 201 1 539 | 121 121 1 540 | 121 175 1 541 | 123 189 1 542 | 123 196 1 543 | 123 183 1 544 | 123 123 1 545 | 123 125 1 546 | 123 176 2 547 | 123 138 1 548 | 123 171 1 549 | 123 163 1 550 | 124 9 1 551 | 124 167 1 552 | 124 124 2 553 | 125 177 1 554 | 125 197 1 555 | 125 181 1 556 | 125 107 1 557 | 125 190 1 558 | 126 55 2 559 | 126 126 7 560 | 126 91 1 561 | 126 139 1 562 | 126 95 1 563 | 126 172 1 564 | 127 151 1 565 | 127 189 1 566 | 127 66 1 567 | 127 127 3 568 | 128 141 1 569 | 128 128 2 570 | 128 197 3 571 | 128 177 1 572 | 128 192 1 573 | 128 61 1 574 | 128 123 1 575 | 128 187 1 576 | 128 7 1 577 | 128 82 1 578 | 128 162 1 579 | 129 120 1 580 | 129 172 1 581 | 118 199 1 582 | 118 178 1 583 | 118 190 1 584 | 118 132 1 585 | 118 134 1 586 | 118 177 2 587 | 118 205 1 588 | 118 166 1 589 | 118 92 1 590 | 118 171 1 591 | 134 191 1 592 | 134 112 1 593 | 59 59 1 594 | 59 157 1 595 | 59 156 1 596 | 59 141 1 597 | 59 155 1 598 | 59 37 1 599 | 59 148 1 600 | 59 88 1 601 | 59 130 1 602 | 59 111 1 603 | 59 187 2 604 | 59 90 1 605 | 59 80 1 606 | 59 92 1 607 | 59 106 1 608 | 59 188 1 609 | 58 201 2 610 | 58 100 1 611 | 58 52 1 612 | 132 97 1 613 | 132 170 1 614 | 55 102 1 615 | 55 131 1 616 | 55 111 1 617 | 55 68 1 618 | 55 80 6 619 | 55 139 3 620 | 55 22 1 621 | 55 46 1 622 | 55 172 3 623 | 55 1 1 624 | 55 183 1 625 | 55 3 1 626 | 55 189 2 627 | 55 146 1 628 | 55 143 4 629 | 55 140 3 630 | 55 99 2 631 | 55 122 1 632 | 55 126 3 633 | 55 127 1 634 | 55 91 2 635 | 55 129 1 636 | 55 93 4 637 | 55 95 4 638 | 55 94 3 639 | 55 55 8 640 | 55 18 1 641 | 55 57 2 642 | 55 37 1 643 | 54 150 1 644 | 54 138 1 645 | 54 155 1 646 | 54 109 1 647 | 54 156 1 648 | 54 160 1 649 | 54 44 1 650 | 54 173 3 651 | 54 176 2 652 | 54 35 1 653 | 54 108 1 654 | 54 89 1 655 | 54 110 1 656 | 54 112 1 657 | 54 90 1 658 | 54 92 1 659 | 54 86 1 660 | 54 118 1 661 | 54 169 1 662 | 57 146 1 663 | 57 151 1 664 | 57 172 1 665 | 57 204 1 666 | 57 140 1 667 | 57 55 2 668 | 57 57 5 669 | 57 136 1 670 | 57 41 1 671 | 57 99 2 672 | 57 65 1 673 | 57 66 2 674 | 57 80 1 675 | 57 189 1 676 | 57 95 3 677 | 57 94 2 678 | 57 171 1 679 | 57 78 1 680 | 56 11 1 681 | 56 25 1 682 | 56 56 1 683 | 56 51 1 684 | 56 52 1 685 | 56 132 1 686 | 56 131 1 687 | 56 74 1 688 | 56 137 1 689 | 56 136 1 690 | 56 7 1 691 | 56 69 2 692 | 56 41 1 693 | 56 119 1 694 | 56 118 2 695 | 51 10 1 696 | 51 153 1 697 | 51 183 1 698 | 51 49 1 699 | 51 46 1 700 | 51 56 1 701 | 51 177 1 702 | 51 41 1 703 | 51 120 2 704 | 51 128 1 705 | 51 62 1 706 | 51 73 1 707 | 51 91 1 708 | 51 100 1 709 | 51 107 1 710 | 51 85 1 711 | 50 25 1 712 | 50 139 1 713 | 50 17 1 714 | 50 181 4 715 | 50 51 1 716 | 50 29 1 717 | 50 41 1 718 | 50 190 2 719 | 50 128 1 720 | 50 75 1 721 | 50 2 1 722 | 50 73 1 723 | 50 169 1 724 | 50 9 1 725 | 50 83 1 726 | 50 80 1 727 | 50 105 2 728 | 50 106 1 729 | 50 118 1 730 | 50 74 1 731 | 50 50 6 732 | 53 120 1 733 | 53 54 1 734 | 53 106 1 735 | 53 53 1 736 | 53 149 1 737 | 52 153 1 738 | 52 152 1 739 | 52 178 1 740 | 52 191 1 741 | 52 131 1 742 | 52 137 1 743 | 52 110 1 744 | 52 113 1 745 | 52 134 1 746 | 52 81 1 747 | 52 85 1 748 | 52 41 1 749 | 52 162 1 750 | 52 170 1 751 | 52 204 1 752 | 52 77 1 753 | 52 123 4 754 | 52 73 1 755 | 52 72 1 756 | 52 71 1 757 | 52 128 1 758 | 52 92 1 759 | 52 160 1 760 | 52 161 1 761 | 52 97 1 762 | 52 163 1 763 | 52 88 1 764 | 52 109 1 765 | 198 198 1 766 | 200 200 1 767 | 194 194 1 768 | 191 201 1 769 | 191 191 3 770 | 190 181 1 771 | 190 118 1 772 | 193 200 1 773 | 193 193 10 774 | 192 194 1 775 | 115 133 1 776 | 115 153 1 777 | 115 141 1 778 | 114 132 1 779 | 88 11 1 780 | 88 116 1 781 | 88 178 1 782 | 88 40 1 783 | 88 115 1 784 | 88 108 1 785 | 88 74 1 786 | 88 89 1 787 | 88 148 1 788 | 88 112 1 789 | 88 102 1 790 | 88 186 1 791 | 88 118 2 792 | 116 174 1 793 | 116 137 1 794 | 111 203 1 795 | 111 145 2 796 | 111 154 1 797 | 111 46 1 798 | 111 158 1 799 | 111 56 1 800 | 111 28 1 801 | 111 117 2 802 | 111 75 1 803 | 111 111 5 804 | 111 194 1 805 | 111 68 1 806 | 111 166 2 807 | 111 175 1 808 | 111 196 1 809 | 111 191 1 810 | 110 77 1 811 | 110 188 1 812 | 110 179 1 813 | 113 153 1 814 | 113 166 1 815 | 113 113 1 816 | 113 185 1 817 | 112 201 1 818 | 112 140 2 819 | 112 191 1 820 | 112 183 1 821 | 112 186 1 822 | 112 178 1 823 | 112 128 1 824 | 112 163 1 825 | 112 170 1 826 | 82 201 1 827 | 82 153 1 828 | 82 196 1 829 | 82 57 1 830 | 82 45 1 831 | 82 182 2 832 | 82 184 1 833 | 82 82 2 834 | 83 146 1 835 | 83 10 1 836 | 83 205 1 837 | 83 119 1 838 | 83 36 1 839 | 83 40 1 840 | 83 169 1 841 | 83 179 1 842 | 83 112 1 843 | 83 176 1 844 | 83 189 1 845 | 83 106 1 846 | 83 187 1 847 | 83 96 1 848 | 80 151 1 849 | 80 167 1 850 | 80 59 2 851 | 80 21 1 852 | 80 55 2 853 | 80 189 2 854 | 80 126 1 855 | 80 35 1 856 | 80 117 1 857 | 80 148 1 858 | 80 66 1 859 | 80 91 3 860 | 80 80 1 861 | 80 92 1 862 | 80 95 1 863 | 80 139 1 864 | 80 143 1 865 | 81 200 1 866 | 81 145 1 867 | 81 159 1 868 | 81 194 1 869 | 81 148 1 870 | 81 88 1 871 | 81 116 1 872 | 81 81 2 873 | 81 106 1 874 | 86 204 1 875 | 86 86 1 876 | 86 196 1 877 | 86 180 1 878 | 86 63 2 879 | 86 111 1 880 | 86 169 1 881 | 86 168 1 882 | 86 160 2 883 | 86 170 1 884 | 87 155 1 885 | 87 157 1 886 | 87 185 1 887 | 87 183 2 888 | 87 131 1 889 | 87 125 1 890 | 87 178 1 891 | 87 106 1 892 | 87 87 1 893 | 84 126 1 894 | 84 84 3 895 | 84 79 1 896 | 84 71 1 897 | 85 147 1 898 | 85 205 3 899 | 85 157 1 900 | 85 150 1 901 | 85 191 1 902 | 85 190 1 903 | 85 192 1 904 | 85 77 1 905 | 85 117 1 906 | 85 89 2 907 | 85 186 1 908 | 85 179 1 909 | 85 178 1 910 | 85 128 1 911 | 85 103 3 912 | 108 102 1 913 | 108 108 1 914 | 108 152 1 915 | 108 136 1 916 | 172 142 1 917 | 172 172 3 918 | 172 126 1 919 | 3 133 2 920 | 3 131 2 921 | 3 129 1 922 | 3 24 1 923 | 3 25 6 924 | 3 26 8 925 | 3 27 3 926 | 3 20 4 927 | 3 21 4 928 | 3 22 2 929 | 3 23 3 930 | 3 28 8 931 | 3 4 4 932 | 3 8 3 933 | 3 122 4 934 | 3 128 1 935 | 3 2 8 936 | 3 59 1 937 | 3 55 1 938 | 3 57 2 939 | 3 56 2 940 | 3 51 2 941 | 3 50 1 942 | 3 52 3 943 | 3 199 1 944 | 3 191 1 945 | 3 111 5 946 | 3 113 2 947 | 3 82 1 948 | 3 81 1 949 | 3 84 2 950 | 3 3 27 951 | 3 7 17 952 | 3 102 2 953 | 3 100 1 954 | 3 101 1 955 | 3 104 2 956 | 3 39 1 957 | 3 33 2 958 | 3 32 4 959 | 3 31 1 960 | 3 30 4 961 | 3 37 10 962 | 3 35 3 963 | 3 34 2 964 | 3 60 1 965 | 3 67 1 966 | 3 175 8 967 | 3 181 1 968 | 3 6 8 969 | 3 188 1 970 | 3 168 1 971 | 3 160 3 972 | 3 161 2 973 | 3 97 1 974 | 3 11 3 975 | 3 10 7 976 | 3 13 3 977 | 3 12 2 978 | 3 15 7 979 | 3 14 2 980 | 3 17 5 981 | 3 16 2 982 | 3 19 8 983 | 3 18 7 984 | 3 151 1 985 | 3 150 1 986 | 3 154 2 987 | 3 49 2 988 | 3 46 3 989 | 3 47 1 990 | 3 44 1 991 | 3 45 6 992 | 3 42 6 993 | 3 43 3 994 | 3 40 3 995 | 3 41 2 996 | 3 1 8 997 | 3 5 2 998 | 3 9 5 999 | 3 145 3 1000 | 3 143 1 1001 | 3 148 4 1002 | 3 74 1 1003 | 3 73 1 1004 | 3 72 2 1005 | 3 70 2 1006 | 7 151 1 1007 | 7 42 2 1008 | 7 191 2 1009 | 7 193 1 1010 | 7 50 1 1011 | 7 61 2 1012 | 7 88 2 1013 | 7 89 1 1014 | 7 111 2 1015 | 7 66 2 1016 | 7 69 1 1017 | 7 80 2 1018 | 7 52 2 1019 | 7 86 1 1020 | 7 175 1 1021 | 7 85 1 1022 | 7 27 1 1023 | 7 20 3 1024 | 7 21 5 1025 | 7 48 1 1026 | 7 23 6 1027 | 7 47 1 1028 | 7 44 1 1029 | 7 45 1 1030 | 7 28 5 1031 | 7 43 2 1032 | 7 40 1 1033 | 7 41 1 1034 | 7 1 5 1035 | 7 3 2 1036 | 7 2 5 1037 | 7 5 2 1038 | 7 4 3 1039 | 7 7 23 1040 | 7 6 1 1041 | 7 9 1 1042 | 7 8 2 1043 | 7 188 1 1044 | 7 200 1 1045 | 7 145 2 1046 | 7 121 1 1047 | 7 39 1 1048 | 7 76 1 1049 | 7 73 1 1050 | 7 72 1 1051 | 7 102 2 1052 | 7 90 1 1053 | 7 95 2 1054 | 7 163 1 1055 | 7 11 3 1056 | 7 10 6 1057 | 7 13 2 1058 | 7 12 6 1059 | 7 15 1 1060 | 7 14 1 1061 | 7 17 1 1062 | 7 16 1 1063 | 7 19 1 1064 | 7 18 2 1065 | 7 57 4 1066 | 7 30 2 1067 | 7 37 10 1068 | 7 36 3 1069 | 7 35 1 1070 | 7 34 1 1071 | 7 33 2 1072 | 7 55 2 1073 | 7 54 1 1074 | 7 114 1 1075 | 7 124 1 1076 | 7 125 3 1077 | 92 153 1 1078 | 92 198 2 1079 | 92 195 1 1080 | 92 51 1 1081 | 92 40 1 1082 | 92 118 1 1083 | 92 130 1 1084 | 92 186 1 1085 | 92 165 1 1086 | 92 106 1 1087 | 92 107 1 1088 | 92 162 1 1089 | 178 178 1 1090 | 178 204 1 1091 | 178 105 1 1092 | 181 181 8 1093 | 181 190 1 1094 | 109 159 1 1095 | 109 93 1 1096 | 109 43 1 1097 | 109 162 1 1098 | 102 146 1 1099 | 102 199 1 1100 | 102 48 1 1101 | 102 51 1 1102 | 102 190 2 1103 | 102 117 1 1104 | 102 111 1 1105 | 102 137 1 1106 | 102 160 1 1107 | 102 96 1 1108 | 103 200 1 1109 | 103 153 1 1110 | 103 55 1 1111 | 103 197 1 1112 | 103 61 1 1113 | 103 181 1 1114 | 103 179 1 1115 | 103 103 4 1116 | 103 188 2 1117 | 103 189 1 1118 | 100 115 1 1119 | 100 121 1 1120 | 100 100 1 1121 | 100 145 1 1122 | 100 195 1 1123 | 101 101 2 1124 | 101 152 1 1125 | 101 148 1 1126 | 106 200 1 1127 | 106 123 2 1128 | 106 155 1 1129 | 106 176 1 1130 | 106 192 1 1131 | 106 183 1 1132 | 106 130 1 1133 | 106 90 1 1134 | 106 83 1 1135 | 106 161 1 1136 | 107 62 1 1137 | 107 74 1 1138 | 104 197 2 1139 | 104 106 1 1140 | 104 198 1 1141 | 104 104 1 1142 | 105 153 1 1143 | 105 199 1 1144 | 105 40 1 1145 | 105 89 1 1146 | 105 186 1 1147 | 105 205 1 1148 | 105 84 1 1149 | 105 163 1 1150 | 39 39 2 1151 | 39 141 1 1152 | 39 134 2 1153 | 39 158 1 1154 | 39 44 1 1155 | 39 56 1 1156 | 39 51 1 1157 | 39 190 1 1158 | 39 35 1 1159 | 39 34 1 1160 | 39 77 1 1161 | 39 102 1 1162 | 39 131 1 1163 | 39 74 1 1164 | 39 73 1 1165 | 39 70 1 1166 | 39 82 1 1167 | 39 186 1 1168 | 39 177 1 1169 | 39 191 1 1170 | 38 152 1 1171 | 38 196 1 1172 | 38 115 1 1173 | 38 131 1 1174 | 38 64 3 1175 | 38 65 1 1176 | 38 82 1 1177 | 38 87 1 1178 | 38 25 2 1179 | 38 21 1 1180 | 38 49 1 1181 | 38 28 1 1182 | 38 41 1 1183 | 38 99 1 1184 | 38 98 1 1185 | 38 108 2 1186 | 38 124 1 1187 | 38 125 1 1188 | 38 70 2 1189 | 38 106 1 1190 | 38 97 1 1191 | 38 96 1 1192 | 38 59 1 1193 | 38 76 1 1194 | 38 55 1 1195 | 38 57 1 1196 | 38 51 1 1197 | 38 75 1 1198 | 33 20 1 1199 | 33 21 1 1200 | 33 54 1 1201 | 33 98 1 1202 | 33 63 1 1203 | 33 7 1 1204 | 33 155 1 1205 | 33 161 1 1206 | 33 104 1 1207 | 32 164 1 1208 | 32 157 1 1209 | 32 191 1 1210 | 32 60 2 1211 | 32 102 1 1212 | 32 88 1 1213 | 32 89 1 1214 | 32 69 1 1215 | 32 86 2 1216 | 32 118 1 1217 | 32 26 1 1218 | 32 27 1 1219 | 32 23 1 1220 | 32 47 3 1221 | 32 87 1 1222 | 32 183 1 1223 | 32 187 1 1224 | 32 7 1 1225 | 32 185 1 1226 | 32 145 1 1227 | 32 142 1 1228 | 32 120 1 1229 | 32 128 1 1230 | 32 106 1 1231 | 32 10 1 1232 | 32 18 2 1233 | 32 56 2 1234 | 32 37 1 1235 | 32 36 1 1236 | 32 131 1 1237 | 32 32 1 1238 | 31 150 1 1239 | 31 155 1 1240 | 31 37 1 1241 | 31 191 1 1242 | 31 133 1 1243 | 31 114 1 1244 | 31 62 2 1245 | 31 117 2 1246 | 31 67 1 1247 | 31 82 1 1248 | 31 138 1 1249 | 31 173 1 1250 | 31 85 1 1251 | 31 27 1 1252 | 31 49 1 1253 | 31 42 1 1254 | 31 41 1 1255 | 31 1 1 1256 | 31 5 1 1257 | 31 6 1 1258 | 31 9 2 1259 | 31 188 1 1260 | 31 202 1 1261 | 31 142 1 1262 | 31 77 1 1263 | 31 108 1 1264 | 31 102 2 1265 | 31 90 2 1266 | 31 92 1 1267 | 31 106 1 1268 | 31 11 2 1269 | 31 120 1 1270 | 31 39 1 1271 | 31 38 1 1272 | 31 31 2 1273 | 31 51 1 1274 | 31 50 1 1275 | 31 53 1 1276 | 31 103 1 1277 | 30 144 1 1278 | 30 27 1 1279 | 30 141 1 1280 | 30 33 1 1281 | 30 54 1 1282 | 30 30 1 1283 | 30 50 1 1284 | 30 41 1 1285 | 30 60 1 1286 | 30 132 1 1287 | 30 108 1 1288 | 30 145 1 1289 | 30 111 1 1290 | 30 113 1 1291 | 30 194 1 1292 | 30 83 1 1293 | 30 100 1 1294 | 30 86 1 1295 | 30 29 1 1296 | 30 166 1 1297 | 30 163 1 1298 | 37 152 4 1299 | 37 155 1 1300 | 37 113 1 1301 | 37 60 1 1302 | 37 128 1 1303 | 37 88 1 1304 | 37 89 2 1305 | 37 111 1 1306 | 37 53 2 1307 | 37 66 1 1308 | 37 112 1 1309 | 37 80 2 1310 | 37 173 1 1311 | 37 27 1 1312 | 37 21 1 1313 | 37 23 1 1314 | 37 45 1 1315 | 37 42 2 1316 | 37 29 1 1317 | 37 40 2 1318 | 37 2 1 1319 | 37 186 1 1320 | 37 7 2 1321 | 37 185 1 1322 | 37 204 1 1323 | 37 148 1 1324 | 37 98 1 1325 | 37 74 3 1326 | 37 124 1 1327 | 37 125 4 1328 | 37 70 1 1329 | 37 102 1 1330 | 37 100 1 1331 | 37 106 3 1332 | 37 97 1 1333 | 37 163 1 1334 | 37 11 1 1335 | 37 10 1 1336 | 37 38 1 1337 | 37 14 1 1338 | 37 55 1 1339 | 37 31 1 1340 | 37 49 1 1341 | 37 37 6 1342 | 37 50 1 1343 | 37 35 1 1344 | 37 52 3 1345 | 37 135 1 1346 | 37 116 1 1347 | 36 194 1 1348 | 36 112 1 1349 | 36 192 1 1350 | 36 60 1 1351 | 36 128 1 1352 | 36 62 1 1353 | 36 130 2 1354 | 36 53 3 1355 | 36 117 1 1356 | 36 67 2 1357 | 36 108 1 1358 | 36 83 1 1359 | 36 81 1 1360 | 36 87 3 1361 | 36 85 1 1362 | 36 123 1 1363 | 36 49 1 1364 | 36 46 1 1365 | 36 44 1 1366 | 36 28 1 1367 | 36 41 1 1368 | 36 3 1 1369 | 36 186 1 1370 | 36 6 1 1371 | 36 145 1 1372 | 36 205 1 1373 | 36 204 1 1374 | 36 141 2 1375 | 36 148 1 1376 | 36 120 1 1377 | 36 75 1 1378 | 36 74 1 1379 | 36 118 1 1380 | 36 102 2 1381 | 36 103 1 1382 | 36 166 1 1383 | 36 106 1 1384 | 36 94 2 1385 | 36 162 1 1386 | 36 11 1 1387 | 36 115 1 1388 | 36 38 1 1389 | 36 132 1 1390 | 36 54 2 1391 | 36 37 1 1392 | 36 36 2 1393 | 36 35 1 1394 | 36 34 1 1395 | 36 109 1 1396 | 36 88 2 1397 | 36 110 1 1398 | 35 151 1 1399 | 35 155 2 1400 | 35 113 1 1401 | 35 157 1 1402 | 35 163 1 1403 | 35 190 2 1404 | 35 193 1 1405 | 35 50 1 1406 | 35 60 1 1407 | 35 61 1 1408 | 35 63 1 1409 | 35 66 1 1410 | 35 178 1 1411 | 35 82 1 1412 | 35 205 1 1413 | 35 80 1 1414 | 35 174 1 1415 | 35 87 2 1416 | 35 84 1 1417 | 35 27 1 1418 | 35 21 1 1419 | 35 48 2 1420 | 35 108 1 1421 | 35 42 2 1422 | 35 29 1 1423 | 35 41 1 1424 | 35 169 2 1425 | 35 181 2 1426 | 35 116 1 1427 | 35 104 1 1428 | 35 133 1 1429 | 35 8 1 1430 | 35 105 1 1431 | 35 166 1 1432 | 35 146 1 1433 | 35 200 1 1434 | 35 145 2 1435 | 35 142 1 1436 | 35 109 2 1437 | 35 191 2 1438 | 35 120 4 1439 | 35 98 1 1440 | 35 75 2 1441 | 35 74 2 1442 | 35 73 1 1443 | 35 72 1 1444 | 35 102 2 1445 | 35 165 1 1446 | 35 93 1 1447 | 35 92 1 1448 | 35 106 4 1449 | 35 94 1 1450 | 35 97 1 1451 | 35 96 1 1452 | 35 118 1 1453 | 35 115 2 1454 | 35 39 1 1455 | 35 12 1 1456 | 35 59 1 1457 | 35 55 1 1458 | 35 32 1 1459 | 35 31 1 1460 | 35 30 1 1461 | 35 51 2 1462 | 35 36 3 1463 | 35 35 2 1464 | 35 167 1 1465 | 35 89 3 1466 | 35 54 1 1467 | 35 57 1 1468 | 35 125 3 1469 | 34 195 1 1470 | 34 62 1 1471 | 34 29 1 1472 | 34 88 2 1473 | 34 63 2 1474 | 34 135 1 1475 | 34 176 1 1476 | 34 80 1 1477 | 34 138 1 1478 | 34 87 1 1479 | 34 85 1 1480 | 34 25 1 1481 | 34 44 1 1482 | 34 43 1 1483 | 34 7 1 1484 | 34 34 1 1485 | 34 77 1 1486 | 34 74 1 1487 | 34 124 2 1488 | 34 128 1 1489 | 34 167 1 1490 | 34 160 2 1491 | 34 162 1 1492 | 34 36 2 1493 | 34 117 2 1494 | 34 52 1 1495 | 34 50 1 1496 | 162 163 1 1497 | 176 176 3 1498 | 176 205 1 1499 | 60 32 1 1500 | 60 88 2 1501 | 60 130 1 1502 | 60 59 1 1503 | 60 67 1 1504 | 61 203 1 1505 | 61 58 1 1506 | 61 109 1 1507 | 61 141 2 1508 | 61 57 1 1509 | 61 61 1 1510 | 61 88 1 1511 | 61 74 1 1512 | 61 125 1 1513 | 61 184 1 1514 | 61 128 1 1515 | 61 85 1 1516 | 62 146 1 1517 | 62 61 1 1518 | 62 131 1 1519 | 62 74 1 1520 | 62 86 1 1521 | 62 105 1 1522 | 63 144 1 1523 | 63 199 1 1524 | 63 48 1 1525 | 63 54 1 1526 | 63 149 1 1527 | 63 88 2 1528 | 63 169 1 1529 | 63 184 1 1530 | 63 185 1 1531 | 63 128 1 1532 | 63 85 1 1533 | 64 124 1 1534 | 64 92 1 1535 | 64 64 1 1536 | 64 143 1 1537 | 64 183 1 1538 | 65 151 1 1539 | 65 205 1 1540 | 65 140 1 1541 | 65 191 1 1542 | 65 99 2 1543 | 65 65 13 1544 | 65 66 1 1545 | 65 80 1 1546 | 65 94 1 1547 | 66 39 1 1548 | 66 55 1 1549 | 66 139 2 1550 | 66 66 1 1551 | 66 127 3 1552 | 66 91 2 1553 | 66 129 2 1554 | 66 80 1 1555 | 66 189 2 1556 | 66 87 1 1557 | 66 104 1 1558 | 67 11 1 1559 | 67 151 1 1560 | 67 152 1 1561 | 67 191 2 1562 | 67 42 1 1563 | 67 117 1 1564 | 67 108 1 1565 | 67 109 1 1566 | 67 67 1 1567 | 67 131 1 1568 | 67 97 1 1569 | 68 201 1 1570 | 68 195 1 1571 | 68 102 1 1572 | 68 117 1 1573 | 68 123 1 1574 | 68 68 1 1575 | 68 90 1 1576 | 68 163 1 1577 | 68 160 1 1578 | 68 105 1 1579 | 69 24 1 1580 | 69 25 1 1581 | 69 6 1 1582 | 69 114 1 1583 | 69 70 1 1584 | 69 165 2 1585 | 69 87 1 1586 | 174 174 2 1587 | 174 205 1 1588 | 173 204 1 1589 | 177 128 1 1590 | 177 177 1 1591 | 171 190 1 1592 | 170 180 1 1593 | 183 156 1 1594 | 180 180 1 1595 | 2 56 3 1596 | 2 81 1 1597 | 2 42 1 1598 | 2 112 1 1599 | 2 191 1 1600 | 2 50 1 1601 | 2 133 1 1602 | 2 88 1 1603 | 2 49 2 1604 | 2 66 1 1605 | 2 67 1 1606 | 2 82 1 1607 | 2 69 1 1608 | 2 139 2 1609 | 2 52 2 1610 | 2 87 2 1611 | 2 84 1 1612 | 2 25 3 1613 | 2 26 9 1614 | 2 27 2 1615 | 2 20 3 1616 | 2 21 1 1617 | 2 22 1 1618 | 2 23 2 1619 | 2 46 4 1620 | 2 44 2 1621 | 2 45 1 1622 | 2 28 5 1623 | 2 29 4 1624 | 2 40 2 1625 | 2 41 1 1626 | 2 1 8 1627 | 2 3 12 1628 | 2 2 7 1629 | 2 5 3 1630 | 2 4 2 1631 | 2 7 15 1632 | 2 6 5 1633 | 2 9 1 1634 | 2 8 3 1635 | 2 51 5 1636 | 2 147 2 1637 | 2 39 1 1638 | 2 120 2 1639 | 2 122 1 1640 | 2 38 1 1641 | 2 124 1 1642 | 2 71 1 1643 | 2 70 1 1644 | 2 102 1 1645 | 2 166 1 1646 | 2 192 1 1647 | 2 107 1 1648 | 2 97 1 1649 | 2 96 2 1650 | 2 11 4 1651 | 2 10 7 1652 | 2 13 3 1653 | 2 12 7 1654 | 2 15 4 1655 | 2 16 2 1656 | 2 19 2 1657 | 2 18 4 1658 | 2 31 4 1659 | 2 30 2 1660 | 2 37 6 1661 | 2 36 3 1662 | 2 53 2 1663 | 2 34 2 1664 | 2 55 5 1665 | 2 32 5 1666 | 2 136 1 1667 | 187 202 1 1668 | 6 150 1 1669 | 6 56 1 1670 | 6 155 1 1671 | 6 113 2 1672 | 6 200 1 1673 | 6 112 1 1674 | 6 191 1 1675 | 6 115 1 1676 | 6 62 1 1677 | 6 111 2 1678 | 6 65 1 1679 | 6 66 1 1680 | 6 67 1 1681 | 6 83 1 1682 | 6 81 1 1683 | 6 87 2 1684 | 6 198 1 1685 | 6 24 1 1686 | 6 25 2 1687 | 6 26 1 1688 | 6 27 4 1689 | 6 21 1 1690 | 6 22 3 1691 | 6 23 1 1692 | 6 46 1 1693 | 6 106 1 1694 | 6 28 1 1695 | 6 29 2 1696 | 6 40 2 1697 | 6 107 1 1698 | 6 1 3 1699 | 6 3 1 1700 | 6 5 1 1701 | 6 7 3 1702 | 6 6 3 1703 | 6 9 2 1704 | 6 8 1 1705 | 6 146 1 1706 | 6 147 1 1707 | 6 120 2 1708 | 6 77 1 1709 | 6 123 1 1710 | 6 73 1 1711 | 6 70 2 1712 | 6 164 1 1713 | 6 90 1 1714 | 6 93 2 1715 | 6 95 2 1716 | 6 94 1 1717 | 6 104 1 1718 | 6 78 1 1719 | 6 11 2 1720 | 6 10 1 1721 | 6 13 3 1722 | 6 12 3 1723 | 6 201 1 1724 | 6 14 1 1725 | 6 17 5 1726 | 6 16 1 1727 | 6 33 3 1728 | 6 18 4 1729 | 6 31 3 1730 | 6 49 1 1731 | 6 37 3 1732 | 6 36 1 1733 | 6 35 2 1734 | 6 55 4 1735 | 6 57 1 1736 | 6 133 1 1737 | 188 90 1 1738 | 188 188 2 1739 | 189 189 1 1740 | 189 127 1 1741 | 179 179 2 1742 | 99 151 1 1743 | 99 120 1 1744 | 99 143 1 1745 | 99 99 5 1746 | 99 111 1 1747 | 99 102 1 1748 | 99 139 1 1749 | 98 144 1 1750 | 98 205 1 1751 | 98 195 1 1752 | 98 98 1 1753 | 98 109 1 1754 | 98 72 2 1755 | 98 163 1 1756 | 168 167 1 1757 | 91 55 2 1758 | 91 99 1 1759 | 91 136 1 1760 | 91 126 2 1761 | 91 91 4 1762 | 91 129 1 1763 | 91 139 1 1764 | 91 94 1 1765 | 90 200 1 1766 | 90 205 1 1767 | 90 67 1 1768 | 90 191 2 1769 | 90 193 1 1770 | 90 60 1 1771 | 90 63 1 1772 | 90 168 2 1773 | 90 72 1 1774 | 90 134 3 1775 | 90 90 1 1776 | 90 188 2 1777 | 90 174 1 1778 | 90 173 1 1779 | 166 166 4 1780 | 166 38 1 1781 | 166 188 1 1782 | 167 182 1 1783 | 167 167 2 1784 | 167 111 1 1785 | 160 203 1 1786 | 160 155 1 1787 | 160 204 1 1788 | 160 86 1 1789 | 160 29 1 1790 | 160 128 1 1791 | 160 160 1 1792 | 97 152 1 1793 | 97 204 1 1794 | 97 156 1 1795 | 97 44 1 1796 | 97 190 1 1797 | 97 183 1 1798 | 97 186 1 1799 | 97 110 1 1800 | 97 178 1 1801 | 97 138 1 1802 | 97 106 1 1803 | 97 197 1 1804 | 96 57 1 1805 | 96 101 1 1806 | 96 88 1 1807 | 96 193 1 1808 | 11 133 2 1809 | 11 131 2 1810 | 11 165 1 1811 | 11 138 1 1812 | 11 26 1 1813 | 11 27 1 1814 | 11 21 1 1815 | 11 22 1 1816 | 11 28 6 1817 | 11 29 1 1818 | 11 8 1 1819 | 11 123 1 1820 | 11 124 1 1821 | 11 128 2 1822 | 11 129 2 1823 | 11 54 3 1824 | 11 57 2 1825 | 11 56 1 1826 | 11 50 1 1827 | 11 53 2 1828 | 11 52 5 1829 | 11 199 1 1830 | 11 115 1 1831 | 11 88 1 1832 | 11 89 1 1833 | 11 111 4 1834 | 11 110 1 1835 | 11 113 1 1836 | 11 112 3 1837 | 11 83 3 1838 | 11 80 1 1839 | 11 86 2 1840 | 11 87 6 1841 | 11 84 1 1842 | 11 85 1 1843 | 11 108 2 1844 | 11 109 1 1845 | 11 102 1 1846 | 11 103 1 1847 | 11 100 1 1848 | 11 106 1 1849 | 11 105 1 1850 | 11 38 1 1851 | 11 33 1 1852 | 11 32 4 1853 | 11 31 1 1854 | 11 37 3 1855 | 11 36 1 1856 | 11 35 3 1857 | 11 34 2 1858 | 11 66 1 1859 | 11 67 1 1860 | 11 173 2 1861 | 11 181 3 1862 | 11 2 1 1863 | 11 167 1 1864 | 11 6 2 1865 | 11 98 1 1866 | 11 168 1 1867 | 11 164 1 1868 | 11 90 1 1869 | 11 92 1 1870 | 11 95 1 1871 | 11 163 1 1872 | 11 11 3 1873 | 11 10 1 1874 | 11 12 3 1875 | 11 15 5 1876 | 11 17 2 1877 | 11 19 1 1878 | 11 18 1 1879 | 11 204 1 1880 | 11 155 1 1881 | 11 154 1 1882 | 11 156 1 1883 | 11 49 2 1884 | 11 47 1 1885 | 11 44 1 1886 | 11 42 3 1887 | 11 43 2 1888 | 11 40 1 1889 | 11 41 4 1890 | 11 200 1 1891 | 11 142 5 1892 | 11 143 1 1893 | 11 141 1 1894 | 11 77 2 1895 | 11 75 3 1896 | 11 74 4 1897 | 11 72 1 1898 | 11 71 1 1899 | 11 70 1 1900 | 11 79 2 1901 | 11 78 1 1902 | 10 56 1 1903 | 10 154 1 1904 | 10 42 2 1905 | 10 191 1 1906 | 10 43 2 1907 | 10 60 1 1908 | 10 137 1 1909 | 10 52 1 1910 | 10 171 1 1911 | 10 85 1 1912 | 10 25 1 1913 | 10 26 6 1914 | 10 27 1 1915 | 10 20 1 1916 | 10 22 1 1917 | 10 46 5 1918 | 10 47 1 1919 | 10 28 1 1920 | 10 29 1 1921 | 10 40 1 1922 | 10 2 1 1923 | 10 5 1 1924 | 10 7 2 1925 | 10 6 2 1926 | 10 8 2 1927 | 10 51 1 1928 | 10 200 1 1929 | 10 142 1 1930 | 10 109 1 1931 | 10 77 1 1932 | 10 75 1 1933 | 10 38 1 1934 | 10 73 1 1935 | 10 70 1 1936 | 10 102 1 1937 | 10 59 1 1938 | 10 167 1 1939 | 10 101 1 1940 | 10 162 1 1941 | 10 11 1 1942 | 10 10 2 1943 | 10 13 1 1944 | 10 12 4 1945 | 10 15 1 1946 | 10 16 2 1947 | 10 55 2 1948 | 10 18 1 1949 | 10 57 1 1950 | 10 30 1 1951 | 10 37 4 1952 | 10 36 1 1953 | 10 34 1 1954 | 10 74 1 1955 | 13 155 1 1956 | 13 195 1 1957 | 13 115 1 1958 | 13 164 1 1959 | 13 88 3 1960 | 13 130 2 1961 | 13 64 1 1962 | 13 66 2 1963 | 13 67 1 1964 | 13 69 2 1965 | 13 81 1 1966 | 13 86 1 1967 | 13 87 1 1968 | 13 84 2 1969 | 13 24 4 1970 | 13 25 1 1971 | 13 26 2 1972 | 13 27 2 1973 | 13 20 5 1974 | 13 23 1 1975 | 13 45 1 1976 | 13 28 1 1977 | 13 43 1 1978 | 13 40 1 1979 | 13 181 1 1980 | 13 5 1 1981 | 13 171 1 1982 | 13 184 1 1983 | 13 6 2 1984 | 13 201 1 1985 | 13 145 1 1986 | 13 142 1 1987 | 13 143 1 1988 | 13 174 2 1989 | 13 148 1 1990 | 13 99 1 1991 | 13 76 2 1992 | 13 73 1 1993 | 13 70 1 1994 | 13 102 2 1995 | 13 93 1 1996 | 13 167 4 1997 | 13 161 1 1998 | 13 79 1 1999 | 13 11 1 2000 | 13 10 1 2001 | 13 13 1 2002 | 13 38 2 2003 | 13 15 1 2004 | 13 121 1 2005 | 13 19 1 2006 | 13 54 2 2007 | 13 31 1 2008 | 13 51 1 2009 | 13 36 3 2010 | 13 53 1 2011 | 13 33 1 2012 | 13 124 1 2013 | 12 150 1 2014 | 12 132 1 2015 | 12 155 1 2016 | 12 42 1 2017 | 12 196 1 2018 | 12 191 1 2019 | 12 60 1 2020 | 12 82 1 2021 | 12 89 1 2022 | 12 110 1 2023 | 12 66 1 2024 | 12 68 1 2025 | 12 69 1 2026 | 12 139 1 2027 | 12 138 1 2028 | 12 152 1 2029 | 12 87 2 2030 | 12 84 1 2031 | 12 85 1 2032 | 12 123 1 2033 | 12 27 2 2034 | 12 21 2 2035 | 12 22 1 2036 | 12 23 2 2037 | 12 44 1 2038 | 12 28 3 2039 | 12 29 1 2040 | 12 130 1 2041 | 12 1 1 2042 | 12 2 2 2043 | 12 7 1 2044 | 12 6 2 2045 | 12 188 1 2046 | 12 96 1 2047 | 12 51 1 2048 | 12 141 1 2049 | 12 98 1 2050 | 12 75 2 2051 | 12 38 3 2052 | 12 73 1 2053 | 12 125 1 2054 | 12 70 2 2055 | 12 102 1 2056 | 12 103 1 2057 | 12 166 1 2058 | 12 107 2 2059 | 12 78 1 2060 | 12 11 5 2061 | 12 10 1 2062 | 12 13 3 2063 | 12 12 1 2064 | 12 59 1 2065 | 12 48 2 2066 | 12 16 1 2067 | 12 55 2 2068 | 12 54 1 2069 | 12 31 1 2070 | 12 30 2 2071 | 12 37 3 2072 | 12 35 1 2073 | 12 116 2 2074 | 12 114 1 2075 | 15 152 1 2076 | 15 155 1 2077 | 15 42 2 2078 | 15 191 2 2079 | 15 29 1 2080 | 15 60 2 2081 | 15 132 1 2082 | 15 62 1 2083 | 15 130 1 2084 | 15 111 1 2085 | 15 117 1 2086 | 15 69 2 2087 | 15 138 2 2088 | 15 118 1 2089 | 15 84 1 2090 | 15 85 1 2091 | 15 24 2 2092 | 15 25 1 2093 | 15 26 1 2094 | 15 20 1 2095 | 15 23 2 2096 | 15 47 1 2097 | 15 28 1 2098 | 15 43 1 2099 | 15 161 1 2100 | 15 1 2 2101 | 15 2 1 2102 | 15 7 1 2103 | 15 170 1 2104 | 15 166 1 2105 | 15 204 1 2106 | 15 39 1 2107 | 15 181 1 2108 | 15 38 1 2109 | 15 100 1 2110 | 15 106 1 2111 | 15 107 1 2112 | 15 13 1 2113 | 15 12 1 2114 | 15 15 4 2115 | 15 55 2 2116 | 15 18 2 2117 | 15 31 1 2118 | 15 49 2 2119 | 15 51 1 2120 | 15 88 1 2121 | 15 34 1 2122 | 15 74 1 2123 | 15 32 2 2124 | 14 159 1 2125 | 14 88 1 2126 | 14 63 1 2127 | 14 64 1 2128 | 14 179 1 2129 | 14 69 2 2130 | 14 139 1 2131 | 14 86 1 2132 | 14 175 1 2133 | 14 27 2 2134 | 14 21 1 2135 | 14 22 1 2136 | 14 42 2 2137 | 14 43 1 2138 | 14 3 1 2139 | 14 9 1 2140 | 14 8 1 2141 | 14 203 1 2142 | 14 202 1 2143 | 14 98 1 2144 | 14 169 1 2145 | 14 71 1 2146 | 14 13 1 2147 | 14 38 1 2148 | 14 14 2 2149 | 14 17 1 2150 | 14 19 1 2151 | 14 18 1 2152 | 14 57 1 2153 | 14 56 1 2154 | 14 50 2 2155 | 14 53 1 2156 | 14 52 1 2157 | 14 131 1 2158 | 14 48 1 2159 | 14 111 2 2160 | 17 146 1 2161 | 17 200 1 2162 | 17 12 1 2163 | 17 22 2 2164 | 17 38 1 2165 | 17 18 1 2166 | 17 112 1 2167 | 17 191 4 2168 | 17 190 1 2169 | 17 35 1 2170 | 17 41 1 2171 | 17 108 1 2172 | 17 2 1 2173 | 17 73 1 2174 | 17 84 1 2175 | 17 7 3 2176 | 17 6 1 2177 | 17 145 2 2178 | 17 149 1 2179 | 17 79 1 2180 | 17 105 1 2181 | 16 201 1 2182 | 16 70 1 2183 | 16 62 1 2184 | 16 67 2 2185 | 16 82 1 2186 | 16 27 2 2187 | 16 20 1 2188 | 16 21 1 2189 | 16 29 1 2190 | 16 40 1 2191 | 16 146 1 2192 | 16 147 1 2193 | 16 76 1 2194 | 16 127 1 2195 | 16 93 1 2196 | 16 101 1 2197 | 16 15 1 2198 | 16 58 1 2199 | 16 16 1 2200 | 16 19 1 2201 | 16 31 1 2202 | 16 56 1 2203 | 16 51 1 2204 | 16 35 2 2205 | 16 52 1 2206 | 19 153 1 2207 | 19 191 1 2208 | 19 50 1 2209 | 19 104 1 2210 | 19 89 1 2211 | 19 111 1 2212 | 19 82 1 2213 | 19 69 1 2214 | 19 175 1 2215 | 19 118 1 2216 | 19 84 1 2217 | 19 170 1 2218 | 19 24 1 2219 | 19 20 1 2220 | 19 21 1 2221 | 19 22 1 2222 | 19 160 1 2223 | 19 44 1 2224 | 19 43 2 2225 | 19 181 1 2226 | 19 97 1 2227 | 19 7 1 2228 | 19 192 1 2229 | 19 148 1 2230 | 19 121 1 2231 | 19 72 1 2232 | 19 70 2 2233 | 19 128 1 2234 | 19 100 3 2235 | 19 101 1 2236 | 19 106 3 2237 | 19 79 2 2238 | 19 38 1 2239 | 19 19 1 2240 | 19 18 1 2241 | 19 31 2 2242 | 19 56 1 2243 | 19 37 1 2244 | 19 36 1 2245 | 19 34 1 2246 | 19 55 1 2247 | 19 116 2 2248 | 19 57 1 2249 | 19 125 1 2250 | 18 150 1 2251 | 18 154 1 2252 | 18 157 1 2253 | 18 54 1 2254 | 18 28 2 2255 | 18 178 1 2256 | 18 191 1 2257 | 18 50 1 2258 | 18 115 1 2259 | 18 132 1 2260 | 18 116 1 2261 | 18 111 2 2262 | 18 53 3 2263 | 18 113 1 2264 | 18 83 1 2265 | 18 82 1 2266 | 18 69 1 2267 | 18 80 1 2268 | 18 52 1 2269 | 18 87 1 2270 | 18 25 6 2271 | 18 27 1 2272 | 18 20 1 2273 | 18 49 3 2274 | 18 108 1 2275 | 18 175 3 2276 | 18 42 1 2277 | 18 29 3 2278 | 18 41 2 2279 | 18 181 1 2280 | 18 187 1 2281 | 18 7 1 2282 | 18 194 1 2283 | 18 9 1 2284 | 18 188 1 2285 | 18 163 1 2286 | 18 51 1 2287 | 18 203 1 2288 | 18 184 1 2289 | 18 120 2 2290 | 18 148 1 2291 | 18 77 1 2292 | 18 76 1 2293 | 18 75 1 2294 | 18 74 2 2295 | 18 72 2 2296 | 18 129 1 2297 | 18 102 1 2298 | 18 103 1 2299 | 18 107 1 2300 | 18 79 1 2301 | 18 96 2 2302 | 18 10 1 2303 | 18 12 1 2304 | 18 14 2 2305 | 18 98 3 2306 | 18 55 1 2307 | 18 18 2 2308 | 18 31 2 2309 | 18 37 1 2310 | 18 36 3 2311 | 18 35 1 2312 | 18 34 2 2313 | 18 32 2 2314 | 18 57 1 2315 | 117 139 1 2316 | 117 75 1 2317 | 117 111 2 2318 | 117 117 1 2319 | 117 193 1 2320 | 89 173 1 2321 | 89 160 2 2322 | 89 191 1 2323 | 89 120 1 2324 | 89 75 1 2325 | 89 89 1 2326 | 89 73 1 2327 | 89 176 1 2328 | 89 80 1 2329 | 89 86 1 2330 | 204 204 2 2331 | 151 151 1 2332 | 151 140 1 2333 | 150 147 1 2334 | 150 203 1 2335 | 150 187 1 2336 | 153 153 3 2337 | 153 193 1 2338 | 153 192 1 2339 | 153 183 2 2340 | 153 164 1 2341 | 153 97 1 2342 | 152 74 1 2343 | 152 37 1 2344 | 152 178 1 2345 | 155 155 1 2346 | 155 195 1 2347 | 155 193 2 2348 | 155 192 1 2349 | 155 185 1 2350 | 155 164 1 2351 | 154 42 1 2352 | 154 154 1 2353 | 156 198 1 2354 | 156 57 1 2355 | 156 118 1 2356 | 156 156 1 2357 | 158 158 1 2358 | 48 27 1 2359 | 48 48 4 2360 | 48 119 1 2361 | 48 191 1 2362 | 48 50 2 2363 | 48 40 1 2364 | 48 98 1 2365 | 48 123 1 2366 | 48 111 1 2367 | 48 152 1 2368 | 48 7 1 2369 | 48 165 1 2370 | 48 194 1 2371 | 48 83 1 2372 | 48 106 2 2373 | 48 118 1 2374 | 48 104 2 2375 | 48 173 1 2376 | 49 120 3 2377 | 49 155 1 2378 | 49 22 1 2379 | 49 49 1 2380 | 49 197 1 2381 | 49 35 1 2382 | 49 77 1 2383 | 49 183 1 2384 | 49 62 1 2385 | 49 111 1 2386 | 49 148 2 2387 | 49 7 1 2388 | 49 165 1 2389 | 49 103 1 2390 | 49 175 2 2391 | 49 106 1 2392 | 49 87 1 2393 | 49 171 1 2394 | 46 77 2 2395 | 46 192 1 2396 | 46 65 1 2397 | 46 66 4 2398 | 46 139 1 2399 | 46 119 1 2400 | 46 80 1 2401 | 46 46 2 2402 | 46 187 1 2403 | 46 6 1 2404 | 46 189 2 2405 | 46 144 1 2406 | 46 143 1 2407 | 46 140 2 2408 | 46 99 1 2409 | 46 76 1 2410 | 46 74 1 2411 | 46 126 3 2412 | 46 91 5 2413 | 46 129 6 2414 | 46 93 2 2415 | 46 95 2 2416 | 46 120 1 2417 | 46 55 3 2418 | 46 57 2 2419 | 46 50 1 2420 | 47 147 1 2421 | 47 143 1 2422 | 47 57 1 2423 | 47 77 1 2424 | 47 132 1 2425 | 47 111 1 2426 | 47 66 1 2427 | 47 107 1 2428 | 44 153 1 2429 | 44 62 1 2430 | 44 60 1 2431 | 44 88 2 2432 | 44 113 1 2433 | 44 108 2 2434 | 44 80 1 2435 | 44 174 1 2436 | 44 119 1 2437 | 44 84 1 2438 | 44 27 1 2439 | 44 46 2 2440 | 44 44 1 2441 | 44 2 1 2442 | 44 186 1 2443 | 44 7 4 2444 | 44 163 1 2445 | 44 201 1 2446 | 44 75 1 2447 | 44 74 1 2448 | 44 72 1 2449 | 44 160 1 2450 | 44 104 1 2451 | 44 105 2 2452 | 44 54 1 2453 | 44 56 1 2454 | 44 131 1 2455 | 44 34 1 2456 | 44 169 2 2457 | 45 12 1 2458 | 45 154 1 2459 | 45 50 1 2460 | 45 35 1 2461 | 45 98 1 2462 | 45 124 1 2463 | 45 136 1 2464 | 45 175 1 2465 | 45 81 1 2466 | 45 161 2 2467 | 42 67 2 2468 | 42 191 6 2469 | 42 192 1 2470 | 42 110 1 2471 | 42 112 1 2472 | 42 139 1 2473 | 42 81 1 2474 | 42 175 1 2475 | 42 85 1 2476 | 42 24 1 2477 | 42 174 1 2478 | 42 44 1 2479 | 42 42 1 2480 | 42 43 1 2481 | 42 40 1 2482 | 42 205 1 2483 | 42 120 1 2484 | 42 98 2 2485 | 42 72 1 2486 | 42 71 1 2487 | 42 102 1 2488 | 42 100 1 2489 | 42 167 1 2490 | 42 77 1 2491 | 42 76 1 2492 | 42 54 1 2493 | 42 37 1 2494 | 43 25 1 2495 | 43 142 1 2496 | 43 179 1 2497 | 43 200 1 2498 | 43 54 2 2499 | 43 196 1 2500 | 43 37 1 2501 | 43 43 2 2502 | 43 61 1 2503 | 43 131 1 2504 | 43 116 1 2505 | 43 205 1 2506 | 43 7 1 2507 | 43 128 1 2508 | 43 83 1 2509 | 43 180 1 2510 | 43 86 1 2511 | 43 118 1 2512 | 43 163 2 2513 | 40 150 1 2514 | 40 159 1 2515 | 40 37 1 2516 | 40 197 1 2517 | 40 191 1 2518 | 40 115 2 2519 | 40 61 2 2520 | 40 130 1 2521 | 40 137 1 2522 | 40 178 1 2523 | 40 82 1 2524 | 40 52 1 2525 | 40 85 2 2526 | 40 26 2 2527 | 40 47 1 2528 | 40 40 4 2529 | 40 180 1 2530 | 40 5 1 2531 | 40 189 1 2532 | 40 142 1 2533 | 40 120 2 2534 | 40 75 1 2535 | 40 109 1 2536 | 40 72 3 2537 | 40 70 1 2538 | 40 100 1 2539 | 40 167 1 2540 | 40 97 1 2541 | 40 31 1 2542 | 40 51 1 2543 | 40 35 2 2544 | 40 34 1 2545 | 40 74 3 2546 | 40 125 1 2547 | 41 153 1 2548 | 41 113 1 2549 | 41 157 1 2550 | 41 133 2 2551 | 41 164 1 2552 | 41 62 1 2553 | 41 116 2 2554 | 41 66 1 2555 | 41 67 1 2556 | 41 175 1 2557 | 41 81 1 2558 | 41 173 2 2559 | 41 84 1 2560 | 41 138 1 2561 | 41 21 1 2562 | 41 160 2 2563 | 41 40 2 2564 | 41 41 2 2565 | 41 7 1 2566 | 41 188 1 2567 | 41 105 2 2568 | 41 146 1 2569 | 41 145 1 2570 | 41 142 1 2571 | 41 204 1 2572 | 41 140 1 2573 | 41 120 1 2574 | 41 75 3 2575 | 41 109 2 2576 | 41 124 2 2577 | 41 72 2 2578 | 41 102 1 2579 | 41 90 2 2580 | 41 100 1 2581 | 41 106 1 2582 | 41 96 1 2583 | 41 11 1 2584 | 41 15 1 2585 | 41 33 1 2586 | 41 32 1 2587 | 41 31 2 2588 | 41 37 1 2589 | 41 131 2 2590 | 41 35 2 2591 | 41 55 1 2592 | 41 123 2 2593 | 41 54 4 2594 | 41 169 1 2595 | 1 56 1 2596 | 1 116 1 2597 | 1 135 1 2598 | 1 125 1 2599 | 1 54 1 2600 | 1 42 1 2601 | 1 50 1 2602 | 1 60 1 2603 | 1 61 1 2604 | 1 63 2 2605 | 1 89 1 2606 | 1 49 1 2607 | 1 66 1 2608 | 1 67 1 2609 | 1 82 1 2610 | 1 69 6 2611 | 1 80 1 2612 | 1 85 1 2613 | 1 24 2 2614 | 1 25 2 2615 | 1 26 7 2616 | 1 27 3 2617 | 1 20 5 2618 | 1 21 2 2619 | 1 22 1 2620 | 1 23 1 2621 | 1 46 8 2622 | 1 28 3 2623 | 1 29 2 2624 | 1 1 9 2625 | 1 3 14 2626 | 1 2 9 2627 | 1 5 2 2628 | 1 7 11 2629 | 1 6 7 2630 | 1 9 2 2631 | 1 8 7 2632 | 1 105 1 2633 | 1 170 1 2634 | 1 83 1 2635 | 1 148 1 2636 | 1 98 1 2637 | 1 38 2 2638 | 1 72 1 2639 | 1 71 1 2640 | 1 91 1 2641 | 1 115 2 2642 | 1 101 1 2643 | 1 106 1 2644 | 1 79 1 2645 | 1 78 1 2646 | 1 11 6 2647 | 1 10 5 2648 | 1 13 4 2649 | 1 12 2 2650 | 1 15 4 2651 | 1 14 4 2652 | 1 17 1 2653 | 1 16 5 2654 | 1 33 2 2655 | 1 18 2 2656 | 1 31 2 2657 | 1 30 3 2658 | 1 37 1 2659 | 1 36 1 2660 | 1 167 1 2661 | 1 74 1 2662 | 1 157 1 2663 | 1 32 1 2664 | 1 111 3 2665 | 1 154 1 2666 | 1 57 1 2667 | 1 65 1 2668 | 5 135 1 2669 | 5 51 1 2670 | 5 191 1 2671 | 5 43 1 2672 | 5 115 1 2673 | 5 111 1 2674 | 5 66 1 2675 | 5 67 1 2676 | 5 175 3 2677 | 5 87 1 2678 | 5 154 1 2679 | 5 27 3 2680 | 5 20 2 2681 | 5 21 2 2682 | 5 47 1 2683 | 5 44 1 2684 | 5 45 1 2685 | 5 29 2 2686 | 5 1 2 2687 | 5 3 3 2688 | 5 2 2 2689 | 5 5 2 2690 | 5 7 4 2691 | 5 6 2 2692 | 5 9 2 2693 | 5 8 1 2694 | 5 170 1 2695 | 5 149 2 2696 | 5 75 1 2697 | 5 124 1 2698 | 5 70 1 2699 | 5 102 1 2700 | 5 100 1 2701 | 5 161 1 2702 | 5 97 1 2703 | 5 11 1 2704 | 5 13 1 2705 | 5 38 2 2706 | 5 15 1 2707 | 5 16 2 2708 | 5 33 1 2709 | 5 18 1 2710 | 5 31 2 2711 | 5 30 2 2712 | 5 37 1 2713 | 5 34 1 2714 | 9 191 1 2715 | 9 111 1 2716 | 9 65 2 2717 | 9 112 1 2718 | 9 80 1 2719 | 9 84 1 2720 | 9 25 1 2721 | 9 27 1 2722 | 9 47 1 2723 | 9 42 3 2724 | 9 1 1 2725 | 9 186 1 2726 | 9 7 4 2727 | 9 8 2 2728 | 9 182 1 2729 | 9 148 1 2730 | 9 75 1 2731 | 9 70 1 2732 | 9 166 1 2733 | 9 107 1 2734 | 9 97 1 2735 | 9 105 1 2736 | 9 11 2 2737 | 9 13 1 2738 | 9 38 1 2739 | 9 15 1 2740 | 9 16 3 2741 | 9 19 1 2742 | 9 32 2 2743 | 9 57 1 2744 | 9 56 1 2745 | 9 37 3 2746 | 9 33 1 2747 | 9 54 1 2748 | 146 146 1 2749 | 146 168 1 2750 | 146 204 1 2751 | 147 50 1 2752 | 147 178 1 2753 | 145 199 1 2754 | 145 190 1 2755 | 145 163 1 2756 | 142 196 2 2757 | 142 160 1 2758 | 143 203 1 2759 | 143 143 4 2760 | 143 140 1 2761 | 140 205 1 2762 | 140 140 4 2763 | 140 191 1 2764 | 140 103 1 2765 | 140 90 1 2766 | 140 139 1 2767 | 141 102 1 2768 | 141 174 1 2769 | 141 141 1 2770 | 148 153 1 2771 | 148 148 1 2772 | 77 61 1 2773 | 77 195 1 2774 | 77 56 1 2775 | 77 148 1 2776 | 77 114 1 2777 | 77 75 1 2778 | 77 167 1 2779 | 77 171 1 2780 | 77 85 1 2781 | 76 146 1 2782 | 76 182 1 2783 | 76 201 1 2784 | 76 7 1 2785 | 76 115 1 2786 | 76 130 1 2787 | 76 135 1 2788 | 75 201 1 2789 | 75 196 1 2790 | 75 48 1 2791 | 75 96 1 2792 | 75 195 1 2793 | 75 44 1 2794 | 75 156 1 2795 | 75 192 1 2796 | 75 115 1 2797 | 75 75 2 2798 | 75 186 1 2799 | 75 70 1 2800 | 75 176 2 2801 | 75 100 1 2802 | 75 92 3 2803 | 75 162 1 2804 | 75 85 1 2805 | 74 198 1 2806 | 74 194 1 2807 | 74 196 2 2808 | 74 37 2 2809 | 74 52 1 2810 | 74 133 1 2811 | 74 98 2 2812 | 74 88 2 2813 | 74 74 4 2814 | 74 163 2 2815 | 74 113 1 2816 | 74 128 1 2817 | 74 69 1 2818 | 74 81 1 2819 | 74 160 3 2820 | 74 105 1 2821 | 73 71 1 2822 | 73 197 1 2823 | 73 56 1 2824 | 73 183 1 2825 | 73 108 1 2826 | 73 73 1 2827 | 73 169 1 2828 | 73 113 1 2829 | 73 177 1 2830 | 73 94 1 2831 | 72 198 1 2832 | 72 203 1 2833 | 72 155 1 2834 | 72 184 1 2835 | 72 158 1 2836 | 72 193 1 2837 | 72 52 1 2838 | 72 98 1 2839 | 72 88 1 2840 | 72 109 1 2841 | 72 186 1 2842 | 72 149 1 2843 | 72 194 1 2844 | 72 205 1 2845 | 72 105 1 2846 | 72 160 1 2847 | 72 85 2 2848 | 71 10 1 2849 | 71 138 1 2850 | 71 199 1 2851 | 71 125 1 2852 | 71 71 2 2853 | 71 205 1 2854 | 71 80 1 2855 | 71 85 1 2856 | 71 188 1 2857 | 71 110 1 2858 | 70 124 1 2859 | 70 201 1 2860 | 70 203 1 2861 | 70 143 1 2862 | 70 197 1 2863 | 70 192 1 2864 | 70 132 1 2865 | 70 108 1 2866 | 70 111 1 2867 | 70 4 1 2868 | 70 112 1 2869 | 70 83 1 2870 | 70 100 1 2871 | 70 173 1 2872 | 70 172 1 2873 | 70 104 1 2874 | 79 200 1 2875 | 79 65 1 2876 | 79 140 1 2877 | 78 198 1 2878 | 78 189 1 2879 | 78 65 1 2880 | 78 78 1 2881 | 144 193 1 2882 | --------------------------------------------------------------------------------