├── .gitignore ├── README.pod ├── cpan_deps_graph.pl ├── cpanfile ├── lib └── CPANDepsGraph │ └── Command │ └── cache.pm ├── public ├── favicon.ico └── graph.js └── templates ├── graph.html.ep └── layouts └── main.html.ep /.gitignore: -------------------------------------------------------------------------------- 1 | log/* 2 | cpan_deps_graph.conf 3 | hypnotoad.pid 4 | -------------------------------------------------------------------------------- /README.pod: -------------------------------------------------------------------------------- 1 | =head1 NAME 2 | 3 | CPAN Dependencies Graph 4 | 5 | =head1 SETUP 6 | 7 | To run this in a local environment, you need a Redis server, 8 | and to create a F pointing to it, for example: 9 | 10 | { 11 | redis_url => 'redis://localhost/', 12 | } 13 | 14 | Install the dependencies from the F: 15 | 16 | $ cpanm --installdeps . 17 | 18 | Then load the cache with at least one module's data: 19 | 20 | $ ./cpan_deps_graph.pl cache --deeply GraphViz2 21 | 22 | Then start the L: 23 | 24 | $ ./cpan_deps_graph.pl daemon 25 | 26 | =head1 COPYRIGHT AND LICENSE 27 | 28 | This software is Copyright (c) 2019 by Dan Book. 29 | 30 | This is free software, licensed under: 31 | 32 | The Artistic License 2.0 (GPL Compatible) 33 | 34 | Bundled Perl 5 Raptor Copyright (c) 2012, Sebastian Riedel. 35 | 36 | Licensed under the CC-SA License, Version 4.0 37 | L. 38 | -------------------------------------------------------------------------------- /cpan_deps_graph.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use 5.020; 3 | use Mojolicious::Lite -signatures; 4 | use CPAN::DistnameInfo; 5 | use HTTP::Simple 'getjson'; 6 | use List::UtilsBy 'uniq_by'; 7 | use MetaCPAN::Client; 8 | use Module::CoreList; 9 | use Mojo::JSON qw(from_json to_json); 10 | use Mojo::Redis; 11 | use Mojo::URL; 12 | use Syntax::Keyword::Try; 13 | use version; 14 | use lib::relative 'lib'; 15 | 16 | our $VERSION = 'v1.1.1'; 17 | helper app_version => sub ($c) { $VERSION }; 18 | 19 | plugin 'Config' => {file => app->home->child('cpan_deps_graph.conf')}; 20 | 21 | if (defined(my $logfile = app->config->{logfile})) { 22 | app->log->with_roles('+Clearable')->path($logfile); 23 | } 24 | 25 | push @{app->commands->namespaces}, 'CPANDepsGraph::Command'; 26 | 27 | my $mcpan = MetaCPAN::Client->new; 28 | helper mcpan => sub ($c) { $mcpan }; 29 | 30 | my $url = app->config->{redis_url}; 31 | my $redis = Mojo::Redis->new($url); 32 | helper redis => sub ($c) { $redis }; 33 | 34 | helper phases => sub ($c) { +{map { ($_ => 1) } qw(configure build test runtime develop)} }; 35 | helper relationships => sub ($c) { +{map { ($_ => 1) } qw(requires recommends suggests)} }; 36 | 37 | helper retrieve_dist_deps => sub ($c, $dist, $dist_version = undef) { 38 | return {} if $dist eq 'Acme-DependOnEverything'; # not happening 39 | my $mcpan = $c->mcpan; 40 | my $release; 41 | try { 42 | $release = $mcpan->release({ 43 | all => [ 44 | { distribution => $dist }, 45 | length($dist_version) ? { version => $dist_version } : { status => 'latest' }, 46 | ], 47 | }); 48 | $release = $release->next; 49 | } catch { return {} } 50 | return {} unless my @deps = @{ ($release && $release->dependency) || [] }; 51 | my %deps_by_module; 52 | foreach my $dep (@deps) { 53 | next if $dep->{module} eq 'perl'; 54 | next unless exists $c->phases->{$dep->{phase}}; 55 | next unless exists $c->relationships->{$dep->{relationship}}; 56 | push @{$deps_by_module{$dep->{module}}}, $dep; 57 | } 58 | my @modules = keys %deps_by_module; 59 | my @package_data; 60 | while (my @chunk = splice @modules, 0, 100) { 61 | my $url = Mojo::URL->new('https://cpanmeta.grinnz.com/api/v2/packages') 62 | ->query(module => \@chunk); 63 | push @package_data, @{getjson("$url")->{data}}; 64 | } 65 | my %deps; 66 | foreach my $package (@package_data) { 67 | my $module = $package->{module} // next; 68 | my $path = $package->{path} // next; 69 | my $distname = CPAN::DistnameInfo->new($path)->dist; 70 | next if $distname eq 'perl'; 71 | push @{$deps{$_->{phase}}{$_->{relationship}}}, {dist => $distname, module => $module, version => $_->{version}} for @{$deps_by_module{$module}}; 72 | } 73 | return \%deps; 74 | }; 75 | 76 | helper cache_dist_deps => sub ($c, $dist, $deps = undef) { 77 | $deps //= $c->retrieve_dist_deps($dist); 78 | my $redis = $c->redis->db; 79 | $redis->multi; 80 | foreach my $phase (keys %{$c->phases}) { 81 | foreach my $relationship (keys %{$c->relationships}) { 82 | my $key = "cpandeps:$dist:$phase:$relationship"; 83 | $redis->del($key); 84 | my $modules = $deps->{$phase}{$relationship} // []; 85 | $redis->set($key, to_json $modules) if @$modules; 86 | } 87 | } 88 | $redis->set('cpandeps:last-update', time); 89 | $redis->exec; 90 | }; 91 | 92 | helper cache_dist_deeply => sub ($c, $dist) { 93 | my %seen; 94 | my @to_check = $dist; 95 | while (defined(my $dist = shift @to_check)) { 96 | next if $seen{$dist}++; 97 | my $deps = $c->retrieve_dist_deps($dist); 98 | $c->cache_dist_deps($dist, $deps); 99 | foreach my $phase (keys %$deps) { 100 | foreach my $relationship (keys %{$deps->{$phase}}) { 101 | my $modules = $deps->{$phase}{$relationship}; 102 | my %dists; 103 | $dists{$_->{dist}} = 1 for @$modules; 104 | push @to_check, keys %dists; 105 | } 106 | } 107 | } 108 | }; 109 | 110 | helper get_dist_deps => sub ($c, $dist, $phases, $relationships, $perl_version, $dist_version = undef) { 111 | $perl_version = $perl_version->numify; 112 | my $redis = $c->redis->db; 113 | my %all_deps; 114 | my $versioned_deps = length($dist_version) ? $c->retrieve_dist_deps($dist, $dist_version) : undef; 115 | foreach my $phase (@$phases) { 116 | foreach my $relationship (@$relationships) { 117 | my $deps; 118 | if ($versioned_deps) { 119 | $deps = $versioned_deps->{$phase}{$relationship} // []; 120 | } else { 121 | my $key = "cpandeps:$dist:$phase:$relationship"; 122 | my $deps_json = $redis->get($key) // next; 123 | try { $deps = from_json $deps_json } catch { next } 124 | } 125 | foreach my $dep (@$deps) { 126 | try { 127 | next if Module::CoreList::is_core $dep->{module}, $dep->{version}, $perl_version; 128 | } catch {} 129 | $all_deps{$dep->{dist}} = 1; 130 | } 131 | } 132 | } 133 | return \%all_deps; 134 | }; 135 | 136 | helper dist_dep_tree => sub ($c, $dist, $phases, $relationships, $perl_version, $dist_version = undef) { 137 | my %seen; 138 | my %deps; 139 | my @to_check = {dist => $dist, version => $dist_version}; # version only for initial 140 | while (defined(my $check = shift @to_check)) { 141 | my ($dist, $d_v) = @$check{qw(dist version)}; 142 | next if $seen{$dist}++; 143 | $deps{$dist} = {}; 144 | my $dist_deps = $c->get_dist_deps($dist, $phases, $relationships, $perl_version, $d_v); 145 | foreach my $dist_dep (keys %$dist_deps) { 146 | $deps{$dist}{$dist_dep} = 1; 147 | push @to_check, {dist => $dist_dep}; 148 | } 149 | } 150 | return \%deps; 151 | }; 152 | 153 | helper dist_dep_graph => sub ($c, $dist, $phases, $relationships, $perl_version, $dist_version = undef) { 154 | my $tree = $c->dist_dep_tree($dist, $phases, $relationships, $perl_version, $dist_version); 155 | my @nodes = map { 156 | {distribution => $_, children => [sort keys %{$tree->{$_}}]} 157 | } sort keys %$tree; 158 | return \@nodes; 159 | }; 160 | 161 | helper dist_dep_table => sub ($c, $dist, $phases, $relationships, $perl_version, $dist_version = undef) { 162 | my $tree = $c->dist_dep_tree($dist, $phases, $relationships, $perl_version, $dist_version); 163 | my %seen; 164 | my @to_check = {dist => $dist, level => 1}; 165 | my @table; 166 | while (defined(my $dep = shift @to_check)) { 167 | my ($dist, $level) = @$dep{'dist','level'}; 168 | push @table, {dist => $dist, level => $level}; 169 | next if $seen{$dist}++; 170 | my @deps = sort keys %{$tree->{$dist}}; 171 | unshift @to_check, map { +{dist => $_, level => $level+1} } @deps; 172 | } 173 | return \@table; 174 | }; 175 | 176 | get '/api/v1/deps' => sub ($c) { 177 | my $dist = $c->req->param('dist'); 178 | my $dist_version = $c->req->param('dist_version'); 179 | my $phases = $c->req->every_param('phase'); 180 | $phases = ['runtime'] unless @$phases; 181 | my $relationships = $c->req->every_param('relationship'); 182 | $relationships = ['requires'] unless @$relationships; 183 | my $perl_version = $c->req->param('perl_version') // "$]"; 184 | try { $perl_version = version->parse($perl_version) } catch { $perl_version = version->parse("$]") } 185 | $c->render(json => $c->dist_dep_graph($dist, $phases, $relationships, $perl_version, $dist_version)); 186 | }; 187 | 188 | my @perl_versions = uniq_by { $_->normal } grep { $_ < '5.006' or !($_->{version}[1] % 2) } 189 | map { version->parse($_) } sort {$b <=> $a} keys %Module::CoreList::released; 190 | 191 | get '/' => sub ($c) { 192 | $c->stash(perl_versions => \@perl_versions); 193 | $c->stash(dist => my $dist = $c->req->param('dist')); 194 | $c->stash(dist_version => my $dist_version = $c->req->param('dist_version')); 195 | if (length $dist and $dist =~ m/::/) { 196 | my $mcpan = $c->mcpan; 197 | try { 198 | my $module = $mcpan->module($dist, {fields => ['distribution']}); 199 | return $c->redirect_to($c->url_with->query({dist => $module->distribution})); 200 | } catch {} 201 | } 202 | $c->stash(style => my $style = $c->req->param('style')); 203 | $c->stash(phase => my $phase = $c->req->param('phase')); 204 | $c->stash(recommends => my $recommends = $c->req->param('recommends')); 205 | $c->stash(suggests => my $suggests = $c->req->param('suggests')); 206 | my $perl_version = $c->req->param('perl_version') || "$]"; 207 | try { $perl_version = version->parse($perl_version) } catch { $perl_version = version->parse("$]") } 208 | $c->stash(perl_version => $perl_version); 209 | if (($style // '') eq 'table' and length $dist) { 210 | my $phases = ['runtime']; 211 | $phase //= 'runtime'; 212 | if ($phase eq 'build') { 213 | push @$phases, 'configure', 'build'; 214 | } elsif ($phase eq 'test') { 215 | push @$phases, 'configure', 'build', 'test'; 216 | } elsif ($phase eq 'configure') { 217 | $phases = ['configure']; 218 | } 219 | my $relationships = ['requires']; 220 | push @$relationships, 'recommends' if $recommends; 221 | push @$relationships, 'suggests' if $suggests; 222 | $c->stash(deps => $c->dist_dep_table($dist, $phases, $relationships, $perl_version, $dist_version)); 223 | } 224 | $c->render; 225 | } => 'graph'; 226 | 227 | app->start; 228 | -------------------------------------------------------------------------------- /cpanfile: -------------------------------------------------------------------------------- 1 | requires 'perl' => '5.020'; 2 | requires 'lib::relative'; 3 | requires 'version'; 4 | requires 'CPAN::DistnameInfo'; 5 | requires 'Cpanel::JSON::XS' => '4.11'; 6 | requires 'IO::Socket::SSL' => '1.56'; 7 | requires 'HTTP::Simple' => '0.004'; 8 | requires 'List::Util'; 9 | requires 'List::UtilsBy'; 10 | requires 'MetaCPAN::Client'; 11 | requires 'Module::CoreList' => '2.99'; 12 | requires 'Mojolicious' => '8.0'; 13 | requires 'Mojo::Log::Clearable'; 14 | requires 'Mojo::Redis'; 15 | requires 'Syntax::Keyword::Try'; 16 | requires 'Time::Piece'; 17 | requires 'Time::Seconds'; 18 | recommends 'Protocol::Redis::XS' => '0.06'; 19 | -------------------------------------------------------------------------------- /lib/CPANDepsGraph/Command/cache.pm: -------------------------------------------------------------------------------- 1 | package CPANDepsGraph::Command::cache; 2 | 3 | use 5.020; 4 | use Mojo::Base 'Mojolicious::Command', -signatures; 5 | use Mojo::Util 'getopt'; 6 | use Time::Piece; 7 | use Time::Seconds; 8 | 9 | sub run ($self, @args) { 10 | getopt \@args, 11 | 'all|a' => \my $all, 12 | 'since|s=s' => \my $since, 13 | 'random|r:i' => \my $random, 14 | 'deeply|d' => \my $deeply; 15 | $deeply = 0 if $all; 16 | 17 | my @dists = @args; 18 | 19 | if ($all) { 20 | my $mcpan = $self->app->mcpan; 21 | my $dists_rs = $mcpan->all('distributions', {fields => ['name']}); 22 | @dists = (); 23 | while (my $dist = $dists_rs->next) { 24 | push @dists, $dist->name; 25 | } 26 | } elsif (defined $since) { 27 | if ($since eq 'last') { 28 | my $redis = $self->app->redis->db; 29 | my $last_epoch = $redis->get('cpandeps:last-update') // time - ONE_DAY; 30 | $since = gmtime($last_epoch - 3 * ONE_HOUR)->datetime; 31 | } 32 | my $mcpan = $self->app->mcpan; 33 | my $releases_rs = $mcpan->all('releases', { 34 | fields => ['distribution'], 35 | es_filter => {and => [ 36 | {range => {date => {gte => $since}}}, 37 | {term => {status => 'latest'}}, 38 | ]}, 39 | }); 40 | @dists = (); 41 | while (my $release = $releases_rs->next) { 42 | push @dists, $release->distribution; 43 | } 44 | } elsif (defined $random) { 45 | my $mcpan = $self->app->mcpan; 46 | my $dists_rs = $mcpan->all('distributions', {fields => ['name']}); 47 | my $total = $dists_rs->total; 48 | $random = 1 + int rand $total unless $random; 49 | my %indexes = map { +int(rand $total) => 1 } 1..$random; 50 | @dists = (); 51 | my $i = 0; 52 | while (my $dist = $dists_rs->next) { 53 | last unless keys %indexes; 54 | push @dists, $dist->name if delete $indexes{$i}; 55 | } continue { $i++ } 56 | } 57 | 58 | foreach my $dist (@dists) { 59 | if ($deeply) { 60 | $self->app->cache_dist_deeply($dist); 61 | } else { 62 | $self->app->cache_dist_deps($dist); 63 | } 64 | print "Cached dependencies for $dist\n"; 65 | } 66 | } 67 | 68 | 1; 69 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Grinnz/cpan-deps-graph/6fcb1faa2d27695c8616e3a4beecda5e5ec38970/public/favicon.ico -------------------------------------------------------------------------------- /public/graph.js: -------------------------------------------------------------------------------- 1 | // Update the application $VERSION when deploying a new version of this file 2 | function populate_graph(data) { 3 | var nodes = []; 4 | var edges = []; 5 | data.forEach(function(elem) { 6 | var dist = elem.distribution; 7 | nodes.push({data: {id: dist, label: dist}}); 8 | elem.children.forEach(function(child) { 9 | edges.push({data: {source: dist, target: child}}); 10 | }); 11 | }); 12 | return {nodes: nodes, edges: edges}; 13 | } 14 | 15 | function create_graph(elements, graphstyle, root) { 16 | var layout; 17 | if (graphstyle === 'topdown') { 18 | layout = { 19 | name: 'breadthfirst', 20 | directed: true, 21 | spacingFactor: 1, 22 | roots: '#' + root 23 | }; 24 | } else if (graphstyle === 'concentric') { 25 | layout = { 26 | name: 'breadthfirst', 27 | circle: true, 28 | directed: true, 29 | spacingFactor: 1, 30 | roots: '#' + root 31 | }; 32 | } else if (graphstyle === 'circle') { 33 | layout = { 34 | name: 'circle', 35 | spacingFactor: 0.5 36 | }; 37 | } else { 38 | layout = { 39 | name: graphstyle 40 | }; 41 | } 42 | var cy = cytoscape({ 43 | container: document.getElementById('deps'), 44 | elements: elements, 45 | minZoom: 0.1, 46 | maxZoom: 2, 47 | wheelSensitivity: 0.5, 48 | style: [ 49 | { 50 | selector: 'node', 51 | style: { 52 | label: 'data(label)', 53 | 'background-color': '#eeeeee', 54 | width: 'label', 55 | shape: 'round-rectangle', 56 | 'text-valign': 'center' 57 | } 58 | }, 59 | { 60 | selector: 'edge', 61 | style: { 62 | width: 1.5, 63 | 'curve-style': 'straight', 64 | 'target-arrow-shape': 'vee', 65 | 'arrow-scale': 1.5 66 | }, 67 | } 68 | ], 69 | layout: layout 70 | }); 71 | cy.on('tap', 'node', function(event) { 72 | var distname = event.target.data('label'); 73 | document.getElementById('form-dist-name').setAttribute('value', distname); 74 | document.getElementById('form-dist-version').setAttribute('value', ''); 75 | }); 76 | } 77 | 78 | function retrieve_graph() { 79 | var params = new URLSearchParams(window.location.search.substring(1)); 80 | var dist = params.get('dist'); 81 | var dist_version = params.get('dist_version'); 82 | if (dist === null || dist === '') { return null; } 83 | var graphstyle = params.get('style'); 84 | var phase = params.get('phase'); 85 | var recommends = params.get('recommends'); 86 | var suggests = params.get('suggests'); 87 | var perl_version = params.get('perl_version'); 88 | 89 | var deps_url = new URL('/api/v1/deps', window.location.href); 90 | deps_url.searchParams.set('dist', dist); 91 | if (dist_version !== null && dist_version !== '') 92 | deps_url.searchParams.set('dist_version', dist_version); 93 | deps_url.searchParams.set('phase', 'runtime'); 94 | if (phase === 'build') { 95 | deps_url.searchParams.append('phase', 'configure'); 96 | deps_url.searchParams.append('phase', 'build'); 97 | } else if (phase === 'test') { 98 | deps_url.searchParams.append('phase', 'configure'); 99 | deps_url.searchParams.append('phase', 'build'); 100 | deps_url.searchParams.append('phase', 'test'); 101 | } else if (phase === 'configure') { 102 | deps_url.searchParams.set('phase', 'configure'); 103 | } 104 | deps_url.searchParams.set('relationship', 'requires'); 105 | if (recommends) { deps_url.searchParams.append('relationship', 'recommends'); } 106 | if (suggests) { deps_url.searchParams.append('relationship', 'suggests'); } 107 | if (perl_version !== null && perl_version !== '') { 108 | deps_url.searchParams.set('perl_version', perl_version); 109 | } 110 | fetch(deps_url).then(function(response) { 111 | if (response.ok) { 112 | return response.json(); 113 | } else { 114 | throw new Error(response.status + ' ' + response.statusText); 115 | } 116 | }).then(function(data) { 117 | if (graphstyle === null || graphstyle === 'auto') { 118 | graphstyle = data.every(function(elem) { return elem.children.length <= 10 ? true : false }) ? 'topdown' : 'concentric'; 119 | } 120 | var elements = populate_graph(data); 121 | create_graph(elements, graphstyle, dist); 122 | }).catch(function(error) { 123 | console.log('Error retrieving dependencies', error); 124 | }); 125 | } 126 | 127 | retrieve_graph(); 128 | -------------------------------------------------------------------------------- /templates/graph.html.ep: -------------------------------------------------------------------------------- 1 | % layout 'main'; 2 | % if (($style // '') eq 'table') { 3 | % my $deps = stash('deps'); 4 |
5 |
6 | 7 | % use List::Util 'max'; 8 | % my $maxlevel = @$deps ? max map { $_->{level} } @$deps : 0; 9 | 10 | % my %query = (style => $style, phase => $phase, perl_version => $perl_version); 11 | % $query{recommends} = $recommends if $recommends; 12 | % $query{suggests} = $suggests if $suggests; 13 | % my $query = join '&', map { "$_=$query{$_}" } sort keys %query; 14 | % foreach my $row (@$deps) { 15 | 16 | % foreach my $i (1..$row->{level}-1) { 17 | 18 | % } 19 | 20 | 21 | % } 22 | 23 |
{level} > 0) { %> colspan="<%= $maxlevel - $row->{level} + 1 %>"<% } %>><%= $row->{dist} %>
24 |
25 |
26 | % } else { 27 |
28 |
29 |
30 | 31 | 32 | 33 | 34 | % } 35 | -------------------------------------------------------------------------------- /templates/layouts/main.html.ep: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <% if (length $dist) { %><%= $dist %> - <% } %>CPAN Dependencies Graph 6 | 7 | % if (defined config->{google_analytics_tracking_id}) { 8 | 9 | 10 | 17 | % } 18 | 19 | 20 |
21 | <% if (($style // '') ne 'table') { %>
<% } %> 22 |
23 |
24 | 25 | 26 | 27 | 28 | 33 |
34 | checked<% } %>> 35 | 36 |
37 |
38 | checked<% } %>> 39 | 40 |
41 | 42 | 47 | 55 | 56 | GitHub 57 |
58 |
59 | <% if (($style // '') ne 'table') { %>
<% } %> 60 | %= content 61 |
62 | 63 | 64 | --------------------------------------------------------------------------------