├── .gitignore
├── pkgupdate-count.sh
├── pkgupdate-diff.sh
├── pkgupdate-watch.sh
├── shared
    ├── make-index.pl
    ├── config.pm
    ├── relman.cfg.example
    ├── patch-index.pl
    └── functions.sh
├── pkgupdate-build.sh
└── pkgupdate-report.pl


/.gitignore:
--------------------------------------------------------------------------------
1 | .cache
2 | shared/relman.cfg
3 | 


--------------------------------------------------------------------------------
/pkgupdate-count.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | [ -f shared/functions.sh ] || {
 4 | 	echo "Please execute as ./${0##*/}" >&2
 5 | 	exit 1
 6 | }
 7 | 
 8 | watch -n 5 '
 9 | 	. ./shared/functions.sh
10 | 
11 | 	for target in $(fetch_remote_targets); do
12 | 		echo -n "$target: "
13 | 		find "$CACHE_DIR/repo-local/$target" -name "*.ipk" 2>/dev/null | wc -l
14 | 	done
15 | '
16 | 


--------------------------------------------------------------------------------
/pkgupdate-diff.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | [ -f shared/functions.sh ] || {
 4 | 	echo "Please execute as ./${0##*/}" >&2
 5 | 	exit 1
 6 | }
 7 | 
 8 | . ./shared/functions.sh
 9 | 
10 | for target in $(fetch_remote_targets); do
11 | 	for feed in $(fetch_remote_feeds "$target"); do
12 | 		R="$CACHE_DIR/mirror/$target/packages/$feed"
13 | 		L="$CACHE_DIR/repo-local/$target/packages/$feed"
14 | 
15 | 		if [ -s "$L/Packages" -a -s "$R/Packages.gz" ]; then
16 | 			echo -en "\nTarget $target Feed $feed\n\n"
17 | 			zcat "$R/Packages.gz" > "$R/Packages"
18 | 			diff -u "$R/Packages" "$L/Packages"
19 | 			rm "$R/Packages"
20 | 		fi
21 | 	done
22 | done | less
23 | 


--------------------------------------------------------------------------------
/pkgupdate-watch.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | [ -f shared/functions.sh ] || {
 4 | 	echo "Please execute as ./${0##*/}" >&2
 5 | 	exit 1
 6 | }
 7 | 
 8 | . ./shared/functions.sh
 9 | 
10 | trap 'clear; exit 0' SIGINT SIGTERM
11 | 
12 | clear
13 | 
14 | while true; do
15 | 	echo -en "\033[0;0f"
16 | 	fetch_remote_targets | while read target; do (
17 | 		if [ -d "$CACHE_DIR/sdk/$target" ]; then
18 | 			cd "$CACHE_DIR/sdk/$target"
19 | 			log="$(find logs/ -type f -name compile.txt -printf '%C@ %h\n' 2>/dev/null | \
20 | 				sort -nr | sed -ne '1s/^[0-9.]\+ //p')"
21 | 
22 | 			if [ -d "$log" ]; then
23 | 				d1=$(date +%s)
24 | 				d2=$(date +%s -r "$log/compile.txt")
25 | 
26 | 				if [ $(($d1 - $d2)) -gt 5 ]; then
27 | 					log="- idle -"
28 | 					msg=""
29 | 				else
30 | 					msg="$(tail -n1 "$log/compile.txt")"
31 | 					if [ ${#msg} -gt 80 ]; then
32 | 						msg="${msg:0:80}"
33 | 					fi
34 | 				fi
35 | 			else
36 | 				log="- pending -"
37 | 				msg=""
38 | 			fi
39 | 		else
40 | 			log="- uninitialized -"
41 | 			msg=""
42 | 		fi
43 | 
44 | 		printf "\033[K%-20s %-16s %s\n" "[$target]" "[${log##*/}]" "$msg"
45 | 	); done
46 | 	sleep 1
47 | done
48 | 


--------------------------------------------------------------------------------
/shared/make-index.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | 
 3 | use strict;
 4 | use warnings;
 5 | use POSIX;
 6 | 
 7 | my $dir = $ARGV[0];
 8 | 
 9 | die "Usage: $0 <package directory>\n" unless -d $dir;
10 | 
11 | setlocale(LC_ALL, "C");
12 | 
13 | sub pkg_metadata
14 | {
15 | 	my $file = shift || return;
16 | 
17 | 	return undef unless -f $file;
18 | 
19 | 	my $size = -s $file;
20 | 	my $md5sum;
21 | 	my $sha256sum;
22 | 
23 | 	if (open MD5, "md5sum $file |")
24 | 	{
25 | 		if (defined(my $line = readline MD5))
26 | 		{
27 | 			($md5sum) = $line =~ /^([0-9a-fA-F]{32})/;
28 | 		}
29 | 
30 | 		close MD5;
31 | 	}
32 | 
33 | 	if (open SHA256, "openssl dgst -sha256 $file |")
34 | 	{
35 | 		if (defined(my $line = readline SHA256))
36 | 		{
37 | 			($sha256sum) = $line =~ /([0-9a-fA-f]{64})$/;
38 | 		}
39 | 
40 | 		close SHA256;
41 | 	}
42 | 
43 | 	return undef unless $md5sum && $sha256sum;
44 | 
45 | 	my $meta = '';
46 | 
47 | 	if (open TAR, "tar -xzOf $file ./control.tar.gz | tar -xzOf - ./control |")
48 | 	{
49 | 		while (defined(my $line = readline TAR))
50 | 		{
51 | 			if ($line =~ /^Description:/)
52 | 			{
53 | 				$meta .= sprintf "Size: %d\n", $size;
54 | 				$meta .= sprintf "MD5Sum: %s\n", $md5sum;
55 | 				$meta .= sprintf "SHA256sum: %s\n", $sha256sum;
56 | 			}
57 | 
58 | 			$meta .= $line;
59 | 		}
60 | 
61 | 		$meta .= "\n";
62 | 
63 | 		close TAR;
64 | 	}
65 | 
66 | 	return $meta;
67 | }
68 | 
69 | 
70 | my @packages;
71 | 
72 | if (opendir D, $dir)
73 | {
74 | 	while (defined(my $e = readdir D))
75 | 	{
76 | 		next unless -f "$dir/$e" && $e =~ m{\.ipk$};
77 | 		push @packages, $e;
78 | 	}
79 | 
80 | 	closedir D;
81 | }
82 | 
83 | foreach my $package (sort @packages)
84 | {
85 | 	print pkg_metadata("$dir/$package");
86 | }
87 | 


--------------------------------------------------------------------------------
/shared/config.pm:
--------------------------------------------------------------------------------
 1 | package config;
 2 | 
 3 | use Exporter 'import';
 4 | 
 5 | @EXPORT = qw(%conf);
 6 | 
 7 | our %conf;
 8 | 
 9 | if (open CONF, "< shared/relman.cfg")
10 | {
11 | 	while (defined(my $line = readline CONF))
12 | 	{
13 | 		chomp $line;
14 | 
15 | 		if ($line =~ m!^\s*(\w+)(\{\w+\}|\[\d+\])?\s*=\s*(\S|\S.*\S)\s*$!)
16 | 		{
17 | 			my ($key, $idx, $val) = ($1, $2, $3);
18 | 
19 | 			$val =~ s!\$(\w+|\{\w+\})!
20 | 				exists($conf{$1}) ? $conf{$1} :
21 | 					(exists($ENV{$1}) ? $ENV{$1} : '')
22 | 			!eg;
23 | 
24 | 			if ($idx && $idx =~ m!^\{(\w+)\}$!)
25 | 			{
26 | 				$conf{$key} = { }
27 | 					unless exists($conf{$key}) && ref($conf{$key}) eq 'HASH';
28 | 
29 | 				$conf{$key}{$1} = $val;
30 | 			}
31 | 			elsif ($idx && $idx =~ m!^\[(\d+)\]$!)
32 | 			{
33 | 				$conf{$key} = [ ]
34 | 					unless exists($conf{$key}) && ref($conf{$key}) eq 'ARRAY';
35 | 
36 | 				$conf{$key}[int($1)] = $val;
37 | 			}
38 | 			else
39 | 			{
40 | 				$conf{$key} = $val;
41 | 			}
42 | 		}
43 | 	}
44 | 
45 | 	close CONF;
46 | }
47 | else
48 | {
49 | 	die "Unable to open configuration: $!\n";
50 | }
51 | 
52 | sub sq($)
53 | {
54 | 	my $s = $_[0];
55 | 	   $s =~ s/'/'"'"'/g;
56 | 
57 | 	return $s;
58 | }
59 | 
60 | unless (caller)
61 | {
62 | 	foreach my $key (sort keys %conf)
63 | 	{
64 | 		if (ref($conf{$key}) eq 'HASH')
65 | 		{
66 | 			foreach my $skey (sort keys %{$conf{$key}})
67 | 			{
68 | 				printf("%s_%s='%s'\n",
69 | 					uc($key), uc($skey), sq($conf{$key}{$skey}));
70 | 			}
71 | 		}
72 | 		elsif (ref($conf{$key}) eq 'ARRAY')
73 | 		{
74 | 			printf("%s='", uc($key));
75 | 
76 | 			my $first = 1;
77 | 			foreach my $val (@{$conf{$key}})
78 | 			{
79 | 				printf("%s%s", $first ? '' : ' ', sq($val));
80 | 				$first = 0;
81 | 			}
82 | 
83 | 			printf("'\n");
84 | 		}
85 | 		else
86 | 		{
87 | 			printf("%s='%s'\n", uc($key), sq($conf{$key}));
88 | 		}
89 | 	}
90 | }
91 | 
92 | 1;
93 | 


--------------------------------------------------------------------------------
/shared/relman.cfg.example:
--------------------------------------------------------------------------------
 1 | # This is an example configuration for the release management scripts.
 2 | # Rename the file to `relman.cfg` and adjust the values as needed.
 3 | 
 4 | # Common release properties
 5 | release_tag  = CC 15.05
 6 | release_name = Chaos Calmer 15.05
 7 | release_feed = http://git.openwrt.org/15.05/openwrt.git
 8 | release_url  = openwrt@downloads.openwrt.org:chaos_calmer/15.05
 9 | # release_url = https://downloads.openwrt.org/chaos_calmer/15.05
10 | # release_url = file:///CC/sync/chaos_calmer/15.05
11 | 
12 | # Cache/build directory
13 | cache_dir = $PWD/.cache
14 | 
15 | # SSH identity (used for rsync)
16 | ssh_ident = $HOME/.ssh/id_rsa_openwrt_rsync
17 | 
18 | # File glob patterns (used for rsync/lftp)
19 | pattern{sdk}  = OpenWrt-SDK-*.tar.bz2
20 | pattern{feed} = Packages.*
21 | 
22 | # Recipient list for package update notifications
23 | recipients_standard[0] = openwrt-devel@lists.openwrt.org
24 | recipients_standard[1] = openwrt-users@lists.openwrt.org
25 | 
26 | # Recipient list for security critical updates
27 | recipients_security[0] = openwrt-security-announce@lists.openwrt.org
28 | recipients_security[1] = openwrt-devel@lists.openwrt.org
29 | recipients_security[2] = openwrt-users@lists.openwrt.org
30 | 
31 | # Mapping of feed names to respective online source browser
32 | commitlink{base}       = http://git.openwrt.org/?p=15.05/openwrt.git;a=commit;h=%s
33 | commitlink{luci}       = https://github.com/openwrt/luci/commit/%s
34 | commitlink{management} = https://github.com/openwrt-management/packages/commit/%s
35 | commitlink{packages}   = https://github.com/openwrt/packages/commit/%s
36 | commitlink{routing}    = https://github.com/openwrt-routing/packages/commit/%s
37 | commitlink{telephony}  = https://github.com/openwrt/telephony/commit/%s
38 | 
39 | # SMTP settings for mail notifications
40 | smtp{from} = example@openwrt.org
41 | smtp{host} = mail.example.org
42 | smtp{user} = example
43 | smtp{pass} = secret
44 | 


--------------------------------------------------------------------------------
/pkgupdate-build.sh:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env bash
  2 | 
  3 | [ -f shared/functions.sh ] || {
  4 | 	echo "Please execute as ./${0##*/}" >&2
  5 | 	exit 1
  6 | }
  7 | 
  8 | . ./shared/functions.sh
  9 | 
 10 | terminate() {
 11 | 	local jobs="$(jobs -p)"
 12 | 
 13 | 	echo "Terminating..."
 14 | 
 15 | 	pkill -P $$
 16 | 	exit 1
 17 | }
 18 | 
 19 | trap terminate INT TERM
 20 | 
 21 | do_clean=0
 22 | do_build=0
 23 | do_index=0
 24 | do_rsync=0
 25 | do_dependants=0
 26 | do_update=0
 27 | 
 28 | num_jobs=$(grep processor /proc/cpuinfo | wc -l)
 29 | 
 30 | use_targets=""
 31 | use_packages=""
 32 | use_sources=""
 33 | 
 34 | usage() {
 35 | 	cat <<-EOT
 36 | $0 {-a|-c|-i|-r} [-j jobs] -p package [-p package ...]
 37 | 
 38 |   -a
 39 |      Perform all build steps (compile, rebuild, rsync)
 40 | 
 41 |   -c
 42 |      Perform clean step in SDK.
 43 | 
 44 |   -b
 45 |      Compile packages for all architectures.
 46 | 
 47 |   -i
 48 |      Rebuild indexes for all architectures.
 49 | 
 50 |   -r
 51 |      Rsync files for all architecures.
 52 | 
 53 |   -d
 54 |      Process dependant packages as well (useful for libraries)
 55 | 
 56 |   -u
 57 |      Update remote metadata.
 58 | 
 59 |   -j jobs
 60 |      Use the given number of jobs when compiling packages.
 61 |      Default is $num_jobs jobs.
 62 | 
 63 |   -p package[:feed]
 64 |      Add given package to the queue. If a feed suffix is
 65 |      given then prefer this feed name when installing the
 66 |      package into the SDK.
 67 | 
 68 |   -t target
 69 |      Restrict operation to given targets.
 70 | 
 71 |   -D
 72 |      Print download URL and exit.
 73 | 
 74 |   -U
 75 |      Print upload URL and exit.
 76 | 
 77 | 	EOT
 78 | 	exit 1
 79 | }
 80 | 
 81 | while getopts ":abcirduj:p:s:t:" opt; do
 82 | 	case "$opt" in
 83 | 		a)
 84 | 			do_clean=1
 85 | 			do_build=1
 86 | 			do_index=1
 87 | 			do_rsync=1
 88 | 		;;
 89 | 		c) do_clean=1 ;;
 90 | 		b) do_build=1 ;;
 91 | 		i) do_index=1 ;;
 92 | 		r) do_rsync=1 ;;
 93 | 		d) do_dependants=1 ;;
 94 | 		u) do_update=1 ;;
 95 | 		j) num_jobs="$OPTARG" ;;
 96 | 		p) use_packages="${use_packages:+$use_packages }$OPTARG" ;;
 97 | 		s) use_sources="${use_sources:+$use_sources }$OPTARG" ;;
 98 | 		t) use_targets="${use_targets:+$use_targets }$OPTARG" ;;
 99 | 	esac
100 | done
101 | 
102 | [ -z "$use_packages" ] && [ -z "$use_sources" ] && [ $do_update -lt 1 ] && usage
103 | 
104 | if [ $do_clean -gt 0 ]; then
105 | 	echo "* Purging local cache"
106 | 	rm -rf "$CACHE_DIR/repo-local"
107 | fi
108 | 
109 | if [ $do_update -gt 0 ] || [ ! -d "$CACHE_DIR/mirror" ]; then
110 | 	mkdir -p "$CACHE_DIR"
111 | 	echo "* Preparing metadata"
112 | 	rm -f "$CACHE_DIR/.mirrored"
113 | 	rm -rf "$CACHE_DIR/repo-remote"/*/*/packages
114 | 	fetch_remote_index
115 | fi
116 | 
117 | if [ -n "$use_sources" ]; then
118 | 	for src in $use_sources; do
119 | 		for pkg in $(find_source_provided_pkgs "$src"); do
120 | 			echo "* Selecting package $pkg for source $src"
121 | 			use_packages="${use_packages:+$use_packages }$pkg"
122 | 		done
123 | 	done
124 | fi
125 | 
126 | [ -n "$use_packages" ] && run_jobs $use_packages
127 | 


--------------------------------------------------------------------------------
/shared/patch-index.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/perl
  2 | 
  3 | use strict;
  4 | use warnings;
  5 | use Getopt::Long;
  6 | use POSIX;
  7 | 
  8 | my $index;
  9 | my @add;
 10 | my @remove;
 11 | 
 12 | GetOptions(
 13 | 	"index=s"  => \$index,
 14 | 	"add=s"    => \@add,
 15 | 	"remove=s" => \@remove
 16 | );
 17 | 
 18 | 
 19 | unless (defined($index) && -f $index && (@add || @remove))
 20 | {
 21 | 	die "Usage: $0 --index <pkg index> --add <pkg> --remove <pkg>\n";
 22 | }
 23 | 
 24 | setlocale(LC_ALL, "C");
 25 | 
 26 | sub pkg_basename
 27 | {
 28 | 	my $file = shift || return;
 29 | 
 30 | 	return undef unless -f $file;
 31 | 
 32 | 	my ($name) = $file =~ m!^(?:.+/)?([^_/]+)[^/]+\.ipk$!;
 33 | 
 34 | 	return $name;
 35 | }
 36 | 
 37 | sub pkg_metadata
 38 | {
 39 | 	my $file = shift || return;
 40 | 
 41 | 	return undef unless -f $file;
 42 | 
 43 | 	my $size = -s $file;
 44 | 	my $md5sum;
 45 | 	my $sha256sum;
 46 | 
 47 | 	if (open MD5, "md5sum $file |")
 48 | 	{
 49 | 		if (defined(my $line = readline MD5))
 50 | 		{
 51 | 			($md5sum) = $line =~ /^([0-9a-fA-F]{32})/;
 52 | 		}
 53 | 
 54 | 		close MD5;
 55 | 	}
 56 | 
 57 | 	if (open SHA256, "openssl dgst -sha256 $file |")
 58 | 	{
 59 | 		if (defined(my $line = readline SHA256))
 60 | 		{
 61 | 			($sha256sum) = $line =~ /([0-9a-fA-f]{64})$/;
 62 | 		}
 63 | 
 64 | 		close SHA256;
 65 | 	}
 66 | 
 67 | 	return undef unless $md5sum && $sha256sum;
 68 | 
 69 | 	my $meta = '';
 70 | 	my ($name) = $file =~ m!([^/]+)$!;
 71 | 
 72 | 	if (open TAR, "tar -xzOf $file ./control.tar.gz | tar -xzOf - ./control |")
 73 | 	{
 74 | 		while (defined(my $line = readline TAR))
 75 | 		{
 76 | 			if ($line =~ /^Description:/)
 77 | 			{
 78 | 				$meta .= sprintf "Filename: %s\n", $name;
 79 | 				$meta .= sprintf "Size: %d\n", $size;
 80 | 				$meta .= sprintf "MD5Sum: %s\n", $md5sum;
 81 | 				$meta .= sprintf "SHA256sum: %s\n", $sha256sum;
 82 | 			}
 83 | 
 84 | 			$meta .= $line;
 85 | 		}
 86 | 
 87 | 		$meta .= "\n";
 88 | 
 89 | 		close TAR;
 90 | 	}
 91 | 
 92 | 	# Fix up source
 93 | 	$meta =~ s!^Source: .+/(feeds/.+)$!Source: $1!m;
 94 | 	$meta =~ s!^Source: feeds/base/!Source: !m;
 95 | 
 96 | 	return $meta;
 97 | }
 98 | 
 99 | 
100 | my %addpkg;
101 | my %rempkg;
102 | 
103 | foreach my $pkg (@add)
104 | {
105 | 	my $name = pkg_basename($pkg);
106 | 	my $meta = pkg_metadata($pkg);
107 | 
108 | 	if (!defined $name || !defined $meta)
109 | 	{
110 | 		warn "$pkg is not a valid .ipk file\n";
111 | 		next;
112 | 	}
113 | 
114 | 	$addpkg{$name} = $meta;
115 | 	$rempkg{$name} = 1;
116 | }
117 | 
118 | my ($cur_pkg, $cur_meta);
119 | my $cat = ($index =~ /\.gz$/) ? "zcat" : "cat";
120 | 
121 | @rempkg{@remove} = (1) x @remove;
122 | 
123 | if (open IDX, "$cat $index |")
124 | {
125 | 	while (1)
126 | 	{
127 | 		my $line = readline IDX;
128 | 
129 | 		if (defined($line) && $line =~ /^Package: (.+)$/)
130 | 		{
131 | 			$cur_pkg = $1;
132 | 			$cur_meta = $line;
133 | 		}
134 | 		elsif (defined($line) && $line !~ /^$/ && defined($cur_meta))
135 | 		{
136 | 			$cur_meta .= $line;
137 | 		}
138 | 		else
139 | 		{
140 | 			foreach my $add (sort keys %addpkg)
141 | 			{
142 | 				if (!defined($cur_pkg) || $add le $cur_pkg)
143 | 				{
144 | 					print $addpkg{$add};
145 | 					delete $addpkg{$add};
146 | 
147 | 					#$cur_pkg = $add;					
148 | 				}
149 | 			}
150 | 			
151 | 			if (defined($cur_pkg) && !$rempkg{$cur_pkg})
152 | 			{
153 | 				print $cur_meta, "\n";
154 | 			}
155 | 
156 | 			undef $cur_pkg;
157 | 			undef $cur_meta;
158 | 		}
159 | 
160 | 		last unless defined $line;
161 | 	}
162 | 
163 | 	close IDX;
164 | }
165 | 


--------------------------------------------------------------------------------
/pkgupdate-report.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env perl
  2 | 
  3 | use strict;
  4 | use warnings;
  5 | 
  6 | use lib "shared";
  7 | use config;
  8 | 
  9 | use Cwd;
 10 | use Encode;
 11 | use MIME::Lite;
 12 | use File::Basename;
 13 | 
 14 | use Getopt::Long;
 15 | 
 16 | my $text_width = 72;
 17 | 
 18 | my $do_check   = 0;
 19 | my $do_preview = 0;
 20 | my $do_send    = 0;
 21 | my $do_pull    = 0;
 22 | 
 23 | my $changetime;
 24 | my @use_packages;
 25 | my @use_cve;
 26 | my @use_ref;
 27 | 
 28 | GetOptions(
 29 | 	'check'     => \$do_check,
 30 | 	'preview'   => \$do_preview,
 31 | 	'send'      => \$do_send,
 32 | 	'pull'      => \$do_pull,
 33 | 	'since=s'   => \$changetime,
 34 | 	'package=s' => \@use_packages,
 35 | 	'cve=s'     => \@use_cve,
 36 | 	'ref=s'     => \@use_ref,
 37 | );
 38 | 
 39 | unless ($do_check || $do_preview || $do_send)
 40 | {
 41 | 	die <<EOT;
 42 | 
 43 | Check for updates:
 44 | 
 45 | 	$0 [--pull] [--since ...] [--package ...] --check
 46 | 
 47 | Generate mail report:
 48 | 
 49 | 	$0 [--cve ...] [--ref ...] [--since ...] [--package ...] {--preview|--send}
 50 | 
 51 | Actions:
 52 | 
 53 | 	--check    Test for package changes since last sync or the time specified
 54 | 	           by `--since`
 55 | 
 56 | 	--preview  Display the mail report that would be sent
 57 | 
 58 | 	--send     Send mail report
 59 | 
 60 | Options:
 61 | 
 62 | 	--pull     Perform a `git pull` in the feed clones before checking changes
 63 | 
 64 | 	--since <TIME>
 65 | 	           Specify the start time when checking changes
 66 | 
 67 | 	--package <NAME>
 68 | 	           Only consider given source package, multiple allowed
 69 | 
 70 | 	--cve <CVE>
 71 | 	           Include given CVE number in report
 72 | 
 73 | 	--ref <URL>
 74 | 	           Include given url in report
 75 | 	
 76 | EOT
 77 | }
 78 | 
 79 | sub last_change_time {
 80 | 	my $changetime = 0;
 81 | 
 82 | 	if (open my $find, 'find .cache/repo-remote -type f -name Packages.gz -printf "%C@\n" |')
 83 | 	{
 84 | 		while (defined(my $timestamp = readline $find))
 85 | 		{
 86 | 			$timestamp = int $timestamp;
 87 | 			$changetime = $timestamp if ($timestamp > $changetime);
 88 | 		}
 89 | 
 90 | 		close $find;
 91 | 	}
 92 | 
 93 | 	return $changetime;
 94 | }
 95 | 
 96 | sub wrap_paragraph {
 97 | 	my $pad1 = shift;
 98 | 	my $pad2 = shift;
 99 | 	my @words = map { split /\s+/ } @_;
100 | 	my $line = '';
101 | 	my @out;
102 | 
103 | 	foreach my $word (@words)
104 | 	{
105 | 		my $line_len = length($line);
106 | 
107 | 		if (($line_len + length($word) + 1) > $text_width)
108 | 		{
109 | 			push @out, $pad1 . (@out > 0 ? $pad2 : '') . $line;
110 | 			$line = $word;
111 | 		}
112 | 		else
113 | 		{
114 | 			$line .= ($line_len ? ' ' : '') . $word;
115 | 		}
116 | 	}
117 | 
118 | 	push @out, $pad1 . (@out > 0 ? $pad2 : '') . $line;
119 | 	return @out;
120 | }
121 | 
122 | sub format_paragraph {
123 | 	my $para = shift || return '';
124 | 
125 | 	my @lines = split /\n/, $para;
126 | 	my @part;
127 | 	my @out;
128 | 
129 | 	foreach my $line (@lines)
130 | 	{
131 | 		if ($line =~ /^\s*([*-]|\d+\))\s+/)
132 | 		{
133 | 			my $pad = ' ' x (length($1) + 1);
134 | 
135 | 			if (@part > 0)
136 | 			{
137 | 				push @out, wrap_paragraph('', '', @part);
138 | 				@part = ();
139 | 			}
140 | 
141 | 			push @out, wrap_paragraph(' ', $pad, $line);
142 | 		}
143 | 		else
144 | 		{
145 | 			push @part, $line;
146 | 		}
147 | 	}
148 | 
149 | 	if (@part > 0)
150 | 	{
151 | 		push @out, wrap_paragraph('', '', @part);
152 | 	}
153 | 
154 | 	return join "\n", @out;
155 | }
156 | 
157 | sub get_pkg_name {
158 | 	my $pkg = shift || return undef;
159 | 
160 | 	if ($pkg->{is_package} && defined($pkg->{PKG_NAME}))
161 | 	{
162 | 		return $pkg->{PKG_NAME};
163 | 	}
164 | 
165 | 	return undef;
166 | }
167 | 
168 | sub get_pkg_version {
169 | 	my $pkg = shift || return undef;
170 | 
171 | 	if (defined($pkg->{PKG_VERSION}) && defined($pkg->{PKG_RELEASE}))
172 | 	{
173 | 		return sprintf '%s-%s', $pkg->{PKG_VERSION}, $pkg->{PKG_RELEASE};
174 | 	}
175 | 	elsif (defined($pkg->{PKG_VERSION}))
176 | 	{
177 | 		return $pkg->{PKG_VERSION};
178 | 	}
179 | 	elsif (defined($pkg->{PKG_RELEASE}))
180 | 	{
181 | 		return $pkg->{PKG_RELEASE};
182 | 	}
183 | 
184 | 	return undef;
185 | }
186 | 
187 | sub parse_pkg_info {
188 | 	my ($hash, $pkgdir) = @_;
189 | 	my $pkg = { };
190 | 
191 | 	if (open my $fd, "git show $hash:$pkgdir/Makefile |")
192 | 	{
193 | 		while (defined(my $line = readline $fd))
194 | 		{
195 | 			if ($line =~ /^\s*(\w+)\s*(?:\?=|:=|=)\s*(\S.*)$/)
196 | 			{
197 | 				$pkg->{$1} = $2;
198 | 			}
199 | 			elsif ($line =~ /call +BuildPackage/)
200 | 			{
201 | 				$pkg->{is_package} = 1;
202 | 			}
203 | 		}
204 | 
205 | 		close $fd;
206 | 	}
207 | 
208 | 	foreach my $key (keys %$pkg)
209 | 	{
210 | 		$pkg->{$key} =~ s!\$\((\w+)\)!exists($pkg->{$1}) ? $pkg->{$1} : ''!eg;
211 | 	}
212 | 
213 | 	return $pkg;
214 | }
215 | 
216 | sub generate_pkg_log {
217 | 	my ($feedname, $changetime, $pkgdir) = @_;
218 | 
219 | 	my ($prev_pkg, $curr_pkg, $prev_hash, $curr_hash);
220 | 	my (@log, %cve);
221 | 	my ($subj, $date, @body, @refs);
222 | 
223 | 	if (open my $fd, "git log --since='$changetime' --format='%H' -- '$pkgdir' |")
224 | 	{
225 | 		my @hashes;
226 | 
227 | 		while (defined(my $hash = readline $fd))
228 | 		{
229 | 			chomp $hash;
230 | 			push @hashes, $hash;
231 | 		}
232 | 
233 | 		if (exists $conf{commitlink}{$feedname})
234 | 		{
235 | 			foreach my $hash (reverse @hashes)
236 | 			{
237 | 				push @refs, sprintf $conf{commitlink}{$feedname}, $hash;
238 | 			}
239 | 		}
240 | 
241 | 		close $fd;
242 | 
243 | 		$prev_hash = pop @hashes;
244 | 		$curr_hash = shift @hashes || $prev_hash;
245 | 
246 | 		if (defined($prev_hash) && open my $fd, "git log --format='%H' $prev_hash~1 -- '$pkgdir' |")
247 | 		{
248 | 			if (defined(my $hash = readline $fd))
249 | 			{
250 | 				chomp $hash;
251 | 				$prev_hash = $hash;
252 | 			}
253 | 
254 | 			close $fd;
255 | 		}
256 | 
257 | 		$prev_pkg = parse_pkg_info($prev_hash, $pkgdir);
258 | 		$curr_pkg = parse_pkg_info($curr_hash, $pkgdir);
259 | 	}
260 | 
261 | 	if (!defined($curr_pkg) || !defined($prev_pkg) ||
262 | 	    !$curr_pkg->{is_package} || !$prev_pkg->{is_package})
263 | 	{
264 | 		return;
265 | 	}
266 | 
267 | 	if (open my $fd, "git log --since='$changetime' --format='\@S:%s%n\@D:%cD %h%n\@B:%b%n\@\@' -- '$pkgdir' |")
268 | 	{
269 | 		@cve{@use_cve} = (1) x @use_cve;
270 | 
271 | 		while (defined(my $line = readline $fd))
272 | 		{
273 | 			chomp $line;
274 | 
275 | 			foreach my $cve ($line =~ m!\bCVE-\d{4}-\d{4,}\b!g)
276 | 			{
277 | 				$cve{$cve}++;
278 | 			}
279 | 
280 | 			if ($line =~ m!^\@S:(.+)$!)
281 | 			{
282 | 				$subj = $1;
283 | 				$subj =~ s/^\S+: //;
284 | 				$subj =~ s/^\[[^\[\]]+\][: ]+//;
285 | 			}
286 | 			elsif ($line =~ m!\@D:(.+)$! && defined($subj))
287 | 			{
288 | 				$date = $1;
289 | 			}
290 | 			elsif ($line =~ m!\@B:(.*)$! && defined($date) && !@body)
291 | 			{
292 | 				@body = ($1);
293 | 			}
294 | 			elsif ($line =~ m!\@\@! && @body > 0)
295 | 			{
296 | 				@body = grep {
297 | 					$_ !~ m!^Signed-off-by: !    &&
298 | 			        $_ !~ m!^Acked-by: !         &&
299 | 					$_ !~ m!^ ?Backport of r\d+! &&
300 | 			        $_ !~ m!^git-svn-id: !;
301 | 				} @body;
302 | 
303 | 				my $body = join "\n", @body;
304 | 				   $body =~ s/\s+$//;
305 | 
306 | 				$body = ucfirst($subj) . "\n\n" . $body
307 | 					unless $body =~ /\n/;
308 | 
309 | 				$body = join "\n\n",
310 | 						map { format_paragraph($_) }
311 | 						split /\n\n/, $body;
312 | 
313 | 				push @log, sprintf("[%s]\n\n%s", $date, $body);
314 | 
315 | 				undef $subj;
316 | 				undef $date;
317 | 				undef @body;
318 | 			}
319 | 			elsif (@body > 0)
320 | 			{
321 | 				push @body, $line;
322 | 			}
323 | 		}
324 | 
325 | 		close $fd;
326 | 	}
327 | 
328 | 	my @cve = sort keys %cve;
329 | 	my ($sub, $out);
330 | 
331 | 	if (@cve > 0)
332 | 	{
333 | 		if (@cve > 1)
334 | 		{
335 | 			$sub = sprintf("%s: Security update (%d CVEs)\n\n",
336 | 						   $curr_pkg->{PKG_NAME}, @cve + 0);
337 | 
338 | 			$out = join "\n", wrap_paragraph('', '', sprintf(
339 | 				"The %s package has been rebuilt and was uploaded to the %s repository due to multiple security issues.\n",
340 | 				$curr_pkg->{PKG_NAME}, $conf{release_name}
341 | 			));
342 | 		}
343 | 		else
344 | 		{
345 | 			$sub = sprintf("%s: Security update (%s)\n\n",
346 | 						   $curr_pkg->{PKG_NAME}, $cve[0]);
347 | 
348 | 			$out = join "\n", wrap_paragraph('', '', sprintf(
349 | 				"The %s package has been rebuilt and was uploaded to the %s repository due to a reported security issue.\n",
350 | 				$curr_pkg->{PKG_NAME}, $conf{release_name}
351 | 			));
352 | 		}
353 | 	}
354 | 	else
355 | 	{
356 | 		$sub = sprintf("%s: Update\n\n",
357 | 					   $curr_pkg->{PKG_NAME});
358 | 
359 | 		$out = join "\n", wrap_paragraph('', '', sprintf(
360 | 			"The %s package has been rebuilt and was uploaded to the %s repository.\n",
361 | 			$curr_pkg->{PKG_NAME}, $conf{release_name}
362 | 		));
363 | 	}
364 | 
365 | 	$out .= "\n\n";
366 | 
367 | 	my $prev_version = get_pkg_version($prev_pkg);
368 | 	my $curr_version = get_pkg_version($curr_pkg);
369 | 
370 | 	if ($prev_version && $curr_version)
371 | 	{
372 | 		$out .= sprintf("\nVERSION\n\n%s\n\n", join "\n",
373 | 						wrap_paragraph('', '  ', "$prev_version => $curr_version"));
374 | 	}
375 | 
376 | 	$out .= sprintf("\nCHANGELOG\n\n");
377 | 
378 | 	foreach my $entry (@log)
379 | 	{
380 | 		$out .= sprintf("%s\n\n", $entry);
381 | 	}
382 | 
383 | 	$out .= sprintf("\nCHANGES\n\n");
384 | 
385 | 	if (open my $fd, "git diff --stat=$text_width $prev_hash..$curr_hash -- '$pkgdir' |")
386 | 	{
387 | 		while (defined(my $line = readline $fd))
388 | 		{
389 | 			chomp $line;
390 | 			$out .= sprintf("%s\n", $line);
391 | 		}
392 | 
393 | 		close $fd;
394 | 
395 | 		$out .= "\n";
396 | 	}
397 | 
398 | 	if (@use_ref > 0 || @refs > 0 || @cve > 0)
399 | 	{
400 | 		$out .= sprintf("\nREFERENCES\n\n");
401 | 
402 | 		foreach my $cve (@cve)
403 | 		{
404 | 			$out .= sprintf(" * http://cve.mitre.org/cgi-bin/cvename.cgi?name=%s\n", $cve);
405 | 		}
406 | 
407 | 		foreach my $link (@use_ref, @refs)
408 | 		{
409 | 			$out .= sprintf(" * %s\n", $link);
410 | 		}
411 | 	}
412 | 
413 | 	return ($sub, $out, @cve > 0);
414 | }
415 | 
416 | sub generate_mail_subject
417 | {
418 | 	my $s = sprintf("[%s] %s", $conf{release_tag}, $_[0]);
419 | 
420 | 	utf8::decode($s);
421 | 
422 | 	return encode('MIME-Q', $s);
423 | }
424 | 
425 | my $workdir = Cwd::getcwd();
426 | my @source_pkgs;
427 | 
428 | $changetime = last_change_time()
429 | 	unless defined $changetime;
430 | 
431 | if (open my $find, "find $workdir/.cache/feeds/ -type d -name .git -printf '%h\n' |")
432 | {
433 | 	while (defined(my $feeddir = readline $find))
434 | 	{
435 | 		chomp $feeddir;
436 | 		chdir $feeddir || next;
437 | 
438 | 		my $feedname = File::Basename::basename($feeddir);
439 | 		my %packages;
440 | 
441 | 		if ($do_pull)
442 | 		{
443 | 			system('git pull --ff >/dev/null 2>/dev/null');
444 | 		}
445 | 
446 | 		if (open my $gitlog, "git log --since='$changetime' --name-only --format='#%h' |")
447 | 		{
448 | 			while (defined(my $line = readline $gitlog))
449 | 			{
450 | 				next unless $line =~ m!^(.+)/Makefile$!;
451 | 
452 | 				my $pkg_dir  = $1;
453 | 				my $pkg_name = File::Basename::basename($pkg_dir);
454 | 
455 | 				if (!@use_packages || grep { $pkg_name eq $_ } @use_packages)
456 | 				{
457 | 					$packages{$pkg_dir}++;
458 | 				}
459 | 			}
460 | 
461 | 			close $gitlog;
462 | 		}
463 | 
464 | 		foreach my $package (sort keys %packages)
465 | 		{
466 | 			my ($subject, $body, $security) = generate_pkg_log($feedname, $changetime, $package);
467 | 
468 | 			next unless $subject && $body;
469 | 
470 | 			if ($do_send || $do_preview)
471 | 			{
472 | 				my $to = $security ? $conf{recipients_security} : $conf{recipients_standard};
473 | 				my $msg = MIME::Lite->new(
474 | 					From     => $conf{smtp}{from},
475 | 					To       => join(', ', @$to),
476 | 					Subject  => generate_mail_subject($subject),
477 | 					Data     => $body,
478 | 					Type     => 'text/plain; charset="UTF-8"'
479 | 				);
480 | 
481 | 				if ($do_send)
482 | 				{
483 | 					$msg->send('smtp', $conf{smtp}{host},
484 | 						Timeout => 10,
485 | 						AuthUser => $conf{smtp}{user},
486 | 						AuthPass => $conf{smtp}{pass}
487 | 					);
488 | 				}
489 | 				else
490 | 				{
491 | 					print $msg->as_string, "\n===\n";
492 | 				}
493 | 			}
494 | 			else
495 | 			{
496 | 				warn "$subject\n";
497 | 
498 | 				push @source_pkgs, File::Basename::basename($package);
499 | 			}
500 | 		}
501 | 	}
502 | 
503 | 	close $find;
504 | }
505 | 
506 | if ($do_check)
507 | {
508 | 	if (!@source_pkgs)
509 | 	{
510 | 		printf("No updates.\n");
511 | 		exit(0);
512 | 	}
513 | 
514 | 	printf("./pkgupdate-build.sh -ubi -s %s\n", join(' -s ', @source_pkgs));
515 | 	printf("./pkgupdate-report.pl --since %s%s --send\n",
516 | 		$changetime,
517 | 		@use_packages ? ' --package ' . join(' --package ', @use_packages) : '');
518 | }
519 | 


--------------------------------------------------------------------------------
/shared/functions.sh:
--------------------------------------------------------------------------------
  1 | [ -f shared/relman.cfg ] || {
  2 | 	echo "Please create shared/relman.cfg" >&2
  3 | 	exit 1
  4 | }
  5 | 
  6 | eval $(perl shared/config.pm)
  7 | 
  8 | N="
  9 | "
 10 | 
 11 | call_rsync() {
 12 | 	LC_ALL=C rsync --bwlimit=8000 \
 13 | 		${SSH_IDENT+-e "ssh -o IdentitiesOnly=yes -o IdentityFile=$SSH_IDENT -T -c arcfour -o Compression=no"} "$@"
 14 | }
 15 | 
 16 | mirror_rsync() {
 17 | 	if [ ! -d "$CACHE_DIR/mirror" ] || [ $do_update -gt 0 -a ! -e "$CACHE_DIR/.mirrored" ]; then
 18 | 		mkdir -p "$CACHE_DIR/mirror"
 19 | 		call_rsync -avz --delete -m \
 20 | 			--include='*/' --include="**/$PATTERN_SDK" --include="**/$PATTERN_FEED" --exclude='*' \
 21 | 			"$RELEASE_URL/" "$CACHE_DIR/mirror/"
 22 | 
 23 | 		touch "$CACHE_DIR/.mirrored"
 24 | 	fi
 25 | }
 26 | 
 27 | mirror_http() {
 28 | 	if [ ! -d "$CACHE_DIR/mirror" ] || [ $do_update -gt 0 -a ! -e "$CACHE_DIR/.mirrored" ]; then
 29 | 		mkdir -p "$CACHE_DIR/mirror"
 30 | 		lftp -e "open $RELEASE_URL/ && mirror -P 2 -vvv --use-cache --only-newer --no-empty-dirs --delete -I '$PATTERN_SDK' -I '$PATTERN_FEED' -x logs/ . $CACHE_DIR/mirror/ && exit"
 31 | 		touch "$CACHE_DIR/.mirrored"
 32 | 	fi
 33 | }
 34 | 
 35 | mirror_file() {
 36 | 	if [ ! -d "$CACHE_DIR/mirror" ] || [ $do_update -gt 0 -a ! -e "$CACHE_DIR/.mirrored" ]; then
 37 | 		mkdir -p "$CACHE_DIR/mirror"
 38 | 		rsync -av --delete -m \
 39 | 			--include='*/' --include="**/$PATTERN_SDK" --include="**/$PATTERN_FEED" --exclude='*' \
 40 | 			"${RELEASE_URL#file:}/" "$CACHE_DIR/mirror/"
 41 | 
 42 | 		touch "$CACHE_DIR/.mirrored"
 43 | 	fi
 44 | }
 45 | 
 46 | fetch_remote_targets() {
 47 | 	local target
 48 | 
 49 | 	if [ -n "$use_targets" ]; then
 50 | 		for target in $use_targets; do
 51 | 			echo "$target"
 52 | 		done
 53 | 		return 0
 54 | 	fi
 55 | 
 56 | 	find "$CACHE_DIR/mirror/" -type f -name "$PATTERN_SDK" -printf "%h\n" | while read target; do
 57 | 		echo "${target##*/mirror/}"
 58 | 	done | sort -u
 59 | 	return 0
 60 | }
 61 | 
 62 | fetch_remote_feeds() {
 63 | 	local feed target="$1"
 64 | 
 65 | 	find "$CACHE_DIR/mirror/$target/" -type f -name "$PATTERN_FEED" -printf "%h\n" | while read feed; do
 66 | 		echo "${feed##*/}"
 67 | 	done | sort -u
 68 | 	return 0
 69 | }
 70 | 
 71 | fetch_remote_index() {
 72 | 	local target feed
 73 | 
 74 | 	echo "Fetching remote package indizes..."
 75 | 
 76 | 	case "$RELEASE_URL" in
 77 | 		file:*)
 78 | 			mirror_file
 79 | 		;;
 80 | 		http:*|https:*|ftp:*)
 81 | 			mirror_http
 82 | 		;;
 83 | 		*)
 84 | 			mirror_rsync
 85 | 		;;
 86 | 	esac
 87 | 
 88 | 	fetch_remote_targets | while read target; do
 89 | 		echo -n "* $target:"
 90 | 		fetch_remote_feeds "$target" | while read feed; do
 91 | 			if [ ! -s "$CACHE_DIR/repo-remote/$target/packages/$feed/Packages.gz" ]; then
 92 | 				echo -n " $feed"
 93 | 				mkdir -p "$CACHE_DIR/repo-remote/$target/packages/$feed"
 94 | 				cp -a "$CACHE_DIR/mirror/$target/packages/$feed/Packages.gz" \
 95 | 					"$CACHE_DIR/repo-remote/$target/packages/$feed/Packages.gz"
 96 | 			fi
 97 | 		done
 98 | 		echo ""
 99 | 	done
100 | }
101 | 
102 | prepare_sdk() {
103 | 	local target="$1"
104 | 
105 | 	if [ ! -d "$CACHE_DIR/sdk/$target/.git" ]; then
106 | 		echo " * [$slot:$target] Initializing SDK"
107 | 
108 | 		local sdk
109 | 		for sdk in "$CACHE_DIR/mirror/$target/"$PATTERN_SDK; do
110 | 			if [ ! -f "$sdk" ]; then
111 | 				echo " * [$slot:$target] $sdk - MISSING!"
112 | 				exit 0
113 | 			fi
114 | 		done
115 | 
116 | 		rm -rf "$CACHE_DIR/sdk/$target"
117 | 		mkdir -p "$CACHE_DIR/sdk/$target"
118 | 		tar --strip-components=1 -C "$CACHE_DIR/sdk/$target" -xjf "$sdk"
119 | 
120 | 		mkdir -p "$CACHE_DIR/dl"
121 | 		rm -rf "$CACHE_DIR/sdk/$target/dl"
122 | 		ln -sf "$CACHE_DIR/dl" "$CACHE_DIR/sdk/$target/dl"
123 | 
124 | 		mkdir -p "$CACHE_DIR/feeds"
125 | 		rm -rf "$CACHE_DIR/sdk/$target/feeds"
126 | 		ln -sf "$CACHE_DIR/feeds" "$CACHE_DIR/sdk/$target/feeds"
127 | 
128 | 		mkdir -p "$CACHE_DIR/ccache"
129 | 
130 | 		local dir
131 | 		for dir in "$CACHE_DIR/sdk/$target/staging_dir/host" \
132 | 		           "$CACHE_DIR/sdk/$target/staging_dir/target-"*; do
133 | 			rm -rf "$dir/ccache"
134 | 			ln -sf "$CACHE_DIR/ccache" "$dir/ccache"
135 | 		done
136 | 
137 | 		(
138 | 			cd "$CACHE_DIR/sdk/$target"
139 | 			git init .
140 | 			find . -maxdepth 1 | xargs git add
141 | 			git commit -m "Snapshot"
142 | 		) >/dev/null
143 | 	elif [ $do_clean -gt 0 ]; then
144 | 		echo " * [$slot:$target] Resetting SDK"
145 | 
146 | 		(
147 | 			cd "$CACHE_DIR/sdk/$target"
148 | 			git reset --hard HEAD
149 | 			git clean -f -d
150 | 		) >/dev/null
151 | 	fi
152 | }
153 | 
154 | find_pkg_dependant_sources() {
155 | 	local pkg
156 | 
157 | 	find_pkg_dependant_ipks "$@" | while read pkg; do
158 | 		sed -ne "s!^package-\$(CONFIG_PACKAGE_${pkg}) += .\+/!!p" "$CACHE_DIR/sdk/$target/tmp/.packagedeps"
159 | 	done | sort -u
160 | }
161 | 
162 | find_pkg_dependant_ipks() {
163 | 	local target="$1" pkg="$2" deps="" dep
164 | 
165 | 	if [ $do_dependants -gt 0 ]; then
166 | 		for dep in $(zcat "$CACHE_DIR/repo-remote/$target/packages"/*/Packages.gz | \
167 | 			grep -B2 -E "^Depends:.* ${pkg%%:*}(,|\$)" | sed -ne 's!^Package: !!p'); do
168 | 			deps="$deps$N$dep"
169 | 		done
170 | 	fi
171 | 
172 | 	echo "${pkg%%:*}$deps" | sort -u
173 | }
174 | 
175 | find_source_provided_pkgs() {
176 | 	local pkg="$1"
177 | 
178 | 	find "$CACHE_DIR/repo-remote/" -name Packages.gz | xargs zcat | \
179 | 		grep -B3 -E "^Source: (.+/)?$pkg\$" | sed -ne 's!^Package: !!p' | \
180 | 		sort -u
181 | }
182 | 
183 | install_sdk_feeds() {
184 | 	local pkg feed target="$1"; shift
185 | 
186 | 	echo " * [$slot:$target] Installing packages"
187 | 
188 | 	(
189 | 		flock -x 8
190 | 
191 | 		cd "$CACHE_DIR/sdk/$target"
192 | 
193 | 		if [ ! -s "feeds.conf" ]; then
194 | 			if ! grep -sq " base " "feeds.conf.default" && [ -n "$RELEASE_FEED" ]; then
195 | 				echo "src-git-full base $RELEASE_FEED" > feeds.conf
196 | 			fi
197 | 			sed -e 's#^src-git #src-git-full #g' feeds.conf.default >> feeds.conf
198 | 		fi
199 | 
200 | 		./scripts/feeds update >/dev/null
201 | 
202 | 		echo " * [$slot:$target] feeds install"
203 | 		for pkg in "$@"; do
204 | 			case "$pkg" in
205 | 				*:*) feed="${pkg#*:}"; pkg="${pkg%%:*}" ;;
206 | 				*) feed="" ;;
207 | 			esac
208 | 
209 | 			find_pkg_dependant_ipks "$target" "$pkg" | while read pkg; do
210 | 				#echo " * [$slot:$target] feeds install $pkg"
211 | 				#./scripts/feeds install ${feed:+ -p "$feed"} "$pkg" >/dev/null
212 | 				echo "$pkg"
213 | 			done
214 | 		done | sort -u | xargs ./scripts/feeds install >/dev/null
215 | 
216 | 		sed -i -e "/CONFIG_PACKAGE_/d" .config
217 | 		echo "CONFIG_ALL=y" >> .config
218 | 		make defconfig >/dev/null
219 | 	) 8>"$CACHE_DIR/feeds.lock" 2>/dev/null
220 | }
221 | 
222 | compile_sdk_packages() {
223 | 	local pkg feed target="$1"; shift
224 | 
225 | 	echo " * [$slot:$target] Compiling packages"
226 | 
227 | 	for pkg in "$@"; do
228 | 		find_pkg_dependant_sources "$target" "$pkg"
229 | 	done | sort -u | while read pkg; do
230 | 		echo " * [$slot:$target] make package/$pkg/download"
231 | 		(
232 | 			cd "$CACHE_DIR/sdk/$target"
233 | 			if ! flock "$CACHE_DIR/download.lock" make "package/$pkg/download" BUILD_LOG=1 >/dev/null 2>/dev/null; then
234 | 				echo " * [$slot:$target] make package/$pkg/download - FAILED!"
235 | 			fi
236 | 		) 2>/dev/null
237 | 
238 | 		echo " * [$slot:$target] make package/$pkg/compile"
239 | 		(
240 | 			cd "$CACHE_DIR/sdk/$target"
241 | 			if ! make "package/$pkg/compile" \
242 | 				BUILD_LOG=1 IGNORE_ERRORS=y \
243 | 				CCACHE_COMPILERCHECK="%compiler% -dumpmachine; %compiler% -dumpversion" \
244 | 				>/dev/null 2>/dev/null;
245 | 			then
246 | 				echo " * [$slot:$target] make package/$pkg/compile - FAILED!"
247 | 			fi
248 | 		)
249 | 	done
250 | 
251 | 	for pkg in "$@"; do
252 | 		find_pkg_dependant_ipks "$target" "$pkg"
253 | 	done | sort -u | while read pkg; do
254 | 		for pkg in "$CACHE_DIR/sdk/$target/bin"/*/packages/*/"${pkg}"_[^_]*_[^_]*.ipk; do
255 | 			if [ -s "$pkg" ]; then
256 | 				feed="${pkg%/*}"; feed="${feed##*/}"
257 | 				mkdir -p "$CACHE_DIR/repo-local/$target/packages/$feed"
258 | 				cp -a "$pkg" "$CACHE_DIR/repo-local/$target/packages/$feed/"
259 | 			else
260 | 				echo " * [$slot:$target] $pkg - MISSING!"
261 | 			fi
262 | 		done
263 | 	done
264 | }
265 | 
266 | find_remote_pkg_feed() {
267 | 	local feed target="$1" pkg="$2"
268 | 
269 | 	for feed in $(fetch_remote_feeds "$target"); do
270 | 		if zcat "$CACHE_DIR/repo-remote/$target/packages/$feed/Packages.gz" | grep -qE "^Package: ${pkg%%:*}\$"; then
271 | 			echo "$feed"
272 | 			return 0
273 | 		fi
274 | 	done
275 | 
276 | 	return 1
277 | }
278 | 
279 | find_local_pkg_feed() {
280 | 	local feed file target="$1" pkg="$2"
281 | 
282 | 	for feed in $(fetch_remote_feeds "$target"); do
283 | 		for file in "$CACHE_DIR/repo-local/$target/packages/$feed/${pkg%%:*}"_[^_]*_[^_]*.ipk; do
284 | 			if [ -s "$file" ]; then
285 | 				echo "$feed"
286 | 				return 0
287 | 			fi
288 | 		done
289 | 	done
290 | 
291 | 	return 1
292 | }
293 | 
294 | patch_index_cmd() {
295 | 	local target="$1" feed="$2"; shift; shift
296 | 	local idir="$CACHE_DIR/repo-remote/$target/packages/$feed"
297 | 	local odir="$CACHE_DIR/repo-local/$target/packages/$feed"
298 | 
299 | 	if [ ! -s "$odir/Packages" ]; then
300 | 		mkdir -p "$odir"
301 | 		zcat "$idir/Packages.gz" > "$odir/Packages"
302 | 	fi
303 | 
304 | 	./shared/patch-index.pl --index "$odir/Packages" "$@" > "$odir/Packages.$$"
305 | 
306 | 	mv "$odir/Packages.$$" "$odir/Packages"
307 | }
308 | 
309 | sign_index_cmd() {
310 | 	local target="$1" feed="$2"; shift; shift
311 | 	local odir="$CACHE_DIR/repo-local/$target/packages/$feed"
312 | 
313 | 	if [ -x "$CACHE_DIR/usign.elf" ] && [ -s "$odir/Packages" ]; then (
314 | 		cd "$odir"
315 | 		{
316 | 			echo "untrusted comment: signing key"
317 | 			echo "$SIGNING_KEY"
318 | 		} | "$CACHE_DIR/usign.elf" -S -m Packages -s -
319 | 	); fi
320 | }
321 | 
322 | patch_indexes() {
323 | 	local target="$1" feed pkg dir; shift
324 | 
325 | 	echo " * [$slot:$target] Patching repository index"
326 | 
327 | 	for pkg in "$@"; do
328 | 		find_pkg_dependant_ipks "$target" "$pkg" | while read pkg; do
329 | 			feed="$(find_remote_pkg_feed "$target" "$pkg")"
330 | 			[ -n "$feed" ] && patch_index_cmd "$target" "$feed" \
331 | 				--remove "${pkg%%:*}"
332 | 
333 | 			feed="$(find_local_pkg_feed "$target" "$pkg")"
334 | 			[ -n "$feed" ] && patch_index_cmd "$target" "$feed" \
335 | 				--add "$CACHE_DIR/repo-local/$target/packages/$feed/${pkg%%:*}"_*.ipk
336 | 		done
337 | 	done
338 | 
339 | 	for feed in $(fetch_remote_feeds "$target"); do
340 | 		dir="$CACHE_DIR/repo-local/$target/packages/$feed"
341 | 		if [ -s "$dir/Packages" ]; then
342 | 			gzip -c -9 "$dir/Packages" > "$dir/Packages.gz"
343 | 		fi
344 | 		if [ -n "$SIGNING_KEY" ]; then
345 | 			sign_index_cmd "$target" "$feed"
346 | 		fi
347 | 	done
348 | }
349 | 
350 | rsync_delete_remote() {
351 | 	local target="$1" feed name pkg dep include line; shift
352 | 
353 | 	for feed in $(fetch_remote_feeds "$target"); do
354 | 		include=""
355 | 
356 | 		for pkg in "$@"; do
357 | 			for dep in $(find_pkg_dependant_ipks "$target" "$pkg"); do
358 | 				name="$(zcat "$CACHE_DIR/repo-remote/$target/packages/$feed/Packages.gz" | \
359 | 					sed -ne "s/Filename: \\(${dep%%:*}_.\\+\\.ipk\\)\$/\1/p")"
360 | 
361 | 				include="${include:+$include }${name:+--include=$name}"
362 | 			done
363 | 		done
364 | 
365 | 		if [ -n "$include" ]; then
366 | 			mkdir -p "$CACHE_DIR/empty"
367 | 			call_rsync -rv --delete $include --exclude="*" "$CACHE_DIR/empty/" "${RELEASE_URL#file:}/$target/packages/$feed/" 2>&1 | \
368 | 				grep "deleting " | while read line; do
369 | 					echo " * [$slot:$target] rsync: $line"
370 | 				done
371 | 		fi
372 | 	done
373 | }
374 | 
375 | rsync_files() {
376 | 	local target="$1" line; shift
377 | 
378 | 	case "$RELEASE_URL" in
379 | 		http:*|https:*|ftp:*)
380 | 			echo "* HTTP/FTP upload not supported!"
381 | 			exit 0
382 | 		;;
383 | 	esac
384 | 
385 | 	echo " * [$slot:$target] Syncing files"
386 | 
387 | 	rsync_delete_remote "$target" "$@"
388 | 	call_rsync -rv "$CACHE_DIR/repo-local/$target/packages/" "${RELEASE_URL#file:}/$target/packages/" 2>&1 | \
389 | 		grep "/" | while read line; do
390 | 			echo " * [$slot:$target] rsync: $line"
391 | 		done
392 | 
393 | 	rm -r "$CACHE_DIR/repo-local/$target/packages/"
394 | }
395 | 
396 | prepare_usign() {
397 | 	if [ ! -x "$CACHE_DIR/usign.elf" ]; then
398 | 		echo "* Extracting usign executable..."
399 | 
400 | 		find "$CACHE_DIR/mirror/" -name "$PATTERN_SDK" | head -n1 | \
401 | 			xargs tar -Ox --wildcards '*/staging_dir/host/bin/usign' -jf \
402 | 				> "$CACHE_DIR/usign.elf"
403 | 
404 | 		if [ -s "$CACHE_DIR/usign.elf" ] && \
405 | 			chmod 0700 "$CACHE_DIR/usign.elf" && \
406 | 			"$CACHE_DIR/usign.elf" 2>&1 | grep -q secret;
407 | 		then
408 | 			return 0
409 | 		fi
410 | 
411 | 		rm -f "$CACHE_DIR/usign.elf"
412 | 		echo "* Unable to extract a working usign executable!" >&2
413 | 		exit 1
414 | 	fi
415 | }
416 | 
417 | test_signing() {
418 | 	local has_signing="$(find "$CACHE_DIR/mirror/" -type f -name Packages.sig | head -n1)"
419 | 
420 | 	if [ -z "$has_signing" ]; then
421 | 		return
422 | 	fi
423 | 
424 | 	prepare_usign
425 | 
426 | 	echo "+--------------------------------------------------+"
427 | 	echo "| SIGNING KEY REQUIRED                             |"
428 | 	echo "| Paste the key below and press enter to continue! |"
429 | 	echo "+--------------------------------------------------+"
430 | 
431 | 	read -p "Signing key > " -s SIGNING_KEY
432 | 	echo ""
433 | 
434 | 	if [ -z "$SIGNING_KEY" ]; then
435 | 		echo "Key not provided, aborting." >&2
436 | 		exit 1
437 | 	fi
438 | 
439 | 	local fingerprint="$({
440 | 		echo "untrusted comment: signing key"
441 | 		echo "$SIGNING_KEY"
442 | 	} | "$CACHE_DIR/usign.elf" -F -s - 2>/dev/null)"
443 | 
444 | 	if [ -z "$fingerprint" ]; then
445 | 		echo "Invalid key provided, aborting." >&2
446 | 		exit 1
447 | 	fi
448 | 
449 | 	echo "Using key with fingerprint $fingerprint"
450 | }
451 | 
452 | run_jobs() {
453 | 	local targets=$(fetch_remote_targets)
454 | 	local target slot count job
455 | 
456 | 	#echo "* Compiling packages"
457 | 
458 | 	if [ $do_index -gt 0 ]; then
459 | 		test_signing
460 | 	fi
461 | 
462 | 	for slot in $(seq 0 $((num_jobs-1))); do (
463 | 		count=1; for target in $targets; do
464 | 			if [ $((count++ % $num_jobs)) -eq $slot ]; then
465 | 				if [ $do_build -gt 0 ]; then
466 | 					prepare_sdk "$target"
467 | 					install_sdk_feeds "$target" "$@"
468 | 					compile_sdk_packages "$target" "$@"
469 | 				fi
470 | 
471 | 				if [ $do_index -gt 0 ]; then
472 | 					patch_indexes "$target" "$@"
473 | 				fi
474 | 
475 | 				if [ $do_rsync -gt 0 ]; then
476 | 					rsync_files "$target" "$@"
477 | 				fi
478 | 			fi
479 | 		done
480 | 	) & done
481 | 
482 | 	for job in $(jobs -p); do
483 | 		wait "$job"
484 | 		echo "* Job $job completed"
485 | 	done
486 | }
487 | 


--------------------------------------------------------------------------------