├── lib └── nixpkgs_monitor │ ├── distro_packages.rb │ ├── migrations │ ├── 002_rename_git_updater.rb │ └── 001_initial.rb │ ├── package_updaters.rb │ ├── package_updaters │ ├── repository │ │ ├── xorg.rb │ │ ├── gnu.rb │ │ ├── npmjs.rb │ │ ├── rubygems.rb │ │ ├── pypi.rb │ │ ├── hackage.rb │ │ ├── gnome.rb │ │ ├── kde.rb │ │ ├── sf.rb │ │ └── cpan.rb │ ├── repository.rb │ ├── gentoo_distfiles.rb │ ├── distro.rb │ ├── git.rb │ └── base.rb │ ├── reports.rb │ ├── distro_packages │ ├── debian.rb │ ├── gentoo.rb │ ├── base.rb │ ├── arch.rb │ └── nix.rb │ ├── build_log.rb │ └── security_advisories.rb ├── debian-watchfiles ├── unpack_watchfiles.rb ├── get_urls.rb ├── watchfiles.md └── update.pl ├── test └── updater_version_handling.rb ├── default.nix ├── help.md ├── unmaintained └── comparepackages.rb ├── README.md ├── service.nix └── bin ├── nixpkgs-monitor └── nixpkgs-monitor-site /lib/nixpkgs_monitor/distro_packages.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/distro_packages/arch' 2 | require 'nixpkgs_monitor/distro_packages/debian' 3 | require 'nixpkgs_monitor/distro_packages/gentoo' 4 | require 'nixpkgs_monitor/distro_packages/nix' 5 | -------------------------------------------------------------------------------- /debian-watchfiles/unpack_watchfiles.rb: -------------------------------------------------------------------------------- 1 | Dir['**/*debian.tar.gz'].reject{ |f| File.directory?(f) }.each do |f| 2 | # puts f 3 | watchfile = %x(tar xvf #{f} debian/watch -O) 4 | File.write("watchfiles/#{File.basename(f)[/(.*)\.debian\.tar\.gz/, 1]}.watch", watchfile) if watchfile.length>0 5 | puts "no watch in #{f}" unless watchfile.length>0 6 | end 7 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/migrations/002_rename_git_updater.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | 4 | # rename tables to match updater renames 5 | DB.rename_table(:repository_fetchgit, :git_fetchgit) 6 | DB.rename_table(:repository_github, :git_github) 7 | DB.rename_table(:repository_metagit, :git_metagit) 8 | 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /debian-watchfiles/get_urls.rb: -------------------------------------------------------------------------------- 1 | Dir['watchfiles/*.watch'].reject{ |f| File.directory?(f) }.each do |f| 2 | puts f 3 | urls = %x(perl update.pl --watchfile #{f} --package a --upstream-version 0) 4 | File.write("deb_urls/#{File.basename(f)[/(.*)\.watch/, 1]}.urls", urls) if urls.length>0 5 | STDERR.puts "no watch in #{f}" unless urls.length>0 6 | puts "no watch in #{f}" unless urls.length>0 7 | end 8 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/package_updaters/distro' 2 | require 'nixpkgs_monitor/package_updaters/gentoo_distfiles' 3 | require 'nixpkgs_monitor/package_updaters/git' 4 | require 'nixpkgs_monitor/package_updaters/repository' 5 | 6 | module NixPkgsMonitor module PackageUpdaters 7 | 8 | Updaters = Distro::Updaters + 9 | Git::Updaters + 10 | Repository::Updaters + 11 | [ GentooDistfiles ] 12 | 13 | end end 14 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/xorg.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/package_updaters/base' 2 | 3 | module NixPkgsMonitor module PackageUpdaters module Repository 4 | 5 | # handles X.org packages hosted at mirror://xorg/ 6 | class Xorg < NixPkgsMonitor::PackageUpdaters::Base 7 | 8 | def self.tarballs 9 | @tarballs ||= tarballs_from_dir_recursive("http://xorg.freedesktop.org/releases/individual/") 10 | end 11 | 12 | def self.covers?(pkg) 13 | pkg.url and pkg.url.start_with? "mirror://xorg/" and usable_version?(pkg.version) 14 | end 15 | 16 | def self.newest_versions_of(pkg) 17 | return nil unless covers?(pkg) 18 | new_tarball_versions(pkg, tarballs) 19 | end 20 | 21 | end 22 | 23 | end end end 24 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/gnu.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/package_updaters/base' 2 | 3 | module NixPkgsMonitor module PackageUpdaters module Repository 4 | 5 | # handles GNU packages hosted at mirror://gnu/ 6 | class GNU < NixPkgsMonitor::PackageUpdaters::Base 7 | 8 | def self.tarballs 9 | @tarballs ||= Hash.new{|h, path| h[path] = tarballs_from_dir("http://ftpmirror.gnu.org#{path}") } 10 | end 11 | 12 | def self.covers?(pkg) 13 | pkg.url =~ %r{^mirror://gnu(/[^/]*)/[^/]*$} and usable_version?(pkg.version) 14 | end 15 | 16 | def self.newest_versions_of(pkg) 17 | return nil unless %r{^mirror://gnu(?/[^/]*)/[^/]*$} =~ pkg.url 18 | new_tarball_versions(pkg, tarballs[path]) 19 | end 20 | 21 | end 22 | 23 | end end end 24 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/package_updaters/repository/cpan' 2 | require 'nixpkgs_monitor/package_updaters/repository/gnome' 3 | require 'nixpkgs_monitor/package_updaters/repository/gnu' 4 | require 'nixpkgs_monitor/package_updaters/repository/hackage' 5 | require 'nixpkgs_monitor/package_updaters/repository/kde' 6 | require 'nixpkgs_monitor/package_updaters/repository/npmjs' 7 | require 'nixpkgs_monitor/package_updaters/repository/pypi' 8 | require 'nixpkgs_monitor/package_updaters/repository/rubygems' 9 | require 'nixpkgs_monitor/package_updaters/repository/sf' 10 | require 'nixpkgs_monitor/package_updaters/repository/xorg' 11 | 12 | module NixPkgsMonitor module PackageUpdaters module Repository 13 | 14 | Updaters = [ CPAN, GNOME, GNU, Hackage, KDE, NPMJS, Pypi, Rubygems, SF, Xorg ] 15 | 16 | end end end 17 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/npmjs.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require 'nixpkgs_monitor/package_updaters/base' 3 | 4 | module NixPkgsMonitor module PackageUpdaters module Repository 5 | 6 | # handles Node.JS packages hosted at npmjs.org 7 | class NPMJS < NixPkgsMonitor::PackageUpdaters::Base 8 | 9 | def self.metadata 10 | @metadata ||= Hash.new{|h, pkgname| h[pkgname] = JSON.parse(http_agent.get("http://registry.npmjs.org/#{pkgname}/").body) } 11 | end 12 | 13 | def self.covers?(pkg) 14 | pkg.url and pkg.url.start_with?("http://registry.npmjs.org/") and usable_version?(pkg.version) 15 | end 16 | 17 | def self.newest_version_of(pkg) 18 | return nil unless %r{http://registry.npmjs.org/(?[^\/]*)/} =~ pkg.url 19 | new_ver = metadata[pkgname]["dist-tags"]["latest"] 20 | return nil unless usable_version?(new_ver) and usable_version?(pkg.version) 21 | ( is_newer?(new_ver, pkg.version) ? new_ver : nil ) 22 | end 23 | 24 | end 25 | 26 | end end end 27 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/rubygems.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require 'nixpkgs_monitor/package_updaters/base' 3 | 4 | module NixPkgsMonitor module PackageUpdaters module Repository 5 | 6 | # handles Ruby gems hosted at http://rubygems.org/ 7 | class Rubygems < NixPkgsMonitor::PackageUpdaters::Base 8 | 9 | def self.tarballs 10 | @tarballs ||= Hash.new do |h, pkgname| 11 | h[pkgname] = JSON.parse(http_agent.get("http://rubygems.org/api/v1/versions/#{pkgname}.json").body) 12 | .map{|v| v["number"]} 13 | end 14 | end 15 | 16 | def self.covers?(pkg) 17 | pkg.url and pkg.url.include? 'rubygems.org/downloads/' and usable_version?(pkg.version) 18 | end 19 | 20 | def self.newest_versions_of(pkg) 21 | return nil unless covers?(pkg) 22 | (package_name, file_version) = parse_tarball_from_url(pkg.url) 23 | return nil unless package_name 24 | new_tarball_versions(pkg, tarballs[package_name]) 25 | end 26 | 27 | end 28 | 29 | end end end 30 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/reports.rb: -------------------------------------------------------------------------------- 1 | module NixPkgsMonitor module Reports 2 | 3 | class Timestamps 4 | 5 | def self.done(action, message = nil) 6 | DB.transaction do 7 | if 1 != DB[:timestamps].where(:action => action.to_s).update(:timestamp => Time.now, :message => message) 8 | DB[:timestamps] << { :action => action.to_s, :timestamp => Time.now, :message => message } 9 | end 10 | end 11 | end 12 | 13 | def self.all 14 | DB[:timestamps].all 15 | end 16 | end 17 | 18 | 19 | class Logs 20 | 21 | def initialize(logtype, clear_log = true) 22 | @logtype = logtype 23 | clear! if clear_log 24 | end 25 | 26 | def pkg(pkg_attr) 27 | DB.transaction do 28 | unless DB[@logtype][:pkg_attr => pkg_attr] 29 | DB[@logtype] << { :pkg_attr => pkg_attr } 30 | end 31 | end 32 | end 33 | 34 | def clear! 35 | DB.transaction do 36 | DB[@logtype].delete if DB.table_exists?(@logtype) 37 | end 38 | end 39 | 40 | end 41 | 42 | end end 43 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/pypi.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require 'nixpkgs_monitor/package_updaters/base' 3 | 4 | module NixPkgsMonitor module PackageUpdaters module Repository 5 | 6 | # handles Python packages hosted at http://pypi.python.org/ 7 | class Pypi < NixPkgsMonitor::PackageUpdaters::Base 8 | 9 | def self.releases 10 | @releases ||= Hash.new do |h, pkgname| 11 | h[pkgname] = JSON.parse(http_agent.get("http://pypi.python.org/pypi/#{pkgname}/json") 12 | .body)["releases"].keys 13 | end 14 | end 15 | 16 | def self.covers?(pkg) 17 | pkg.url =~ %r{^(https?://pypi.python.org/packages/source|mirror://pypi)/./[^/]*/[^/]*$} and usable_version?(pkg.version) 18 | end 19 | 20 | def self.newest_versions_of(pkg) 21 | return nil unless %r{^(https?://pypi.python.org/packages/source|mirror://pypi)/./(?[^/]*)/[^/]*$} =~ pkg.url 22 | new_versions(pkg.version.downcase, releases[pkgname], pkg.internal_name) 23 | end 24 | 25 | end 26 | 27 | end end end 28 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/distro_packages/debian.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/distro_packages/base' 2 | 3 | module NixPkgsMonitor module DistroPackages 4 | 5 | class Debian < NixPkgsMonitor::DistroPackages::Base 6 | @cache_name = "debian" 7 | 8 | def version 9 | @version.sub(/^\d:/,"").sub(/-\d$/,"").sub(/\+dfsg.*$/,"") 10 | end 11 | 12 | def self.generate_list 13 | deb_list = {} 14 | 15 | puts "Downloading repository metadata" 16 | %x(curl http://ftp.debian.org/debian/dists/sid/main/source/Sources.xz -o debian-main.xz) 17 | %x(curl http://ftp.debian.org/debian/dists/sid/contrib/source/Sources.xz -o debian-contrib.xz) 18 | %x(curl http://ftp.debian.org/debian/dists/sid/non-free/source/Sources.xz -o debian-non-free.xz) 19 | 20 | %x(xzcat debian-main.xz debian-contrib.xz debian-non-free.xz) 21 | .split("\n\n").each do |pkgmeta| 22 | next unless (/Package:\s*(?.+)/ =~ pkgmeta and 23 | /Version:\s*(?.+)/ =~ pkgmeta) 24 | deb_list[pkg_name] = Debian.new(pkg_name, pkg_name, pkg_version) 25 | end 26 | 27 | serialize_list(deb_list.values) 28 | end 29 | 30 | end 31 | 32 | end end 33 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/hackage.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems/package' 2 | require 'zlib' 3 | require 'nixpkgs_monitor/package_updaters/base' 4 | 5 | module NixPkgsMonitor module PackageUpdaters module Repository 6 | 7 | # handles Haskell packages hosted at http://hackage.haskell.org/ 8 | class Hackage < NixPkgsMonitor::PackageUpdaters::Base 9 | 10 | def self.tarballs 11 | unless @tarballs 12 | @tarballs = Hash.new{|h,k| h[k] = Array.new } 13 | index_gz = http_agent.get('https://hackage.haskell.org/packages/index.tar.gz').body 14 | tgz = Zlib::GzipReader.new(StringIO.new(index_gz)).read 15 | tar = Gem::Package::TarReader.new(StringIO.new(tgz)) 16 | tar.each do |entry| 17 | log.warn "failed to parse #{entry.full_name}" unless %r{^(?[^/]+)/(?[^/]+)/} =~ entry.full_name 18 | @tarballs[pkg] << ver 19 | end 20 | tar.close 21 | end 22 | @tarballs 23 | end 24 | 25 | def self.covers?(pkg) 26 | pkg.url and pkg.url.start_with? 'mirror://hackage/' and usable_version?(pkg.version) 27 | end 28 | 29 | def self.newest_versions_of(pkg) 30 | return nil unless covers?(pkg) 31 | new_tarball_versions(pkg, tarballs) 32 | end 33 | 34 | end 35 | 36 | end end end 37 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/gnome.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require 'nixpkgs_monitor/package_updaters/base' 3 | 4 | module NixPkgsMonitor module PackageUpdaters module Repository 5 | 6 | # handles GNOME packages hosted at mirror://gnome/ 7 | class GNOME < NixPkgsMonitor::PackageUpdaters::Base 8 | 9 | def self.tarballs 10 | @tarballs ||= Hash.new{|h, path| h[path] = JSON.parse(http_agent.get("http://download.gnome.org#{path}cache.json").body) } 11 | end 12 | 13 | def self.covers?(pkg) 14 | pkg.url =~ %r{^mirror://gnome(/sources/[^/]*/)[^/]*/[^/]*$} and usable_version?(pkg.version) 15 | end 16 | 17 | def self.find_tarball(pkg, version) 18 | return nil if pkg.url.to_s.empty? or version.to_s.empty? or pkg.version.to_s.empty? 19 | (package_name, file_version) = parse_tarball_from_url(pkg.url) 20 | return nil unless package_name 21 | repo = tarballs["/sources/#{package_name}/"][1][package_name][version] 22 | return nil unless repo 23 | file ||= repo["tar.xz"] || repo["tar.bz2"] || repo["tar.gz"] 24 | return (file ? "mirror://gnome/sources/#{package_name}/#{file}" : nil ) 25 | end 26 | 27 | def self.newest_versions_of(pkg) 28 | return nil unless %r{^mirror://gnome(?/sources/[^/]*/)[^/]*/[^/]*$} =~ pkg.url 29 | new_tarball_versions(pkg, tarballs[path][2]) 30 | end 31 | 32 | end 33 | 34 | end end end 35 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/kde.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/package_updaters/base' 2 | 3 | module NixPkgsMonitor module PackageUpdaters module Repository 4 | 5 | # handles KDE stable packages hosted at mirror://kde/stable/ 6 | class KDE < NixPkgsMonitor::PackageUpdaters::Base 7 | 8 | def self.tarballs 9 | unless @tarballs 10 | @tarballs = Hash.new{|h,k| h[k] = Array.new } 11 | dirs = http_agent.get("http://download.kde.org/ls-lR").body.split("\n\n") 12 | dirs.each do |dir| 13 | lines = dir.split("\n") 14 | next unless lines[0].include? '/stable' 15 | next if lines[0].include? '/win32:' 16 | lines.delete_at(0) 17 | lines.each do |line| 18 | next if line[0] == 'd' or line [0] == 'l' 19 | tarball = line.split(' ').last 20 | next if ['.xdelta', '.sha1', '.md5', '.CHANGELOG', '.sha256', '.patch', '.diff'].index{ |s| tarball.include? s} 21 | (package_name, file_version) = parse_tarball_name(tarball) 22 | if file_version and package_name 23 | @tarballs[package_name] << file_version 24 | end 25 | end 26 | end 27 | end 28 | @tarballs 29 | end 30 | 31 | def self.covers?(pkg) 32 | pkg.url and pkg.url.start_with? 'mirror://kde/stable/' and usable_version?(pkg.version) 33 | end 34 | 35 | def self.newest_versions_of(pkg) 36 | return nil unless covers?(pkg) 37 | new_tarball_versions(pkg, tarballs) 38 | end 39 | 40 | end 41 | 42 | end end end 43 | -------------------------------------------------------------------------------- /test/updater_version_handling.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'nixpkgs_monitor/package_updaters/base' 3 | 4 | class TestSimple < Test::Unit::TestCase 5 | 6 | Updater = NixPkgsMonitor::PackageUpdaters::Base 7 | 8 | def test_usable_version 9 | assert(Updater.usable_version?('1')); 10 | assert(Updater.usable_version?('4.5')); 11 | assert(!Updater.usable_version?('4.5.')); 12 | assert(!Updater.usable_version?('4..5')); 13 | assert(!Updater.usable_version?('.4.5')); 14 | assert(!Updater.usable_version?('4.-5')); 15 | assert(Updater.usable_version?('2.4.0')); 16 | assert(Updater.usable_version?('1.3.0.pre')); 17 | assert(Updater.usable_version?('1.3.0.rc.3')); 18 | assert(Updater.usable_version?('2.4.0a')); 19 | assert(!Updater.usable_version?('2-4-0')); 20 | assert(Updater.usable_version?('2.4.3.5')); 21 | end 22 | 23 | 24 | def test_version_parsing 25 | assert_equal([1, 3, 0, -85, 3, -1, -1, -1, -1, -1], 26 | Updater.tokenize_version('1.3.0.pre.3')) 27 | assert_equal([2, 4, 0, -100, -1, -1, -1, -1, -1, -1], 28 | Updater.tokenize_version('2.4.0A')) 29 | end 30 | 31 | 32 | def test_version_comparison 33 | assert(Updater.is_newer?('2','1')) 34 | assert(Updater.is_newer?('2.2','2.1')) 35 | assert(Updater.is_newer?('2.1','2')) 36 | assert(!Updater.is_newer?('2', '2.0')) 37 | assert(!Updater.is_newer?('2.1.3','2.1.03')) 38 | assert(!Updater.is_newer?('2.1.3','2.1.4')) 39 | 40 | assert(Updater.is_newer?('1.3.0.1', '1.1.0.3')) 41 | assert(!Updater.is_newer?('1.1.0.3', '1.3.0.1')) 42 | assert(Updater.tokenize_version('1.3.0.1')) 43 | end 44 | 45 | end 46 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/sf.rb: -------------------------------------------------------------------------------- 1 | require 'nokogiri' 2 | require 'nixpkgs_monitor/package_updaters/base' 3 | 4 | module NixPkgsMonitor module PackageUpdaters module Repository 5 | 6 | # FIXME: nixpkgs has lots of urls which don't use mirror and instead have direct links :( 7 | # handles packages hosted at SourceForge 8 | class SF < NixPkgsMonitor::PackageUpdaters::Base 9 | 10 | def self.tarballs 11 | @tarballs ||= Hash.new do |h, sf_project| 12 | tarballs = Hash.new{|h,k| h[k] = Array.new } 13 | 14 | begin 15 | data = http_agent.get("http://sourceforge.net/projects/#{sf_project}/rss").body 16 | Nokogiri.XML(data).xpath('rss/channel/item/title').each do |v| 17 | next if v.inner_text.end_with?('.asc', '.exe', '.dmg', '.sig', '.sha1', '.patch', '.patch.gz', '.patch.bz2', '.diff', '.diff.bz2', '.xdelta') 18 | (name, version) = parse_tarball_from_url(v.inner_text) 19 | tarballs[name] << version if name and version 20 | end 21 | rescue Net::HTTPForbidden, Mechanize::ResponseCodeError 22 | log.warn "failed to fetch http://sourceforge.net/projects/#{sf_project}/rss" 23 | end 24 | 25 | h[sf_project] = tarballs 26 | end 27 | end 28 | 29 | def self.covers?(pkg) 30 | pkg.url =~ %r{^mirror://sourceforge/(?:project/)?([^/]+).*?/([^/]+)$} and usable_version?(pkg.version) 31 | end 32 | 33 | def self.newest_versions_of(pkg) 34 | return nil unless %r{^mirror://sourceforge/(?:project/)?(?[^/]+).*?/([^/]+)$} =~ pkg.url 35 | new_tarball_versions(pkg, tarballs[sf_project]) 36 | end 37 | 38 | end 39 | 40 | end end end 41 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? (import {}), stdenv ? pkgs.stdenv }: 2 | 3 | let 4 | monitor_runtime_deps = with pkgs; [ 5 | ruby_1_9 git patch curl bzip2 gzip gnutar gnugrep coreutils gnused bash file 6 | ]; 7 | tame_nix = pkgs.lib.overrideDerivation pkgs.nixUnstable (a: { 8 | patches = [ ./build/expose-attrs.patch ./build/extra-meta.patch ]; 9 | }); 10 | 11 | in stdenv.mkDerivation rec { 12 | name = "nixpkgs-monitor-dev"; 13 | 14 | src = ./.; 15 | 16 | env = pkgs.bundlerEnv { 17 | name = "nixpkgs-monitor-dev"; 18 | ruby = pkgs.ruby_1_9; 19 | gemfile = ./build/Gemfile; 20 | lockfile = ./build/Gemfile.lock; 21 | gemset = ./build/gemset.nix; 22 | }; 23 | 24 | buildInputs = [ pkgs.makeWrapper env.ruby pkgs.bundler ]; 25 | 26 | doCheck = true; 27 | checkPhase = "RUBYLIB=lib:${env}/${env.ruby.gemPath} GEM_PATH=${env}/${env.ruby.gemPath} test/all.rb"; 28 | 29 | installPhase = '' 30 | mkdir -p $out 31 | cp -r lib $out 32 | cp -r bin $out 33 | 34 | wrapProgram "$out/bin/nixpkgs-monitor" \ 35 | ${stdenv.lib.concatMapStrings (x: "--prefix PATH : ${x}/bin ") monitor_runtime_deps} \ 36 | --prefix PATH : "${env.ruby}/bin:${tame_nix}/bin" \ 37 | --prefix GEM_PATH : "${env}/${env.ruby.gemPath}" \ 38 | --prefix RUBYLIB : "${env}/${env.ruby.gemPath}:$out/lib" \ 39 | --set RUBYOPT rubygems 40 | 41 | wrapProgram "$out/bin/nixpkgs-monitor-site" \ 42 | --set PATH "${env.ruby}/bin:${pkgs.diffutils}/bin:${pkgs.which}/bin" \ 43 | --prefix GEM_PATH : "${env}/${env.ruby.gemPath}" \ 44 | --prefix RUBYLIB : "${env}/${env.ruby.gemPath}:$out/lib" \ 45 | --set RUBYOPT rubygems 46 | 47 | ''; 48 | } 49 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/gentoo_distfiles.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/package_updaters/base' 2 | 3 | require 'nixpkgs_monitor/package_updaters/repository/cpan' 4 | require 'nixpkgs_monitor/package_updaters/repository/hackage' 5 | require 'nixpkgs_monitor/package_updaters/repository/pypi' 6 | require 'nixpkgs_monitor/package_updaters/repository/rubygems' 7 | 8 | module NixPkgsMonitor module PackageUpdaters 9 | 10 | class GentooDistfiles < NixPkgsMonitor::PackageUpdaters::Base 11 | 12 | def self.covers?(pkg) 13 | return false if Repository::CPAN.covers?(pkg) or Repository::Pypi.covers?(pkg) or 14 | Repository::Rubygems.covers?(pkg) or Repository::Hackage.covers?(pkg) 15 | (package_name, file_version) = parse_tarball_from_url(pkg.url) 16 | 17 | package_name and file_version and distfiles[package_name] and 18 | usable_version?(pkg.version) and usable_version?(file_version) 19 | end 20 | 21 | def self.distfiles 22 | unless @distfiles 23 | @distfiles = Hash.new{|h,k| h[k] = Array.new } 24 | files = http_agent.get('http://distfiles.gentoo.org/distfiles/').links.map(&:href) 25 | files.each do |tarball| 26 | (name, version) = parse_tarball_name(tarball) 27 | if name and name != "v" 28 | name = name.downcase 29 | version = version.downcase 30 | unless version.include? 'patch' or version.include? 'diff' 31 | @distfiles[name] << version 32 | end 33 | end 34 | end 35 | end 36 | @distfiles 37 | end 38 | 39 | 40 | def self.newest_versions_of(pkg) 41 | return nil unless covers?(pkg) 42 | new_tarball_versions(pkg, distfiles) 43 | end 44 | 45 | end 46 | 47 | end end 48 | -------------------------------------------------------------------------------- /debian-watchfiles/watchfiles.md: -------------------------------------------------------------------------------- 1 | # Debian watchfiles playground 2 | 3 | This was an attempt at estimating the quality of update 4 | coverage provided by Debian watchfiles. 5 | 6 | 7 | ## How to replicate the experiment 8 | 9 | All these tools expect a debian repository checkout: 10 | wget -r -np http://ftp.debian.org/debian/pool/ -A '*debian*' 11 | 12 | update.pl is a hacked version of Debian's uscan tool modified to 13 | produce the list of all available tarballs and not only the most recent one. 14 | 15 | unpack_watchfiles.rb extracts all available watchfiles from debian repos to watchfiles dir 16 | 17 | get_urls.rb obtains all available tarballs using watchfiles and puts the lists into deb_urls dir 18 | 19 | 20 | ## Some stats 21 | 22 | Total packages: 23 | find ftp.debian.org/ -iname '*.debian.tar.gz'|wc -l 24 | 23007 25 | 26 | Packages may not be unique, that is several versions 27 | of the same package may be in the repo. 28 | 29 | Total watchfiles(extracted using a simple script): 30 | find watchfiles/|wc -l 31 | 20347 32 | 33 | Watchfiles that returned some URLs: 34 | find deb_urls/|wc -l 35 | 16400 36 | 37 | Watchfiles tend to be present for more popular/important packages, 38 | which also tend to have several versions in the repos at once. 39 | Due to this skew, the sheer number of watchfiles covers less 40 | unique packages than it seems at first. 41 | 42 | 43 | ## Why debian watchfiles suck. Lessons learned 44 | 45 | * Maintainers are (lazy) people 46 | * Maintainers probably have other ways to watch for release such as RSS, MLs 47 | or personal contacts. debian is huge and probably can afford a nontechnical 48 | solution to this problem. 49 | * Writing resilient and reliable watchfiles requires skill and understanding of 50 | what can break. It can be practically obtained only when you deal with a large 51 | sample of tarball names. 52 | * Expecting hundreds of maintainers acquire this knowledge independently is 53 | not reasonable. 54 | * Upstream needs to actually be aware of the fact that the releases are 55 | watched by software and take care to not break it. 56 | * Educating upstream is even less practical thus watchfiles themselves are 57 | subject to bit rot, especially for long-tail packages. -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/distro_packages/gentoo.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/distro_packages/base' 2 | require 'json' 3 | 4 | module NixPkgsMonitor module DistroPackages 5 | 6 | class Gentoo < NixPkgsMonitor::DistroPackages::Base 7 | 8 | attr_accessor :version_overlay, :version_upstream 9 | @cache_name = "gentoo" 10 | 11 | def version 12 | return version_upstream if version_upstream 13 | return version_overlay if version_overlay and not(version_overlay.end_with?('9999')) 14 | return @version 15 | end 16 | 17 | 18 | def serialize 19 | super.merge({:version_overlay => @version_overlay, 20 | :version_upstream => @version_upstream}) 21 | end 22 | 23 | 24 | def self.deserialize(val) 25 | pkg = super(val) 26 | pkg.version_overlay = val[:version_overlay] 27 | pkg.version_upstream = val[:version_upstream] 28 | return pkg 29 | end 30 | 31 | 32 | def self.generate_list 33 | gentoo_list = {} 34 | 35 | categories_json = http_agent.get('http://euscan.gentooexperimental.org/api/1.0/categories.json').body 36 | JSON.parse(categories_json)["categories"].each do |cat| 37 | puts cat["category"] 38 | packages_json = http_agent.get("http://euscan.gentooexperimental.org/api/1.0/packages/by-category/#{cat["category"]}.json").body 39 | JSON.parse(packages_json)["packages"].each do |pkg| 40 | name = pkg["name"] 41 | gentoo_list[name] = Gentoo.new(cat["category"] + '/' + name, name) unless gentoo_list[name] 42 | if pkg["last_version_gentoo"] and not(pkg["last_version_gentoo"]["version"].include? '9999') 43 | gentoo_list[name].version = pkg["last_version_gentoo"]["version"] 44 | end 45 | if pkg["last_version_overlay"] and not(pkg["last_version_overlay"]["version"].include? '9999') 46 | gentoo_list[name].version_overlay = pkg["last_version_overlay"]["version"] 47 | end 48 | if pkg["last_version_upstream"] 49 | gentoo_list[name].version_upstream = pkg["last_version_upstream"]["version"] 50 | end 51 | end 52 | end 53 | 54 | serialize_list(gentoo_list.values) 55 | end 56 | 57 | end 58 | 59 | end end 60 | -------------------------------------------------------------------------------- /help.md: -------------------------------------------------------------------------------- 1 | # Reports: 2 | 3 | ## Coverage 4 | 5 | The number of different update sources available for a given package. 6 | This report is an estimate. More precise report can only be obtained 7 | during an actual updater run and it isn't being done yet. 8 | 9 | ## Vulnerability 10 | 11 | A list of potential matches against CVE vulnerability database. Unfortunately, 12 | the report is noisy because the matching code has been tuned to give virtually 13 | no false negatives. That is, it tries to never miss a match. 14 | 15 | ## Outdated: 16 | 17 | A list of packages for which newer versions were reported by updaters. The report 18 | generator tries to be smart and guess which version is a major update (most likely 19 | requiring extensive testing) and which one is a minor update(safe to commit if it 20 | compiles). 21 | 22 | ### Flags 23 | 24 | (V) beside the package name: the package is potentially vulnerable 25 | 26 | Version numbers are color coded to indicate whether a patch is available and its status(queued, failed, built). 27 | 28 | ## Package Details 29 | 30 | Clicking a package name shows more details including specific CVE matches, 31 | found tarballs, available patches and corresponding build logs and statuses. 32 | 33 | ## Build logs 34 | 35 | There's a buildlog-lint code which tries to detect subtle problems such as 36 | missing dependencies and add helpful warnings at the top of the log. Currently, 37 | it's a proof-of-concept code which detects typical missing documentation-related 38 | dependencies such as doxygen, missing perl dependencies and failed tests. 39 | 40 | # Suggested Workflow 41 | 42 | Before committing a patch, take a look at its build log to check for irregularities, 43 | even if it compiles and is a minor update. 44 | 45 | A patch can be applied using curl 'patch url'|git am 46 | 47 | It's recommended to re-set the author fields before pushing to nixpkgs. You can do 48 | this for a of a whole bunch of commits in one go: 49 | 50 | git filter-branch --env-filter 'export GIT_AUTHOR_NAME="Joe Doe" GIT_AUTHOR_EMAIL=joe@example.com' origin/master..master 51 | 52 | Alternatively, you can add 'm' parameter to the patch url with a substring of maintainer 53 | name. Usually 2-3 chars is enough to uniquely identify maintainer. 54 | eg http://monitor.nixos.org/patch?p=xxx&v=1.2.3&m=ph is enough to properly set the author field. 55 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/repository/cpan.rb: -------------------------------------------------------------------------------- 1 | require 'zlib' 2 | require 'nixpkgs_monitor/package_updaters/base' 3 | 4 | module NixPkgsMonitor module PackageUpdaters module Repository 5 | 6 | # handles Perl packages hosted at mirror://cpan/ 7 | class CPAN < NixPkgsMonitor::PackageUpdaters::Base 8 | 9 | def self.tarballs 10 | unless @tarballs 11 | @tarballs = Hash.new{|h,k| h[k] = Array.new } 12 | @locations = {} 13 | z = Zlib::GzipReader.new(StringIO.new(http_agent.get("http://www.cpan.org/indices/ls-lR.gz").body)) 14 | unzipped = z.read 15 | dirs = unzipped.split("\n\n") 16 | dirs.each do |dir| 17 | lines = dir.split("\n") 18 | next unless lines[0].include? '/authors/' 19 | #remove dir and total 20 | dir = lines[0][2..-2] 21 | lines.delete_at(0) 22 | lines.delete_at(0) 23 | lines.each do |line| 24 | next if line[0] == 'd' or line [0] == 'l' 25 | tarball = line.split(' ').last 26 | next if tarball.end_with?('.txt', "CHECKSUMS", '.readme', '.meta', '.sig', '.diff', '.patch') 27 | (package_name, file_version) = parse_tarball_name(tarball) 28 | if file_version and package_name 29 | package_name = package_name.downcase 30 | @tarballs[package_name] << file_version 31 | @locations[[package_name, file_version]] = "mirror://cpan/#{dir}/#{tarball}" 32 | else 33 | log.debug "weird #{line}" 34 | end 35 | end 36 | end 37 | end 38 | @tarballs 39 | end 40 | 41 | def self.find_tarball(pkg, version) 42 | return nil if pkg.url.to_s.empty? or version.to_s.empty? or pkg.version.to_s.empty? 43 | (package_name, file_version) = parse_tarball_from_url(pkg.url) 44 | return nil unless package_name 45 | tarballs # workaround to fetch data 46 | @locations[[package_name.downcase, version]] 47 | end 48 | 49 | def self.covers?(pkg) 50 | return( pkg.url and pkg.url.start_with? 'mirror://cpan/' and usable_version?(pkg.version) ) 51 | end 52 | 53 | def self.newest_versions_of(pkg) 54 | return nil unless covers?(pkg) 55 | return new_tarball_versions(pkg, tarballs) 56 | end 57 | 58 | end 59 | 60 | end end end 61 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/build_log.rb: -------------------------------------------------------------------------------- 1 | require 'open-uri' 2 | 3 | module NixPkgsMonitor module BuildLog 4 | 5 | def BuildLog.lint(log) 6 | package_names = [ "gobject-introspection", 7 | 8 | # documentation 9 | "gtkdoc-check", "gtkdoc-rebase", "gtkdoc-mkpdf", 10 | "gtk-doc documentation", "bison", "byacc", "flex", "lex", "pkg-config", 11 | "doxygen", "msgfmt", "gmsgfmt", "xgettext", "msgmerge", "gnome-doc-utils", 12 | "documentation", "manpages", "txt2html", "rst2html", "xmlto", "asciidoc", 13 | 14 | # archives 15 | "lzma", "zlib", "bzlib", 16 | 17 | # TODO: something for gif, jpeg etc 18 | ] 19 | lint = log.lines.select do |line| 20 | linedc = line.downcase 21 | package_names.find{ |pn| linedc =~ /checking .*#{pn}.*\.\.\..*(no|:)/ or linedc =~ /could not find .*#{pn}/ } or 22 | linedc.include? 'not installed' or # perl prequisites 23 | linedc =~ /skipped.* require/ or # perl test dependencies 24 | linedc =~ /skipped.* no.* available/ or # perl test dependencies 25 | linedc.=~ /subroutine .* redefined at/ or# perl warning 26 | linedc =~ /prerequisite .* not found/ or # perl warning 27 | linedc.include? "module not found" or # perl warning 28 | linedc =~ /failed.*test/ or # perl test failure 29 | linedc =~ /skipped:.*only with/ # perl warning 30 | end 31 | 32 | return lint 33 | end 34 | 35 | 36 | def BuildLog.sanitize(log, outpath, substitutes = {}) 37 | sanitized = log.dup 38 | sanitized.gsub!(outpath, 'OUTPATH') unless outpath.to_s.empty? 39 | pkgname = outpath && outpath.partition('-')[2] 40 | sanitized.gsub!(%r{/tmp/nix-build-#{pkgname}.drv-\d*/#{pkgname}},'BUILDPATH') unless pkgname.to_s.empty? 41 | substitutes.each{ |orig, value| sanitized.gsub!(orig, value) } 42 | sanitized.gsub(%r{/nix/store/(\S{32})}, '/nix/store/...') 43 | end 44 | 45 | 46 | def BuildLog.get_hydra_log(outpath) 47 | open("http://hydra.nixos.org/log/#{outpath.sub(%r{^/nix/store/},"")}").read rescue nil 48 | end 49 | 50 | def BuildLog.get_db_log(outpath) 51 | build = DB[:builds][:outpath => outpath] 52 | build && build[:log] 53 | end 54 | 55 | def BuildLog.get_log(outpath) 56 | get_db_log(outpath) || get_hydra_log(outpath) 57 | end 58 | 59 | end end 60 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/distro_packages/base.rb: -------------------------------------------------------------------------------- 1 | require 'mechanize' 2 | 3 | module NixPkgsMonitor module DistroPackages 4 | 5 | # Generic distro package 6 | class Base 7 | 8 | attr_accessor :internal_name, :name, :version, :url, :revision 9 | 10 | def initialize(internal_name, name = internal_name, version = '0', url = nil, revision = nil ) 11 | @internal_name = internal_name 12 | @name = name 13 | @version = version 14 | @url = url 15 | @revision = revision 16 | end 17 | 18 | 19 | def serialize 20 | { 21 | :internal_name =>@internal_name, 22 | :name => @name, 23 | :version => @version, 24 | :url => @url, 25 | :revision => @revision 26 | } 27 | end 28 | 29 | 30 | def self.deserialize(val) 31 | new(val[:internal_name], val[:name], val[:version], val[:url], val[:revision]) 32 | end 33 | 34 | 35 | def self.table_name 36 | "packages_#{@cache_name}".to_sym 37 | end 38 | 39 | 40 | def self.load_from_db(db) 41 | if DB.table_exists?(table_name) 42 | DB[table_name].each do |record| 43 | package = deserialize(record) 44 | @packages << package 45 | @list[package.name.downcase] = package 46 | @by_internal_name[package.internal_name] = package 47 | end 48 | else 49 | STDERR.puts "#{table_name} doesn't exist" 50 | end 51 | end 52 | 53 | 54 | def self.list 55 | unless @list 56 | @list = {} 57 | @by_internal_name = {} 58 | @packages = [] 59 | 60 | load_from_db(DB) 61 | end 62 | 63 | return @list 64 | end 65 | 66 | 67 | def self.refresh 68 | @list = nil 69 | @by_internal_name = nil 70 | @packages = nil 71 | end 72 | 73 | 74 | def self.packages 75 | list unless @packages 76 | @packages 77 | end 78 | 79 | 80 | def self.by_internal_name 81 | list unless @by_internal_name 82 | @by_internal_name 83 | end 84 | 85 | 86 | def self.serialize_to_db(db, list) 87 | db[table_name].delete 88 | list.each do |package| 89 | db[table_name] << package.serialize 90 | end 91 | end 92 | 93 | 94 | def self.serialize_list(list) 95 | DB.transaction do 96 | serialize_to_db(DB, list) 97 | end 98 | end 99 | 100 | 101 | def self.http_agent 102 | agent = Mechanize.new 103 | agent.user_agent = 'NixPkgs software update checker' 104 | return agent 105 | end 106 | 107 | end 108 | 109 | end end -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/distro_packages/arch.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/distro_packages/base' 2 | require 'set' 3 | 4 | module NixPkgsMonitor module DistroPackages 5 | 6 | class GenericArch < NixPkgsMonitor::DistroPackages::Base 7 | 8 | def self.parse_pkgbuild(entry, path) 9 | pkg_data = %x(bash -c 'source #{path} && echo -e $source\\\\n$pkgver\\\\n${pkgname[*]}').split("\n") 10 | url = pkg_data[0].strip 11 | pkg_ver = pkg_data[1].strip 12 | pkg_names = [entry] + pkg_data[2].split(' ') 13 | 14 | if pkg_ver.to_s.empty? 15 | puts "skipping #{entry}: no package version" 16 | return {} 17 | end 18 | if url.to_s.empty? 19 | puts "skipping #{entry}: no url found" 20 | return {} 21 | end 22 | 23 | puts "warning #{entry}: failed to parse package name list" if pkg_names.length <= 1 24 | 25 | Set.new(pkg_names).each_with_object(Hash.new) do |name, pkgs| 26 | pkgs[name] = new(name, name, pkg_ver, url.strip) 27 | end 28 | end 29 | 30 | end 31 | 32 | 33 | class Arch < GenericArch 34 | @cache_name = "arch" 35 | 36 | def self.generate_list 37 | arch_list = {} 38 | 39 | puts "Cloning / pulling repos." 40 | puts %x(git clone git://projects.archlinux.org/svntogit/packages.git) 41 | puts %x(cd packages && git pull --rebase) 42 | puts %x(git clone git://projects.archlinux.org/svntogit/community.git) 43 | puts %x(cd community && git pull --rebase) 44 | 45 | (Dir.entries("packages") + Dir.entries("community")).reject{ |entry| ['.', '..'].include? entry } 46 | .each do |entry| 47 | [ File.join("packages", entry, "repos", "extra-i686", "PKGBUILD"), 48 | File.join("packages", entry, "repos", "core-i686", "PKGBUILD"), 49 | File.join("community", entry, "repos", "community-i686", "PKGBUILD") 50 | ].select { |f| File.exists? f } 51 | .each { |pkgbuild_file| arch_list.merge!(parse_pkgbuild(entry, pkgbuild_file)) } 52 | end 53 | serialize_list(arch_list.values) 54 | end 55 | 56 | end 57 | 58 | 59 | class AUR < GenericArch 60 | @cache_name = "aur" 61 | 62 | def self.generate_list 63 | aur_list = {} 64 | 65 | puts "Cloning AUR repos" 66 | puts %x(curl http://aur3.org/all_pkgbuilds.tar.gz -O) 67 | puts %x(rm -rf aur/*) 68 | puts %x(mkdir aur) 69 | puts %x(tar -xvf all_pkgbuilds.tar.gz --show-transformed --transform s,/PKGBUILD,, --strip-components=1 -C aur) 70 | 71 | puts "Scanning AUR" 72 | Dir.entries("aur").each do |entry| 73 | next if entry == '.' or entry == '..' 74 | 75 | pkgbuild_file = File.join("aur", entry) 76 | aur_list.merge!(parse_pkgbuild(entry, pkgbuild_file)) if File.exists? pkgbuild_file 77 | end 78 | 79 | serialize_list(aur_list.values) 80 | end 81 | 82 | end 83 | 84 | end end 85 | -------------------------------------------------------------------------------- /unmaintained/comparepackages.rb: -------------------------------------------------------------------------------- 1 | # currently defunct 2 | 3 | require 'optparse' 4 | require 'set' 5 | require './distro-package.rb' 6 | 7 | # levenshtein distance 8 | # taken from https://github.com/threedaymonk/text/blob/master/lib/text/levenshtein.rb 9 | def distance(str1, str2) 10 | prepare = 11 | if "ruby".respond_to?(:encoding) 12 | lambda { |str| str.encode(Encoding::UTF_8).unpack("U*") } 13 | else 14 | rule = $KCODE.match(/^U/i) ? "U*" : "C*" 15 | lambda { |str| str.unpack(rule) } 16 | end 17 | 18 | s, t = [str1, str2].map(&prepare) 19 | n = s.length 20 | m = t.length 21 | return m if n.zero? 22 | return n if m.zero? 23 | 24 | d = (0..m).to_a 25 | x = nil 26 | 27 | n.times do |i| 28 | e = i + 1 29 | m.times do |j| 30 | cost = (s[i] == t[j]) ? 0 : 1 31 | x = [ 32 | d[j+1] + 1, # insertion 33 | e + 1, # deletion 34 | d[j] + cost # substitution 35 | ].min 36 | d[j] = e 37 | e = x 38 | end 39 | d[m] = x 40 | end 41 | 42 | return x 43 | end 44 | 45 | def src_distance(str1,str2) 46 | url1 = $1 if str1 =~ /:\/\/[^\/]*(\/.*)/ 47 | url2 = $1 if str2 =~ /:\/\/[^\/]*(\/.*)/ 48 | 49 | return 10000 unless url1 and url2 50 | return distance(url1, url2) 51 | end 52 | 53 | 54 | known_missing = Set.new ["dvdrip","ogle", "lha", "xlogo", "beep", ] 55 | 56 | OptionParser.new do |o| 57 | 58 | o.on("--match-arch", "Try matching Arch packages to Nix packages") do 59 | arch_list = DistroPackage::Arch.list 60 | nix_list = DistroPackage::Nix.list 61 | 62 | missing = (Set.new(arch_list.keys) - Set.new(nix_list.keys)) - known_missing 63 | found = (Set.new(arch_list.keys) - missing) - known_missing 64 | puts "Found #{found.count} packages: #{found.inspect}" 65 | puts "known missing #{known_missing.count} packages : #{known_missing}" 66 | puts "Missing #{missing.count} packages: #{missing.inspect}" 67 | 68 | found.each do |pkg| 69 | puts pkg 70 | arch_url = arch_list[pkg].url 71 | nix_url = nix_list[pkg].url 72 | next if arch_url == 'none' or nix_url == 'none' 73 | puts "#{nix_url} #{arch_url} #{src_distance(nix_url,arch_url)}" 74 | end 75 | 76 | puts "TRYING TO FIND MATCHES BY URL ONLY" 77 | missing.each do |pkg| 78 | puts pkg 79 | arch_url = arch_list[pkg].url 80 | nix_list.each_value do |nixpkg| 81 | nix_url = nixpkg.url 82 | next if arch_url == 'none' or nix_url == 'none' 83 | 84 | if src_distance(nix_url,arch_url)<8 85 | puts " found match #{pkg} #{nixpkg.name} " 86 | end 87 | end 88 | end 89 | 90 | end 91 | 92 | 93 | o.on("--match-deb", "Try matching Nix packages to Debian packages") do 94 | deb_list = DistroPackage::Deb.generate_list 95 | nix_list = DistroPackage::Nix.list 96 | unmatched = (Set.new(nix_list.keys) - Set.new(deb_list.keys)) 97 | matched = Set.new(nix_list.keys)- unmatched 98 | puts "Matched #{matched.count} packages: #{matched.inspect}" 99 | puts "Unmatched #{unmatched.count} packages: #{unmatched.inspect}" 100 | end 101 | 102 | o.on("-h", "--help", "Show this message") do 103 | puts o 104 | exit 105 | end 106 | 107 | o.parse(ARGV) 108 | end 109 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/security_advisories.rb: -------------------------------------------------------------------------------- 1 | require 'nokogiri' 2 | require 'nixpkgs_monitor/distro_packages' 3 | require 'uri' 4 | 5 | module NixPkgsMonitor module SecurityAdvisories 6 | 7 | class CVE 8 | attr_reader :id, :packages 9 | 10 | def self.load_from(file) 11 | result = [] 12 | xml = Nokogiri::XML(File.read(file)) 13 | 14 | xml.xpath('xmlns:nvd/xmlns:entry').each do |entry| 15 | #puts "loading #{entry[:id]}" 16 | packages = [] 17 | entry.xpath('vuln:vulnerable-software-list').each do |list| 18 | 19 | list.xpath('vuln:product').each do |product| 20 | pname = product.inner_text 21 | unless parse_package(pname) 22 | puts "failed to parse #{pname} @ #{entry[:id]}" 23 | else 24 | packages << pname 25 | end 26 | end 27 | 28 | end 29 | result << new(entry[:id], packages) 30 | end 31 | 32 | return result 33 | end 34 | 35 | 36 | def self.update_names 37 | (-4..0).map { |offs| (Time.now.utc.year + offs).to_s } + ["Modified", "Recent"] 38 | end 39 | 40 | def self.fetch_updates 41 | update_names.each do |name| 42 | puts %x(curl -O https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-#{name}.xml.gz) 43 | puts %x(zcat nvdcve-2.0-#{name}.xml.gz > nvdcve-2.0-#{name}.xml) 44 | end 45 | end 46 | 47 | def self.list 48 | @list ||= update_names.map {|n| load_from("nvdcve-2.0-#{n}.xml")}.reduce(:+) 49 | end 50 | 51 | 52 | def initialize(id, packages) 53 | @id = id 54 | @packages = packages 55 | end 56 | 57 | def self.parse_package(package) 58 | return nil unless %r{^cpe:/.:(?[^:]*):(?[^:]*):(?[^:]*)} =~ package 59 | return [ URI.unescape(supplier), URI.unescape(product), URI.unescape(version) ] 60 | end 61 | 62 | end 63 | 64 | 65 | class GLSA 66 | attr_reader :id, :packages 67 | 68 | def self.update_list 69 | end 70 | 71 | def self.parse(file) 72 | glsa = Nokogiri::XML(File.read(file)).xpath('//glsa').first 73 | packages = [] 74 | glsa.xpath('affected/package').each do |package| 75 | packages << package[:name].downcase 76 | end 77 | self.new('GLSA-' + glsa[:id], packages) 78 | end 79 | 80 | def initialize(id, packages) 81 | @id = id 82 | @packages = packages 83 | end 84 | 85 | 86 | def self.list 87 | unless @glsa_list 88 | @glsa_list = [] 89 | Dir.entries('portage/metadata/glsa').each do |entry| 90 | glsa_name = 'portage/metadata/glsa/' + entry 91 | next unless File.file? glsa_name and entry.end_with? ".xml" and entry != 'index.xml' and entry =~ /201[01234]/ 92 | @glsa_list << self.parse(glsa_name) 93 | end 94 | end 95 | 96 | return @glsa_list 97 | end 98 | 99 | 100 | def matching_nixpkgs 101 | return nil unless %r{(?[^/]+)/(?[^/]+)} =~ packages[0] 102 | result = NixPkgsMonitor::DistroPackages::Nix.list[name] 103 | result = NixPkgsMonitor::DistroPackages::Nix.list['ruby-' + name] unless result 104 | result = NixPkgsMonitor::DistroPackages::Nix.list['python-' + name] unless result 105 | result = NixPkgsMonitor::DistroPackages::Nix.list['perl-' + name] unless result 106 | return result 107 | end 108 | 109 | def affected_nixpkgs 110 | end 111 | 112 | end 113 | 114 | end end 115 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/distro.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/distro_packages' 2 | require 'nixpkgs_monitor/package_updaters/base' 3 | 4 | module NixPkgsMonitor module PackageUpdaters module Distro 5 | 6 | class Base < NixPkgsMonitor::PackageUpdaters::Base 7 | 8 | def self.covers?(pkg) 9 | match_nixpkg(pkg) and usable_version?(pkg.version) 10 | end 11 | 12 | def self.newest_version_of(pkg) 13 | return nil unless covers?(pkg) 14 | distro_pkg = match_nixpkg(pkg) 15 | return nil unless usable_version?(distro_pkg.version) 16 | ( is_newer?(distro_pkg.version, pkg.version) ? distro_pkg.version : nil ) 17 | end 18 | 19 | end 20 | 21 | class ArchBase < Base 22 | 23 | def self.match_nixpkg(pkg) 24 | pkgname = pkg.name.downcase 25 | 26 | list[pkgname] or 27 | list['xorg-'+pkgname] or 28 | list['kdeedu-'+pkgname] or 29 | list['kdemultimedia-'+pkgname] or 30 | list['kdeutils-'+pkgname] or 31 | list['kdegames-'+pkgname] or 32 | list['kdebindings-'+pkgname] or 33 | list['kdegraphics-'+pkgname] or 34 | list['kdeaccessibility-'+pkgname] or 35 | list[pkgname.gsub(/^python[0-9\.]*-/, 'python-')] or 36 | list[pkgname.gsub(/^python[0-9\.]*-/, 'python2-')] or 37 | list[pkgname.gsub(/^aspell-dict-/, 'aspell-')] or 38 | list[pkgname.gsub(/^(haskell-.*)-ghc\d+\.\d+\.\d+$/, '\1')] or 39 | list[pkgname.gsub(/^ktp-/, 'telepathy-kde-')] 40 | end 41 | 42 | end 43 | 44 | # checks package versions against Arch Core, Community and Extra repositories 45 | class Arch < ArchBase 46 | 47 | def self.list 48 | NixPkgsMonitor::DistroPackages::Arch.list 49 | end 50 | 51 | end 52 | 53 | # checks package versions against Arch AUR 54 | class AUR < ArchBase 55 | 56 | def self.list 57 | NixPkgsMonitor::DistroPackages::AUR.list 58 | end 59 | 60 | end 61 | 62 | # TODO: checks package versions against Debian Sid 63 | class Debian < Base 64 | 65 | def self.match_nixpkg(pkg) 66 | pkgname = pkg.name.downcase 67 | list = NixPkgsMonitor::DistroPackages::Debian.list 68 | 69 | list[pkgname] or 70 | list[pkgname.gsub(/^python[0-9\.]*-/, '')] or 71 | list[pkgname.gsub(/^perl-(.*)$/, 'lib\1-perl')] or 72 | list[pkgname.gsub(/^(haskell-.*)-ghc\d+\.\d+\.\d+$/, '\1')] or 73 | list[pkgname.gsub(/^xf86-/, 'xserver-xorg-')] or 74 | list[pkgname+"1"] or 75 | list[pkgname+"2"] or 76 | list[pkgname+"3"] or 77 | list[pkgname+"4"] or 78 | list[pkgname+"5"] or 79 | list[pkgname+"6"] 80 | end 81 | 82 | end 83 | 84 | 85 | # checks package versions agains those discovered by http://euscan.iksaif.net, 86 | # which include Gentoo portage, Gentoo developer repositories, euscan-discovered upstream. 87 | class Gentoo < Base 88 | 89 | def self.covers?(pkg) 90 | match_nixpkg(pkg) and usable_version?(pkg.version) and 91 | not Repository::CPAN.covers?(pkg) and not Repository::Hackage.covers?(pkg) 92 | end 93 | 94 | def self.match_nixpkg(pkg) 95 | pkgname = pkg.name.downcase 96 | list = NixPkgsMonitor::DistroPackages::Gentoo.list 97 | 98 | list[pkgname] or 99 | list[pkgname.gsub(/^ruby-/, '')] or 100 | list[pkgname.gsub(/^python[0-9\.]*-/, '')] or 101 | list[pkgname.gsub(/^perl-/, '')] or 102 | list[pkgname.gsub(/^haskell-(.*)-ghc\d+\.\d+\.\d+$/,'\1')] 103 | end 104 | 105 | end 106 | 107 | Updaters = [ Gentoo, Arch, Debian, AUR ] 108 | 109 | end end end 110 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nixpkgs-monitor 2 | 3 | A tool to monitor and improve NixPkgs packages quality, freshness and security. 4 | 5 | ## NixOS module 6 | 7 | After checking out the nixpkgs-monitor repository, add this to configuration.nix: 8 | 9 | require = [ /path/to/nixpkgs-monitor/service.nix ]; 10 | services.nixpkgs-monitor.enable = true; 11 | 12 | This sets up 3 systemd units: 13 | * nixpkgs-monitor-site, a web interface. It will return errors until the first updater run is finished. 14 | * nixpkgs-monitor-updater, a job that does the full update checker, patch generator and builder run. 15 | You need to run it manually or via cron or timers. 16 | * nixpkgs-monitor-updater-drop-negative-cache, a maintenance job that drops failed tarball downloads and builds from the cache. 17 | Should be run from time to time to recover from intermittent failures such as: disk getting full, connectivity troubles, upstream services going down. 18 | 19 | ## How to install and use as a regular user 20 | 21 | Have Nix? Just run: 22 | 23 | nix-env -if . 24 | 25 | ## nixpkgs-monitor executable 26 | 27 | Does all the dirty work: obtains package metada, finds updates, vulnerabilities, downloads tarballs, generates patches and attempts to build them. 28 | 29 | ### A typical workflow 30 | 31 | cd to the working directory. The cache DB will be stored here along with the needed repository checkouts. 32 | 33 | (Re)generate package caches: --list-{arch,deb,nix,gentoo} 34 | Fetch CVE data: --cve-update 35 | 36 | After package cache is populated, generate all essential reports: 37 | --coverage --check-updates --cve-check 38 | Nix package cache should be populated before this step; the rest is optional, but strongly recommended 39 | 40 | Fetch tarballs and generate patches: --tarballs --patches 41 | 42 | Attempt building the generated patches: --build 43 | 44 | All the mentioned actions except for build: --all. 45 | A good idea before running the build step is to trigger a web interface refresh. 46 | 47 | ## nixpkgs-monitor-site executable 48 | 49 | Provides a nice web interface to browse the reports, patches, build logs and such. 50 | By default runs on http://localhost:4567. Must be run from the same directory as nixpkgs-monitor tool. 51 | 52 | The web interface caches some of the data in RAM, and must be kicked by requesting /refresh 53 | or restarted after nixpkgs-monitor run finishes. 54 | 55 | ## database 56 | 57 | nixpkgs-monitor puts package cache, coverage, version mismatch and updater reports into 58 | a database(db.sqlite by default). 59 | 60 | Coverage report is in estimated_coverage table, version mismatch report is in 61 | version_mismatch table and updater reports are in repository_* and distro_* tables. 62 | 63 | Package caches are in packages_* tables. 64 | 65 | Tarball candidates are in tarballs table. 66 | 67 | Tarball hashes or '404' for failed downloads are in tarball_sha256 table. 68 | 404 records can be dropped using nixpkgs-monitor --redownload. 69 | 70 | Generated patches along with derivation paths are in patches table. 71 | 72 | You can extract individual patches by running something like 73 | SELECT patch FROM patches WHERE pkg_attr='package' AND version='version'; 74 | 75 | Build logs and statuses are in builds table. Failed build records can be 76 | dropped using nixpkgs-monitor --rebuild if you suspect intermittent build 77 | failures caused by eg disk being full or network going down. 78 | 79 | Potential CVE matches are in cve_match table. 80 | 81 | ## comparepackages.rb 82 | 83 | Matches packages in one distro to packages in another one. 84 | To be used for experimentation only. Unmaintained. 85 | Probably you don't care that it exists. 86 | 87 | ## Debian watchfiles tools 88 | 89 | Scripts and a writeup of an experiment to see just how useful Debian watchfiles are. 90 | Spoiler: not very useful :( 91 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/migrations/001_initial.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | 4 | # updaters 5 | 6 | [ :repository_cpan, :repository_fetchgit, :repository_github, 7 | :repository_gnome, :repository_gnu, :repository_hackage, 8 | :repository_kde, :repository_metagit, :repository_npmjs, 9 | :repository_pypi, :repository_rubygems, :repository_sf, 10 | :repository_xorg, :gentoodistfiles, 11 | :distro_arch, :distro_aur, :distro_debian, :distro_gentoo, 12 | ].each do |updater| 13 | DB.create_table!(updater) do 14 | String :pkg_attr 15 | String :version 16 | primary_key [ :pkg_attr, :version ] 17 | end 18 | end 19 | 20 | create_table(:estimated_coverage) do 21 | String :pkg_attr, :unique => true, :primary_key => true 22 | Integer :coverage 23 | end 24 | 25 | create_table(:tarball_sha256) do 26 | String :tarball, :unique => true, :primary_key => true 27 | String :sha256 28 | end 29 | 30 | create_table(:tarballs) do 31 | String :pkg_attr 32 | String :version 33 | String :tarball 34 | end 35 | 36 | create_table(:patches) do 37 | String :pkg_attr 38 | String :version 39 | String :tarball 40 | primary_key [ :pkg_attr, :version, :tarball ] 41 | Text :patch 42 | String :drvpath 43 | String :outpath 44 | end 45 | 46 | create_table(:builds) do 47 | String :outpath, :unique => true, :primary_key => true 48 | String :status 49 | String :log 50 | end 51 | 52 | create_table(:cve_match) do 53 | String :pkg_attr#, :primary_key => true 54 | String :product 55 | String :version 56 | String :CVE 57 | end 58 | 59 | create_table(:timestamps) do 60 | String :action, :unique => true, :primary_key => true 61 | Time :timestamp 62 | String :message 63 | end 64 | 65 | # package cache 66 | 67 | create_table!(:packages_nix) do 68 | String :internal_name, :unique => true, :primary_key => true 69 | String :name 70 | String :version 71 | String :repository_git 72 | String :branch 73 | String :url 74 | String :revision 75 | String :sha256 76 | String :position 77 | String :homepage 78 | String :drvpath 79 | String :outpath 80 | end 81 | 82 | create_table!(:nix_maintainers) do 83 | String :internal_name 84 | String :maintainer 85 | end 86 | 87 | create_table!(:packages_gentoo) do 88 | String :internal_name, :unique => true, :primary_key => true 89 | String :name 90 | String :version 91 | String :url 92 | String :version_overlay 93 | String :version_upstream 94 | String :revision 95 | end 96 | 97 | create_table!(:packages_arch) do 98 | String :internal_name, :unique => true, :primary_key => true 99 | String :name 100 | String :version 101 | String :url 102 | String :revision 103 | end 104 | 105 | create_table!(:packages_aur) do 106 | String :internal_name, :unique => true, :primary_key => true 107 | String :name 108 | String :version 109 | String :url 110 | String :revision 111 | end 112 | 113 | create_table!(:packages_debian) do 114 | String :internal_name, :unique => true, :primary_key => true 115 | String :name 116 | String :version 117 | String :url 118 | String :revision 119 | end 120 | 121 | # logs 122 | [ :nixpkgs_failed_name_parse, :nixpkgs_no_sources, :version_mismatch ].each do |log_name| 123 | create_table!(log_name) do 124 | String :pkg_attr, :unique => true, :primary_key => true 125 | end 126 | end 127 | 128 | end 129 | end -------------------------------------------------------------------------------- /service.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | with lib; 4 | 5 | let 6 | 7 | cfg = config.services.nixpkgs-monitor; 8 | 9 | env_db = optionalAttrs (cfg.database != null) { DB = cfg.database; }; 10 | 11 | npmon = import ./default.nix { inherit pkgs; }; 12 | 13 | in 14 | 15 | { 16 | ###### interface 17 | 18 | options = { 19 | 20 | services.nixpkgs-monitor = rec { 21 | 22 | enable = mkOption { 23 | default = false; 24 | description = '' 25 | Whether to run Nixpkgs-monitor services. 26 | ''; 27 | }; 28 | 29 | baseDir = mkOption { 30 | default = "/var/lib/nixpkgs-monitor"; 31 | description = '' 32 | The directory holding configuration, logs and temporary files. 33 | ''; 34 | }; 35 | 36 | user = mkOption { 37 | default = "nixpkgsmon"; 38 | description = '' 39 | The user the Nixpkgs-monitor services should run as. 40 | ''; 41 | }; 42 | 43 | host = mkOption { 44 | default = "localhost"; 45 | description = '' 46 | The IP address to listen at. 47 | ''; 48 | }; 49 | 50 | port = mkOption { 51 | default = 4567; 52 | description = '' 53 | The IP address to listen at. 54 | ''; 55 | }; 56 | 57 | baseUrl = mkOption { 58 | default = null; 59 | description = '' 60 | Base URL at which the monitor should run. 61 | ''; 62 | }; 63 | 64 | database = mkOption { 65 | default = null; 66 | example = "postgres://db_user:db_password@host/db_name"; 67 | description = '' 68 | Use the specified database instead of the default(sqlite) one. 69 | ''; 70 | }; 71 | 72 | builderCount = mkOption { 73 | default = 1; 74 | description = '' 75 | The number of builds to run in parallel 76 | ''; 77 | }; 78 | 79 | }; 80 | 81 | }; 82 | 83 | 84 | ###### implementation 85 | 86 | config = mkIf cfg.enable { 87 | 88 | users.extraUsers = singleton { 89 | name = cfg.user; 90 | description = "Nixpkgs-monitor"; 91 | home = cfg.baseDir; 92 | createHome = true; 93 | useDefaultShell = true; 94 | }; 95 | 96 | systemd.services."nixpkgs-monitor-site" = { 97 | wantedBy = [ "multi-user.target" ]; 98 | 99 | environment = 100 | env_db // 101 | optionalAttrs (cfg.baseUrl != null) { BASE_URL = cfg.baseUrl; }; 102 | 103 | serviceConfig = { 104 | ExecStart = "${npmon}/bin/nixpkgs-monitor-site -p ${toString cfg.port} -o ${cfg.host}"; 105 | User = cfg.user; 106 | Restart = "always"; 107 | WorkingDirectory = cfg.baseDir; 108 | }; 109 | }; 110 | 111 | systemd.services."nixpkgs-monitor-updater" = { 112 | path = [ pkgs.xz ]; 113 | environment = { 114 | NIX_REMOTE = "daemon"; 115 | NIX_CONF_DIR = "/etc/nix"; 116 | OPENSSL_X509_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; 117 | GIT_SSL_CAINFO = "/etc/ssl/certs/ca-certificates.crt"; 118 | CURL_CA_BUNDLE = "/etc/ssl/certs/ca-certificates.crt"; 119 | SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; 120 | NIX_PATH = "/nix/var/nix/profiles/per-user/root/channels/nixos"; # to be able to prefetch mirror:// urls 121 | } // env_db; 122 | 123 | script = '' 124 | ${npmon}/bin/nixpkgs-monitor --all 125 | ${pkgs.curl}/bin/curl ${cfg.host}:${toString cfg.port}/refresh 126 | ${npmon}/bin/nixpkgs-monitor --build --builder-count ${toString cfg.builderCount} 127 | ''; 128 | 129 | serviceConfig = { 130 | User = cfg.user; 131 | WorkingDirectory = cfg.baseDir; 132 | }; 133 | }; 134 | 135 | systemd.services."nixpkgs-monitor-updater-drop-negative-cache" = { 136 | environment = env_db; 137 | serviceConfig = { 138 | ExecStart = "${npmon}/bin/nixpkgs-monitor --redownload --rebuild"; 139 | User = cfg.user; 140 | WorkingDirectory = cfg.baseDir; 141 | }; 142 | }; 143 | 144 | }; 145 | } 146 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/git.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/package_updaters/base' 2 | 3 | module NixPkgsMonitor module PackageUpdaters module Git 4 | 5 | # Generic git-based updater. Discovers new versions using git repository tags. 6 | class Base < NixPkgsMonitor::PackageUpdaters::Base 7 | 8 | def self.ls_remote 9 | @repo_cache ||= Hash.new do |repo_cache, repo| 10 | repo_cache[repo] = %x(GIT_ASKPASS="echo" SSH_ASKPASS= git ls-remote #{repo}) 11 | .force_encoding("iso-8859-1") 12 | .split("\n") 13 | end 14 | end 15 | 16 | # Tries to handle the tag as a tarball name. 17 | # if parsing it as a tarball fails, treats it as a version. 18 | def self.tag_to_version(tag_line) 19 | if %r{refs/tags.*/[vr]?(?\S*?)(\^\{\})?$} =~ tag_line 20 | if tag =~ /^[vr]?\d/ 21 | return tag 22 | else 23 | (name, version) = parse_tarball_name(tag) 24 | return (version ? version : tag) 25 | end 26 | else 27 | return nil 28 | end 29 | end 30 | 31 | def self.repo_contents_to_tags(repo_contents) 32 | repo_contents.select{ |s| s.include? "refs/tags/" } 33 | .map{ |tag| tag_to_version(tag) } 34 | end 35 | 36 | end 37 | 38 | 39 | # Handles fetchgit-based packages. 40 | # Tries to detect which tag the current revision corresponds to. 41 | # Otherwise assumes the package is tracking master because 42 | # there's no easy way to be smarter without checking out the repository. 43 | # Tries to find a newer tag or if tracking master, newest commit. 44 | class FetchGit < Base 45 | 46 | def self.covers?(pkg) 47 | pkg.url and not(pkg.revision.to_s.empty?) and pkg.url.include? "git" 48 | end 49 | 50 | 51 | def self.newest_version_of(pkg) 52 | return nil unless covers?(pkg) 53 | 54 | repo_contents = ls_remote[pkg.url].select{|s| s.include?("refs/tags") or s.include?("refs/heads/master") } 55 | tag_line = repo_contents.index{|line| line.include? pkg.revision } 56 | 57 | log.debug "for #{pkg.revision} found #{tag_line}" 58 | if tag_line # revision refers to a tag? 59 | return nil if repo_contents[tag_line].include?("refs/heads/master") 60 | 61 | current_version = tag_to_version(repo_contents[tag_line]) 62 | 63 | if current_version and usable_version?(current_version) 64 | 65 | versions = repo_contents_to_tags(repo_contents) 66 | max_version = versions.reduce(current_version) do |v1, v2| 67 | ( usable_version?(v2) and is_newer?(v2, v1) ) ? v2 : v1 68 | end 69 | return (max_version != current_version ? max_version : nil) 70 | 71 | else 72 | log.warn "failed to parse tag #{repo_contents[tag_line]} for #{pkg.name}. Assuming tracking master" 73 | end 74 | end 75 | 76 | # assuming tracking master 77 | master_line = repo_contents.index{|line| line.include? "refs/heads/master" } 78 | if master_line 79 | /^(?\S*)/ =~ repo_contents[master_line] 80 | log.info "new master commit #{master_commit} for #{pkg.name}:#{pkg.revision}" 81 | return( master_commit.start_with?(pkg.revision) ? nil : master_commit ) 82 | else 83 | log.warn "failed to find master for #{pkg.name}" 84 | return nil 85 | end 86 | 87 | end 88 | 89 | end 90 | 91 | 92 | # Handles GitHub-provided tarballs. 93 | class GitHub < Base 94 | 95 | def self.covers?(pkg) 96 | pkg.revision.to_s.empty? and pkg.url =~ %r{^https?://github.com/} and usable_version?(pkg.version) 97 | end 98 | 99 | def self.newest_version_of(pkg) 100 | return nil unless covers?(pkg) 101 | return nil unless %r{^https?://github.com/(?:downloads/)?(?[^/]*)/(?[^/]*)/} =~ pkg.url 102 | 103 | available_versions = repo_contents_to_tags( ls_remote["https://github.com/#{owner}/#{repo}.git"] ) 104 | new_versions(pkg.version.downcase, available_versions, pkg.internal_name) 105 | end 106 | 107 | end 108 | 109 | 110 | # Handles packages which specify meta.repositories.git. 111 | class MetaGit < Base 112 | 113 | # if meta.repository.git is the same as src.url, defer to FetchGit updater 114 | def self.covers?(pkg) 115 | not(pkg.repository_git.to_s.empty?) and (pkg.repository_git != pkg.url) and usable_version?(pkg.version) 116 | end 117 | 118 | def self.newest_version_of(pkg) 119 | return nil unless covers?(pkg) 120 | 121 | available_versions = repo_contents_to_tags( ls_remote[pkg.repository_git] ) 122 | new_versions(pkg.version.downcase, available_versions, pkg.internal_name) 123 | end 124 | 125 | end 126 | 127 | Updaters = [ FetchGit, GitHub, MetaGit ] 128 | 129 | end end end 130 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/distro_packages/nix.rb: -------------------------------------------------------------------------------- 1 | require 'nixpkgs_monitor/distro_packages/base' 2 | require 'nixpkgs_monitor/reports' 3 | require 'nokogiri' 4 | 5 | module NixPkgsMonitor module DistroPackages 6 | 7 | class Nix < NixPkgsMonitor::DistroPackages::Base 8 | 9 | attr_accessor :homepage, :repository_git, :branch, :sha256, :maintainers, :position, :outpath, :drvpath 10 | @cache_name = "nix" 11 | 12 | def version 13 | result = @version.gsub(/-gimp-\d.\d.\d+-plugin$/,"") 14 | result.gsub!(/-[34]\.\d(\.|-rc)\d$/,"") if internal_name.include? 'linuxPackages' 15 | return result 16 | end 17 | 18 | 19 | def serialize 20 | super.merge({ :homepage => @homepage, 21 | :repository_git => @repository_git, 22 | :branch => @branch, 23 | :sha256 => @sha256, 24 | :position => @position, 25 | :outpath => @outpath, 26 | :drvpath => @drvpath }) 27 | end 28 | 29 | def instantiate 30 | d_path = %x(nix-instantiate -A '#{internal_name}' ./nixpkgs/).strip 31 | raise "Failed to instantiate #{internal_name} #{drvpath}: [#{d_path}]" unless $? == 0 and d_path.split('!').first == drvpath 32 | end 33 | 34 | def self.deserialize(val) 35 | pkg = super(val) 36 | pkg.homepage = val[:homepage] 37 | pkg.repository_git = val[:repository_git] 38 | pkg.branch = val[:branch] 39 | pkg.sha256 = val[:sha256] 40 | pkg.position = val[:position] 41 | pkg.drvpath = val[:drvpath] 42 | pkg.outpath = val[:outpath] 43 | pkg.maintainers = [] 44 | return pkg 45 | end 46 | 47 | 48 | def self.serialize_to_db(db, list) 49 | super 50 | db[:nix_maintainers].delete 51 | list.each do |package| 52 | package.maintainers.each do |maintainer| 53 | db[:nix_maintainers] << { :internal_name => package.internal_name, :maintainer =>maintainer } 54 | end 55 | end 56 | end 57 | 58 | 59 | def self.load_from_db(db) 60 | super 61 | if DB.table_exists?(:nix_maintainers) 62 | DB[:nix_maintainers].each do |record| 63 | @by_internal_name[record[:internal_name]].maintainers << record[:maintainer] 64 | end 65 | else 66 | STDERR.puts "#{:nix_maintainers} doesn't exist" 67 | end 68 | end 69 | 70 | 71 | def self.repository_path 72 | "nixpkgs" 73 | end 74 | 75 | def self.log_name_parse 76 | @log_name_parse ||= NixPkgsMonitor::Reports::Logs.new(:nixpkgs_failed_name_parse) 77 | end 78 | 79 | def self.log_no_sources 80 | @log_no_sources ||= NixPkgsMonitor::Reports::Logs.new(:nixpkgs_no_sources) 81 | end 82 | 83 | def self.package_from_xml(pkg_xml) 84 | attr = pkg_xml[:attrPath] 85 | name = pkg_xml[:name] 86 | if name and attr 87 | package = ( name =~ /(.*?)-([^A-Za-z].*)/ ? Nix.new(attr, $1, $2) : Nix.new(attr, name, "") ) 88 | log_name_parse.pkg(attr) if package.version.to_s.empty? 89 | 90 | package.drvpath = pkg_xml[:drvPath] 91 | 92 | outpath = pkg_xml.xpath('output[@name="out"]').first 93 | package.outpath = outpath[:path] if outpath 94 | 95 | repository_git = pkg_xml.xpath('meta[@name="repositories.git"]').first 96 | package.repository_git = repository_git[:value] if repository_git 97 | 98 | url = pkg_xml.xpath('meta[@name="src.repo"]').first 99 | package.url = url[:value] if url 100 | 101 | url = pkg_xml.xpath('meta[@name="src.url"]').first 102 | package.url = url[:value] if url 103 | 104 | log_no_sources.pkg(attr) if package.url.to_s.empty? 105 | 106 | rev = pkg_xml.xpath('meta[@name="src.rev"]').first 107 | package.revision = rev[:value] if rev 108 | 109 | sha256 = pkg_xml.xpath('meta[@name="src.sha256"]').first 110 | package.sha256 = sha256[:value] if sha256 111 | 112 | position = pkg_xml.xpath('meta[@name="position"]').first 113 | package.position = position[:value].rpartition('/nixpkgs/')[2] if position 114 | 115 | # if the package file name looks like a version, it is probably a branch, at least for haskell 116 | if package.position and package.internal_name.start_with? 'haskellPackages' 117 | file_name = File.basename(package.position).split('.') 118 | version = (file_name.last.start_with?('nix') ? file_name[0..-2] : file_name) 119 | package.branch = version.join('.') if version.reject{|s| s.to_i.to_s == s }.empty? 120 | end 121 | 122 | homepage = pkg_xml.xpath('meta[@name="homepage"]').first 123 | package.homepage = homepage[:value] if homepage 124 | 125 | branch = pkg_xml.xpath('meta[@name="branch"]').first 126 | package.branch = branch[:value] if branch 127 | 128 | maintainers = pkg_xml.xpath('meta[@name="maintainers"]/string').map{|m| m[:value]} 129 | package.maintainers = ( maintainers ? maintainers : [] ) 130 | return package 131 | end 132 | return nil 133 | end 134 | 135 | 136 | def self.load_package(pkg) 137 | pkgs_xml = Nokogiri.XML(%x(nix-env -qaA '#{pkg}' --attr-path --meta --xml --out-path --drv-path --file ./nixpkgs/)) 138 | entry = pkgs_xml.xpath('items/item').first 139 | return nil unless entry and entry[:attrPath] == pkg 140 | package = package_from_xml(entry) 141 | return package if package and package.internal_name == pkg 142 | end 143 | 144 | 145 | def self.generate_list 146 | nix_list = {} 147 | 148 | puts %x(git clone https://github.com/NixOS/nixpkgs.git) 149 | puts %x(cd #{repository_path} && git checkout master --force && git pull --rebase) 150 | 151 | log_name_parse.clear! 152 | log_no_sources.clear! 153 | 154 | pkgs_xml = Nokogiri.XML(%x(nix-env -qa '*' --attr-path --meta --xml --out-path --drv-path --file ./nixpkgs/)) 155 | raise "nixpkgs evaluation failed" unless $? == 0 156 | pkgs_xml.xpath('items/item').each do|entry| 157 | package = package_from_xml(entry) 158 | if package 159 | pkg_hash = package.sha256 ? package.sha256 : package.internal_name 160 | nix_list[pkg_hash] = ( nix_list.has_key?(pkg_hash) ? 161 | (nix_list[pkg_hash].internal_name > package.internal_name ? nix_list[pkg_hash] : package) : 162 | package ) 163 | else 164 | puts "failed to parse #{entry}" 165 | end 166 | end 167 | 168 | serialize_list(nix_list.values) 169 | end 170 | 171 | end 172 | 173 | end end 174 | -------------------------------------------------------------------------------- /lib/nixpkgs_monitor/package_updaters/base.rb: -------------------------------------------------------------------------------- 1 | require 'mechanize' 2 | 3 | module NixPkgsMonitor module PackageUpdaters 4 | 5 | class Base 6 | 7 | def self.friendly_name 8 | name.gsub(/^NixPkgsMonitor::PackageUpdaters::/,"").gsub("::","_").downcase.to_sym 9 | end 10 | 11 | 12 | def self.log 13 | Log 14 | end 15 | 16 | 17 | def self.http_agent 18 | agent = Mechanize.new 19 | agent.user_agent = 'NixPkgs software update checker' 20 | return agent 21 | end 22 | 23 | 24 | def self.version_cleanup!(version) 25 | 10.times do 26 | # _all was added for p7zip 27 | # add -stable? 28 | version.gsub!(/\.gz|\.Z|\.bz2?|\.tbz|\.tbz2|\.lzma|\.lz|\.zip|\.xz|[-\.]tar$/, "") 29 | version.gsub!(/\.tgz|\.iso|\.dfsg|\.7z|\.gem|\.full|[-_\.]?src|[-_\.]?[sS]ources?$/, "") 30 | version.gsub!(/\.run|\.otf|-dist|\.deb|\.rpm|[-_]linux|-release|-bin|\.el$/, "") 31 | version.gsub!(/[-_\.]i386|-i686|[-\.]orig|\.rpm|\.jar|_all$/, "") 32 | end 33 | end 34 | 35 | def self.parse_tarball_name(tarball) 36 | package_name = file_version = nil 37 | 38 | if tarball =~ /^(.+?)[-_][vV]?([^A-Za-z].*)$/ 39 | package_name = $1 40 | file_version = $2 41 | elsif tarball =~ /^([a-zA-Z]+?)\.?(\d[^A-Za-z].*)$/ 42 | package_name = $1 43 | file_version = $2 44 | else 45 | log.info "failed to parse tarball #{tarball}" 46 | return nil 47 | end 48 | 49 | version_cleanup!(file_version) 50 | 51 | return [ package_name, file_version ] 52 | end 53 | 54 | def self.parse_tarball_from_url(url) 55 | return parse_tarball_name($1) if url =~ %r{/([^/]*)$} 56 | log.info "Failed to parse url #{url}" 57 | return [nil, nil] 58 | end 59 | 60 | # FIXME: add support for X.Y.Z[-_]?(a|b|beta|c|r|rc|pre)?\d* 61 | 62 | # FIXME: add support for the previous case when followed by [-_]?p\d* , 63 | # which usually mentions date, but may be a revision. the easiest way is to detect date by length and some restricitons 64 | # find out what is the order of preference of such packages. 65 | 66 | # FIXME: support for abcd - > a.bc.d versioning scheme. compare package and tarball versions to detect 67 | # FIXME: support date-based versioning: seems to be automatic as long as previous case is handled correctly 68 | 69 | # Returns true if the version format can be parsed and compared against another 70 | def self.usable_version?(version) 71 | !tokenize_version(version).nil? 72 | end 73 | 74 | 75 | def self.tokenize_version(v) 76 | return nil if v.start_with?('.') or v.end_with?('.') 77 | result = [] 78 | 79 | vcp = v.downcase.dup 80 | while vcp.length>0 81 | found = vcp.sub!(/\A(\d+|[a-zA-Z]+)\.?/) { result << $1; "" } 82 | return nil unless found 83 | end 84 | 85 | result.each do |token| 86 | return nil unless token =~ /^(\d+)$/ or ['alpha','beta','pre','rc'].include?(token) or ('a'..'z').include?(token) 87 | end 88 | 89 | result.map! do |token| 90 | token = 'a' if token == 'alpha' 91 | token = 'b' if token == 'beta' 92 | token = 'p' if token == 'pre' 93 | token = 'r' if token == 'rc' 94 | #puts "<#{token}>" 95 | if ('a'..'z').include? token 96 | -100 + token.ord - 'a'.ord 97 | elsif token =~ /^(\d+)$/ 98 | (token ? token.to_i : -1) 99 | else 100 | return nil 101 | end 102 | end 103 | 104 | result.fill(-1,result.length, 10-result.length) 105 | return result 106 | end 107 | 108 | def self.is_newer?(v1, v2) 109 | t_v1 = tokenize_version(v1) 110 | t_v2 = tokenize_version(v2) 111 | 112 | return( (t_v1 <=> t_v2) >0 ) 113 | end 114 | 115 | 116 | # check that package and tarball versions match 117 | def self.versions_match?(pkg) 118 | (package_name, file_version) = parse_tarball_from_url(pkg.url) 119 | 120 | if file_version and package_name and true # test only 121 | v1 = file_version.downcase 122 | # removes haskell suffix, gimp plugin suffix and FIXME: linux version 123 | # FIXME: linux version removal breaks a couple of matches 124 | v2 = pkg.version.downcase 125 | unless (v1 == v2) or (v1.gsub(/[-_]/,".") == v2) or (v1 == v2.gsub(".","")) 126 | log.info "version mismatch: #{package_name} #{file_version} #{pkg.url} #{pkg.name} #{pkg.version}" 127 | return false 128 | end 129 | return true 130 | else 131 | log.info "failed to parse tarball #{pkg.url} #{pkg.internal_name}" 132 | end 133 | return false 134 | end 135 | 136 | 137 | # returns an array of major, minor and fix versions from the available_versions array 138 | def self.new_versions(version, available_versions, package_name) 139 | t_pv = tokenize_version(version) 140 | return nil unless t_pv 141 | 142 | max_version_major = version 143 | max_version_minor = version 144 | max_version_fix = version 145 | available_versions.each do |v| 146 | t_v = tokenize_version(v) 147 | if t_v 148 | #check for and skip 345.gz == v3.4.5 versions for now 149 | if t_v[0]>9 and t_v[1] = -1 and t_pv[1] != -1 and t_v[0]>5*t_pv[0] 150 | log.info "found weird(too high) version of #{package_name} : #{v}. skipping" 151 | else 152 | max_version_major = v if (t_v[0] != t_pv[0]) and is_newer?(v, max_version_major) 153 | max_version_minor = v if (t_v[0] == t_pv[0]) and (t_v[1] != t_pv[1]) and is_newer?(v, max_version_minor) 154 | max_version_fix = v if (t_v[0] == t_pv[0]) and (t_v[1] == t_pv[1]) and (t_v[2] != t_pv[2]) and is_newer?(v, max_version_fix) 155 | end 156 | else 157 | log.info "can't parse update version candidate of #{package_name} : #{v}. skipping" 158 | end 159 | end 160 | 161 | return( (max_version_major != version ? [ max_version_major ] : []) + 162 | (max_version_minor != version ? [ max_version_minor ] : []) + 163 | (max_version_fix != version ? [ max_version_fix ] : []) ) 164 | end 165 | 166 | 167 | def self.new_tarball_versions(pkg, tarballs) 168 | (package_name, file_version) = parse_tarball_from_url(pkg.url) 169 | return nil if file_version.to_s.empty? or package_name.to_s.empty? 170 | 171 | return nil unless versions_match?(pkg) 172 | 173 | vlist = tarballs[package_name.downcase] 174 | return nil unless vlist 175 | 176 | new_versions(pkg.version.downcase, vlist, package_name) 177 | end 178 | 179 | 180 | def self.tarballs_from_dir(dir, tarballs = {}) 181 | begin 182 | 183 | http_agent.get(dir).links.each do |l| 184 | next if l.href.end_with?('.asc', '.exe', '.dmg', '.sig', '.sha1', '.patch', '.patch.gz', '.patch.bz2', '.diff', '.diff.bz2', '.xdelta') 185 | (name, version) = parse_tarball_name(l.href) 186 | if name and version 187 | tarballs[name] = [] unless tarballs[name] 188 | tarballs[name] = tarballs[name] << version 189 | end 190 | end 191 | return tarballs 192 | 193 | rescue Mechanize::ResponseCodeError 194 | log.warn $! 195 | return {} 196 | end 197 | end 198 | 199 | 200 | def self.tarballs_from_dir_recursive(dir) 201 | tarballs = {} 202 | 203 | log.debug "#{dir}" 204 | http_agent.get(dir).links.each do |l| 205 | next if l.href == '..' or l.href == '../' 206 | if l.href =~ %r{^[^/]*/$} 207 | log.debug l.href 208 | tarballs = tarballs_from_dir(dir+l.href, tarballs) 209 | end 210 | end 211 | 212 | return tarballs 213 | end 214 | 215 | 216 | def self.newest_versions_of(pkg) 217 | v = newest_version_of(pkg) 218 | (v ? [v] : nil) 219 | end 220 | 221 | 222 | def self.find_tarball(pkg, version) 223 | return nil if pkg.url.to_s.empty? or version.to_s.empty? or pkg.version.to_s.empty? 224 | new_url = (pkg.url.include?(pkg.version) ? pkg.url.gsub(pkg.version, version) : nil ) 225 | return nil unless new_url 226 | bz_url = new_url.sub(/\.tar\.gz$/, ".tar.bz2") 227 | xz_url = bz_url.sub(/\.tar\.bz2$/, ".tar.xz") 228 | [ xz_url, bz_url, new_url ] 229 | end 230 | 231 | end 232 | 233 | end end 234 | -------------------------------------------------------------------------------- /bin/nixpkgs-monitor: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'optparse' 4 | require 'mechanize' 5 | require 'logger' 6 | require 'nixpkgs_monitor/distro_packages' 7 | require 'nixpkgs_monitor/package_updaters' 8 | require 'nixpkgs_monitor/security_advisories' 9 | require 'nixpkgs_monitor/reports' 10 | require 'sequel' 11 | require 'set' 12 | 13 | 14 | STDOUT.sync = true; 15 | STDERR.sync = true; 16 | 17 | 18 | log = Logger.new(STDOUT) 19 | log.level = Logger::WARN 20 | log.formatter = proc { |severity, datetime, progname, msg| 21 | "#{severity}: #{msg}\n" 22 | } 23 | 24 | Log = log 25 | 26 | actions = Set.new 27 | pkg_names_to_check = [] 28 | builds_ignore_negative = false 29 | builder_count = 1 30 | 31 | DB = (ENV["DB"] && Sequel.connect(ENV["DB"])) || Sequel.sqlite('./db.sqlite') 32 | 33 | distros_to_update = Set.new 34 | 35 | updaters = NixPkgsMonitor::PackageUpdaters::Updaters 36 | 37 | OptionParser.new do |o| 38 | o.on("-v", "Verbose output. Can be specified multiple times") do 39 | log.level -= 1 40 | end 41 | 42 | o.on("--list-arch", "List Arch packages") do 43 | distros_to_update << NixPkgsMonitor::DistroPackages::Arch 44 | end 45 | 46 | o.on("--list-aur", "List AUR packages") do 47 | distros_to_update << NixPkgsMonitor::DistroPackages::AUR 48 | end 49 | 50 | o.on("--list-nix", "List nixpkgs packages") do 51 | distros_to_update << NixPkgsMonitor::DistroPackages::Nix 52 | end 53 | 54 | o.on("--list-deb", "List Debian packages") do 55 | distros_to_update << NixPkgsMonitor::DistroPackages::Debian 56 | end 57 | 58 | o.on("--list-gentoo", "List Gentoo packages") do 59 | distros_to_update << NixPkgsMonitor::DistroPackages::Gentoo 60 | end 61 | 62 | o.on("--check-pkg-version-match", "List Nix packages for which either tarball can't be parsed or its version doesn't match the package version") do 63 | actions << :check_pkg_version_match 64 | end 65 | 66 | o.on("--updater UPDATER", "Check for updates using only UPDATER. Accepts partial names.") do |uname| 67 | updaters = NixPkgsMonitor::PackageUpdaters::Updaters.select { |u| u.friendly_name.to_s.include? uname.downcase } 68 | end 69 | 70 | o.on("--check-updates", "list NixPkgs packages which have updates available") do 71 | actions << :check_updates 72 | end 73 | 74 | o.on("--check-package PACKAGE", "Check what updates are available for PACKAGE") do |pkgname| 75 | actions << :check_updates 76 | pkg_names_to_check << pkgname 77 | end 78 | 79 | o.on("--tarballs", "Try downloading all the candidate tarballs to the nix store") do 80 | actions << :tarballs 81 | end 82 | 83 | o.on("--redownload", "Try downloading missing tarballs again on the next --tarballs run") do 84 | actions << :drop_negative_tarball_cache 85 | end 86 | 87 | o.on("--patches", "Generate patches for packages updates") do 88 | actions << :patches 89 | end 90 | 91 | o.on("--build", "Try building patches") do 92 | actions << :build 93 | end 94 | 95 | o.on("--rebuild", "Try building patches marked as failed again on the next --build run") do 96 | actions << :drop_negative_build_cache 97 | end 98 | 99 | o.on("--builder-count NUMBER", Integer, "Number of packages to build in parallel") do |bc| 100 | builder_count = bc 101 | abort "builder count must be in 1..100 range" unless (1..100).include? bc 102 | end 103 | 104 | o.on("--find-unmatched-advisories", "Find security advisories which don't map to a Nix package(don't touch yet)") do 105 | actions << :find_unmatched_advisories 106 | end 107 | 108 | o.on("--cve-update", "Fetch CVE updates") do 109 | actions << :cve_update 110 | end 111 | 112 | o.on("--cve-check", "Check NixPkgs against CVE database") do 113 | actions << :cve_check 114 | end 115 | 116 | o.on("--coverage", "list NixPkgs packages which have (no) update coverage") do 117 | actions << :coverage 118 | end 119 | 120 | o.on("--all", "Update package definitions, check for updates, vulnerabilities, tarballs and write patches") do 121 | actions.merge([ :coverage, :check_updates, :cve_check, :cve_update, :tarballs, :patches ]) 122 | distros_to_update.merge([ NixPkgsMonitor::DistroPackages::Arch, 123 | NixPkgsMonitor::DistroPackages::Nix, 124 | NixPkgsMonitor::DistroPackages::Debian, 125 | NixPkgsMonitor::DistroPackages::Gentoo ]) 126 | end 127 | 128 | o.on("-h", "--help", "Show this message") do 129 | puts o 130 | exit 131 | end 132 | 133 | begin 134 | o.parse(ARGV) 135 | rescue 136 | abort "Wrong parameters: #{$!}. See --help for more information." 137 | end 138 | end 139 | 140 | abort "No action requested. See --help for more information." unless distros_to_update.count > 0 or actions.count > 0 141 | 142 | Sequel.extension :migration 143 | Sequel::Migrator.run(DB, File.join(File.dirname(__FILE__), '..', 'lib', 'nixpkgs_monitor', 'migrations') ) 144 | 145 | distros_to_update.each do |distro| 146 | begin 147 | log.debug distro.generate_list.inspect 148 | NixPkgsMonitor::Reports::Timestamps.done("fetch_#{distro.name.split('::').last.downcase}", 149 | "found #{distro.packages.count} packages") 150 | rescue Exception => e 151 | NixPkgsMonitor::Reports::Timestamps.done("fetch_#{distro.name.split('::').last.downcase}", 152 | "error: #{e}") 153 | log.error "Error: #{e}" 154 | raise if distro == NixPkgsMonitor::DistroPackages::Nix 155 | end 156 | end 157 | 158 | 159 | if actions.include? :coverage 160 | 161 | DB.transaction do 162 | DB[:estimated_coverage].delete 163 | 164 | NixPkgsMonitor::DistroPackages::Nix.packages.each do |pkg| 165 | DB[:estimated_coverage] << { 166 | :pkg_attr => pkg.internal_name, 167 | :coverage => NixPkgsMonitor::PackageUpdaters::Updaters.count{ |updater| updater.covers?(pkg) } 168 | } 169 | end 170 | 171 | NixPkgsMonitor::Reports::Timestamps.done(:coverage) 172 | end 173 | 174 | end 175 | if actions.include? :check_updates 176 | 177 | pkgs_to_check = ( pkg_names_to_check.empty? ? 178 | NixPkgsMonitor::DistroPackages::Nix.packages : 179 | pkg_names_to_check.map{ |pkgname| NixPkgsMonitor::DistroPackages::Nix.list[pkgname] } 180 | ) 181 | 182 | updaters.each do |updater| 183 | begin 184 | DB.transaction do 185 | DB[updater.friendly_name].delete 186 | log.warn "running #{updater.friendly_name}" 187 | 188 | pkgs_to_check.each do |pkg| 189 | new_ver = updater.newest_versions_of(pkg).to_a.flatten.reject(&:nil?) 190 | unless new_ver.empty? 191 | puts "#{pkg.internal_name}/#{pkg.name}:#{pkg.version} " + 192 | "has new version(s) #{new_ver} according to #{updater.friendly_name}" 193 | new_ver.each do |version| 194 | DB[updater.friendly_name] << { 195 | :pkg_attr => pkg.internal_name, 196 | :version => version, 197 | } 198 | end 199 | end 200 | end 201 | 202 | NixPkgsMonitor::Reports::Timestamps.done("updater_#{updater.friendly_name}", 203 | "Found #{DB[updater.friendly_name].count} updates") 204 | end 205 | rescue => e 206 | NixPkgsMonitor::Reports::Timestamps.done("updater_#{updater.friendly_name}", "Error: #{e}") 207 | log.error "Error: #{e}" 208 | end 209 | end 210 | 211 | NixPkgsMonitor::Reports::Timestamps.done(:updaters) 212 | 213 | end 214 | if actions.include? :drop_negative_tarball_cache 215 | 216 | DB[:tarball_sha256].where(:sha256 => "404").delete 217 | 218 | end 219 | if actions.include? :tarballs 220 | 221 | def fetch_tarball(tarball) 222 | hash = DB[:tarball_sha256][:tarball => tarball] 223 | unless hash 224 | (sha256, path) = %x(NIX_PATH=. PRINT_PATH="1" nix-prefetch-url '#{tarball}').split.map(&:strip) 225 | if $? == 0 and sha256 and sha256 != "" 226 | mimetype = %x(file -b --mime-type #{path}).strip 227 | raise "failed to determine mimetype for #{path}" unless $? == 0 228 | 229 | if tarball.end_with?(".gz", ".bz2", ".xz", ".lzma", ".zip", ".7z", ".jar", 230 | ".deb", ".rpm",".tgz", ".tbz", ".tbz2", ".lz") and 231 | ["application/xml", "text/html", "application/xhtml+xml" "text/plain"].include?(mimetype) 232 | sha256 = "404" 233 | puts "#{path} is most likely a malformed error page. Assuming error 404." 234 | else 235 | puts "found #{sha256} hash for #{path}" 236 | end 237 | else 238 | puts "tarball #{tarball} not found" 239 | sha256 = "404" 240 | end 241 | DB.transaction do 242 | if 1 != DB[:tarball_sha256].where(:tarball => tarball).update(:sha256 => sha256) 243 | DB[:tarball_sha256] << { :tarball => tarball, :sha256 => sha256 } 244 | end 245 | end 246 | hash = (sha256 == "404" ? nil : sha256) 247 | end 248 | return hash 249 | end 250 | 251 | DB.transaction do 252 | DB[:tarballs].delete 253 | 254 | updaters.each do |updater| 255 | DB[updater.friendly_name].all.each do |row| 256 | pkg = NixPkgsMonitor::DistroPackages::Nix.by_internal_name[row[:pkg_attr]] 257 | next unless pkg 258 | 259 | tarballs = [ updater.find_tarball(pkg, row[:version]) ].flatten 260 | tarballs.each do |tarball| 261 | hash = tarball ? fetch_tarball(tarball) : nil 262 | DB[:tarballs] << { 263 | :pkg_attr => row[:pkg_attr], 264 | :version => row[:version], 265 | :tarball => tarball 266 | } if hash 267 | end 268 | end 269 | end 270 | 271 | NixPkgsMonitor::Reports::Timestamps.done(:tarballs) 272 | end 273 | 274 | end 275 | if actions.include? :patches 276 | 277 | puts %x(cd #{NixPkgsMonitor::DistroPackages::Nix.repository_path} && git checkout master --force) 278 | 279 | DB.transaction do 280 | 281 | DB[:patches].delete 282 | 283 | # this is the biggest and ugliest collection of hacks 284 | 285 | DB[:tarballs].join(:tarball_sha256,:tarball => :tarball).exclude(:sha256 => "404").distinct.all.each do |row| 286 | nixpkg = NixPkgsMonitor::DistroPackages::Nix.by_internal_name[row[:pkg_attr]] 287 | next unless nixpkg 288 | 289 | file_name = nixpkg.position && File.join(NixPkgsMonitor::DistroPackages::Nix.repository_path, 290 | nixpkg.position.rpartition(':')[0]) 291 | original_content = file_name && File.readlines(file_name) 292 | sha256_location = original_content && original_content.index{ |l| l.include? nixpkg.sha256 } 293 | unless sha256_location 294 | #puts "failed to find the original hash value in the file reported to contain the derivation for #{row[:pkg_attr]}. Grepping for it instead" 295 | file_name = %x(grep -ir '#{nixpkg.sha256}' -rl #{File.join(NixPkgsMonitor::DistroPackages::Nix.repository_path, 'pkgs')}).split("\n")[0] 296 | original_content = File.readlines(file_name) 297 | 298 | sha256_location = original_content.index{ |l| l.include? nixpkg.sha256 } 299 | unless sha256_location 300 | puts "failed to find the original hash value to replace for #{row[:pkg_attr]}" 301 | next 302 | end 303 | end 304 | patched = original_content.map(&:dup) # deep copy 305 | patched[sha256_location].sub!(nixpkg.sha256, row[:sha256]) 306 | # upgrade hash to sha256 if necessary 307 | patched[sha256_location].sub!(/md5\s*=/, "sha256 =") 308 | patched[sha256_location].sub!(/sha1\s*=/, "sha256 =") 309 | 310 | src_url_location = patched.index{ |l| l =~ /url\s*=.*;/ and l.include? nixpkg.url } 311 | patched[src_url_location].sub!(nixpkg.url, row[:tarball]) if src_url_location 312 | 313 | # a stupid heuristic targetting name = "..."; version = "..."; and such 314 | version_locations = patched.map. 315 | with_index{ |l, i| (l =~ /[nv].*=.*".*".*;/ and l.include? nixpkg.version) ? i : nil }. 316 | reject(&:nil?). 317 | sort_by{|l| (l-sha256_location).abs } 318 | 319 | unless version_locations.size>0 320 | puts "failed to find the original version value to replace for #{row[:pkg_attr]}" 321 | next 322 | end 323 | 324 | patch_index = version_locations.index do |version_location| 325 | patched_v = patched.map(&:dup) # deep copy 326 | patched_v[version_location].sub!(nixpkg.version, row[:version]) 327 | 328 | File.write(file_name, patched_v.join) 329 | patch = %x(cd #{NixPkgsMonitor::DistroPackages::Nix.repository_path} && git diff) 330 | new_pkg = NixPkgsMonitor::DistroPackages::Nix.load_package(row[:pkg_attr]) 331 | 332 | success = (new_pkg and new_pkg.sha256 == row[:sha256] and new_pkg.version == row[:version]) 333 | if success and new_pkg.url == row[:tarball] 334 | new_pkg.instantiate 335 | DB[:patches] << { 336 | :pkg_attr => row[:pkg_attr], 337 | :version => row[:version], 338 | :tarball => row[:tarball], 339 | :patch => patch, 340 | :drvpath => new_pkg.drvpath, 341 | :outpath => new_pkg.outpath 342 | } 343 | elsif success 344 | puts "trying advanced patching techniques for #{row[:pkg_attr]}" 345 | # todo: handle a separate but rare case where src_url_location is not nil, but we still failed to change src 346 | src_url_locations = patched_v.map.with_index do |l, i| 347 | l =~ /url\s*=.*;/ and (l. 348 | gsub("${name}", "#{nixpkg.name.sub(/^perl-/,"")}-#{nixpkg.version}"). 349 | gsub("${version}", nixpkg.version). 350 | include? nixpkg.url) ? i : nil 351 | end. 352 | reject(&:nil?). 353 | sort_by{|l| (l-sha256_location).abs } 354 | 355 | patch_src_index = src_url_locations.index do |src_url_location| 356 | patched_s = patched_v.map(&:dup) # deep copy 357 | new_url = row[:tarball].dup 358 | 359 | new_url.gsub!("#{nixpkg.name.sub(/^perl-/,"")}-#{row[:version]}", "${name}") if patched_s[src_url_location].include? "${name}" 360 | new_url.gsub!(row[:version], "${version}") if patched_s[src_url_location].include? "${version}" 361 | patched_s[src_url_location].sub!(/url\s*=\s*"([^"]*)"/, %{url = "#{new_url}"}) 362 | 363 | File.write(file_name, patched_s.join) 364 | patch = %x(cd #{NixPkgsMonitor::DistroPackages::Nix.repository_path} && git diff) 365 | new_pkg = NixPkgsMonitor::DistroPackages::Nix.load_package(row[:pkg_attr]) 366 | 367 | s_success = (new_pkg and new_pkg.url == row[:tarball] and new_pkg.sha256 == row[:sha256] and new_pkg.version == row[:version]) 368 | if s_success 369 | puts "made an advanced patch! #{row[:pkg_attr]}" 370 | new_pkg.instantiate 371 | DB[:patches] << { 372 | :pkg_attr => row[:pkg_attr], 373 | :version => row[:version], 374 | :tarball => row[:tarball], 375 | :patch => patch, 376 | :drvpath => new_pkg.drvpath, 377 | :outpath => new_pkg.outpath 378 | } 379 | end 380 | s_success 381 | end 382 | puts "failed advanced patching for #{row[:pkg_attr]}" unless patch_src_index 383 | end 384 | success 385 | end 386 | 387 | puts "patch failed to change version, url or hash for #{row[:pkg_attr]}" unless patch_index 388 | 389 | File.write(file_name, original_content.join) 390 | end 391 | 392 | NixPkgsMonitor::Reports::Timestamps.done(:patches) 393 | end 394 | 395 | end 396 | if actions.include? :drop_negative_build_cache 397 | 398 | DB[:builds].exclude(:status => "ok").delete 399 | 400 | end 401 | if actions.include? :build 402 | 403 | queue = Queue.new 404 | 405 | DB[:patches].distinct.all.each do |row| 406 | outpath = row[:outpath] 407 | build = DB[:builds][:outpath => outpath] 408 | queue << row unless build 409 | end 410 | builder_count.times{ queue << nil } # add end of queue "job" x builder count 411 | 412 | (1..builder_count).map do |n| 413 | Thread.new(n) do |builder_id| 414 | 415 | while (row = queue.pop) 416 | if File.exist? row[:drvpath] 417 | 418 | log.warn "Builder #{builder_id} building: #{row[:drvpath]}" 419 | %x(nix-store --realise #{row[:drvpath]} --log-type flat --timeout #{6*3600} 2>&1) 420 | status = ($? == 0 ? "ok" : "failed") 421 | 422 | log_path = row[:drvpath].sub(%r{^/nix/store/}, "") 423 | log_path = "/nix/var/log/nix/drvs/#{log_path[0,2]}/#{log_path[2,100]}.bz2" 424 | status = "dep failed" if not(File.exist?(log_path)) and status == "failed" 425 | build_log = ( status == "dep failed" ? 426 | "" : 427 | %x(bzcat #{log_path}).encode("us-ascii", :invalid=>:replace, :undef => :replace) 428 | ) 429 | 430 | log.warn "Builder #{builder_id} finished building: #{row[:drvpath]}" 431 | 432 | DB.transaction do 433 | if 1 != DB[:builds].where(:outpath => row[:outpath]).update(:status => status, :log => build_log) 434 | DB[:builds] << { :outpath => row[:outpath], :status => status, :log => build_log } 435 | end 436 | end 437 | 438 | else 439 | puts "derivation #{row[:drvpath]} seems to have been garbage-collected" 440 | end 441 | end 442 | 443 | end 444 | end. 445 | each(&:join) # wait for threads to finish 446 | 447 | NixPkgsMonitor::Reports::Timestamps.done(:builds) 448 | 449 | end 450 | if actions.include? :check_pkg_version_match 451 | 452 | DB.transaction do 453 | version_mismatch = NixPkgsMonitor::Reports::Logs.new(:version_mismatch) 454 | DB[:version_mismatch].delete 455 | 456 | NixPkgsMonitor::DistroPackages::Nix.packages. 457 | reject{|pkg| NixPkgsMonitor::PackageUpdaters::Base.versions_match?(pkg)}. 458 | each{|pkg| version_mismatch.pkg(pkg.internal_name)} 459 | end 460 | 461 | end 462 | if actions.include? :find_unmatched_advisories 463 | 464 | known_safe = [ 465 | # these advisories don't apply because they have been checked to refer to packages that don't exist in nixpgs 466 | "GLSA-201210-02", 467 | ] 468 | NixPkgsMonitor::SecurityAdvisories::GLSA.list.each do |glsa| 469 | nixpkgs = glsa.matching_nixpkgs 470 | if nixpkgs 471 | log.info "Matched #{glsa.id} to #{nixpkgs.internal_name}" 472 | elsif known_safe.include? glsa.id 473 | log.info "Skipping #{glsa.id} as known safe" 474 | else 475 | log.warn "Failed to match #{glsa.id} #{glsa.packages}" 476 | end 477 | end 478 | end 479 | 480 | 481 | NixPkgsMonitor::SecurityAdvisories::CVE.fetch_updates if actions.include? :cve_update 482 | 483 | 484 | if actions.include? :cve_check 485 | 486 | def sorted_hash_to_s(tokens) 487 | tokens.keys.sort{|x,y| tokens[x] <=> tokens[y] }.map{|t| "#{t}: #{tokens[t]}"}.join("\n") 488 | end 489 | 490 | list = NixPkgsMonitor::SecurityAdvisories::CVE.list 491 | 492 | products = {} 493 | product_to_cve = {} 494 | list.each do |entry| 495 | entry.packages.each do |pkg| 496 | (supplier, product, version) = NixPkgsMonitor::SecurityAdvisories::CVE.parse_package(pkg) 497 | pname = "#{product}" 498 | products[pname] = Set.new unless products[pname] 499 | products[pname] << version 500 | 501 | fullname = "#{product}:#{version}" 502 | product_to_cve[fullname] = Set.new unless product_to_cve[fullname] 503 | product_to_cve[fullname] << entry.id 504 | end 505 | end 506 | log.debug "products #{products.count}: #{products.keys.join("\n")}" 507 | 508 | products.each_pair do |product, versions| 509 | versions.each do |version| 510 | log.warn "can't parse version #{product} : #{version}" unless version =~ /^\d+\.\d+\.\d+\.\d+/ or version =~ /^\d+\.\d+\.\d+/ or version =~ /^\d+\.\d+/ or version =~ /^\d+/ 511 | end 512 | end 513 | 514 | tokens = {} 515 | products.keys.each do |product| 516 | product.scan(/(?:[a-zA-Z]+)|(?:\d+)/).each do |token| 517 | tokens[token] = ( tokens[token] ? (tokens[token] + 1) : 1 ) 518 | end 519 | end 520 | log.debug "token counts \n #{sorted_hash_to_s(tokens)} \n\n" 521 | 522 | selectivity = {} 523 | tokens.keys.each do |token| 524 | selectivity[token] = NixPkgsMonitor::DistroPackages::Nix.packages.count do |pkg| 525 | pkg.internal_name.include? token or pkg.name.include? token 526 | end 527 | end 528 | log.debug "token selectivity \n #{sorted_hash_to_s(selectivity)} \n\n" 529 | 530 | false_positive_impact = tokens.keys. 531 | each_with_object(Hash.new()) { |t, impact| 532 | impact[t] = tokens[t] * selectivity[t] 533 | } 534 | log.debug "false positive impact \n #{sorted_hash_to_s(false_positive_impact)} \n\n" 535 | 536 | common_prefixes = NixPkgsMonitor::DistroPackages::Nix.list.keys. 537 | map{ |name| ((%r{^(?[^-_]*)[-_]} =~ name) and (prefix.length >2)) ? prefix : nil }. 538 | reject(&:nil?). 539 | each_with_object(Hash.new(0)){ |prefix, counts| counts[prefix] += 1 }. 540 | reject{ |prefix, count| count < 30 } 541 | log.debug "common name prefixes\n #{common_prefixes.inspect}\n" 542 | 543 | def normalize_name(name, common_prefixes) 544 | prefix = common_prefixes.find{ |prefix| name.start_with?(prefix+"-", prefix+"_") } 545 | (prefix ? name.sub(prefix, '') : name).downcase.gsub(%r{[^a-z]}, '') 546 | end 547 | 548 | def normalize_attrpath(attrpath) 549 | ((%r{(?[^.]*)$} =~ attrpath) ? last : attrpath).downcase.gsub(%r{[^a-z]}, '') 550 | end 551 | 552 | def sparse_contains(str, substr) 553 | p = -1 554 | substr.chars.all? {|c| p = str.index(c, p+1) } 555 | end 556 | 557 | DB.transaction do 558 | 559 | DB[:cve_match].delete 560 | 561 | # if a nix package is named like this, match it only against a cve product with an identical name 562 | # this prevents eg ruby nix package from being matched to ruby on rails cve product 563 | exact_name_match = ['perl', 'python', 'ruby'] 564 | 565 | # match whole versions for these products 566 | # use sparingly for packages that produce too many false positives due to version suffixes 567 | exact_version_match = ['openssl'] 568 | 569 | products.each_pair do |product, versions| 570 | tk = product.scan(/(?:[a-zA-Z]+)|(?:\d+)/).select do |token| 571 | token.size != 1 and not(['the','and','in','on','of','for'].include? token) 572 | end 573 | 574 | pkgs = 575 | NixPkgsMonitor::DistroPackages::Nix.packages.select do |pkg| 576 | score = tk.reduce(0) do |score, token| 577 | res = ((pkg.internal_name.include? token or pkg.name.include? token) ? 1 : 0) 578 | res *= ( selectivity[token]>20 ? 0.51 : 1 ) 579 | score + res 580 | end 581 | ( score >= 1 or ( tk.size == 1 and score >= 0.3 ) ) 582 | end.to_set 583 | 584 | versions.each do |version| 585 | version =~ /^\d+\.\d+\.\d+\.\d+/ or version =~ /^\d+\.\d+\.\d+/ or version =~ /^\d+\.\d+/ or version =~ /^\d+/ 586 | v = exact_version_match.include?(product) ? version.downcase : $& 587 | next unless v 588 | 589 | pkgs.each do |pkg| 590 | next if exact_name_match.include?(pkg.name) and not exact_name_match.include?(product) 591 | pkg.version =~ /^\d+\.\d+\.\d+\.\d+/ or pkg.version =~ /^\d+\.\d+\.\d+/ or pkg.version =~ /^\d+\.\d+/ or pkg.version =~ /^\d+/ 592 | v2 = exact_version_match.include?(product) ? pkg.version.downcase : $& 593 | next unless v2 594 | 595 | #if (pkg.version == v) or (pkg.version.start_with? v and not( ('0'..'9').include? pkg.version[v.size])) 596 | fullname = "#{product}:#{version}" 597 | if (v == v2) and ( 598 | sparse_contains(product, normalize_attrpath(pkg.internal_name)) or 599 | sparse_contains(product, normalize_name(pkg.name, common_prefixes.keys))) 600 | product_to_cve[fullname].each do |cve| 601 | DB[:cve_match] << { 602 | :pkg_attr => pkg.internal_name, 603 | :product => product, 604 | :version => version, 605 | :CVE => cve 606 | } 607 | end 608 | log.warn "match #{product_to_cve[fullname].inspect}: #{product}:#{version} = #{pkg.internal_name}/#{pkg.name}:#{pkg.version}" 609 | elsif v == v2 610 | log.debug "weak match #{product_to_cve[fullname].inspect}: #{product}:#{version} = #{pkg.internal_name}/#{pkg.name}:#{pkg.version}" 611 | end 612 | end 613 | end 614 | end 615 | 616 | NixPkgsMonitor::Reports::Timestamps.done(:cve_check) 617 | end 618 | 619 | end 620 | -------------------------------------------------------------------------------- /bin/nixpkgs-monitor-site: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'sinatra' 4 | require 'haml' 5 | require 'sequel' 6 | require 'nixpkgs_monitor/distro_packages' 7 | require 'nixpkgs_monitor/package_updaters' 8 | require 'nixpkgs_monitor/build_log' 9 | require 'nixpkgs_monitor/reports' 10 | require 'cgi' 11 | require 'rss' 12 | require 'diffy' 13 | require 'uri' 14 | 15 | DB = (ENV["DB"] && Sequel.connect(ENV["DB"])) || Sequel.sqlite('./db.sqlite') 16 | 17 | set :base_url, (ENV["BASE_URL"] || "http://monitor.nixos.org/") 18 | set :server, 'webrick' 19 | 20 | def cache 21 | @@cache ||= {} 22 | end 23 | 24 | def maintainers_with_email 25 | @@maintainers_with_email ||= DB[:nix_maintainers].select(:maintainer).distinct.order(:maintainer).map(:maintainer). 26 | map{ |m| m.strip } 27 | end 28 | 29 | def maintainers 30 | @@maintainers ||= maintainers_with_email.map{ |m| m.gsub(/<.*>/,"").strip } 31 | end 32 | 33 | def coverage 34 | @@coverage ||= DB[:estimated_coverage].all. 35 | each_with_object({}){ |c, coverage| coverage[c[:pkg_attr]] = c[:coverage] } 36 | end 37 | 38 | def coverage_stats 39 | @@coverage_stats ||= coverage.values. 40 | each_with_object(Hash.new(0)){ |c, cstat| cstat[c] += 1 } 41 | end 42 | 43 | def maintainer_stats 44 | @@maintainer_stats ||= NixPkgsMonitor::DistroPackages::Nix.packages. 45 | each_with_object(Hash.new(0)){ |pkg, mstat| mstat[pkg.maintainers.count] += 1 } 46 | end 47 | 48 | def outdated 49 | @@outdated ||= NixPkgsMonitor::PackageUpdaters::Updaters.each. 50 | with_object(Hash.new{|h,k| h[k] = Hash.new{|h,k| h[k] = Array.new } }) do |updater, data| 51 | DB[updater.friendly_name].all.each do |row| 52 | pkg = NixPkgsMonitor::DistroPackages::Nix.by_internal_name[row[:pkg_attr]] 53 | if pkg.nil? or pkg.branch.to_s.empty? or is_branch_update?(pkg, row[:version]) 54 | data[row[:pkg_attr]][row[:version]] << updater.friendly_name.to_s 55 | end 56 | end 57 | end 58 | end 59 | 60 | def is_branch_update?(pkg, new_ver) 61 | return false if pkg.branch.to_s.empty? 62 | branch_t = NixPkgsMonitor::PackageUpdaters::Base.tokenize_version(pkg.branch) 63 | nv_t = NixPkgsMonitor::PackageUpdaters::Base.tokenize_version(new_ver) 64 | return (nv_t.nil? or branch_t.nil? or (nv_t[0..branch_t.size-1] == branch_t) ) 65 | end 66 | 67 | def is_major_update?(pkg, new_ver) 68 | return not(is_branch_update?(pkg, new_ver)) unless pkg.branch.to_s.empty? 69 | v_t = NixPkgsMonitor::PackageUpdaters::Base.tokenize_version(pkg.version) 70 | nv_t = NixPkgsMonitor::PackageUpdaters::Base.tokenize_version(new_ver) 71 | return ( 72 | not(v_t) or not(nv_t) or (v_t[0] != nv_t[0]) or 73 | (v_t[2]>=0 and (v_t[1] != nv_t[1])) 74 | ) 75 | end 76 | 77 | def vulnerable 78 | unless @vulnerable 79 | @vulnerable = Hash.new{|h,k| h[k] = Hash.new{|h,k| h[k] = Set.new} } 80 | DB[:cve_match].all.each do |match| 81 | cve_product = "#{match[:product]}:#{match[:version]}" 82 | products = @vulnerable[match[:pkg_attr]] 83 | products[cve_product] << match[:CVE] 84 | end 85 | end 86 | return @vulnerable 87 | end 88 | 89 | def refresh 90 | @@vulnerable = nil 91 | @@outdated = nil 92 | @@maintainer_stats = nil 93 | @@coverage_stats = nil 94 | @@coverage = nil 95 | @@maintainers = nil 96 | @@maintainers_with_email = nil 97 | NixPkgsMonitor::DistroPackages::Nix.refresh 98 | cache.clear 99 | 100 | vulnerable 101 | outdated 102 | maintainer_stats 103 | coverage_stats 104 | coverage 105 | maintainers 106 | end 107 | 108 | 109 | def filter_packages(filter) 110 | 111 | have_patches = ( filter[:haspatch] ? Set.new( DB[:patches].select(:pkg_attr).distinct.map(:pkg_attr) ) : nil ) 112 | 113 | good_patches = (filter[:hasgoodpatch] ? 114 | Set.new( DB[:patches].join(:builds, :outpath => :outpath).where(:status => "ok"). 115 | select(:pkg_attr).distinct.map(:pkg_attr) ) : 116 | nil ) 117 | 118 | Set.new( NixPkgsMonitor::DistroPackages::Nix.by_internal_name.select { |pkg, nixpkg| 119 | # maintainer count 120 | (filter[:mc].to_s.empty? or filter[:mc].to_i == nixpkg.maintainers.count) and 121 | # coverage 122 | (filter[:c].to_s.empty? or filter[:c].to_i == coverage[pkg]) and 123 | #vulnerable 124 | (not filter[:vulnerable] or vulnerable.has_key?(pkg)) and 125 | # has a patch that builds 126 | (not filter[:hasgoodpatch] or good_patches.include?(pkg) ) and 127 | # has a patch 128 | (not filter[:haspatch] or have_patches.include?(pkg) ) and 129 | # outdated which has a minor update 130 | (not filter[:outdated_minor] or (outdated.has_key?(pkg) and outdated[pkg].keys.find{|v| not is_major_update?(nixpkg, v)}) ) and 131 | # outdated which has a major update 132 | (not filter[:outdated_major] or (outdated.has_key?(pkg) and outdated[pkg].keys.find{|v| is_major_update?(nixpkg, v)}) ) and 133 | #outdated 134 | (not filter[:outdated] or outdated.has_key?(pkg)) and 135 | # has maintainer 136 | (filter[:m].to_s.empty? or (nixpkg.maintainers.index{ |m| m.downcase.include? filter[:m].downcase })) 137 | # to be continued 138 | }.keys ) 139 | end 140 | 141 | 142 | def render_checked(value) 143 | (value ? "checked" : "") 144 | end 145 | 146 | def render_selected(value) 147 | (value ? "selected" : "") 148 | end 149 | 150 | def render_filter(params) 151 | %{ 152 |
153 |
154 | Maintainers:
161 | Coverage: 168 |
169 |
170 | Vulnerable
171 | With patch 172 | (that builds)
173 | Outdated: 174 | Minor 175 | Major 176 |
177 |
178 | Maintainer:
179 | 180 |
181 |
182 | } 183 | end 184 | 185 | def render_versions(pkg, versions, patches, for_m) 186 | versions.map do |version, updaters| 187 | build_record = patches[ [ pkg, version ] ] 188 | patch_label = (build_record and build_record[0]) ? (build_record[0] == "ok" ? "label-success" : "label-danger" ) : "label-primary" 189 | 190 | if patches.key? ([ pkg, version ]) 191 | "#{version}" 192 | else 193 | "#{version}" 194 | end 195 | end. 196 | join(' ') 197 | end 198 | 199 | def render_cve(cve) 200 | %{#{cve.upcase}} 201 | end 202 | 203 | def render_pkgname(pkg, options = []) 204 | options = [ options ].flatten 205 | nixpkg = NixPkgsMonitor::DistroPackages::Nix.by_internal_name[pkg] 206 | %{#{pkg}#{ 207 | nixpkg.branch ? "[#{nixpkg.branch}]" : "" 208 | }#{ 209 | options.include?(:with_version) ? ":#{nixpkg.version}" : "" 210 | }} 211 | end 212 | 213 | 214 | helpers do 215 | 216 | def personalize_for 217 | (params[:for_m] && params[:for_m].downcase) || 218 | (request.cookies['maintainer'] && request.cookies['maintainer'].downcase) 219 | end 220 | 221 | def personalize_list 222 | (["I'm..."] + maintainers).map{|m| ""}.join 225 | end 226 | 227 | def rss_url 228 | uri = URI.parse(request.fullpath) 229 | new_query_ar = URI.decode_www_form(uri.query || []) << ["rss", "on"] 230 | uri.query = URI.encode_www_form(new_query_ar) 231 | return uri.to_s 232 | end 233 | 234 | def rss_link 235 | @rss_available && %{
  • rss
    • } 236 | end 237 | 238 | def rss_meta_link 239 | @rss_available && %{} 240 | end 241 | 242 | end 243 | 244 | 245 | set(:rss_requested) { |value| condition { not(params[:rss].nil?) == value } } 246 | 247 | 248 | get '/' do 249 | cache[:coverage_report] ||= coverage_stats. 250 | sort_by{ |cnum, ccnt| cnum }. 251 | map{ |c, cs| %{#{c}#{cs}} }. 252 | join 253 | 254 | cache[:maintainer_report] ||= maintainer_stats. 255 | sort_by{ |mnum, pcnt| mnum }. 256 | map{ |mc, ms| %{#{mc}#{ms}} }. 257 | join 258 | 259 | cache[:needsattention_report] ||= %{ 260 | Potentially vulnerable#{vulnerable.count} 261 | Unmaintained not covered#{ 262 | filter_packages({:c => 0, :mc => 0}).count 263 | } 264 | Outdated unmaintained#{ 265 | filter_packages({:mc =>0, :outdated => true}).count 266 | } 267 | Outdated#{outdated.count} 268 | } 269 | 270 | patches = DB[:patches].left_join(:builds, :outpath => :outpath) 271 | patch_stats = { 272 | :ok => patches.where(:status => "ok").count, 273 | :failed => patches.exclude(:status => nil, :status => "ok").count, 274 | :queued => patches.where(:status => nil).count, 275 | :total => DB[:patches].count, 276 | } 277 | 278 | cache[:task_report] = NixPkgsMonitor::Reports::Timestamps.all. 279 | map{|ts| %{ 280 | #{ts[:action]} 281 | #{ts[:timestamp].utc.strftime("%Y-%m-%d %H:%M:%S")} 282 | #{ts[:message]} 283 | } }. 284 | join 285 | 286 | cache[:permaintainer_report] ||= maintainers. 287 | map{ |maintainer| %{ 288 | 289 | #{maintainer} 290 | #{filter_packages({ :m => maintainer }).count} 291 | #{filter_packages({ :m => maintainer, :outdated => true }).count} 292 | #{filter_packages({ :m => maintainer, :c => 0 }).count} 293 | #{filter_packages({ :m => maintainer, :vulnerable => true }).count} 294 | 295 | } }. 296 | join 297 | 298 | haml :dashboard, :layout => :layout, :locals => { 299 | :coverage_report => cache[:coverage_report], 300 | :maintainer_report => cache[:maintainer_report], 301 | :needsattention_report => cache[:needsattention_report], 302 | :patch_stats => patch_stats, 303 | :permaintainer_report => cache[:permaintainer_report], 304 | :task_report => cache[:task_report], 305 | } 306 | end 307 | 308 | 309 | get '/coverage' do 310 | filtered = filter_packages(params) 311 | report = coverage. 312 | select{ |pkg, c| filtered.include? pkg }. 313 | sort_by{ |pkg, c| c }. 314 | map{ |pkg, c| %{ 315 | #{render_pkgname(pkg)}#{c} 316 | #{NixPkgsMonitor::DistroPackages::Nix.by_internal_name[pkg].maintainers.count} 317 | 318 | } }. 319 | join 320 | 321 | haml :coverage, :layout => :layout, 322 | :locals => { :coverage_report => report, :filter => render_filter(params) } 323 | end 324 | 325 | 326 | before '/outdated' do 327 | filtered = filter_packages(params) 328 | @dataset = outdated. 329 | select{ |pkg, v| filtered.include? pkg }. 330 | sort_by{ |pkg, v| pkg } 331 | @patches = DB[:patches].left_join(:builds, :outpath => :outpath).select_hash_groups([:pkg_attr, :version], :status) 332 | end 333 | 334 | get '/outdated', :rss_requested => true do 335 | content_type "application/rss+xml", :charset => 'utf-8' 336 | 337 | RSS::Maker.make("2.0") do |maker| 338 | maker.channel.author = "Nixpkgs Monitor" 339 | maker.channel.updated = Time.now.to_s 340 | maker.channel.title = "Outdated packages" 341 | maker.channel.description = "Automatically detected updated versions of Nix packages" 342 | maker.channel.link = settings.base_url + "outdated" 343 | 344 | @dataset.each do |pkg, versions| 345 | versions.keys.each do |version| 346 | if params[:hasgoodpatch] 347 | build_record = @patches[ [ pkg, version ] ] 348 | next unless build_record and build_record[0] == "ok" 349 | end 350 | 351 | nixpkg = NixPkgsMonitor::DistroPackages::Nix.by_internal_name[pkg] 352 | maker.items.new_item do |item| 353 | item.title = "#{pkg}#{ 354 | nixpkg.branch ? "[#{nixpkg.branch}]" : "" 355 | }:#{nixpkg.version} has a new version #{version}" 356 | item.description = "#{render_pkgname(pkg, [:with_version, :with_base])} has a new version #{version}" 357 | end 358 | end 359 | end 360 | end.to_s 361 | end 362 | 363 | get '/outdated', :rss_requested => false do 364 | @rss_available = true 365 | 366 | report = @dataset.map do |pkg, v| 367 | nixpkg = NixPkgsMonitor::DistroPackages::Nix.by_internal_name[pkg] 368 | %{ 369 | #{render_pkgname(pkg)}#{ 370 | vulnerable.has_key?(pkg) ? 371 | "(V)" : "" 372 | } 373 | 374 | #{nixpkg.version} 375 | #{ 376 | render_versions(pkg, v.select { |version, updaters| not is_major_update?(nixpkg, version) }, @patches, personalize_for) 377 | }#{ 378 | render_versions(pkg, v.select { |version, updaters| is_major_update?(nixpkg, version) }, @patches, personalize_for) 379 | } 380 | 381 | } 382 | end. 383 | join 384 | 385 | haml :outdated, :layout => :layout, 386 | :locals => { :outdated_report => report, :filter => render_filter(params.merge({:outdated => true})) } 387 | end 388 | 389 | 390 | get '/patch' do 391 | patch_record = DB[:patches][:pkg_attr => params[:p], :version => params[:v]] 392 | halt(404, 'no matching patch found') unless patch_record 393 | content_type 'text/plain', :charset => 'utf-8' 394 | %{ 395 | From: #{maintainers_with_email.select{|m| params[:m] and m.downcase.include? (params[:m].downcase)}.first || 396 | "Nixpkgs Monitor " } 397 | Subject: #{params[:p]}: #{NixPkgsMonitor::DistroPackages::Nix.by_internal_name[params[:p]].version} -> #{params[:v]}#{ 398 | vulnerable.has_key?(params[:p]) ? ", potentially fixes #{vulnerable[params[:p]].values.map{|s| s.to_a}.flatten.join(', ')}": "" 399 | } 400 | 401 | #{patch_record[:patch]} 402 | } 403 | end 404 | 405 | 406 | before '/vulnerable' do 407 | filtered = filter_packages(params) 408 | @dataset = vulnerable. 409 | select{ |pkg, v| filtered.include? pkg }. 410 | sort_by{ |pkg, v| pkg } 411 | end 412 | 413 | get '/vulnerable', :rss_requested => true do 414 | content_type "application/rss+xml", :charset => 'utf-8' 415 | 416 | RSS::Maker.make("2.0") do |maker| 417 | maker.channel.author = "Nixpkgs Monitor" 418 | maker.channel.updated = Time.now.to_s 419 | maker.channel.title = "Vulnerable packages" 420 | maker.channel.description = "Automatically detected matches of Nix packages against CVE vulnerability database" 421 | maker.channel.link = settings.base_url + "vulnerable" 422 | 423 | @dataset.each do |pkg, candidates| 424 | candidates.each do |prod, cves| 425 | cves.each do |cve| 426 | maker.items.new_item do |item| 427 | item.title = "#{pkg}:#{NixPkgsMonitor::DistroPackages::Nix.by_internal_name[pkg].version} matches #{cve} as #{prod}" 428 | item.description = "#{render_pkgname(pkg, [:with_version, :with_base])} matches #{render_cve(cve)} as #{prod}" 429 | end 430 | end 431 | end 432 | end 433 | end.to_s 434 | end 435 | 436 | get '/vulnerable', :rss_requested => false do 437 | @rss_available = true 438 | 439 | report = @dataset. 440 | map do |pkg, candidates| 441 | candidates.map do |prod, cves_raw| 442 | cves = cves_raw.sort.reverse 443 | %{ 444 | #{render_pkgname(pkg, :with_version)} 445 | #{prod} 446 | #{cves[0..(cves.size>3 ? 1 : 2)].map{|cve| render_cve(cve)}.join(', ')} 447 | #{cves.size>3 ? "... #{cves.size} total" : ""} 448 | 449 | } 450 | end. 451 | join 452 | end. 453 | join 454 | 455 | haml :vulnerable, :layout => :layout, 456 | :locals => { :vulnerable_report => report, :filter => render_filter(params.merge({:vulnerable => true})) } 457 | end 458 | 459 | 460 | get '/pd' do 461 | halt "package not specified" unless params[:p] 462 | pkg = NixPkgsMonitor::DistroPackages::Nix.by_internal_name[params[:p]] 463 | halt "package not found" unless pkg 464 | %{ 465 | #{ request.xhr? ? "" : 466 | %{ 467 | 468 | 469 | 470 | 471 | 472 | #{params[:p]} 473 | 474 | 475 | 476 | } 477 | } 478 |

    Information:

    479 | 480 | 481 | 482 | 483 | 484 | 485 |
    Package:#{pkg.internal_name}
    Name:#{pkg.name}
    Version:#{pkg.version}
    Source:#{pkg.url}
    Vulnerable:#{vulnerable.has_key?(pkg.internal_name) ? "YES" : "no"}
    486 | #{ vulnerable.has_key?(pkg.internal_name) ? 487 | %{ 488 | 489 | 490 | #{ 491 | vulnerable[pkg.internal_name]. 492 | map do |prod, cves| 493 | %{ 494 | 495 | 496 | 501 | 502 | } 503 | end. 504 | join 505 | } 506 |
    Matches toCVEs
    #{prod}#{cves.sort.reverse. 497 | map{|cve| render_cve(cve) }. 498 | join(', ') 499 | } 500 |
    507 | } : "" 508 | } 509 | 510 | #{ outdated.has_key?(pkg.internal_name) ? 511 | %{ 512 |

    Available updates:

    513 | 514 | 515 | #{ 516 | outdated[params[:p]].map do |version, updaters| 517 | patch = DB[:patches][:pkg_attr => pkg.internal_name, :version => version] 518 | build = (patch ? DB[:builds][:outpath => patch[:outpath]] : nil) 519 | tarball = DB[:tarballs].join(:tarball_sha256,:tarball => :tarball). 520 | exclude(:sha256 => "404")[:pkg_attr => pkg.internal_name, :version => version] 521 | url = (tarball ? tarball [:tarball] : "") 522 | %{ 523 | 524 | 525 | 526 | 527 | 528 | 529 | } 530 | end. 531 | join 532 | } 533 | } : "" 534 | } 535 |
    VersionReported byTarballPatchLog
    #{version}#{updaters.join ", "}#{url}#{ patch ? "yes" : "" }#{ build ? "#{build[:status]}(diff)" : "" }
    536 | #{ request.xhr? ? "" : "" } 537 | } 538 | end 539 | 540 | 541 | get '/buildlog' do 542 | halt 404, "derivation not specified" unless params[:outpath] 543 | log = NixPkgsMonitor::BuildLog.get_log(params[:outpath]) 544 | halt 404, "log not found" unless log 545 | content_type 'text/plain', :charset => 'utf-8' 546 | 547 | lint = NixPkgsMonitor::BuildLog.lint(log) 548 | 549 | %{ 550 | #{ lint.empty? ? "" : 551 | %{An INCOMPLETE list of issues: 552 | #{lint.join} 553 | 554 | } 555 | } 556 | #{log} 557 | } 558 | end 559 | 560 | 561 | get '/builddiff' do 562 | halt 404, "derivations not specified" unless params[:old] and params[:new] 563 | oldlog = NixPkgsMonitor::BuildLog.get_log(params[:old]) 564 | newlog = NixPkgsMonitor::BuildLog.get_log(params[:new]) 565 | halt 404, "log not found" unless oldlog and newlog 566 | 567 | "" + 568 | Diffy::Diff.new(NixPkgsMonitor::BuildLog.sanitize(oldlog, params[:old]), 569 | NixPkgsMonitor::BuildLog.sanitize(newlog, params[:new]), 570 | :context => 2).to_s(:html) 571 | end 572 | 573 | 574 | get '/report' do 575 | available_reports = { 576 | :version_mismatch => 577 | "Nix packages for which either tarball name can't be parsed or its version doesn't match the package version", 578 | :nixpkgs_failed_name_parse => 579 | "Nix packages for which it was impossible to determine version from name string", 580 | :nixpkgs_no_sources => 581 | "Nix packages for which no sources were specified in Nixpkgs", 582 | } 583 | 584 | report_name = params[:name] ? params[:name].to_sym : nil 585 | if report_name and not report_name.empty? and DB.table_exists?(report_name) 586 | report_header = DB[report_name].columns!.map{ |r_name| "#{r_name}" }.join 587 | report_body = DB[report_name].map do |row| %{ 588 | #{row.values.map{|v| "#{v}"}.join} } 589 | end. 590 | join 591 | 592 | haml :report, :layout => :layout, :locals => 593 | { :report_header => report_header, :report_body => report_body } 594 | else 595 | reports = available_reports. 596 | map do |r_name, r_desc| %{ 597 | #{r_name} 598 | #{DB.table_exists?(r_name) ? DB[r_name].count : 0} 599 | #{r_desc} } 600 | end. 601 | join 602 | 603 | haml :reports, :layout => :layout, :locals => { :reports => reports } 604 | end 605 | end 606 | 607 | 608 | get '/refresh' do 609 | refresh 610 | redirect back 611 | end 612 | 613 | 614 | get '/rawdb' do 615 | send_file 'db.sqlite' 616 | end 617 | 618 | 619 | get '/default.css' do 620 | content_type 'text/css', :charset => 'utf-8' 621 | <