├── jobs ├── PortIndex2MySQL ├── porttree_tarball.sh ├── current_versions.tcl ├── org.macports.mprsyncup.plist ├── org.macports.PortIndex2MySQL.plist ├── README ├── mirror_macports.sh ├── GuideRegen.sh ├── port_binary_distributable.tcl ├── build_deploy_prbot.sh ├── portfile_mirror.pl ├── delete_old_archives.py ├── portfile_lint.pl ├── portindex2postgres.tcl ├── delete_old_distfiles.tcl ├── mprsyncup-alt ├── mprsyncup ├── distributable_lib.tcl ├── PortIndex2PGSQL.tcl └── PortIndex2MySQL.tcl ├── .github └── FUNDING.yml ├── .gitattributes └── containers └── paste ├── Dockerfile └── deploy.sh /jobs/PortIndex2MySQL: -------------------------------------------------------------------------------- 1 | PortIndex2MySQL.tcl -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | open_collective: macports 2 | github: macports 3 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Normalize EOLs by default, to avoid falling back on committers' 2 | # "core.autocrlf" settings. 3 | * text=auto 4 | -------------------------------------------------------------------------------- /jobs/porttree_tarball.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd /var/rsync/release 4 | /bin/tar zcf /tmp/ports.tar.gz.new ports 5 | cp /tmp/ports.tar.gz{.new,.html} 6 | #chcon -t httpd_sys_content_t /tmp/ports.tar.gz.html 7 | mv -f /tmp/ports.tar.gz.html /var/www/html/distfiles/ports.tar.gz 8 | mv -f /tmp/ports.tar.gz.new /var/rsync/release/ports.tar.gz 9 | 10 | 11 | -------------------------------------------------------------------------------- /jobs/current_versions.tcl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env port-tclsh 2 | 3 | package require macports 4 | mportinit 5 | 6 | if {[catch {set res [mportlistall]} result]} { 7 | puts stderr "$::errorInfo" 8 | error "listing all ports failed: $result" 9 | } 10 | 11 | foreach {name dictionary} $res { 12 | array unset portinfo 13 | array set portinfo $dictionary 14 | puts "${name} $portinfo(version)_$portinfo(revision)" 15 | } 16 | 17 | mportshutdown 18 | -------------------------------------------------------------------------------- /containers/paste/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3 2 | 3 | RUN apk add --no-cache \ 4 | perl \ 5 | perl-io-socket-ssl \ 6 | perl-app-cpanminus \ 7 | perl-mojolicious \ 8 | perl-text-csv \ 9 | git 10 | 11 | RUN git clone https://github.com/jhthorsen/app-mojopaste /app-mojopaste \ 12 | && git -C /app-mojopaste checkout 1.05 13 | 14 | VOLUME /app/data 15 | 16 | ENV MOJO_MODE production 17 | ENV PASTE_DIR /app/data 18 | ENV PASTE_ENABLE_CHARTS 0 19 | EXPOSE 8080 20 | 21 | USER nobody:nobody 22 | ENTRYPOINT ["/usr/bin/perl", "/app-mojopaste/script/mojopaste", "prefork", "-l", "http://*:8080"] 23 | -------------------------------------------------------------------------------- /jobs/org.macports.mprsyncup.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Label 6 | org.macports.mpryncup 7 | ProgramArguments 8 | 9 | /opt/local/share/macports/resources/portmgr/mprsyncup 10 | 11 | StartCalendarInterval 12 | 13 | 14 | Minute 15 | 00 16 | 17 | 18 | Minute 19 | 30 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /jobs/org.macports.PortIndex2MySQL.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Label 6 | org.macports.PortIndex2MySQL 7 | ProgramArguments 8 | 9 | /opt/local/share/macports/resources/portmgr/PortIndex2MySQL 10 | 11 | StartCalendarInterval 12 | 13 | 14 | Hour 15 | 4 16 | Minute 17 | 15 18 | 19 | 20 | Hour 21 | 16 22 | Minute 23 | 15 24 | 25 | 26 | UserName 27 | mysql 28 | 29 | 30 | -------------------------------------------------------------------------------- /jobs/README: -------------------------------------------------------------------------------- 1 | 2 | This file explains the automated jobs that live in this directory and keep MacPorts running at the server level, 3 | appart from the basic mail, mailman, www, trac & svn services: 4 | 5 | 1) mprsyncup: 6 | Runs on Mac OS Forge servers every 30 minutes, on the hour and on the half hour, off launchd through the org.macports.mprsyncup.plist provided. 7 | Repopulates the rsync modules that feed the "sync" and "selfupdate" operations for both the ports tree and MacPorts sources, freshly off the svn sources. 8 | Updates the port indexes stored on the rsync server. 9 | 10 | 3) PortIndex2MySQL: 11 | Running Mac OS Forge servers twice a day every 12 hours off launchd through the org.macports.PortIndex2MySQL.plist provided, 15 minutes after the 12 | PortIndexRegen job. This jobs repopulates the database that feeds the www.macports.org/ports.php page off the regenerated PortIndex. 13 | 14 | 4) GuideRegen: 15 | Running on Mac OS Forge servers as an svn post-commit hook after commits to the /trunk/doc-new dir take place. Regenerates the project guide found at 16 | https://guide.macports.org 17 | 18 | 5) Website: 19 | There's actually no script for this one, as the job itself is really simple: a post-commit svn hook attached to the /trunk/www dir 20 | updates the copy of our website on the Mac OS Forge web server whenever a commit to that directory occurs. 21 | 22 | -------------------------------------------------------------------------------- /jobs/mirror_macports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### 3 | # Run "port mirror" for each variant of each port 4 | # Created by William Siegrist, 5 | # e-mail: wsiegrist@apple.com 6 | # $Id$ 7 | #### 8 | 9 | # regexp of ports that we do not mirror 10 | EXCLUSIONS='^(molden|metis)$' 11 | 12 | # macports really wants this, so lets appease it 13 | export COLUMNS=80 14 | export LINES=24 15 | 16 | # send all output here 17 | LOG="/admin/var/log/macports.log" 18 | 19 | PORT="/opt/local/bin/port" 20 | CUT=/usr/bin/cut 21 | GREP=/usr/bin/grep 22 | XARGS=/usr/bin/xargs 23 | EGREP=/usr/bin/egrep 24 | 25 | exec >> $LOG 2>&1 26 | 27 | echo "------------------------------ 28 | Beginning mirror run 29 | ------------------------------"; 30 | 31 | # for each port 32 | for P in `$PORT list | $CUT -f 1 -d " " | $EGREP -v $EXCLUSIONS`; 33 | do 34 | 35 | NOW=`/bin/date`; 36 | echo "TIME: ${NOW}"; 37 | 38 | # mirror with no variants 39 | echo "Mirroring ${P}"; 40 | $PORT clean $P; 41 | $PORT mirror $P; 42 | 43 | # for each variant 44 | for V in `$PORT -q variants $P | $CUT -c 4- | $CUT -d ":" -f 1 | $GREP -v universal | $XARGS`; 45 | do 46 | # mirror with each variant 47 | echo "Mirroring ${P} +${V}"; 48 | $PORT clean $P; 49 | $PORT mirror $P +$V; 50 | done 51 | 52 | # mirror with each platform (can exclude the one the server is running) 53 | for VERS in 8 9; 54 | do 55 | for ARCH in i386 powerpc; 56 | do 57 | echo "Mirroring ${P} with platform darwin ${VERS} ${ARCH}" 58 | $PORT mirror $P os.platform=darwin os.subplatform=macosx os.major=${VERS} os.arch=${ARCH} 59 | done 60 | done 61 | for VERS in 10 11 12 13 14; 62 | do 63 | echo "Mirroring ${P} with platform darwin ${VERS} i386" 64 | $PORT mirror $P os.platform=darwin os.subplatform=macosx os.major=${VERS} os.arch=i386 65 | done 66 | 67 | # clean up the work area 68 | $PORT clean --work $P; 69 | 70 | done 71 | 72 | # record the last time we mirrored 73 | /bin/date > /rsync/macports-san/distfiles/TIMESTAMP 74 | 75 | echo "------------------------------ 76 | End of mirror run 77 | ------------------------------"; 78 | 79 | -------------------------------------------------------------------------------- /jobs/GuideRegen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #### 4 | # Guide regen automation script. 5 | # Created by Daniel J. Luke 6 | # e-mail: dluke@geeklair.net 7 | # Based on IndexRegen.sh 8 | # $Id$ 9 | #### 10 | 11 | # Configuration 12 | LOCKFILE=/tmp/.mp_svn_guide_regen.lock 13 | # ROOT directory, where everything is. This needs to exist! 14 | ROOT=/var/tmp/macports/ 15 | # e-mail address to spam in case of failure. 16 | #SPAM_LOVERS=example@hostname.com 17 | 18 | # Other settings (probably don't need to be changed). 19 | SVN_CONFIG_DIR=${ROOT}/svnconfig 20 | REPO_BASE=https://svn.macports.org/repository/macports 21 | SVN="/opt/local/bin/svn -q --non-interactive --config-dir $SVN_CONFIG_DIR" 22 | # Where to checkout the source code. This needs to exist! 23 | SRCTREE=${ROOT}/source 24 | # Log for the e-mail in case of failure. 25 | FAILURE_LOG=${ROOT}/guide_failure.log 26 | # The date. 27 | DATE=$(/bin/date +'%A %Y-%m-%d at %H:%M:%S') 28 | 29 | # Where to find the binaries we need 30 | MAIL=/usr/bin/mail 31 | RM=/bin/rm 32 | TOUCH=/usr/bin/touch 33 | MAKE=/usr/bin/make 34 | MKDIR=/bin/mkdir 35 | 36 | # Function to spam people in charge if something goes wrong during guide regen. 37 | bail () { 38 | $MAIL -s "Guide Regen Failure on ${DATE}" $SPAM_LOVERS < $FAILURE_LOG 39 | cleanup; exit 1 40 | } 41 | 42 | # Cleanup fuction for runtime files. 43 | cleanup () { 44 | $RM -f $FAILURE_LOG $LOCKFILE 45 | } 46 | 47 | 48 | if [ ! -e $LOCKFILE ]; then 49 | $TOUCH $LOCKFILE 50 | else 51 | echo "Guide Regen lockfile found, is another regen job running?" > $FAILURE_LOG; bail 52 | fi 53 | 54 | # Checkout/update the doc tree 55 | if [ -d ${SRCTREE}/doc-new ]; then 56 | $SVN update ${SRCTREE}/doc-new > $FAILURE_LOG 2>&1 \ 57 | || { echo "Updating the doc tree from $REPO_BASE/trunk/doc-new failed." >> $FAILURE_LOG; bail ; } 58 | else 59 | $MKDIR -p ${SRCTREE}/doc-new 60 | $SVN checkout ${REPO_BASE}/trunk/doc-new ${SRCTREE}/doc-new > $FAILURE_LOG 2>&1 \ 61 | || { echo "Checking out the doc tree from $REPO_BASE/trunk/doc-new failed." >> $FAILURE_LOG; bail ; } 62 | fi 63 | 64 | # build single html version 65 | { cd ${SRCTREE}/doc-new && $MAKE guide > $FAILURE_LOG 2>&1 ; } \ 66 | || { echo "make failed." >> $FAILURE_LOG ; bail ; } 67 | 68 | # build chunked version 69 | { cd ${SRCTREE}/doc-new && $MAKE guide-chunked > $FAILURE_LOG 2>&1 ; } \ 70 | || { echo "make failed." >> $FAILURE_LOG ; bail ; } 71 | 72 | # At this point the guide was regen'd successfuly, so we cleanup before we exit. 73 | cleanup && exit 0 74 | -------------------------------------------------------------------------------- /containers/paste/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ## 4 | # This script downloads and builds a docker container from the Dockerfile in 5 | # the current directory. 6 | # 7 | # 1. update the git repository this script resides in 8 | # 2. rebuild the container if 9 | # 2.1 the Dockerfile or any of the files in its directory changed, or 10 | # 2.2 the container is older than a week, to update dependencies. 11 | # 3. retag the container as paste:latest 12 | # 4. kill the current container, so that systemd will restart it 13 | 14 | set -euo pipefail 15 | 16 | # The number of old containers to keep for this Dockerfile 17 | KEEP_OLD_VERSIONS=5 18 | 19 | THISDIR=$(cd "$(dirname "$0")" && pwd) 20 | 21 | git -C "$THISDIR" fetch --quiet || true # Ignore network problems, assuming they are temporary 22 | OLDREV=$(git -C "$THISDIR" log -1 --pretty=%H "$THISDIR") 23 | git -C "$THISDIR" reset --quiet --hard origin/master 24 | NEWREV=$(git -C "$THISDIR" log -1 --pretty=%H "$THISDIR") 25 | 26 | if [ "$OLDREV" != "$NEWREV" ]; then 27 | # The container's folder changed. Since this potentially also affects this 28 | # script, re-execute the script itself. 29 | printf "Revision changed from %s to %s, re-executing...\n" "$OLDREV" "$NEWREV" 30 | exec "$0" "$@" 31 | exit 1 32 | fi 33 | 34 | cd "$THISDIR" 35 | CONTAINERNAME=$(basename "$(readlink -f .)") 36 | TIMESTAMP=$(date +%G-%V) 37 | 38 | # Check whether the current container was already built for this version 39 | IMAGES=$(docker images --format "{{.ID}}" "$CONTAINERNAME:$NEWREV-$TIMESTAMP" | wc -l) 40 | if [ "$IMAGES" -gt 0 ]; then 41 | printf "Container %s is already the newest version. Nothing to do\n" "$CONTAINERNAME" 42 | exit 0 43 | fi 44 | 45 | printf "Rebuilding container %s with tag %s\n" "$CONTAINERNAME" "$NEWREV-$TIMESTAMP" 46 | docker build --pull --no-cache -t "$CONTAINERNAME:$NEWREV-$TIMESTAMP" . 2>&1 47 | 48 | printf "Rebuild successful, tagging as %s:latest\n" "$CONTAINERNAME" 49 | docker tag "$CONTAINERNAME:$NEWREV-$TIMESTAMP" "$CONTAINERNAME:latest" 50 | 51 | # Stop currently running container to force systemd to restart it 52 | RUNNING_CONTAINER_ID=$(docker container inspect --format "{{.ID}}" "$CONTAINERNAME" 2>/dev/null || true) 53 | if [ -n "$RUNNING_CONTAINER_ID" ]; then 54 | printf "Stopping running instance %s of container %s\n" "$RUNNING_CONTAINER_ID" "$CONTAINERNAME" 55 | docker stop "$RUNNING_CONTAINER_ID" 56 | fi 57 | 58 | # Cleanup old images 59 | CLEANUP_IMAGES_STRING=$(docker images --format "{{.ID}}" --filter "before=$CONTAINERNAME:$NEWREV-$TIMESTAMP" "$CONTAINERNAME" | sed "1,${KEEP_OLD_VERSIONS}d" | tr '\n' ' ') 60 | if [ -n "$CLEANUP_IMAGES_STRING" ]; then 61 | IFS=' ' read -r -a CLEANUP_IMAGES <<<"$CLEANUP_IMAGES_STRING" 62 | docker rmi "${CLEANUP_IMAGES[@]}" 63 | fi 64 | 65 | printf "Updated %s to %s\n" "$CONTAINERNAME" "$NEWREV-$TIMESTAMP" 66 | exit 0 67 | -------------------------------------------------------------------------------- /jobs/port_binary_distributable.tcl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env port-tclsh 2 | # -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4 3 | # 4 | # Check that binaries of a port are distributable by looking at its license 5 | # and the licenses of its dependencies. 6 | # 7 | # Expected format: A {B C} means the license is A plus either B or C. 8 | # 9 | # Exit status: 10 | # 0: distributable 11 | # 1: non-distributable 12 | # 2: error 13 | 14 | set MY_VERSION 0.1 15 | 16 | source [file join [file dirname [info script]] distributable_lib.tcl] 17 | 18 | proc printUsage {} { 19 | puts "Usage: $::argv0 \[-d dir\] \[-hvV\] port-name \[variants...\]" 20 | puts " -d dir Use directory 'dir' for persistent data storage" 21 | puts " -h This help" 22 | puts " -v verbose output" 23 | puts " -V show version and MacPorts version being used" 24 | puts "" 25 | puts "port-name is the name of a port to check" 26 | puts "variants is the list of variants to enable/disable: +one -two..." 27 | } 28 | 29 | set verbose 0 30 | set showVersion 0 31 | set dbdir "" 32 | 33 | while {[string index [lindex $::argv 0] 0] eq "-"} { 34 | switch [string range [lindex $::argv 0] 1 end] { 35 | d { 36 | if {[llength $::argv] < 2} { 37 | printUsage 38 | exit 2 39 | } 40 | set dbdir [lindex $::argv 1] 41 | set ::argv [lrange $::argv 1 end] 42 | } 43 | h { 44 | printUsage 45 | exit 0 46 | } 47 | v { 48 | set verbose 1 49 | } 50 | V { 51 | set showVersion 1 52 | } 53 | default { 54 | puts stderr "Unknown option [lindex $::argv 0]" 55 | printUsage 56 | exit 2 57 | } 58 | } 59 | set ::argv [lrange $::argv 1 end] 60 | } 61 | 62 | package require macports 63 | mportinit 64 | 65 | if {$showVersion} { 66 | puts "Version $MY_VERSION" 67 | puts "MacPorts version [macports::version]" 68 | exit 0 69 | } 70 | 71 | if {[llength $::argv] == 0} { 72 | puts stderr "Error: missing port-name" 73 | printUsage 74 | exit 2 75 | } 76 | set portName [lindex $::argv 0] 77 | set ::argv [lrange $::argv 1 end] 78 | 79 | if {$dbdir ne ""} { 80 | init_license_db $dbdir 81 | } 82 | 83 | array set variantInfo {} 84 | foreach variantSetting $::argv { 85 | set variant [split_variants $variantSetting] 86 | foreach {variantName flag} $variant { 87 | set variantInfo($variantName) $flag 88 | } 89 | } 90 | 91 | set results [check_licenses $portName [array get variantInfo]] 92 | if {$dbdir ne ""} { 93 | write_license_db $dbdir 94 | } 95 | if {$verbose} { 96 | puts [lindex $results 1] 97 | } 98 | exit [lindex $results 0] 99 | -------------------------------------------------------------------------------- /jobs/build_deploy_prbot.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ## 4 | # This script downloads and builds the pull request handling bot developed by 5 | # l2dy in Google Summer of Code 2017. It will 6 | # 7 | # 1. clone the git repository if it does not exist yet and update it to the 8 | # latest version if it exists already 9 | # 2. check whether an update is available, and if there is one 10 | # 3. download and update the necessary go dependencies 11 | # 4. build the pull request bot 12 | # 5. update a symlink that will always point to the latest version 13 | # 14 | # To use this script, set $GOPATH in your environment to the absolute path of 15 | # the go workspace you want to use. 16 | # 17 | # Additionally, you can set the $GO variable to the path of the go utility. 18 | # This may be helpful if you are trying to use a different version of Go than 19 | # the one that comes with your operating system. 20 | 21 | set -euo pipefail 22 | 23 | if [ -z "${GOPATH:-}" ]; then 24 | printf >&2 "You must set \$GOPATH to the go workspace you want to use before calling this script.\n" 25 | exit 1 26 | fi 27 | 28 | # Allow overriding the path to the go executable 29 | GO=${GO:-go} 30 | 31 | MPBOT_GITHUB_URL=https://github.com/macports/mpbot-github 32 | MPBOT_GITHUB_SRC=/var/www/prbot/mpbot-github 33 | MPBOT_GITHUB_SUBFOLDER=pr/prbot 34 | PRBOT_CURRENT=$GOPATH/bin/prbot-current 35 | PRBOT_NEXT=$GOPATH/bin/prbot-next 36 | 37 | # Set up GOPATH, fetch or update source 38 | mkdir -p "$GOPATH" 39 | if [ -d "$MPBOT_GITHUB_SRC" ]; then 40 | git -C "$MPBOT_GITHUB_SRC" fetch --quiet || true # Ignore network problems assuming they are temporary 41 | git -C "$MPBOT_GITHUB_SRC" reset --quiet --hard origin/master 42 | else 43 | git clone --quiet "$MPBOT_GITHUB_URL" "$MPBOT_GITHUB_SRC" 44 | fi 45 | 46 | # Find out whether there are new changes to be deployed 47 | HEADREV=$(git -C "$MPBOT_GITHUB_SRC" rev-parse HEAD) 48 | if [ -z "$HEADREV" ]; then 49 | printf >&2 "Could not determine head revision of Git repository %s\n" "$MPBOT_GITHUB_SRC" 50 | exit 1 51 | fi 52 | 53 | CURRENTREV="" 54 | if [[ $(readlink "$PRBOT_CURRENT") =~ .*-([0-9a-f]+)$ ]]; then 55 | CURRENTREV=${BASH_REMATCH[1]} 56 | fi 57 | 58 | if [ "$HEADREV" = "$CURRENTREV" ]; then 59 | printf "Revision %s is already the newest revision. Nothing to do.\n" "$CURRENTREV" 60 | exit 0 61 | fi 62 | 63 | # Get dependencies 64 | (cd "$MPBOT_GITHUB_SRC/$MPBOT_GITHUB_SUBFOLDER" && "$GO" get) 65 | # Install 66 | (cd "$MPBOT_GITHUB_SRC/$MPBOT_GITHUB_SUBFOLDER" && "$GO" install) 67 | 68 | # Update symlink 69 | mv "$GOPATH/bin/prbot" "$GOPATH/bin/prbot-$HEADREV" 70 | rm -f "$PRBOT_NEXT" 71 | ln -s "prbot-$HEADREV" "$PRBOT_NEXT" 72 | mv -f "$PRBOT_NEXT" "$PRBOT_CURRENT" 73 | 74 | # Killing currently running service to force systemd to restart it 75 | pkill prbot-current || true 76 | 77 | printf "Updated from %s to %s\n" "$CURRENTREV" "$HEADREV" 78 | exit 0 79 | -------------------------------------------------------------------------------- /jobs/portfile_mirror.pl: -------------------------------------------------------------------------------- 1 | #!/opt/local/bin/perl -w 2 | ## 3 | # Run "port mirror" for all Portfiles changed in a given revision 4 | # Created by William Siegrist, 5 | # e-mail: wsiegrist@apple.com 6 | # $Id$ 7 | ## 8 | use strict; 9 | use Mail::Sendmail; 10 | 11 | my $EXCLUSIONS = ('molden'); 12 | 13 | my $REPOPATH = "/svn/repositories/macports/"; 14 | my $REPOHOST = "https://svn.macports.org/repository/macports"; 15 | my $SVNLOOK = "/opt/local/bin/svnlook"; 16 | my $PORTCMD = "/opt/local/bin/port"; 17 | my $SVN = "/opt/local/bin/svn -q --non-interactive"; 18 | my $MKDIR = "/bin/mkdir -p"; 19 | 20 | my $rev = $ARGV[0] or usage(); 21 | my $TMPROOT = "/tmp/mp_mirror/$rev"; 22 | 23 | my @changes = `$SVNLOOK changed $REPOPATH -r $rev | grep '/Portfile' | grep -vE '^[ ]+D'`; 24 | 25 | foreach my $change (@changes) { 26 | if ($change =~ /Portfile/) { 27 | # remove svn status and whitespace 28 | chop($change); 29 | $change =~ s/\w\s+([\/\w]+)/$1/g; 30 | # extract the portname from parent dir of Portfile 31 | my $port = $change; 32 | $port =~ s/^.*\/([^\/]+)\/Portfile$/$1/g; 33 | 34 | if (in_array($port, $EXCLUSIONS)) { 35 | die("Port exclusion: $port \n"); 36 | } 37 | 38 | # get the group directory 39 | my $group = $change; 40 | $group =~ s/^.*\/([^\/]+)\/[^\/]+\/Portfile$/$1/g; 41 | 42 | # make a temporary work area 43 | `$MKDIR $TMPROOT/$group/$port`; 44 | chdir("$TMPROOT/$group/$port") or die("Failed to change dir for port: $port"); 45 | `$SVN co $REPOHOST/trunk/dports/$group/$port/ .`; 46 | # test the port 47 | _mirror($port); 48 | } 49 | } 50 | 51 | 52 | # 53 | # Subroutines 54 | # 55 | 56 | sub _mirror { 57 | my ($port) = @_; 58 | my $errors = `sudo $PORTCMD -qc mirror`; 59 | 60 | if ($errors) { 61 | my $maintainers = `$PORTCMD -q info --maintainer $port`; 62 | # strip everything but the email addresses 63 | $maintainers =~ s/maintainer: //; 64 | $maintainers =~ s/openmaintainer\@macports.org//; 65 | $maintainers =~ s/nomaintainer\@macports.org//; 66 | chop($maintainers); 67 | 68 | _mail($port, $maintainers, $errors); 69 | } 70 | } 71 | 72 | sub _mail { 73 | my ($port, $maintainers, $errors) = @_; 74 | 75 | my %mail = ( 76 | To => $maintainers, 77 | From => 'noreply@macports.org', 78 | Subject => "[MacPorts Mirror] Portfile Mirror Errors for: $port", 79 | Message => "Portfile: $port \n\n\n Errors: $errors \n\n", 80 | smtp => 'relay.apple.com', 81 | ); 82 | 83 | sendmail(%mail) or die $Mail::Sendmail::error; 84 | } 85 | 86 | sub usage { 87 | print "usage: portfile_mirror.pl \n"; 88 | exit(); 89 | } 90 | 91 | sub in_array { 92 | my ($needle, @haystack) = @_; 93 | 94 | foreach my $element (@haystack) { 95 | if ($element eq $needle) { 96 | return 1; 97 | } 98 | } 99 | return 0; 100 | } 101 | 102 | 103 | -------------------------------------------------------------------------------- /jobs/delete_old_archives.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | greatestAge = 0 4 | greatestSize = 0 5 | totalSize = 0 6 | 7 | class archiveFile(object): 8 | """An archive file being considered for deletion""" 9 | def __init__(self, path, age, size): 10 | self.path = path 11 | self.age = age 12 | self.size = size 13 | 14 | def weightedValue(age, size): 15 | """Combine an age and a size into a value out of 100""" 16 | weightedAge = (float(age) / float(greatestAge)) * 50.0 17 | weightedSize = (float(size) / float(greatestSize)) * 50.0 18 | return weightedAge + weightedSize 19 | 20 | def weightedKey(archive): 21 | """Key extraction function for sorting using weightedValue""" 22 | return weightedValue(archive.age, archive.size) 23 | 24 | import sys 25 | 26 | versionFile = 'current_versions.txt' 27 | rootDir = '.' 28 | 29 | if len(sys.argv) > 1: 30 | rootDir = sys.argv[1] 31 | if len(sys.argv) > 2: 32 | versionFile = sys.argv[2] 33 | 34 | import re 35 | # patterns to match against for archives that are the current version 36 | currentVersions = {} 37 | if versionFile == '-': 38 | fd = sys.stdin 39 | else: 40 | fd = open(versionFile, 'r') 41 | for line in fd: 42 | name, version = line.split() 43 | currentVersions[name] = re.compile(name+'-'+version+'[.+]') 44 | fd.close() 45 | 46 | import time 47 | now = time.time() 48 | fileList = [] 49 | 50 | import os 51 | for portdir in os.listdir(rootDir): 52 | portDirPath = os.path.join(rootDir, portdir) 53 | if os.path.isdir(portDirPath): 54 | for archiveFilename in os.listdir(portDirPath): 55 | try: 56 | if archiveFilename.endswith('.rmd160') or currentVersions[portdir].match(archiveFilename): 57 | continue 58 | except KeyError: 59 | pass 60 | archivePath = os.path.join(portDirPath, archiveFilename) 61 | if os.path.isfile(archivePath): 62 | thisAge = now - os.path.getmtime(archivePath) 63 | thisSize = os.path.getsize(archivePath) 64 | thisArchiveFile = archiveFile(archivePath, thisAge, thisSize) 65 | fileList.append(thisArchiveFile) 66 | if thisArchiveFile.age > greatestAge: 67 | greatestAge = thisArchiveFile.age 68 | if thisArchiveFile.size > greatestSize: 69 | greatestSize = thisArchiveFile.size 70 | totalSize += thisSize 71 | 72 | fileList.sort(key=weightedKey, reverse=True) 73 | 74 | for f in fileList: 75 | sys.stderr.write(f.path+' '+str(f.age)+' '+str(f.size)+': weighted value = '+str(weightedValue(f.age, f.size))+'\n') 76 | 77 | # trim files until the total size of non-current archives remaining is this or less 78 | targetSize = 200 * 10**9 79 | 80 | for f in fileList: 81 | if totalSize <= targetSize: 82 | break 83 | print (f.path) 84 | sigpath = f.path+'.rmd160' 85 | if os.path.isfile(sigpath): 86 | print (sigpath) 87 | totalSize -= f.size 88 | -------------------------------------------------------------------------------- /jobs/portfile_lint.pl: -------------------------------------------------------------------------------- 1 | #!/opt/local/bin/perl -w 2 | 3 | #### 4 | # Run "port lint" for all Portfiles changed in a given revision 5 | # Created by William Siegrist, 6 | # e-mail: wsiegrist@apple.com 7 | # $Id$ 8 | #### 9 | 10 | use strict; 11 | use Mail::Sendmail; 12 | 13 | $ENV{'HOME'} = '/tmp/mp_lint/'; 14 | $ENV{'LANG'} = 'en_US.UTF-8'; 15 | 16 | my $REPOPATH = "/svn/repositories/macports/"; 17 | my $REPOHOST = "https://svn.macports.org/repository/macports"; 18 | my $SVNLOOK = "/opt/local/bin/svnlook"; 19 | 20 | # use a release (non-trunk) version of port 21 | my $PORTCMD = "/opt/local/bin/port"; 22 | 23 | my $SVN = "/opt/local/bin/svn -Nq --non-interactive"; 24 | my $MKDIR = "/bin/mkdir -p"; 25 | 26 | 27 | my $rev = $ARGV[0] or usage(); 28 | my $TMPROOT = "/tmp/mp_lint/$rev"; 29 | 30 | my @changes = `$SVNLOOK changed $REPOPATH -r $rev`; 31 | 32 | my $author = `$SVNLOOK author $REPOPATH -r $rev`; 33 | chomp($author); 34 | 35 | _log("Rev: $rev"); 36 | 37 | foreach my $change (@changes) { 38 | if ($change =~ /[AU][\sU]\s\s[\/\w\-_]+Portfile$/) { 39 | # remove svn status and whitespace 40 | chop($change); 41 | $change =~ s/[ADU_][\sU]\s\s([\/\w\-_]+)/$1/g; 42 | # extract the portname from parent dir of Portfile 43 | my $port = $change; 44 | $port =~ s/^.*\/([^\/]+)\/Portfile$/$1/g; 45 | 46 | # get the group directory 47 | my $group = $change; 48 | $group =~ s/^.*\/([^\/]+)\/[^\/]+\/Portfile$/$1/g; 49 | 50 | # get the parent directory of the Portfile 51 | my $parent = $change; 52 | $parent =~ s/Portfile//; 53 | 54 | _log("Port: $group / $port "); 55 | 56 | # make a temporary work area 57 | `$MKDIR $TMPROOT/$group/$port`; 58 | chdir("$TMPROOT/$group/$port") or die("Failed to change dir for port: $port"); 59 | `$SVN co -r $rev $REPOHOST/$parent .`; 60 | # test the port 61 | _lint($port); 62 | } 63 | } 64 | 65 | 66 | # 67 | # Subroutines 68 | # 69 | 70 | sub _lint { 71 | my ($port) = @_; 72 | my $errors = `$PORTCMD -qc lint 2>&1`; 73 | 74 | if ($errors) { 75 | _log("Error: $errors "); 76 | my $maintainers = `$PORTCMD -q info --maintainer $port`; 77 | # strip everything but the email addresses 78 | $maintainers =~ s/maintainer: //; 79 | $maintainers =~ s/openmaintainer\@macports.org//; 80 | $maintainers =~ s/nomaintainer\@macports.org//; 81 | chop($maintainers); 82 | 83 | _log("Maintainers: $maintainers "); 84 | 85 | _mail($port, $maintainers, $errors); 86 | } 87 | } 88 | 89 | sub _mail { 90 | my ($port, $maintainers, $errors) = @_; 91 | 92 | # remove duplicates, such as a maintainer being the author of the commit 93 | $maintainers =~ s/$author//g; 94 | 95 | my %mail = ( 96 | To => "$author, $maintainers", 97 | From => 'noreply@macports.org', 98 | Subject => "[$rev] $port Lint Report", 99 | Message => "Change: https://trac.macports.org/changeset/$rev\nPortfile: $port\n\n$errors \n\n", 100 | smtp => 'relay.apple.com', 101 | ); 102 | 103 | _log("Mailto: $maintainers "); 104 | 105 | sendmail(%mail) or die $Mail::Sendmail::error; 106 | } 107 | 108 | sub _log { 109 | my ($errors) = @_; 110 | open(LOG, ">>$TMPROOT/errors") or return; 111 | print LOG "$errors\n"; 112 | close(LOG); 113 | } 114 | 115 | sub usage { 116 | print "usage: portfile_lint.pl \n"; 117 | exit(); 118 | } 119 | 120 | -------------------------------------------------------------------------------- /jobs/portindex2postgres.tcl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env port-tclsh 2 | 3 | 4 | ### Helper functions 5 | 6 | # SQL string escaping. 7 | proc sql_escape {str} { 8 | regsub -all -- {'} $str {''} str 9 | return $str 10 | } 11 | 12 | 13 | ### main script 14 | 15 | # write sources.conf 16 | set fd [open "sources.conf" "w"] 17 | puts $fd "file://[pwd]/ports \[default\]" 18 | close $fd 19 | 20 | # write macports.conf 21 | set fd [open "macports.conf" "w"] 22 | puts $fd "sources_conf [pwd]/sources.conf" 23 | close $fd 24 | 25 | # export custom configuration 26 | set env(PORTSRC) "[pwd]/macports.conf" 27 | 28 | # Load and initialize MacPorts 29 | package require macports 1.0 30 | 31 | array set ui_options {ports_verbose yes} 32 | mportinit ui_options 33 | 34 | # Open SQL output file 35 | set sqlfd [open "PortIndex.sql" "w"] 36 | 37 | # Start transaction 38 | puts $sqlfd "BEGIN;" 39 | 40 | # Create schema 41 | puts $sqlfd "CREATE TABLE IF NOT EXISTS log (activity VARCHAR(255), activity_time TIMESTAMP);" 42 | puts $sqlfd "CREATE TABLE IF NOT EXISTS portfiles (name VARCHAR(255) PRIMARY KEY NOT NULL, path VARCHAR(255), version VARCHAR(255), description TEXT);" 43 | puts $sqlfd "CREATE TABLE IF NOT EXISTS categories (portfile VARCHAR(255), category VARCHAR(255), is_primary INTEGER);" 44 | puts $sqlfd "CREATE TABLE IF NOT EXISTS maintainers (portfile VARCHAR(255), maintainer VARCHAR(255), is_primary INTEGER);" 45 | puts $sqlfd "CREATE TABLE IF NOT EXISTS dependencies (portfile VARCHAR(255), library VARCHAR(255));" 46 | puts $sqlfd "CREATE TABLE IF NOT EXISTS variants (portfile VARCHAR(255), variant VARCHAR(255));" 47 | puts $sqlfd "CREATE TABLE IF NOT EXISTS platforms (portfile VARCHAR(255), platform VARCHAR(255));" 48 | puts $sqlfd "CREATE TABLE IF NOT EXISTS licenses (portfile VARCHAR(255), license VARCHAR(255));" 49 | 50 | # Truncate existing data 51 | puts $sqlfd "TRUNCATE portfiles;" 52 | puts $sqlfd "TRUNCATE categories;" 53 | puts $sqlfd "TRUNCATE maintainers;" 54 | puts $sqlfd "TRUNCATE dependencies;" 55 | puts $sqlfd "TRUNCATE variants;" 56 | puts $sqlfd "TRUNCATE platforms;" 57 | puts $sqlfd "TRUNCATE licenses;" 58 | 59 | # Get list of all ports 60 | set ports [mportlistall] 61 | 62 | # Iterate over each matching port, extracting its information from the 63 | # portinfo array. 64 | foreach {name array} $ports { 65 | 66 | array unset portinfo 67 | array set portinfo $array 68 | 69 | set portname [sql_escape $portinfo(name)] 70 | if {[info exists portinfo(version)]} { 71 | set portversion [sql_escape $portinfo(version)] 72 | } else { 73 | set portversion "" 74 | } 75 | set portdir [sql_escape $portinfo(portdir)] 76 | if {[info exists portinfo(description)]} { 77 | set description [sql_escape $portinfo(description)] 78 | } else { 79 | set description "" 80 | } 81 | if {[info exists portinfo(categories)]} { 82 | set categories $portinfo(categories) 83 | } else { 84 | set categories "" 85 | } 86 | if {[info exists portinfo(maintainers)]} { 87 | set maintainers $portinfo(maintainers) 88 | } else { 89 | set maintainers "" 90 | } 91 | if {[info exists portinfo(variants)]} { 92 | set variants $portinfo(variants) 93 | } else { 94 | set variants "" 95 | } 96 | if {[info exists portinfo(depends_fetch)]} { 97 | set depends_fetch $portinfo(depends_fetch) 98 | } else { 99 | set depends_fetch "" 100 | } 101 | if {[info exists portinfo(depends_extract)]} { 102 | set depends_extract $portinfo(depends_extract) 103 | } else { 104 | set depends_extract "" 105 | } 106 | if {[info exists portinfo(depends_build)]} { 107 | set depends_build $portinfo(depends_build) 108 | } else { 109 | set depends_build "" 110 | } 111 | if {[info exists portinfo(depends_lib)]} { 112 | set depends_lib $portinfo(depends_lib) 113 | } else { 114 | set depends_lib "" 115 | } 116 | if {[info exists portinfo(depends_run)]} { 117 | set depends_run $portinfo(depends_run) 118 | } else { 119 | set depends_run "" 120 | } 121 | if {[info exists portinfo(platforms)]} { 122 | set platforms $portinfo(platforms) 123 | } else { 124 | set platforms "" 125 | } 126 | if {[info exists portinfo(license)]} { 127 | set licenses $portinfo(license) 128 | } else { 129 | set licenses "" 130 | } 131 | 132 | puts $sqlfd "INSERT INTO portfiles VALUES ('$portname', '$portdir', '$portversion', '$description');" 133 | 134 | set primary 1 135 | foreach category $categories { 136 | set category [sql_escape $category] 137 | puts $sqlfd "INSERT INTO categories VALUES ('$portname', '$category', $primary);" 138 | set primary 0 139 | } 140 | 141 | set primary 1 142 | foreach maintainer $maintainers { 143 | set maintainer [sql_escape $maintainer] 144 | puts $sqlfd "INSERT INTO maintainers VALUES ('$portname', '$maintainer', $primary);" 145 | set primary 0 146 | } 147 | 148 | foreach fetch_dep $depends_fetch { 149 | set fetch_dep [sql_escape $fetch_dep] 150 | puts $sqlfd "INSERT INTO dependencies VALUES ('$portname', '$fetch_dep');" 151 | } 152 | 153 | foreach extract_dep $depends_extract { 154 | set extract_dep [sql_escape $extract_dep] 155 | puts $sqlfd "INSERT INTO dependencies VALUES ('$portname', '$extract_dep');" 156 | } 157 | 158 | foreach build_dep $depends_build { 159 | set build_dep [sql_escape $build_dep] 160 | puts $sqlfd "INSERT INTO dependencies VALUES ('$portname', '$build_dep');" 161 | } 162 | 163 | foreach lib $depends_lib { 164 | set lib [sql_escape $lib] 165 | puts $sqlfd "INSERT INTO dependencies VALUES ('$portname', '$lib');" 166 | } 167 | 168 | foreach run_dep $depends_run { 169 | set run_dep [sql_escape $run_dep] 170 | puts $sqlfd "INSERT INTO dependencies VALUES ('$portname', '$run_dep');" 171 | } 172 | 173 | foreach variant $variants { 174 | set variant [sql_escape $variant] 175 | puts $sqlfd "INSERT INTO variants VALUES ('$portname', '$variant');" 176 | } 177 | 178 | foreach platform $platforms { 179 | set platform [sql_escape $platform] 180 | puts $sqlfd "INSERT INTO platforms VALUES ('$portname', '$platform');" 181 | } 182 | 183 | foreach license $licenses { 184 | set license [sql_escape $license] 185 | puts $sqlfd "INSERT INTO licenses VALUES ('$portname', '$license');" 186 | } 187 | 188 | } 189 | 190 | # Insert timestamp of last update 191 | puts $sqlfd "INSERT INTO log VALUES ('update', NOW());" 192 | 193 | # End transaction 194 | puts $sqlfd "COMMIT;" 195 | 196 | # Close SQL output file 197 | close $sqlfd 198 | 199 | exit 0 200 | -------------------------------------------------------------------------------- /jobs/delete_old_distfiles.tcl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env port-tclsh 2 | 3 | # Find all distfiles that exist under distfiles_root and are not needed by 4 | # the current version of any port or by a version existing as a package 5 | # under packages_root. 6 | 7 | set start_time [clock seconds] 8 | 9 | if {[llength $argv] == 4 && [lindex $argv 0] eq "-k"} { 10 | set keepfile [lindex $argv 1] 11 | set argv [lrange $argv 2 end] 12 | } 13 | if {[llength $argv] != 2} { 14 | puts "Usage: $argv0 [-k keepfile] distfiles_root packages_root" 15 | exit 1 16 | } 17 | set distfiles_root [lindex $argv 0] 18 | set packages_root [lindex $argv 1] 19 | set archive_type tbz2 20 | if {[info exists env(TMPDIR)]} { 21 | set tmpdir $env(TMPDIR) 22 | } else { 23 | set tmpdir /tmp 24 | } 25 | set workdir [file normalize [pwd]] 26 | 27 | proc get_variants {portinfovar} { 28 | upvar $portinfovar portinfo 29 | if {![info exists portinfo(vinfo)]} { 30 | return {} 31 | } 32 | set variants [list] 33 | array set vinfo $portinfo(vinfo) 34 | foreach v [array names vinfo] { 35 | array unset variant 36 | array set variant $vinfo($v) 37 | if {![info exists variant(is_default)] || $variant(is_default) ne "+"} { 38 | lappend variants $v 39 | } 40 | } 41 | return $variants 42 | } 43 | 44 | package require macports 45 | mportinit 46 | 47 | set platforms [list 9 powerpc 9 i386] 48 | foreach vers {10 11 12 13 14 15 16 17 18 19} { 49 | if {${macports::os_major} != $vers} { 50 | lappend platforms $vers i386 51 | } 52 | } 53 | foreach vers {20 21 22 23 24} { 54 | if {${macports::os_major} != $vers} { 55 | lappend platforms $vers arm $vers i386 56 | } elseif {${macports::os_arch} eq "i386"} { 57 | lappend platforms $vers arm 58 | } else { 59 | lappend platforms $vers i386 60 | } 61 | } 62 | # build_arch values that could be considered "native" on platforms 63 | # where 'uname -p' says 'i386' 64 | set i386_archs [list x86_64 noarch i386] 65 | 66 | if {[catch {set res [mportlistall]} result]} { 67 | puts stderr "$::errorInfo" 68 | error "listing all ports failed: $result" 69 | } 70 | 71 | proc get_distfiles {porturl subport check_platforms} { 72 | set portname_distfiles [list] 73 | if {[catch {mportopen $porturl [list subport $subport] {}} mport]} { 74 | ui_error "mportopen $porturl failed: $mport" 75 | #error "couldn't open portfile for $subport" 76 | return $portname_distfiles 77 | } 78 | set workername [ditem_key $mport workername] 79 | if {![catch {$workername eval {portfetch::fetch_init; return $all_dist_files}} all_dist_files]} { 80 | # has distfiles, add them to the list 81 | lappend portname_distfiles {*}$all_dist_files 82 | } 83 | 84 | array set portinfo [mportinfo $mport] 85 | mportclose $mport 86 | 87 | set variants [get_variants portinfo] 88 | foreach variant $variants { 89 | #ui_msg "$subport +${variant}" 90 | if {[catch {mportopen $porturl [list subport $subport] [list $variant +]} mport]} { 91 | ui_error "mportopen $porturl failed: $mport" 92 | # unfortunately quite a few ports have variants that fail 93 | continue 94 | } 95 | set workername [ditem_key $mport workername] 96 | if {![catch {$workername eval {portfetch::fetch_init; return $all_dist_files}} all_dist_files]} { 97 | lappend portname_distfiles {*}$all_dist_files 98 | } 99 | mportclose $mport 100 | } 101 | 102 | foreach {os_major os_arch} $check_platforms { 103 | #ui_msg "$subport with platform 'darwin $os_major $os_arch'" 104 | if {[catch {mportopen $porturl [list subport $subport os_major $os_major os_arch $os_arch] {}} mport]} { 105 | ui_error "mportopen $porturl failed: $mport" 106 | # sometimes whole subports are not defined on certain platforms 107 | continue 108 | } 109 | set workername [ditem_key $mport workername] 110 | if {![catch {$workername eval {portfetch::fetch_init; return $all_dist_files}} all_dist_files]} { 111 | lappend portname_distfiles {*}$all_dist_files 112 | } 113 | mportclose $mport 114 | } 115 | 116 | return $portname_distfiles 117 | } 118 | 119 | filemap create distfiles_to_keep 120 | if {[info exists keepfile]} { 121 | set fd [open $keepfile r] 122 | while {[gets $fd line] != -1} { 123 | filemap set distfiles_to_keep $line 1 124 | } 125 | close $fd 126 | } else { 127 | # generate set of desired distfiles 128 | set portfile_dir [file join ${tmpdir} from_archive] 129 | file delete -force ${portfile_dir} 130 | file mkdir ${portfile_dir} 131 | foreach {portname info_list} $result { 132 | array unset portinfo 133 | array set portinfo $info_list 134 | if {[lsearch -exact -nocase $portinfo(license) "nomirror"] >= 0} { 135 | # shouldn't be mirrored, so don't keep it if it is somehow there 136 | continue 137 | } 138 | foreach f [get_distfiles $portinfo(porturl) $portname $platforms] { 139 | filemap set distfiles_to_keep $f 1 140 | } 141 | foreach archive [glob -nocomplain -directory $packages_root ${portname}/*.${archive_type}] { 142 | exec -ignorestderr tar -xjq -C ${portfile_dir} -f $archive +PORTFILE 143 | file rename -force ${portfile_dir}/+PORTFILE ${portfile_dir}/Portfile 144 | # figure out the platform from the filename 145 | set segments [split [file tail $archive] .] 146 | set archs [split [lindex $segments end-1] -] 147 | set major [lindex [split [lindex $segments end-2] _] end] 148 | if {$major eq "any"} { 149 | set major ${macports::os_major} 150 | } 151 | set this_platforms [list] 152 | foreach arch $archs { 153 | if {$arch eq "arm64"} { 154 | lappend this_platforms $major arm 155 | } elseif {$arch in $i386_archs} { 156 | lappend this_platforms $major i386 157 | } else { 158 | lappend this_platforms $major powerpc 159 | } 160 | } 161 | foreach f [get_distfiles file://${portfile_dir} $portname $this_platforms] { 162 | filemap set distfiles_to_keep $f 1 163 | } 164 | } 165 | } 166 | 167 | set fd [open [file join $workdir distfiles_keep.txt] w] 168 | puts $fd [join [filemap list distfiles_to_keep 1] \n] 169 | close $fd 170 | file delete -force ${portfile_dir} 171 | } 172 | 173 | # scan actual distfiles 174 | # What we have in $distfiles_to_keep is filenames only, i.e. $dist_subdir is 175 | # not taken into account. This is a lot simpler to deal with, even if it means 176 | # we may keep a few files that could be deleted if there are identically named 177 | # files in different subdirs. This may even be preferable with stealth updates 178 | # since we don't know which version the archives were built from. 179 | set dirlist [list $distfiles_root] 180 | set fd [open [file join $workdir distfiles_delete.txt] w] 181 | while {$dirlist ne ""} { 182 | set dir [lindex $dirlist end] 183 | set dirlist [lreplace ${dirlist}[set dirlist {}] end end] 184 | foreach f [glob -nocomplain -directory $dir *] { 185 | if {[file isfile $f] && [file mtime $f] < $start_time && ![filemap exists distfiles_to_keep [file tail $f]]} { 186 | puts $fd $f 187 | } elseif {[file isdirectory $f]} { 188 | lappend dirlist $f 189 | } 190 | } 191 | } 192 | close $fd 193 | -------------------------------------------------------------------------------- /jobs/mprsyncup-alt: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Alternate method of updating MacPorts rsync files. 4 | 5 | set -e 6 | 7 | echo "Starting update of rsync files at $(date -u -Iseconds)" 8 | 9 | # Read shell fragment with config variables if specified 10 | if [ -f "$MPRSYNCUP_CONFIG" ]; then 11 | . "$MPRSYNCUP_CONFIG" 12 | fi 13 | 14 | # Defaults 15 | OPENSSL=${OPENSSL:-/usr/bin/openssl} 16 | RSYNC=${RSYNC:-/usr/bin/rsync} 17 | SIGNIFY=${SIGNIFY:-/usr/bin/signify-openbsd} 18 | TAR=${TAR:-/usr/bin/tar} 19 | GZIP=${GZIP:-/usr/bin/gzip} 20 | REDIS_CLI=${REDIS_CLI:-/usr/bin/redis-cli} 21 | # signify public key used to verify base releases 22 | BASE_PUBKEY=${BASE_PUBKEY:-/var/keys/macports-base-2024.pub} 23 | 24 | # Path where the public mirrored files should be deployed 25 | if [ -z "$MIRROR_PUBLIC_PATH" ]; then 26 | echo "Must specify MIRROR_PUBLIC_PATH" 27 | exit 1 28 | fi 29 | # Path to where ports trees uploaded by GitHub Actions can be found 30 | if [ -z "$MIRROR_UPLOAD_PATH" ]; then 31 | echo "Must specify MIRROR_UPLOAD_PATH" 32 | exit 1 33 | fi 34 | # Path to openssl private key file used to sign base and ports tarballs 35 | if [ -z "$PRIVKEY" ]; then 36 | echo "Must specify PRIVKEY" 37 | exit 1 38 | fi 39 | 40 | BASEDIR=$(pwd) 41 | 42 | rm -rf "${BASEDIR}/rsyncroot" 43 | mkdir -p "${BASEDIR}/rsyncroot/release/tarballs" 44 | 45 | # Check for a newer base release 46 | NEW_BASE=0 47 | if [ -f "${MIRROR_PUBLIC_PATH}/distfiles/MacPorts/RELEASE_URL" ]; then 48 | NEW_BASE_VERS="$(cat "${MIRROR_PUBLIC_PATH}/distfiles/MacPorts/RELEASE_URL" | sed -E 's/.*v([0-9.]+)$/\1/')" 49 | echo "Latest base version is $NEW_BASE_VERS" 50 | if [ -f "${MIRROR_PUBLIC_PATH}/release/tarballs/base.tar" ]; then 51 | CUR_BASE_VERS="$("$TAR" -xOf "${MIRROR_PUBLIC_PATH}/release/tarballs/base.tar" base/config/macports_version)" 52 | echo "Currently mirrored base version is $CUR_BASE_VERS" 53 | if [ "$CUR_BASE_VERS" != "$NEW_BASE_VERS" ]; then 54 | NEW_BASE=1 55 | fi 56 | else 57 | NEW_BASE=1 58 | fi 59 | fi 60 | 61 | # Update base.tar (for selfupdate with older clients) 62 | if [ "$NEW_BASE" = 1 -a -f "${MIRROR_PUBLIC_PATH}/distfiles/MacPorts/MacPorts-${NEW_BASE_VERS}.tar.bz2" \ 63 | -a -f "${MIRROR_PUBLIC_PATH}/distfiles/MacPorts/MacPorts-${NEW_BASE_VERS}.tar.bz2.sig" ]; then 64 | if "$SIGNIFY" -V -p "$BASE_PUBKEY" \ 65 | -m "${MIRROR_PUBLIC_PATH}/distfiles/MacPorts/MacPorts-${NEW_BASE_VERS}.tar.bz2" \ 66 | -s "${MIRROR_PUBLIC_PATH}/distfiles/MacPorts/MacPorts-${NEW_BASE_VERS}.tar.bz2.sig"; then 67 | echo "Signature verified for MacPorts-${NEW_BASE_VERS}.tar.bz2" 68 | rm -rf "${BASEDIR}/rsyncroot/release/tarballs/base" "${BASEDIR}/rsyncroot/release/tarballs/MacPorts-${NEW_BASE_VERS}" 69 | 70 | echo "Creating base.tar" 71 | "$TAR" -C "${BASEDIR}/rsyncroot/release/tarballs" -xjf "${MIRROR_PUBLIC_PATH}/distfiles/MacPorts/MacPorts-${NEW_BASE_VERS}.tar.bz2" 72 | mv "${BASEDIR}/rsyncroot/release/tarballs/MacPorts-${NEW_BASE_VERS}" "${BASEDIR}/rsyncroot/release/tarballs/base" 73 | "$TAR" -C "${BASEDIR}/rsyncroot/release/tarballs" -cf "${BASEDIR}/rsyncroot/release/tarballs/base.tar" base 74 | rm -rf "${BASEDIR}/rsyncroot/release/tarballs/base" "${BASEDIR}/rsyncroot/release/tarballs/MacPorts-${NEW_BASE_VERS}" 75 | 76 | echo "Signing base.tar" 77 | "$OPENSSL" dgst -ripemd160 -sign "${PRIVKEY}" \ 78 | -out "${BASEDIR}/rsyncroot/release/tarballs/base.tar.rmd160" \ 79 | "${BASEDIR}/rsyncroot/release/tarballs/base.tar" 80 | else 81 | echo "Failed to verify signature for MacPorts-${NEW_BASE_VERS}.tar.bz2!" 82 | fi 83 | fi 84 | 85 | # Prepare ports tree and PortIndex dirs 86 | 87 | NEW_PORTS=0 88 | # Figure out the ports tree tarball corresponding to the latest commit 89 | # recorded as mirrored. This will be what the GitHub Actions job tries 90 | # to download next time it runs, so we can only delete the tarballs 91 | # older than this one. 92 | if [ -x "$REDIS_CLI" ]; then 93 | echo "Getting latest mirrored commit from redis db" 94 | LAST_MIRRORED_COMMIT="$(redis-cli get mirror.last_commit)" 95 | if [ -n "$LAST_MIRRORED_COMMIT" ]; then 96 | echo "Got $LAST_MIRRORED_COMMIT" 97 | LATEST_PORTS_FROM_DB="${MIRROR_UPLOAD_PATH}/portindex-${LAST_MIRRORED_COMMIT}.tar.bz2" 98 | else 99 | echo "No latest mirrored commit found" 100 | fi 101 | else 102 | echo "Can't find redis-cli" 103 | fi 104 | # Find the most recently uploaded ports tree tarball. This is what we 105 | # will actually deploy. It will usually be the same as the latest 106 | # mirrored commit, but not always. 107 | LATEST_PORTS_UPLOAD="$(ls -1t "${MIRROR_UPLOAD_PATH}"/portindex-*.tar.bz2 | head -n1)" 108 | if [ -z "$LATEST_PORTS_FROM_DB" -o ! -f "$LATEST_PORTS_FROM_DB" ]; then 109 | echo "Falling back to assuming latest uploaded ports is latest mirrored" 110 | LATEST_PORTS_FROM_DB="$LATEST_PORTS_UPLOAD" 111 | fi 112 | if [ -z "$LATEST_PORTS_UPLOAD" ]; then 113 | echo "No uploaded ports" 114 | elif [ ! -f "${BASEDIR}/last_deployed_ports" -o \ 115 | "$(basename "$LATEST_PORTS_UPLOAD")" != "$(cat "${BASEDIR}/last_deployed_ports")" ]; then 116 | 117 | # Clean up older uploads 118 | echo "Cleaning up old ports uploads" 119 | find "${MIRROR_UPLOAD_PATH}" -maxdepth 1 -type f -name "portindex-*.tar.bz2" \ 120 | -not -newer "$LATEST_PORTS_FROM_DB" -not -samefile "$LATEST_PORTS_FROM_DB" \ 121 | -delete -printf "Deleted %f\n" 122 | rm -rf "${BASEDIR}/rsyncroot/release/tarballs/extract" 123 | mkdir -p "${BASEDIR}/rsyncroot/release/tarballs/extract" 124 | echo "Extracting latest ports from $LATEST_PORTS_UPLOAD" 125 | "$TAR" -C "${BASEDIR}/rsyncroot/release/tarballs/extract" -xjf "$LATEST_PORTS_UPLOAD" 126 | 127 | # Create ports.tar 128 | echo "Creating release/tarballs/ports.tar(.gz)" 129 | rm -f "${BASEDIR}/rsyncroot/release/tarballs/ports.tar" "${BASEDIR}/rsyncroot/release/tarballs/ports.tar.gz" 130 | "$TAR" -C "${BASEDIR}/rsyncroot/release/tarballs/extract" -cf "${BASEDIR}/rsyncroot/release/tarballs/ports.tar" ports 131 | "$GZIP" --best --keep --rsyncable "${BASEDIR}/rsyncroot/release/tarballs/ports.tar" 132 | 133 | # Create daily tarball 134 | echo "Creating daily tarball" 135 | rm -f "${BASEDIR}/rsyncroot/distfiles/ports.tar.gz" "${BASEDIR}/rsyncroot/release/ports.tar.gz" 136 | mkdir -p "${BASEDIR}/rsyncroot/distfiles" 137 | mv "${BASEDIR}"/rsyncroot/release/tarballs/extract/PortIndex* "${BASEDIR}"/rsyncroot/release/tarballs/extract/ports 138 | "$TAR" -C "${BASEDIR}/rsyncroot/release/tarballs/extract" --exclude 'PortIndex*/PortIndex.json' \ 139 | -c --use-compress-program "$GZIP --best --rsyncable" -f "${BASEDIR}/rsyncroot/release/ports.tar.gz" ports 140 | ln "${BASEDIR}/rsyncroot/release/ports.tar.gz" "${BASEDIR}/rsyncroot/distfiles/ports.tar.gz" 141 | 142 | # Put PortIndex dirs in place 143 | echo "Moving PortIndex dirs" 144 | mv "${BASEDIR}"/rsyncroot/release/tarballs/extract/ports/PortIndex* "${BASEDIR}"/rsyncroot/release/tarballs 145 | echo "Deleting temp extracted ports tree" 146 | rm -rf "${BASEDIR}/rsyncroot/release/tarballs/extract" 147 | 148 | # Sign files 149 | for f in "${BASEDIR}"/rsyncroot/release/tarballs/ports.tar{,.gz} \ 150 | "${BASEDIR}"/rsyncroot/release/tarballs/PortIndex*/PortIndex \ 151 | "${BASEDIR}"/rsyncroot/{release,distfiles}/ports.tar.gz; do 152 | echo "Signing $f" 153 | "$OPENSSL" dgst -ripemd160 -sign "${PRIVKEY}" -out "${f}.rmd160" "${f}" 154 | if [ -n "$PRIVKEY_SIGNIFY" ]; then 155 | "$SIGNIFY" -S -s "$PRIVKEY_SIGNIFY" -x "${f}.sig" -m "${f}" 156 | fi 157 | done 158 | 159 | echo "$(basename "$LATEST_PORTS_UPLOAD")" > "${BASEDIR}/last_deployed_ports" 160 | NEW_PORTS=1 161 | fi 162 | 163 | if [ "$NEW_BASE" = 1 -o "$NEW_PORTS" = 1 ]; then 164 | # Ensure readability 165 | echo "Setting permissions" 166 | chmod -R a+rX "${BASEDIR}/rsyncroot" 167 | 168 | HOSTOS="$(uname -s)" 169 | 170 | # Stop rsyncd once there are no connections 171 | if [ "$HOSTOS" = "Linux" ]; then 172 | WAITED=0 173 | echo "Waiting for rsyncd to be idle" 174 | while [ -n "$(ss -H 'sport = :rsync')" ]; do 175 | sleep 10 176 | WAITED=$(( $WAITED + 10 )) 177 | if [ "$WAITED" -ge 7200 ]; then 178 | echo "Waited $(( $WAITED / 60 )) minutes for rsync connections to close, shutting down anyway" 179 | break 180 | fi 181 | done 182 | echo "Stopping rsyncd" 183 | sudo /usr/bin/systemctl stop rsync 184 | fi 185 | 186 | # Sync files 187 | echo "Rsyncing files" 188 | "$RSYNC" -aHv "${BASEDIR}"/rsyncroot/ "$MIRROR_PUBLIC_PATH" 189 | 190 | # Start rsyncd 191 | if [ "$HOSTOS" = "Linux" ]; then 192 | echo "Starting rsyncd" 193 | sudo /usr/bin/systemctl start rsync 194 | fi 195 | fi 196 | 197 | echo "Finished update of rsync files at $(date -u -Iseconds)" 198 | -------------------------------------------------------------------------------- /jobs/mprsyncup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #### 3 | # Script to checkout/update base sources from both master and the current 4 | # release tag (as determined by the base/config/RELEASE_URL file) and a ports 5 | # tree from master, and then export and sync all of them to the 6 | # ${RSYNCROOT} location, wherefrom the rsync modules are fed to the `sync' 7 | # and `selfupdate' routines in port(1). Read the base/portmgr/rsync.repos 8 | # file for more information on both the necessary rsync modules and filesystem 9 | # level paths, which this script bootstraps. 10 | # 11 | # Whatever server uses this script to mirror the MacPorts rsync repositories 12 | # should simply adapt the ${RSYNCROOT} path variable as necessary (keeping it 13 | # in sync with the equally named variable in base/portmgr/rsync.repos) and 14 | # install it on cron/launchd with a suitable periodicity, previously discussed 15 | # with the portmgr@ team (macports-mgr@lists.macosforge.org). Repositories 16 | # themselves are detailed & served by base/portmgr/rsync.repos, as stated above 17 | # (that is, no manual intervention what-so-ever is needed, other than installing 18 | # this script and adding the repositories detailed in base/portmgr/rsync.repos 19 | # to a local rsyncd.conf file). 20 | # 21 | # Lastly, it is required of every 3rd party mirrors to keep track of this script 22 | # and the base/portmgr/rsync.repos file and always maintain local copies in as 23 | # close sync as possible. 24 | # 25 | # 26 | # Created by fkr@opendarwin.org, jberry@macports.org and yeled@macports.org, 27 | # Updated by jmpp@macports.org 28 | #### 29 | 30 | set -e 31 | set -x 32 | 33 | # Paths we'll work on: 34 | ROOT=/var/tmp/macports 35 | PREFIX=${ROOT}/opt/local 36 | GITROOT=/var/tmp/macports 37 | TBASE=${GITROOT}/trunk/base 38 | RBASE=${GITROOT}/release/base 39 | PORTS=${GITROOT}/release/ports 40 | CONTRIB=${GITROOT}/release/contrib 41 | RSYNCROOT=/rsync/macports 42 | MPTOOLSPREFIX=/opt/local 43 | MPBIN=${MPTOOLSPREFIX}/bin 44 | 45 | # Commands we need. For options to be substituted correctly, these must 46 | # not be substituted within double quotes. Thus, there must not be any 47 | # globbing characters, and the command itself must not contain spaces. 48 | GIT="${MPBIN}/git" 49 | RSYNC="${MPBIN}/rsync -q" 50 | RM="/bin/rm" 51 | MKDIR="/bin/mkdir" 52 | MV="${MPBIN}/gmv" 53 | LN="/bin/ln" 54 | TAR="/usr/bin/tar" 55 | OPENSSL="/usr/bin/openssl" 56 | AWK="/usr/bin/awk" 57 | STAT="${MPBIN}/gstat" 58 | GZIP="${MPBIN}/gzip" 59 | BASENAME="/usr/bin/basename" 60 | TCLSH="${PREFIX}/bin/port-tclsh" 61 | PORTINDEX="${PREFIX}/bin/portindex" 62 | PORTINDEX2JSON="${CONTRIB}/portindex2json/portindex2json.tcl" 63 | 64 | PATH=${PREFIX}/bin:/bin:/usr/bin:/usr/sbin:${MPBIN} 65 | 66 | # Platforms we generate indexes for. This is intentionally split on 67 | # whitespace later. 68 | PLATFORMS="9_powerpc 9_i386 10_i386 11_i386 12_i386 13_i386 14_i386 15_i386 16_i386 17_i386 18_i386 19_i386 20_i386 20_arm 21_i386 21_arm 22_i386 22_arm 23_i386 23_arm 24_i386 24_arm 25_i386 25_arm" 69 | 70 | # Sources information: 71 | BASEURL=https://github.com/macports/macports-base.git 72 | PORTSURL=https://github.com/macports/macports-ports.git 73 | CONTRIBURL=https://github.com/macports/macports-contrib.git 74 | RELEASE_URL_FILE=config/RELEASE_URL 75 | 76 | # private key to use for signing 77 | # XXX set real path 78 | PRIVKEY="" 79 | 80 | # 81 | # Functions 82 | # 83 | 84 | hardlink() { 85 | SOURCE="$1" 86 | TARGET="$2" 87 | [ -f "${SOURCE}" ] && [ -f "${TARGET}" ] && [ "$(${STAT} -c %i "${SOURCE}")" = "$(${STAT} -c %i "${TARGET}")" ] && return 88 | printf "Hard linking %s to %s\n" "${SOURCE}" "${TARGET}" 89 | TMPTARGET="${ROOT}"/"${TARGET##*/}".$$ 90 | ${LN} "${SOURCE}" "${TMPTARGET}" 91 | ${MV} "${TMPTARGET}" "${TARGET}" 92 | } 93 | 94 | sign() { 95 | [ -z "${PRIVKEY}" ] && return 96 | for FILE in "$@"; do 97 | printf "Signing %s\n" "${FILE}" 98 | ${OPENSSL} dgst -ripemd160 -sign "${PRIVKEY}" -out "${FILE}.rmd160" "${FILE}" 99 | done 100 | } 101 | 102 | # 103 | # Update trunk/base 104 | # 105 | 106 | if [ -d "${TBASE}/.git" ]; then 107 | ${GIT} -C "${TBASE}" pull -q 108 | ${GIT} -C "${TBASE}" fetch --tags 109 | else 110 | ${GIT} clone -q "${BASEURL}" "${TBASE}" 111 | fi 112 | 113 | ${MKDIR} -p "${RSYNCROOT}/trunk/base" 114 | ${RSYNC} -aIC --delete "${TBASE}/" "${RSYNCROOT}/trunk/base" 115 | 116 | # 117 | # Update release/base 118 | # 119 | 120 | read -r RELEASE_URL < "${TBASE}/${RELEASE_URL_FILE}" 121 | if [ -z "${RELEASE_URL}" ]; then 122 | echo "no RELEASE_URL specified in git master, bailing out!" 123 | exit 1 124 | fi 125 | hardlink "${RSYNCROOT}/trunk/base/${RELEASE_URL_FILE}" "${RSYNCROOT}/distfiles/MacPorts/$(${BASENAME} ${RELEASE_URL_FILE})" 126 | 127 | RBASE_CHANGED=1 128 | if [ -d "${RBASE}/.git" ]; then 129 | cd "${RBASE}" 130 | ${GIT} fetch --tags 131 | RBASE_OLD_REV="$(${GIT} rev-parse HEAD)" 132 | ${GIT} checkout -q "$(${BASENAME} ${RELEASE_URL})" 133 | RBASE_NEW_REV="$(${GIT} rev-parse HEAD)" 134 | [ "${RBASE_OLD_REV}" = "${RBASE_NEW_REV}" ] && RBASE_CHANGED=0 135 | else 136 | ${GIT} clone -q "${TBASE}" "${RBASE}" 137 | ${GIT} -C "${RBASE}" checkout -q "$(${BASENAME} ${RELEASE_URL})" 138 | fi 139 | 140 | ${MKDIR} -p "${RSYNCROOT}/release/base" 141 | ${RSYNC} -aIC --delete "${RBASE}/" "${RSYNCROOT}/release/base" 142 | 143 | # 144 | # Update release/ports 145 | # 146 | 147 | PORTS_CHANGED=1 148 | if [ -d "${PORTS}/.git" ]; then 149 | cd "${PORTS}" 150 | PORTS_OLD_REV="$(${GIT} rev-parse HEAD)" 151 | ${GIT} pull -q 152 | PORTS_NEW_REV="$(${GIT} rev-parse HEAD)" 153 | [ "${PORTS_OLD_REV}" = "${PORTS_NEW_REV}" ] && PORTS_CHANGED=0 154 | else 155 | ${GIT} clone -q --depth 1 "${PORTSURL}" "${PORTS}" 156 | PORTS_NEW_REV="$(${GIT} rev-parse HEAD)" 157 | fi 158 | 159 | # 160 | # Update release/contrib 161 | # 162 | 163 | if [ -d "${CONTRIB}"/.git ]; then 164 | cd "${CONTRIB}" 165 | ${GIT} pull -q 166 | else 167 | ${GIT} clone -q "${CONTRIBURL}" "${CONTRIB}" 168 | fi 169 | 170 | if [ "${RBASE_CHANGED}" -eq 1 ]; then 171 | PORTS_CHANGED=1 172 | ( 173 | # build MP in a private location for indexing 174 | cd "${RBASE}" 175 | ./configure \ 176 | --prefix="${PREFIX}" \ 177 | --with-install-group="$(id -gn)" \ 178 | --with-install-user="$(id -un)" 179 | make clean 180 | JOBS=1 181 | if [ "$(uname -s)" = "Darwin" ]; then 182 | JOBS="$(sysctl -n hw.activecpu)" 183 | fi 184 | make -j"$JOBS" 185 | make install 186 | make distclean 187 | ) 188 | fi 189 | 190 | if [ "${PORTS_CHANGED}" -eq 1 ]; then 191 | ( 192 | # generate platform-specific indexes 193 | cd "${PORTS}" 194 | # Intentionally split PLATFORMS on whitespace. 195 | for PLATFORM in $PLATFORMS; do 196 | INDEX="PortIndex_darwin_${PLATFORM}" 197 | ${PORTINDEX} -p "macosx_${PLATFORM}" -o "${INDEX}" \ 198 | | ${AWK} '{ print "Updating " idx ":\t" $0 }' idx="$INDEX" \ 199 | | expand -t 40,48,56,64,72,80 200 | done 201 | 202 | # generate json for each platform-specific index 203 | for PLATFORM in $PLATFORMS; do 204 | INDEX="PortIndex_darwin_${PLATFORM}" 205 | ${TCLSH} "${PORTINDEX2JSON}" "${INDEX}"/PortIndex --info commit="${PORTS_NEW_REV}" > "${INDEX}"/PortIndex.json 206 | done 207 | ) 208 | fi 209 | 210 | ${MKDIR} -p "${RSYNCROOT}/release/ports" 211 | ${RSYNC} -aIC --delete "${PORTS}/" "${RSYNCROOT}/release/ports" 212 | 213 | # 214 | # Update trunk/dports 215 | # 216 | 217 | cd "${RSYNCROOT}" 218 | if [ ! -L trunk/dports ]; then 219 | cd trunk 220 | ${RM} -rf dports && ${LN} -s ../release/ports dports 221 | fi 222 | 223 | # 224 | # Update release/tarballs 225 | # 226 | 227 | # Generate and sign tarballs of base and ports and the PortIndex files. 228 | if [ "${RBASE_CHANGED}" -eq 1 ]; then 229 | ${TAR} -C "${RSYNCROOT}"/release/ -cf "${ROOT}"/base.tar base 230 | sign "${ROOT}"/base.tar 231 | fi 232 | if [ "${PORTS_CHANGED}" -eq 1 ]; then 233 | ${TAR} --exclude 'PortIndex*/PortIndex.json' -C "${RSYNCROOT}"/release/ -czf "${ROOT}"/ports.tar.gz ports 234 | 235 | ${TAR} --exclude 'PortIndex*' -C "${RSYNCROOT}"/release/ -cf "${ROOT}"/ports.tar ports 236 | for INDEX_DIR in "${RSYNCROOT}"/release/ports/PortIndex_*; do 237 | INDEX_LINK_DIR="${ROOT}"/"${INDEX_DIR##*/}" 238 | ${MKDIR} -p "${INDEX_LINK_DIR}" 239 | hardlink "${INDEX_DIR}"/PortIndex "${INDEX_LINK_DIR}"/PortIndex 240 | done 241 | sign "${ROOT}"/ports.tar "${ROOT}"/PortIndex_*/PortIndex 242 | if [ -x "${GZIP}" ]; then 243 | ${GZIP} --best --keep --rsyncable --suffix .gz-rsync "${ROOT}"/ports.tar 244 | sign "${ROOT}"/ports.tar.gz-rsync 245 | fi 246 | fi 247 | 248 | # Tarballs used to be a symlink to the real directory tarballs_current. 249 | # Clean up this situation if found. This should only happen once. 250 | [ -L "${RSYNCROOT}"/release/tarballs ] && ${RM} -f "${RSYNCROOT}"/release/tarballs 251 | [ -d "${RSYNCROOT}"/release/tarballs_current ] && ${MV} "${RSYNCROOT}"/release/tarballs_current "${RSYNCROOT}"/release/tarballs 252 | 253 | # Replace files on rsync server as quickly as possible. 254 | # This is not atomic but doing it atomically is difficult. 255 | ${MKDIR} -p "${RSYNCROOT}"/release/tarballs "${RSYNCROOT}"/distfiles 256 | if [ "${RBASE_CHANGED}" -eq 1 ]; then 257 | ${MV} "${ROOT}"/base.tar* "${RSYNCROOT}"/release/tarballs 258 | fi 259 | if [ "${PORTS_CHANGED}" -eq 1 ]; then 260 | ${MV} "${ROOT}"/ports.tar.gz "${RSYNCROOT}"/release 261 | hardlink "${RSYNCROOT}"/release/ports.tar.gz "${RSYNCROOT}"/distfiles/ports.tar.gz 262 | if [ -f "${ROOT}"/ports.tar.gz-rsync ]; then 263 | ${MV} "${ROOT}"/ports.tar.gz-rsync "${ROOT}"/ports.tar.gz 264 | ${MV} "${ROOT}"/ports.tar.gz-rsync.rmd160 "${ROOT}"/ports.tar.gz.rmd160 265 | fi 266 | ${MV} "${ROOT}"/ports.tar* "${RSYNCROOT}"/release/tarballs 267 | for INDEX_DIR in "${RSYNCROOT}"/release/ports/PortIndex_*; do 268 | INDEX_LINK_DIR="${RSYNCROOT}/release/tarballs/${INDEX_DIR##*/}" 269 | ${MKDIR} -p "${INDEX_LINK_DIR}" 270 | hardlink "${INDEX_DIR}"/PortIndex "${INDEX_LINK_DIR}"/PortIndex 271 | hardlink "${INDEX_DIR}"/PortIndex.quick "${INDEX_LINK_DIR}"/PortIndex.quick 272 | hardlink "${ROOT}"/"${INDEX_DIR##*/}"/PortIndex.rmd160 "${INDEX_LINK_DIR}"/PortIndex.rmd160 273 | done 274 | ${RM} -rf "${ROOT}"/PortIndex_* 275 | fi 276 | 277 | printf "Done\n" 278 | -------------------------------------------------------------------------------- /jobs/distributable_lib.tcl: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4 2 | 3 | # Library code for checking if ports are binary distributable. 4 | # Used by the port_binary_distributable tool. 5 | 6 | set check_deptypes [list depends_build depends_lib] 7 | 8 | # Notes: 9 | # 'Restrictive/Distributable' means a non-free license that nonetheless allows 10 | # distributing binaries. 11 | # 'Restrictive' means a non-free license that does not allow distributing 12 | # binaries, and is thus not in the list. 13 | # 'Permissive' is a catchall for other licenses that allow 14 | # modification and distribution of source and binaries. 15 | # 'Copyleft' means a license that requires source code to be made available, 16 | # and derivative works to be licensed the same as the original. 17 | # 'GPLConflict' should be added if the license conflicts with the GPL (and its 18 | # variants like CeCILL and the AGPL) and is not in the list of licenses known 19 | # to do so below. 20 | # 'Noncommercial' means a license that prohibits commercial use. 21 | set good_licenses [list afl agpl apache apsl artistic autoconf beopen bitstreamvera \ 22 | boost bsd bsd-old cc-by cc-by-sa cddl cecill cecill-b cecill-c cnri copyleft \ 23 | cpl curl epl fpll fontconfig freetype gd gfdl gpl \ 24 | gplconflict ibmpl ijg isc jasper lgpl libtool lppl mit \ 25 | mpl ncsa noncommercial openldap openssl permissive php \ 26 | psf public-domain qpl restrictive/distributable ruby \ 27 | sleepycat ssleay tcl/tk vim w3c wtfpl wxwidgets x11 zlib zpl] 28 | foreach lic $good_licenses { 29 | set license_good($lic) 1 30 | } 31 | 32 | # keep these values sorted 33 | array set license_conflicts \ 34 | [list \ 35 | afl [list agpl cecill gpl] \ 36 | agpl [list afl apache-1 apache-1.1 apsl beopen bsd-old cc-by-1 cc-by-2 cc-by-2.5 cc-by-3 cc-by-sa cddl cecill cnri cpl epl gd gpl-1 gpl-2 gplconflict ibmpl lppl mpl noncommercial openssl php qpl restrictive/distributable ruby ssleay zpl-1] \ 37 | agpl-1 [list apache freetype gpl-3 gpl-3+ lgpl-3 lgpl-3+] \ 38 | apache [list agpl-1 cecill gpl-1 gpl-2] \ 39 | apache-1 [list agpl gpl] \ 40 | apache-1.1 [list agpl gpl] \ 41 | apsl [list agpl cecill gpl] \ 42 | beopen [list agpl cecill gpl] \ 43 | bsd-old [list agpl cecill gpl] \ 44 | cc-by-1 [list agpl cecill gpl] \ 45 | cc-by-2 [list agpl cecill gpl] \ 46 | cc-by-2.5 [list agpl cecill gpl] \ 47 | cc-by-3 [list agpl cecill gpl] \ 48 | cc-by-sa [list agpl cecill gpl] \ 49 | cddl [list agpl cecill gpl] \ 50 | cecill [list afl agpl apache apsl beopen bsd-old cc-by-1 cc-by-2 cc-by-2.5 cc-by-3 cc-by-sa cddl cnri cpl epl gd gplconflict ibmpl lppl mpl noncommercial openssl php qpl restrictive/distributable ruby ssleay zpl-1] \ 51 | cnri [list agpl cecill gpl] \ 52 | cpl [list agpl cecill gpl] \ 53 | epl [list agpl cecill gpl] \ 54 | freetype [list agpl-1 gpl-2] \ 55 | gd [list agpl cecill gpl] \ 56 | gpl [list afl apache-1 apache-1.1 apsl beopen bsd-old cc-by-1 cc-by-2 cc-by-2.5 cc-by-3 cc-by-sa cddl cnri cpl epl gd gplconflict ibmpl lppl mpl noncommercial openssl php qpl restrictive/distributable ruby ssleay zpl-1] \ 57 | gpl-1 [list agpl apache gpl-3 gpl-3+ lgpl-3 lgpl-3+] \ 58 | gpl-2 [list agpl apache freetype gpl-3 gpl-3+ lgpl-3 lgpl-3+] \ 59 | gpl-3 [list agpl-1 gpl-1 gpl-2] \ 60 | gpl-3+ [list agpl-1 gpl-1 gpl-2] \ 61 | gplconflict [list agpl cecill gpl] \ 62 | ibmpl [list agpl cecill gpl] \ 63 | lgpl-3 [list agpl-1 gpl-1 gpl-2] \ 64 | lgpl-3+ [list agpl-1 gpl-1 gpl-2] \ 65 | lppl [list agpl cecill gpl] \ 66 | mpl [list agpl cecill gpl] \ 67 | noncommercial [list agpl cecill gpl] \ 68 | openssl [list agpl cecill gpl] \ 69 | php [list agpl cecill gpl] \ 70 | qpl [list agpl cecill gpl] \ 71 | restrictive/distributable [list agpl cecill gpl] \ 72 | ruby [list agpl cecill gpl] \ 73 | ssleay [list agpl cecill gpl] \ 74 | zpl-1 [list agpl cecill gpl] \ 75 | ] 76 | 77 | # map license name indicating exception to regex matching port names it applies to 78 | set license_exceptions(opensslexception) {^openssl[0-9]*$} 79 | 80 | # license database format: 81 | # each line consists of "portname mtime {array}" 82 | # where array is one or more {variant_string {dependencies license installs_libs [license_noconflict]}} 83 | 84 | # load database if it exists 85 | proc init_license_db {dbpath} { 86 | if {[file isfile ${dbpath}/license_db]} { 87 | global license_db 88 | set fd [open ${dbpath}/license_db r] 89 | while {[gets $fd entry] >= 0} { 90 | set license_db([lindex $entry 0]) [lrange $entry 1 end] 91 | } 92 | close $fd 93 | } 94 | } 95 | 96 | # write out database 97 | proc write_license_db {dbpath} { 98 | global license_db 99 | if {![file isdirectory dbpath]} { 100 | file mkdir $dbpath 101 | } 102 | set fd [open ${dbpath}/license_db w] 103 | foreach portname [array names license_db] { 104 | puts $fd [list $portname {*}$license_db($portname)] 105 | } 106 | close $fd 107 | } 108 | 109 | # purge old ports from database 110 | proc cleanup_license_db {dbpath} { 111 | if {[file isfile ${dbpath}/license_db]} { 112 | set fd [open ${dbpath}/license_db r] 113 | set content [read $fd] 114 | close $fd 115 | set fd [open ${dbpath}/license_db w] 116 | foreach entry [split $content \n] { 117 | set portSearchResult [mportlookup [lindex $entry 0]] 118 | if {$portSearchResult ne ""} { 119 | set portInfo [lindex $portSearchResult 1] 120 | set portfile_path [macports::getportdir [dict get $portInfo porturl]]/Portfile 121 | if {[file mtime $portfile_path] == [lindex $entry 1]} { 122 | puts $fd $entry 123 | } 124 | } 125 | } 126 | close $fd 127 | } 128 | } 129 | 130 | # return deps and license for given port 131 | proc infoForPort {portName variantInfo} { 132 | set portSearchResult [mportlookup $portName] 133 | if {[llength $portSearchResult] < 2} { 134 | puts stderr "Warning: port \"$portName\" not found" 135 | return {} 136 | } 137 | lassign $portSearchResult portName portInfo 138 | set porturl [dict get $portInfo porturl] 139 | set portfile_path [macports::getportdir $porturl]/Portfile 140 | set variant_string [normalize_variants $variantInfo] 141 | 142 | # check if the port's info is already in the db 143 | global license_db 144 | if {[info exists license_db($portName)]} { 145 | set info_list $license_db($portName) 146 | if {[file mtime $portfile_path] == [lindex $info_list 0]} { 147 | # keyed by normalized variant string 148 | set info_array [lindex $info_list 1] 149 | if {[dict exists $info_array $variant_string]} { 150 | return [dict get $info_array $variant_string] 151 | } 152 | } else { 153 | unset license_db($portName) 154 | } 155 | } 156 | 157 | set dependencyList [list] 158 | if {[catch {mportopen $porturl [dict create subport $portName] $variantInfo} result]} { 159 | puts stderr "Warning: port \"$portName\" failed to open: $result" 160 | return {} 161 | } else { 162 | set mport $result 163 | } 164 | set portInfo [mportinfo $mport] 165 | # Quicker not to close the mport, but memory use might become 166 | # excessive when processing many ports. 167 | mportclose $mport 168 | 169 | global check_deptypes 170 | foreach dependencyType $check_deptypes { 171 | if {[dict exists $portInfo $dependencyType]} { 172 | foreach dependency [dict get $portInfo $dependencyType] { 173 | lappend dependencyList [string range $dependency [string last ":" $dependency]+1 end] 174 | } 175 | } 176 | } 177 | 178 | set ret [list $dependencyList [dict get $portInfo license]] 179 | if {[dict exists $portInfo installs_libs]} { 180 | lappend ret [dict get $portInfo installs_libs] 181 | } else { 182 | # when in doubt, assume code from the dep is incorporated 183 | lappend ret yes 184 | } 185 | if {[dict exists $portInfo license_noconflict]} { 186 | lappend ret [dict get $portInfo license_noconflict] 187 | } 188 | 189 | # update the db 190 | dict set info_array $variant_string $ret 191 | set license_db($portName) [list [file mtime $portfile_path] $info_array] 192 | 193 | return $ret 194 | } 195 | 196 | # return license with any trailing dash followed by a number and/or plus sign removed 197 | set remove_version_re {[0-9.+]+} 198 | proc remove_version {license} { 199 | global remove_version_re 200 | set dash [string last - $license] 201 | if {$dash != -1 && [regexp $remove_version_re [string range $license $dash+1 end]]} { 202 | return [string range $license 0 $dash-1] 203 | } else { 204 | return $license 205 | } 206 | } 207 | 208 | proc check_licenses {portName variantInfo} { 209 | array set portSeen {} 210 | set top_info [infoForPort $portName $variantInfo] 211 | if {$top_info eq {}} { 212 | return 1 213 | } 214 | set top_license [lindex $top_info 1] 215 | foreach noconflict_port [lindex $top_info 3] { 216 | set noconflict_ports($noconflict_port) 1 217 | } 218 | set top_license_names [list] 219 | # check that top-level port's license(s) are good 220 | if {$top_license eq ""} { 221 | return [list 1 "\"$portName\" is not distributable because its license option is empty"] 222 | } 223 | global license_good 224 | foreach sublist $top_license { 225 | # each element may be a list of alternatives (i.e. only one need apply) 226 | set any_good 0 227 | set lic "" 228 | set sub_names [list] 229 | foreach full_lic $sublist { 230 | # chop off any trailing version number 231 | set lic [remove_version [string tolower $full_lic]] 232 | # add name to the list for later 233 | lappend sub_names $lic 234 | if {[info exists license_good($lic)]} { 235 | set any_good 1 236 | } 237 | } 238 | lappend top_license_names $sub_names 239 | if {!$any_good} { 240 | return [list 1 "\"$portName\" is not distributable because its license \"$lic\" is not known to be distributable"] 241 | } 242 | } 243 | 244 | # start with deps of top-level port 245 | set portList [lindex $top_info 0] 246 | foreach p $portList { 247 | set portSeen($p) 1 248 | } 249 | global license_exceptions license_conflicts 250 | while {[llength $portList] > 0} { 251 | set aPort [lindex $portList 0] 252 | # remove it from the list 253 | set portList [lreplace $portList 0 0] 254 | if {[info exists noconflict_ports($aPort)]} { 255 | continue 256 | } 257 | 258 | set aPortInfo [infoForPort $aPort $variantInfo] 259 | if {$aPortInfo eq {}} { 260 | continue 261 | } 262 | set aPortLicense [lindex $aPortInfo 1] 263 | if {$aPortLicense eq ""} { 264 | return [list 1 "\"$portName\" is not distributable because its dependency \"$aPort\" has an empty license option"] 265 | } 266 | set installs_libs [lindex $aPortInfo 2] 267 | if {!$installs_libs} { 268 | continue 269 | } 270 | foreach sublist $aPortLicense { 271 | set any_good 0 272 | set any_compatible 0 273 | set lic "" 274 | # check that this dependency's license(s) are good 275 | foreach full_lic $sublist { 276 | set lic [remove_version [string tolower $full_lic]] 277 | if {[info exists license_good($lic)]} { 278 | set any_good 1 279 | set conflicting_dep_lic $full_lic 280 | } else { 281 | # no good being compatible with other licenses if it's not distributable itself 282 | continue 283 | } 284 | 285 | # ... and that they don't conflict with the top-level port's 286 | set any_conflict 0 287 | foreach top_sublist [concat $top_license $top_license_names] { 288 | set any_sub_compatible 0 289 | foreach top_lic $top_sublist { 290 | #puts stderr "checking $top_lic with $full_lic in $aPort" 291 | set top_lic_low [string tolower $top_lic] 292 | if {[info exists license_exceptions($top_lic_low)]} { 293 | #puts stderr "exception exists for $top_lic" 294 | if {[regexp $license_exceptions($top_lic_low) $aPort]} { 295 | #puts stderr "exception $top_lic_low exists for $license_exceptions($top_lic_low) which matches $aPort" 296 | set any_sub_compatible 1 297 | break 298 | } else { 299 | #puts stderr "exception $top_lic_low does not apply to $aPort" 300 | continue 301 | } 302 | } elseif {![info exists license_conflicts($top_lic_low)] 303 | || ([lsearch -sorted $license_conflicts($top_lic_low) $lic] == -1 304 | && [lsearch -sorted $license_conflicts($top_lic_low) [string tolower $full_lic]] == -1)} { 305 | #puts stderr "no exception and no conflict exists for $top_lic with $full_lic in $aPort" 306 | set any_sub_compatible 1 307 | break 308 | } 309 | set conflicting_top_lic $top_lic 310 | } 311 | if {!$any_sub_compatible} { 312 | set any_conflict 1 313 | break 314 | } 315 | } 316 | if {!$any_conflict} { 317 | set any_compatible 1 318 | break 319 | } 320 | } 321 | 322 | if {!$any_good} { 323 | return [list 1 "\"$portName\" is not distributable because its dependency \"$aPort\" has license \"$lic\" which is not known to be distributable"] 324 | } 325 | if {!$any_compatible} { 326 | if {[info exists conflicting_top_lic]} { 327 | set display_lic " \"$conflicting_top_lic\" " 328 | } else { 329 | # Only has an exception and not an actual license listed. 330 | # This is a mistake, but let's handle it gracefully anyway. 331 | set display_lic "" 332 | } 333 | return [list 1 "\"$portName\" is not distributable because its license${display_lic}conflicts with license \"$conflicting_dep_lic\" of dependency \"$aPort\""] 334 | } 335 | } 336 | 337 | # skip deps that are explicitly stated to not conflict 338 | array unset aPort_noconflict_ports 339 | foreach noconflict_port [lindex $aPortInfo 3] { 340 | set aPort_noconflict_ports($noconflict_port) 1 341 | } 342 | # add its deps to the list 343 | foreach possiblyNewPort [lindex $aPortInfo 0] { 344 | if {![info exists portSeen($possiblyNewPort)] && ![info exists aPort_noconflict_ports($possiblyNewPort)]} { 345 | lappend portList $possiblyNewPort 346 | set portSeen($possiblyNewPort) 1 347 | } 348 | } 349 | } 350 | 351 | return [list 0 "\"$portName\" is distributable"] 352 | } 353 | 354 | # given a variant string, return an array of variations 355 | set split_variants_re {([-+])([[:alpha:]_]+[\w\.]*)} 356 | proc split_variants {variants} { 357 | global split_variants_re 358 | set result [list] 359 | set l [regexp -all -inline -- $split_variants_re $variants] 360 | foreach { match sign variant } $l { 361 | lappend result $variant $sign 362 | } 363 | return $result 364 | } 365 | 366 | # given an array of variations, return a variant string in normalized form 367 | proc normalize_variants {variations} { 368 | set variant_string {} 369 | foreach vname [lsort -ascii [dict keys $variations]] { 370 | append variant_string [dict get $variations $vname]${vname} 371 | } 372 | return $variant_string 373 | } 374 | -------------------------------------------------------------------------------- /jobs/PortIndex2PGSQL.tcl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env port-tclsh 2 | # -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:filetype=tcl:et:sw=4:ts=4:sts=4 3 | # 4 | # PortIndex2PGSQL.tcl 5 | # Kevin Van Vechten | kevin@opendarwin.org 6 | # 3-Oct-2002 7 | # Juan Manuel Palacios | jmpp@macports.org 8 | # 22-Nov-2007 9 | # Eric Le Lay | elelay@macports.org 10 | # 24-Sep-2013 11 | # $Id$ 12 | # 13 | # Copyright (c) 2013 Eric Le Lay, The Macports Project 14 | # Copyright (c) 2007 Juan Manuel Palacios, The MacPorts Project. 15 | # Copyright (c) 2003 Apple Inc. 16 | # Copyright (c) 2002 Kevin Van Vechten. 17 | # All rights reserved. 18 | # 19 | # Redistribution and use in source and binary forms, with or without 20 | # modification, are permitted provided that the following conditions 21 | # are met: 22 | # 1. Redistributions of source code must retain the above copyright 23 | # notice, this list of conditions and the following disclaimer. 24 | # 2. Redistributions in binary form must reproduce the above copyright 25 | # notice, this list of conditions and the following disclaimer in the 26 | # documentation and/or other materials provided with the distribution. 27 | # 3. Neither the name of Apple Inc. nor the names of its contributors 28 | # may be used to endorse or promote products derived from this software 29 | # without specific prior written permission. 30 | # 31 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 32 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 33 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 34 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 35 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 36 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 37 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 38 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 39 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 40 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 41 | # POSSIBILITY OF SUCH DAMAGE. 42 | 43 | 44 | ##### 45 | # The PortIndex2PGSQL script populates a database with key information extracted 46 | # from the Portfiles in the ports tree pointed to by the sources.conf file in a 47 | # MacPorts installation, found by loading its macports1.0 tcl package and initializing 48 | # it with 'mportinit' below. Main use of the resulting database is providing live 49 | # information to the ports.php page, a client tailored to poll it. For this very reason, 50 | # information fed to the database always has to be kept up to date in order to remain 51 | # meaningful, which is accomplished simply by calling the 'mportsync' proc in macports1.0 52 | # (which updates the ports tree in use) and by installing the script on cron/launchd to be 53 | # run on a timely schedule (not any more frequent than the run of the PortIndexRegen.sh 54 | # script on that creates a new PortIndex file). 55 | # 56 | # Remaining requirement to successfully run this script is performing the necessary 57 | # PGSQL admin tasks on the host box to create the database in the first place and the 58 | # PGSQL user that will be given enough privileges to alter it. Values in the database 59 | # related variables provided below have to be adapted accordingly to match the chosen 60 | # setup. 61 | ##### 62 | 63 | 64 | # Configuration variables. Fill in the blank strings. 65 | # 66 | # Email addresses that get failure notifications - e.g. "admin@macosforge.org" 67 | set SPAM_LOVERS "" 68 | # File to read the db password from - e.g. "/var/macports/script_data" 69 | set passwdfile "" 70 | # Database abstraction variables: 71 | # name of the psql executable 72 | set psql_exe "psql" 73 | # path where the psql executable is located (only needed if not in the default PATH) 74 | set psql_exe_path "" 75 | set sqlfile "/tmp/portsdb.sql" 76 | set portsdb_host localhost 77 | set portsdb_name macports 78 | set portsdb_user macports 79 | 80 | # Runtime information log file and recipient. 81 | set runlog "/tmp/portsdb.log" 82 | set runlog_fd [open $runlog w+] 83 | set lockfile "/tmp/portsdb.lock" 84 | set mailprog "/usr/sbin/sendmail" 85 | set DATE [clock format [clock seconds] -format "%A %Y-%m-%d at %T"] 86 | 87 | set SUBJECT "PortIndex2PGSQL run failure on $DATE" 88 | set FROM noreply@macports.org 89 | set HEADERS "To: $SPAM_LOVERS\r\nFrom: $FROM\r\nSubject: $SUBJECT\r\n\r\n" 90 | 91 | # handle command line arguments 92 | set create_tables true 93 | if {[llength $argv]} { 94 | if {[lindex $argv 0] eq "--create-tables"} { 95 | set create_tables true 96 | } 97 | } 98 | 99 | # House keeping on exit. 100 | proc cleanup {args} { 101 | foreach file_to_clean $args { 102 | upvar $file_to_clean up_file_to_clean 103 | upvar ${file_to_clean}_fd up_file_to_clean_fd 104 | close $up_file_to_clean_fd 105 | file delete -force $up_file_to_clean 106 | } 107 | } 108 | 109 | # What to do when terminating execution, depending on the $exit_status condition. 110 | proc terminate {exit_status} { 111 | global runlog runlog_fd 112 | if {$exit_status} { 113 | global subject SPAM_LOVERS mailprog 114 | seek $runlog_fd 0 start 115 | exec -- $mailprog $SPAM_LOVERS <@ $runlog_fd 116 | } 117 | cleanup runlog 118 | exit $exit_status 119 | } 120 | 121 | # macports1.0 UI instantiation to route information/error messages wherever we want. 122 | # This is a custom ui_channels proc because we want to get reported information on 123 | # channels other than the default stdout/stderr that the macports1.0 API provides, 124 | # namely a log file we can later mail to people in charge if need be. 125 | proc ui_channels {priority} { 126 | global runlog_fd 127 | switch $priority { 128 | debug { 129 | if {[macports::ui_isset ports_debug]} { 130 | return $runlog_fd 131 | } else { 132 | return {} 133 | } 134 | } 135 | info { 136 | if {[macports::ui_isset ports_verbose]} { 137 | return $runlog_fd 138 | } else { 139 | return {} 140 | } 141 | } 142 | msg { 143 | if {[macports::ui_isset ports_quiet]} { 144 | return $runlog_fd 145 | } else { 146 | return {} 147 | } 148 | } 149 | error { 150 | return $runlog_fd 151 | } 152 | default { 153 | return {} 154 | } 155 | } 156 | } 157 | 158 | # Procedure to catch the database password from a protected file. 159 | proc getpasswd {passwdfile} { 160 | if {$passwdfile eq ""} { 161 | global lockfile lockfile_fd 162 | ui_error "passwdfile is empty, did you forget to set it?" 163 | cleanup lockfile 164 | terminate 1 165 | } 166 | if {[catch {open $passwdfile r} passwdfile_fd]} { 167 | global lockfile lockfile_fd 168 | ui_error "${::errorCode}: $passwdfile_fd" 169 | cleanup lockfile 170 | terminate 1 171 | } 172 | if {[gets $passwdfile_fd passwd] <= 0} { 173 | global lockfile lockfile_fd 174 | close $passwdfile_fd 175 | ui_error "No password found in password file $passwdfile!" 176 | cleanup lockfile 177 | terminate 1 178 | } 179 | close $passwdfile_fd 180 | return $passwd 181 | } 182 | 183 | # SQL string escaping. 184 | proc sql_escape {str} { 185 | regsub -all -- {'} $str {''} str 186 | return $str 187 | } 188 | 189 | # We first initialize the runlog with proper mail headers 190 | puts $runlog_fd $HEADERS 191 | 192 | # Check if there are any stray sibling jobs before moving on, bail in such case. 193 | if {[file exists $lockfile]} { 194 | puts $runlog_fd "PortIndex2PGSQL lock file found, is another job running?" 195 | terminate 1 196 | } else { 197 | set lockfile_fd [open $lockfile a] 198 | } 199 | 200 | # Load macports1.0 so that we can use some of its procs and the portinfo array. 201 | if {[catch { package require macports } errstr]} { 202 | puts $runlog_fd "${::errorInfo}" 203 | puts $runlog_fd "Failed to load the macports1.0 Tcl package: $errstr" 204 | cleanup lockfile 205 | terminate 1 206 | } 207 | 208 | # Initialize macports1.0 and its UI, in order to find the sources.conf file 209 | # (which is what will point us to the PortIndex we're gonna use) and use 210 | # the runtime information. 211 | array set ui_options {ports_verbose yes} 212 | if {[catch {mportinit ui_options} errstr]} { 213 | puts $runlog_fd "${::errorInfo}" 214 | puts $runlog_fd "Failed to initialize MacPorts: $errstr" 215 | cleanup lockfile 216 | terminate 1 217 | } 218 | 219 | 220 | set portsdb_passwd [getpasswd $passwdfile] 221 | set portsdb_cmd [macports::findBinary $psql_exe $psql_exe_path] 222 | 223 | 224 | # Flat text file to which sql statements are written. 225 | if {[catch {open $sqlfile w+} sqlfile_fd]} { 226 | ui_error "${::errorCode}: $sqlfile_fd" 227 | cleanup lockfile 228 | terminate 1 229 | } 230 | 231 | 232 | # Call the sync procedure to make sure we always have a fresh ports tree. 233 | if {[catch {mportsync} errstr]} { 234 | ui_error "${::errorInfo}" 235 | ui_error "Failed to update the ports tree, $errstr" 236 | cleanup sqlfile lockfile 237 | terminate 1 238 | } 239 | 240 | # Load every port in the index through a search that matches everything. 241 | if {[catch {set ports [mportlistall]} errstr]} { 242 | ui_error "${::errorInfo}" 243 | ui_error "port search failed: $errstr" 244 | cleanup sqlfile lockfile 245 | terminate 1 246 | } 247 | 248 | if {$create_tables} { 249 | # Initial creation of database tables: log, portfiles, categories, maintainers, dependencies, variants and platforms. 250 | # Do we need any other? 251 | puts $sqlfile_fd "DROP TABLE IF EXISTS log;" 252 | puts $sqlfile_fd "CREATE TABLE log (activity VARCHAR(255), activity_time TIMESTAMP);" 253 | 254 | puts $sqlfile_fd "DROP TABLE IF EXISTS portfiles;" 255 | puts $sqlfile_fd "CREATE TABLE portfiles (name VARCHAR(255) PRIMARY KEY NOT NULL, path VARCHAR(255), version VARCHAR(255), description TEXT);" 256 | 257 | puts $sqlfile_fd "DROP TABLE IF EXISTS categories;" 258 | puts $sqlfile_fd "CREATE TABLE categories (portfile VARCHAR(255), category VARCHAR(255), is_primary INTEGER);" 259 | 260 | puts $sqlfile_fd "DROP TABLE IF EXISTS maintainers;" 261 | puts $sqlfile_fd "CREATE TABLE maintainers (portfile VARCHAR(255), maintainer VARCHAR(255), is_primary INTEGER);" 262 | 263 | puts $sqlfile_fd "DROP TABLE IF EXISTS dependencies;" 264 | puts $sqlfile_fd "CREATE TABLE dependencies (portfile VARCHAR(255), library VARCHAR(255));" 265 | 266 | puts $sqlfile_fd "DROP TABLE IF EXISTS variants;" 267 | puts $sqlfile_fd "CREATE TABLE variants (portfile VARCHAR(255), variant VARCHAR(255));" 268 | 269 | puts $sqlfile_fd "DROP TABLE IF EXISTS platforms;" 270 | puts $sqlfile_fd "CREATE TABLE platforms (portfile VARCHAR(255), platform VARCHAR(255));" 271 | 272 | puts $sqlfile_fd "DROP TABLE IF EXISTS licenses;" 273 | puts $sqlfile_fd "CREATE TABLE licenses (portfile VARCHAR(255), license VARCHAR(255));" 274 | } else { 275 | # if we are not creating tables from scratch, remove the old data 276 | puts $sqlfile_fd "TRUNCATE log;" 277 | puts $sqlfile_fd "TRUNCATE portfiles;" 278 | puts $sqlfile_fd "TRUNCATE categories;" 279 | puts $sqlfile_fd "TRUNCATE maintainers;" 280 | puts $sqlfile_fd "TRUNCATE dependencies;" 281 | puts $sqlfile_fd "TRUNCATE variants;" 282 | puts $sqlfile_fd "TRUNCATE platforms;" 283 | puts $sqlfile_fd "TRUNCATE licenses;" 284 | } 285 | 286 | # Iterate over each matching port, extracting its information from the 287 | # portinfo array. 288 | foreach {name array} $ports { 289 | 290 | array unset portinfo 291 | array set portinfo $array 292 | 293 | set portname [sql_escape $portinfo(name)] 294 | if {[info exists portinfo(version)]} { 295 | set portversion [sql_escape $portinfo(version)] 296 | } else { 297 | set portversion "" 298 | } 299 | set portdir [sql_escape $portinfo(portdir)] 300 | if {[info exists portinfo(description)]} { 301 | set description [sql_escape $portinfo(description)] 302 | } else { 303 | set description "" 304 | } 305 | if {[info exists portinfo(categories)]} { 306 | set categories $portinfo(categories) 307 | } else { 308 | set categories "" 309 | } 310 | if {[info exists portinfo(maintainers)]} { 311 | set maintainers $portinfo(maintainers) 312 | } else { 313 | set maintainers "" 314 | } 315 | if {[info exists portinfo(variants)]} { 316 | set variants $portinfo(variants) 317 | } else { 318 | set variants "" 319 | } 320 | if {[info exists portinfo(depends_fetch)]} { 321 | set depends_fetch $portinfo(depends_fetch) 322 | } else { 323 | set depends_fetch "" 324 | } 325 | if {[info exists portinfo(depends_extract)]} { 326 | set depends_extract $portinfo(depends_extract) 327 | } else { 328 | set depends_extract "" 329 | } 330 | if {[info exists portinfo(depends_build)]} { 331 | set depends_build $portinfo(depends_build) 332 | } else { 333 | set depends_build "" 334 | } 335 | if {[info exists portinfo(depends_lib)]} { 336 | set depends_lib $portinfo(depends_lib) 337 | } else { 338 | set depends_lib "" 339 | } 340 | if {[info exists portinfo(depends_run)]} { 341 | set depends_run $portinfo(depends_run) 342 | } else { 343 | set depends_run "" 344 | } 345 | if {[info exists portinfo(platforms)]} { 346 | set platforms $portinfo(platforms) 347 | } else { 348 | set platforms "" 349 | } 350 | if {[info exists portinfo(license)]} { 351 | set licenses $portinfo(license) 352 | } else { 353 | set licenses "" 354 | } 355 | 356 | puts $sqlfile_fd "INSERT INTO portfiles VALUES ('$portname', '$portdir', '$portversion', '$description');" 357 | 358 | set primary 1 359 | foreach category $categories { 360 | set category [sql_escape $category] 361 | puts $sqlfile_fd "INSERT INTO categories VALUES ('$portname', '$category', $primary);" 362 | set primary 0 363 | } 364 | 365 | set primary 1 366 | foreach maintainer $maintainers { 367 | set maintainer [sql_escape $maintainer] 368 | puts $sqlfile_fd "INSERT INTO maintainers VALUES ('$portname', '$maintainer', $primary);" 369 | set primary 0 370 | } 371 | 372 | foreach fetch_dep $depends_fetch { 373 | set fetch_dep [sql_escape $fetch_dep] 374 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$fetch_dep');" 375 | } 376 | 377 | foreach extract_dep $depends_extract { 378 | set extract_dep [sql_escape $extract_dep] 379 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$extract_dep');" 380 | } 381 | 382 | foreach build_dep $depends_build { 383 | set build_dep [sql_escape $build_dep] 384 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$build_dep');" 385 | } 386 | 387 | foreach lib $depends_lib { 388 | set lib [sql_escape $lib] 389 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$lib');" 390 | } 391 | 392 | foreach run_dep $depends_run { 393 | set run_dep [sql_escape $run_dep] 394 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$run_dep');" 395 | } 396 | 397 | foreach variant $variants { 398 | set variant [sql_escape $variant] 399 | puts $sqlfile_fd "INSERT INTO variants VALUES ('$portname', '$variant');" 400 | } 401 | 402 | foreach platform $platforms { 403 | set platform [sql_escape $platform] 404 | puts $sqlfile_fd "INSERT INTO platforms VALUES ('$portname', '$platform');" 405 | } 406 | 407 | foreach license $licenses { 408 | set license [sql_escape $license] 409 | puts $sqlfile_fd "INSERT INTO licenses VALUES ('$portname', '$license');" 410 | } 411 | 412 | } 413 | 414 | # Mark the db regen as done only once we're done processing all ports: 415 | puts $sqlfile_fd "INSERT INTO log VALUES ('update', NOW());" 416 | 417 | # Pipe the contents of the generated sql file to the database command, 418 | # reading from the file descriptor for the raw sql file to assure completeness. 419 | if {[catch {seek $sqlfile_fd 0 start} errstr]} { 420 | ui_error "${::errorCode}: $errstr" 421 | cleanup sqlfile lockfile 422 | terminate 1 423 | } 424 | 425 | # psql will read the password from the file all by itself... 426 | global env 427 | set env(PGPASSFILE) $passwdfile 428 | if {[catch {exec -- $portsdb_cmd -q --host=$portsdb_host $portsdb_name $portsdb_user <@ $sqlfile_fd} errstr]} { 429 | ui_error "${::errorCode}: $errstr" 430 | cleanup sqlfile lockfile 431 | terminate 1 432 | } 433 | 434 | # done regenerating the database. Cleanup and exit successfully. 435 | cleanup sqlfile lockfile 436 | terminate 0 437 | -------------------------------------------------------------------------------- /jobs/PortIndex2MySQL.tcl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env port-tclsh 2 | # -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:filetype=tcl:et:sw=4:ts=4:sts=4 3 | # 4 | # PortIndex2MySQL.tcl 5 | # Kevin Van Vechten | kevin@opendarwin.org 6 | # 3-Oct-2002 7 | # Juan Manuel Palacios | jmpp@macports.org 8 | # 22-Nov-2007 9 | # $Id$ 10 | # 11 | # Copyright (c) 2007 Juan Manuel Palacios, The MacPorts Project. 12 | # Copyright (c) 2003 Apple Inc. 13 | # Copyright (c) 2002 Kevin Van Vechten. 14 | # All rights reserved. 15 | # 16 | # Redistribution and use in source and binary forms, with or without 17 | # modification, are permitted provided that the following conditions 18 | # are met: 19 | # 1. Redistributions of source code must retain the above copyright 20 | # notice, this list of conditions and the following disclaimer. 21 | # 2. Redistributions in binary form must reproduce the above copyright 22 | # notice, this list of conditions and the following disclaimer in the 23 | # documentation and/or other materials provided with the distribution. 24 | # 3. Neither the name of Apple Inc. nor the names of its contributors 25 | # may be used to endorse or promote products derived from this software 26 | # without specific prior written permission. 27 | # 28 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 29 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 32 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 33 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 34 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 35 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 36 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 37 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 38 | # POSSIBILITY OF SUCH DAMAGE. 39 | 40 | 41 | ##### 42 | # The PortIndex2MySQL script populates a database with key information extracted 43 | # from the Portfiles in the ports tree pointed to by the sources.conf file in a 44 | # MacPorts installation, found by loading its macports1.0 tcl package and initializing 45 | # it with 'mportinit' below. Main use of the resulting database is providing live 46 | # information to the ports.php page, a client tailored to poll it. For this very reason, 47 | # information fed to the database always has to be kept up to date in order to remain 48 | # meaningful, which is accomplished simply by calling the 'mportsync' proc in macports1.0 49 | # (which updates the ports tree in use) and by installing the script on cron/launchd to be 50 | # run on a timely schedule (not any more frequent than the run of the PortIndexRegen.sh 51 | # script on that creates a new PortIndex file). 52 | # 53 | # Remaining requirement to successfully run this script is performing the necessary 54 | # MySQL admin tasks on the host box to create the database in the first place and the 55 | # MySQL user that will be given enough privileges to alter it. Values in the database 56 | # related variables provided below have to be adapted accordingly to match the chosen 57 | # setup. 58 | ##### 59 | 60 | 61 | # Configuration variables. Fill in the blank strings. 62 | # 63 | # Email addresses that get failure notifications - e.g. "admin@macosforge.org" 64 | set SPAM_LOVERS "" 65 | # File to read the db password from - e.g. "/var/macports/script_data" 66 | set passwdfile "" 67 | # Database abstraction variables: 68 | # name of the mysql executable 69 | set mysql_exe "mysql" 70 | # path where the mysql executable is located (only needed if not in the default PATH) 71 | set mysql_exe_path "" 72 | set sqlfile "/tmp/portsdb.sql" 73 | set portsdb_host localhost 74 | set portsdb_name macports 75 | set portsdb_user macports 76 | 77 | # Runtime information log file and recipient. 78 | set runlog "/tmp/portsdb.log" 79 | set runlog_fd [open $runlog w+] 80 | set lockfile "/tmp/portsdb.lock" 81 | set mailprog "/usr/sbin/sendmail" 82 | set DATE [clock format [clock seconds] -format "%A %Y-%m-%d at %T"] 83 | 84 | set SUBJECT "PortIndex2MySQL run failure on $DATE" 85 | set FROM noreply@macports.org 86 | set HEADERS "To: $SPAM_LOVERS\r\nFrom: $FROM\r\nSubject: $SUBJECT\r\n\r\n" 87 | 88 | # handle command line arguments 89 | set create_tables true 90 | if {[llength $argv]} { 91 | if {[lindex $argv 0] eq "--create-tables"} { 92 | set create_tables true 93 | } 94 | } 95 | 96 | # House keeping on exit. 97 | proc cleanup {args} { 98 | foreach file_to_clean $args { 99 | upvar $file_to_clean up_file_to_clean 100 | upvar ${file_to_clean}_fd up_file_to_clean_fd 101 | close $up_file_to_clean_fd 102 | file delete -force $up_file_to_clean 103 | } 104 | } 105 | 106 | # What to do when terminating execution, depending on the $exit_status condition. 107 | proc terminate {exit_status} { 108 | global runlog runlog_fd 109 | if {$exit_status} { 110 | global subject SPAM_LOVERS mailprog 111 | seek $runlog_fd 0 start 112 | exec -- $mailprog $SPAM_LOVERS <@ $runlog_fd 113 | } 114 | cleanup runlog 115 | exit $exit_status 116 | } 117 | 118 | # macports1.0 UI instantiation to route information/error messages wherever we want. 119 | # This is a custom ui_channels proc because we want to get reported information on 120 | # channels other than the default stdout/stderr that the macports1.0 API provides, 121 | # namely a log file we can later mail to people in charge if need be. 122 | proc ui_channels {priority} { 123 | global runlog_fd 124 | switch $priority { 125 | debug { 126 | if {[macports::ui_isset ports_debug]} { 127 | return $runlog_fd 128 | } else { 129 | return {} 130 | } 131 | } 132 | info { 133 | if {[macports::ui_isset ports_verbose]} { 134 | return $runlog_fd 135 | } else { 136 | return {} 137 | } 138 | } 139 | msg { 140 | if {[macports::ui_isset ports_quiet]} { 141 | return $runlog_fd 142 | } else { 143 | return {} 144 | } 145 | } 146 | error { 147 | return $runlog_fd 148 | } 149 | default { 150 | return {} 151 | } 152 | } 153 | } 154 | 155 | # Procedure to catch the database password from a protected file. 156 | proc getpasswd {passwdfile} { 157 | if {$passwdfile eq ""} { 158 | global lockfile lockfile_fd 159 | ui_error "passwdfile is empty, did you forget to set it?" 160 | cleanup lockfile 161 | terminate 1 162 | } 163 | if {[catch {open $passwdfile r} passwdfile_fd]} { 164 | global lockfile lockfile_fd 165 | ui_error "${::errorCode}: $passwdfile_fd" 166 | cleanup lockfile 167 | terminate 1 168 | } 169 | if {[gets $passwdfile_fd passwd] <= 0} { 170 | global lockfile lockfile_fd 171 | close $passwdfile_fd 172 | ui_error "No password found in password file $passwdfile!" 173 | cleanup lockfile 174 | terminate 1 175 | } 176 | close $passwdfile_fd 177 | return $passwd 178 | } 179 | 180 | # SQL string escaping. 181 | proc sql_escape {str} { 182 | regsub -all -- {'} $str {\\'} str 183 | regsub -all -- {"} $str {\\"} str 184 | regsub -all -- {\n} $str {\\n} str 185 | return $str 186 | } 187 | 188 | # We first initialize the runlog with proper mail headers 189 | puts $runlog_fd $HEADERS 190 | 191 | # Check if there are any stray sibling jobs before moving on, bail in such case. 192 | if {[file exists $lockfile]} { 193 | puts $runlog_fd "PortIndex2MySQL lock file found, is another job running?" 194 | terminate 1 195 | } else { 196 | set lockfile_fd [open $lockfile a] 197 | } 198 | 199 | # Load macports1.0 so that we can use some of its procs and the portinfo array. 200 | if {[catch { package require macports } errstr]} { 201 | puts $runlog_fd "${::errorInfo}" 202 | puts $runlog_fd "Failed to load the macports1.0 Tcl package: $errstr" 203 | cleanup lockfile 204 | terminate 1 205 | } 206 | 207 | # Initialize macports1.0 and its UI, in order to find the sources.conf file 208 | # (which is what will point us to the PortIndex we're gonna use) and use 209 | # the runtime information. 210 | array set ui_options {ports_verbose yes} 211 | if {[catch {mportinit ui_options} errstr]} { 212 | puts $runlog_fd "${::errorInfo}" 213 | puts $runlog_fd "Failed to initialize MacPorts: $errstr" 214 | cleanup lockfile 215 | terminate 1 216 | } 217 | 218 | 219 | set portsdb_passwd [getpasswd $passwdfile] 220 | set portsdb_cmd [macports::findBinary $mysql_exe $mysql_exe_path] 221 | 222 | 223 | # Flat text file to which sql statements are written. 224 | if {[catch {open $sqlfile w+} sqlfile_fd]} { 225 | ui_error "${::errorCode}: $sqlfile_fd" 226 | cleanup lockfile 227 | terminate 1 228 | } 229 | 230 | 231 | # Call the sync procedure to make sure we always have a fresh ports tree. 232 | if {[catch {mportsync} errstr]} { 233 | ui_error "${::errorInfo}" 234 | ui_error "Failed to update the ports tree, $errstr" 235 | cleanup sqlfile lockfile 236 | terminate 1 237 | } 238 | 239 | # Load every port in the index through a search that matches everything. 240 | if {[catch {set ports [mportlistall]} errstr]} { 241 | ui_error "${::errorInfo}" 242 | ui_error "port search failed: $errstr" 243 | cleanup sqlfile lockfile 244 | terminate 1 245 | } 246 | 247 | if {$create_tables} { 248 | # Initial creation of database tables: log, portfiles, categories, maintainers, dependencies, variants and platforms. 249 | # Do we need any other? 250 | puts $sqlfile_fd "DROP TABLE IF EXISTS log;" 251 | puts $sqlfile_fd "CREATE TABLE log (activity VARCHAR(255), activity_time TIMESTAMP(14)) DEFAULT CHARSET=utf8;" 252 | 253 | puts $sqlfile_fd "DROP TABLE IF EXISTS portfiles;" 254 | puts $sqlfile_fd "CREATE TABLE portfiles (name VARCHAR(255) PRIMARY KEY NOT NULL, path VARCHAR(255), version VARCHAR(255), description TEXT) DEFAULT CHARSET=utf8;" 255 | 256 | puts $sqlfile_fd "DROP TABLE IF EXISTS categories;" 257 | puts $sqlfile_fd "CREATE TABLE categories (portfile VARCHAR(255), category VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;" 258 | 259 | puts $sqlfile_fd "DROP TABLE IF EXISTS maintainers;" 260 | puts $sqlfile_fd "CREATE TABLE maintainers (portfile VARCHAR(255), maintainer VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;" 261 | 262 | puts $sqlfile_fd "DROP TABLE IF EXISTS dependencies;" 263 | puts $sqlfile_fd "CREATE TABLE dependencies (portfile VARCHAR(255), library VARCHAR(255)) DEFAULT CHARSET=utf8;" 264 | 265 | puts $sqlfile_fd "DROP TABLE IF EXISTS variants;" 266 | puts $sqlfile_fd "CREATE TABLE variants (portfile VARCHAR(255), variant VARCHAR(255)) DEFAULT CHARSET=utf8;" 267 | 268 | puts $sqlfile_fd "DROP TABLE IF EXISTS platforms;" 269 | puts $sqlfile_fd "CREATE TABLE platforms (portfile VARCHAR(255), platform VARCHAR(255)) DEFAULT CHARSET=utf8;" 270 | 271 | puts $sqlfile_fd "DROP TABLE IF EXISTS licenses;" 272 | puts $sqlfile_fd "CREATE TABLE licenses (portfile VARCHAR(255), license VARCHAR(255)) DEFAULT CHARSET=utf8;" 273 | } else { 274 | # if we are not creating tables from scratch, remove the old data 275 | puts $sqlfile_fd "TRUNCATE log;" 276 | puts $sqlfile_fd "TRUNCATE portfiles;" 277 | puts $sqlfile_fd "TRUNCATE categories;" 278 | puts $sqlfile_fd "TRUNCATE maintainers;" 279 | puts $sqlfile_fd "TRUNCATE dependencies;" 280 | puts $sqlfile_fd "TRUNCATE variants;" 281 | puts $sqlfile_fd "TRUNCATE platforms;" 282 | puts $sqlfile_fd "TRUNCATE licenses;" 283 | } 284 | 285 | # Iterate over each matching port, extracting its information from the 286 | # portinfo array. 287 | foreach {name array} $ports { 288 | 289 | array unset portinfo 290 | array set portinfo $array 291 | 292 | set portname [sql_escape $portinfo(name)] 293 | if {[info exists portinfo(version)]} { 294 | set portversion [sql_escape $portinfo(version)] 295 | } else { 296 | set portversion "" 297 | } 298 | set portdir [sql_escape $portinfo(portdir)] 299 | if {[info exists portinfo(description)]} { 300 | set description [sql_escape $portinfo(description)] 301 | } else { 302 | set description "" 303 | } 304 | if {[info exists portinfo(categories)]} { 305 | set categories $portinfo(categories) 306 | } else { 307 | set categories "" 308 | } 309 | if {[info exists portinfo(maintainers)]} { 310 | set maintainers $portinfo(maintainers) 311 | } else { 312 | set maintainers "" 313 | } 314 | if {[info exists portinfo(variants)]} { 315 | set variants $portinfo(variants) 316 | } else { 317 | set variants "" 318 | } 319 | if {[info exists portinfo(depends_fetch)]} { 320 | set depends_fetch $portinfo(depends_fetch) 321 | } else { 322 | set depends_fetch "" 323 | } 324 | if {[info exists portinfo(depends_extract)]} { 325 | set depends_extract $portinfo(depends_extract) 326 | } else { 327 | set depends_extract "" 328 | } 329 | if {[info exists portinfo(depends_build)]} { 330 | set depends_build $portinfo(depends_build) 331 | } else { 332 | set depends_build "" 333 | } 334 | if {[info exists portinfo(depends_lib)]} { 335 | set depends_lib $portinfo(depends_lib) 336 | } else { 337 | set depends_lib "" 338 | } 339 | if {[info exists portinfo(depends_run)]} { 340 | set depends_run $portinfo(depends_run) 341 | } else { 342 | set depends_run "" 343 | } 344 | if {[info exists portinfo(platforms)]} { 345 | set platforms $portinfo(platforms) 346 | } else { 347 | set platforms "" 348 | } 349 | if {[info exists portinfo(license)]} { 350 | set licenses $portinfo(license) 351 | } else { 352 | set licenses "" 353 | } 354 | 355 | puts $sqlfile_fd "INSERT INTO portfiles VALUES ('$portname', '$portdir', '$portversion', '$description');" 356 | 357 | set primary 1 358 | foreach category $categories { 359 | set category [sql_escape $category] 360 | puts $sqlfile_fd "INSERT INTO categories VALUES ('$portname', '$category', $primary);" 361 | set primary 0 362 | } 363 | 364 | set primary 1 365 | foreach maintainer $maintainers { 366 | set maintainer [sql_escape $maintainer] 367 | puts $sqlfile_fd "INSERT INTO maintainers VALUES ('$portname', '$maintainer', $primary);" 368 | set primary 0 369 | } 370 | 371 | foreach fetch_dep $depends_fetch { 372 | set fetch_dep [sql_escape $fetch_dep] 373 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$fetch_dep');" 374 | } 375 | 376 | foreach extract_dep $depends_extract { 377 | set extract_dep [sql_escape $extract_dep] 378 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$extract_dep');" 379 | } 380 | 381 | foreach build_dep $depends_build { 382 | set build_dep [sql_escape $build_dep] 383 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$build_dep');" 384 | } 385 | 386 | foreach lib $depends_lib { 387 | set lib [sql_escape $lib] 388 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$lib');" 389 | } 390 | 391 | foreach run_dep $depends_run { 392 | set run_dep [sql_escape $run_dep] 393 | puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$run_dep');" 394 | } 395 | 396 | foreach variant $variants { 397 | set variant [sql_escape $variant] 398 | puts $sqlfile_fd "INSERT INTO variants VALUES ('$portname', '$variant');" 399 | } 400 | 401 | foreach platform $platforms { 402 | set platform [sql_escape $platform] 403 | puts $sqlfile_fd "INSERT INTO platforms VALUES ('$portname', '$platform');" 404 | } 405 | 406 | foreach license $licenses { 407 | set license [sql_escape $license] 408 | puts $sqlfile_fd "INSERT INTO licenses VALUES ('$portname', '$license');" 409 | } 410 | 411 | } 412 | 413 | # Mark the db regen as done only once we're done processing all ports: 414 | puts $sqlfile_fd "INSERT INTO log VALUES ('update', NOW());" 415 | 416 | # Pipe the contents of the generated sql file to the database command, 417 | # reading from the file descriptor for the raw sql file to assure completeness. 418 | if {[catch {seek $sqlfile_fd 0 start} errstr]} { 419 | ui_error "${::errorCode}: $errstr" 420 | cleanup sqlfile lockfile 421 | terminate 1 422 | } 423 | 424 | if {[catch {exec -- $portsdb_cmd --host=$portsdb_host --user=$portsdb_user --password=$portsdb_passwd --database=$portsdb_name <@ $sqlfile_fd} errstr]} { 425 | ui_error "${::errorCode}: $errstr" 426 | cleanup sqlfile lockfile 427 | terminate 1 428 | } 429 | 430 | # done regenerating the database. Cleanup and exit successfully. 431 | cleanup sqlfile lockfile 432 | terminate 0 433 | --------------------------------------------------------------------------------