├── .gitignore ├── LICENSE ├── README.md ├── add-port.sh ├── import.sh ├── include ├── functions-import.sh ├── functions-sql.sh ├── functions-update.sh └── functions.sh ├── params.sh ├── patches └── Mk-portsdb.patch ├── periodic └── run.sh ├── run-sql ├── sql-misc-queries ├── find-broken-ports-maintained-by-{user}.sql ├── find-distinct-projects.sql ├── find-duplicate-ports.sql ├── find-ports-depending-on-{pkgorigin}.sql ├── find-ports-maintained-by-{user}-with-missing-built-packages.sql ├── find-ports-maintained-by-{user}-with-{uses}.sql ├── find-ports-maintained-by-{user}.sql ├── find-ports-that-have-test-depends.sql ├── find-ports-with-missing-built-packages.sql ├── find-ports-with-{string}-in-comment.sql └── get-ports-list.sql ├── sql ├── fix-default-parent-flavor.sql └── schema.sql ├── update.sh └── util ├── copy-tree.sh ├── number-lines.sh └── squeeze-log-into-line.sh /.gitignore: -------------------------------------------------------------------------------- 1 | ports.sqlite 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2022-2023 Yuri Victorovich 2 | 3 | Redistribution and use in source and binary forms, with or without modification, 4 | are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | 3. Neither the name of the copyright holder nor the names of its contributors 14 | may be used to endorse or promote products derived from this software without 15 | specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 21 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 24 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PortsDB 2 | PortsDB is a program that imports the FreeBSD ports tree into an SQLite database. 3 | 4 | ## Purpose 5 | SQLite is a relational database. In many cases it is easier to query data in a relational database than when the data is in text files. 6 | 7 | ## How to use PortsDB 8 | Just run the command './import.sh' on a FreeBSD system. It will run for several minutes and will create the database 'ports.sqlite'. 9 | Then you can update your ports tree and run './update.sh', which will update the database with all new commits. 10 | 11 | ## What to do with the database produced by PortsDB? 12 | A variety of applications can easily run SQL queries against the PortsDB database to 13 | * list ports or packages 14 | * list port dependencies in both directions 15 | * find the list of ports maintained by a particular maintainer 16 | * search ports by the content of their comment 17 | * any other similar applications 18 | 19 | ## What software does PortsDB depend on? 20 | There are only few dependencies: 21 | * Bourne shell (/bin/sh) which is already present on all systems 22 | * BSD make which is already present on all systems 23 | * SQLite database application which can be installed from the 'sqlite3' package 24 | * git program 25 | * gsed program 26 | 27 | ## Design principles 28 | * *Ease of use:* users only need to run import.sh to import the ports tree into a database. They can also choose to run update.sh to quickly update the database with new commits, instead of re-running import.sh, which takes longer 29 | * *Performance:* import.sh only takes ~15 minutes, depending on the system. 30 | * *No artificial keys:* we didn't introduce any external integer keys into tables. Tables are indexed with PKGORIGIN in the form of *{carteg}/{name}* and flavor. This makes tables easy to understand and easy to query. 31 | 32 | ## How to see what's in PortsDB? 33 | There is an excellent SQLite DB viewer [SQLiteStudio](https://www.sqlitestudio.pl/) that can be installed with ```pkg install SQLiteStudio```. 34 | -------------------------------------------------------------------------------- /add-port.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | 6 | ## 7 | ## set strict mode 8 | ## 9 | 10 | STRICT="set -euo pipefail" 11 | $STRICT 12 | 13 | ## 14 | ## functions 15 | ## 16 | 17 | . $CODEBASE/include/functions-sql.sh 18 | 19 | log() { 20 | $STRICT 21 | #echo "add-port.sh: $1" 22 | } 23 | 24 | wrap_nullable_string() { 25 | $STRICT 26 | if [ -z "$1" ]; then 27 | echo "null" 28 | else 29 | echo "'$1'" 30 | fi 31 | } 32 | wrap_non_nullable_string() { 33 | $STRICT 34 | echo "'$1'" 35 | } 36 | wrap_nullable_integer() { 37 | $STRICT 38 | if [ -z "$1" ]; then 39 | echo "null" 40 | else 41 | echo "$1" 42 | fi 43 | } 44 | wrap_non_nullable_integer() { 45 | $STRICT 46 | echo $1 47 | } 48 | list_begins_with() { 49 | $STRICT 50 | local small="$1" 51 | local large="$2" 52 | local pattern="$small " 53 | 54 | if [ "$small" = "$large" ]; then 55 | echo "YES" 56 | return 57 | fi 58 | 59 | case "$large" in 60 | $pattern*) 61 | echo "YES" 62 | ;; 63 | *) 64 | echo "NO" 65 | ;; 66 | esac 67 | } 68 | escape_special_chars() { 69 | $STRICT 70 | echo "$1" | sed -e "s|'|''|g" 71 | } 72 | expand_dollar_sign() { 73 | $STRICT 74 | echo "$1" | sed -e "s|%%DOLLAR%%|$|g" 75 | } 76 | normalize_makefile() { 77 | $STRICT 78 | # the filter chooses makefiles (1) only in the ports tree (2) not in the current port's directory 79 | echo $1 | xargs realpath | (grep "^$PORTSDIR" || true) | sed -e "s|^$PORTSDIR/||" | (grep -v "^$PKGORIGIN/" || true) 80 | } 81 | 82 | ## 83 | ## read supplied arguments 84 | ## 85 | 86 | for name in DB \ 87 | FLAVOR PKGORIGIN PORTNAME PORTVERSION DISTVERSION DISTVERSIONPREFIX DISTVERSIONSUFFIX PORTREVISION \ 88 | PORTEPOCH CATEGORIES \ 89 | MAINTAINER WWW \ 90 | CPE_STR \ 91 | COMPLETE_OPTIONS_LIST OPTIONS_DEFAULT \ 92 | FLAVORS \ 93 | COMMENT PKGBASE PKGNAME PKGNAMESUFFIX USES \ 94 | PKG_DEPENDS FETCH_DEPENDS EXTRACT_DEPENDS PATCH_DEPENDS BUILD_DEPENDS LIB_DEPENDS RUN_DEPENDS TEST_DEPENDS \ 95 | USE_GITHUB GH_ACCOUNT GH_PROJECT GH_TAGNAME \ 96 | USE_GITLAB GL_SITE GL_ACCOUNT GL_PROJECT GL_COMMIT \ 97 | DEPRECATED EXPIRATION_DATE \ 98 | BROKEN \ 99 | MAKEFILES ; \ 100 | do 101 | eval "$name=\"$1\"" 102 | shift 103 | done 104 | 105 | ## 106 | ## wrap argument values 107 | ## 108 | 109 | PKGORIGINw=$(wrap_non_nullable_string "$PKGORIGIN") 110 | PORTNAMEw=$(wrap_non_nullable_string "$PORTNAME") 111 | PORTVERSIONw=$(wrap_non_nullable_string "$PORTVERSION") 112 | DISTVERSIONw=$(wrap_non_nullable_string "$DISTVERSION") 113 | DISTVERSIONPREFIXw=$(wrap_nullable_string "$DISTVERSIONPREFIX") 114 | DISTVERSIONSUFFIXw=$(wrap_nullable_string "$DISTVERSIONSUFFIX") 115 | PORTREVISIONw=$(wrap_nullable_integer "$PORTREVISION") 116 | PORTEPOCHw=$(wrap_nullable_integer "$PORTEPOCH") 117 | CATEGORIESw=$(wrap_non_nullable_string "$CATEGORIES") 118 | MAINTAINERw=$(wrap_non_nullable_string "$MAINTAINER") 119 | WWWw=$(wrap_non_nullable_string "$WWW") 120 | CPE_STRw=$(expand_dollar_sign "$(wrap_nullable_string "$(escape_special_chars "$CPE_STR")")") 121 | COMPLETE_OPTIONS_LISTw=$(wrap_nullable_string "$COMPLETE_OPTIONS_LIST") 122 | OPTIONS_DEFAULTw=$(wrap_nullable_string "$OPTIONS_DEFAULT") 123 | FLAVORSw=$(wrap_nullable_string "$FLAVORS") 124 | FLAVORw=$(wrap_non_nullable_string "$FLAVOR") 125 | COMMENTw=$(expand_dollar_sign "$(wrap_non_nullable_string "$(escape_special_chars "$COMMENT")")") 126 | PKGBASEw=$(wrap_non_nullable_string "$PKGBASE") 127 | PKGNAMEw=$(wrap_non_nullable_string "$PKGNAME") 128 | PKGNAMESUFFIXw=$(wrap_nullable_string "$PKGNAMESUFFIX") 129 | USESw=$(wrap_nullable_string "$USES") 130 | USE_GITHUBw=$(wrap_non_nullable_string "$USE_GITHUB") 131 | GH_ACCOUNTw=$(wrap_non_nullable_string "$GH_ACCOUNT") 132 | GH_PROJECTw=$(wrap_non_nullable_string "$GH_PROJECT") 133 | GH_TAGNAMEw=$(wrap_non_nullable_string "$GH_TAGNAME") 134 | USE_GITLABw=$(wrap_non_nullable_string "$USE_GITLAB") 135 | GL_SITEw=$(wrap_non_nullable_string "$GL_SITE") 136 | GL_ACCOUNTw=$(wrap_non_nullable_string "$GL_ACCOUNT") 137 | GL_PROJECTw=$(wrap_non_nullable_string "$GL_PROJECT") 138 | GL_COMMITw=$(wrap_nullable_string "$GL_COMMIT") 139 | DEPRECATEDw=$(expand_dollar_sign "$(wrap_non_nullable_string "$(escape_special_chars "$DEPRECATED")")") 140 | EXPIRATION_DATEw=$(wrap_nullable_string "$EXPIRATION_DATE") 141 | BROKENw=$(expand_dollar_sign "$(wrap_non_nullable_string "$(escape_special_chars "$BROKEN")")") 142 | 143 | ## 144 | ## DB functions 145 | ## 146 | 147 | insert_port() { 148 | $STRICT 149 | run_SQL "INSERT INTO Port(PKGORIGIN,PORTNAME,PORTVERSION,DISTVERSION,DISTVERSIONPREFIX,DISTVERSIONSUFFIX,PORTREVISION,PORTEPOCH,CATEGORIES,MAINTAINER,WWW,CPE_STR,COMPLETE_OPTIONS_LIST,OPTIONS_DEFAULT,FLAVORS) VALUES ($PKGORIGINw,$PORTNAMEw,$PORTVERSIONw,$DISTVERSIONw,$DISTVERSIONPREFIXw,$DISTVERSIONSUFFIXw,$PORTREVISIONw,$PORTEPOCHw,$CATEGORIESw,$MAINTAINERw,$WWWw,$CPE_STRw,$COMPLETE_OPTIONS_LISTw,$OPTIONS_DEFAULTw,$FLAVORSw)" 150 | } 151 | insert_flavor() { 152 | $STRICT 153 | run_SQL "INSERT INTO PortFlavor(PKGORIGIN,FLAVOR,COMMENT,PKGBASE,PKGNAME,PKGNAMESUFFIX,USES) VALUES ($PKGORIGINw,$FLAVORw,$COMMENTw,$PKGBASEw,$PKGNAMEw,$PKGNAMESUFFIXw,$USESw)" 154 | } 155 | insert_dependencies() { 156 | $STRICT 157 | local PKGORIGIN="$1" 158 | local FLAVOR="$2" 159 | local DEPENDS="$3" 160 | local KIND="$4" 161 | 162 | local CHILD_PKGORIGINw=$(wrap_non_nullable_string "$PKGORIGIN") 163 | local CHILD_FLAVORw=$(wrap_non_nullable_string "$FLAVOR") 164 | 165 | for DEP in $DEPENDS; do 166 | local PARENT_PKGORIGIN="" 167 | local PARENT_FLAVOR="" 168 | local PARENT_PHASE="" 169 | 170 | # parse dependency expression into shell expression 171 | # the format is ({pkg_ver_spec}|{exe}|{shlib}):{pkgorigin}(|:{parent_stage})(|@{parent_flavor}) - it has 4 parts, out of which 1 part is ignored, and 2 are optional 172 | local expression 173 | expression=$(echo $DEP | gsed -E 's/([^:@]+):([^:@]+)(|:([a-z]+))(|@([a-zA-Z0-9_]+))$/PARENT_PKGORIGIN=\2;PARENT_FLAVOR=\6;PARENT_PHASE=\4/') 174 | # expression wouldn't be equal to $DEP in case the above regex would fail to match the string, and the next line would fail 175 | 176 | # evaluate shell expression 177 | eval $expression 178 | 179 | # wrap values 180 | local PARENT_PKGORIGINw=$(wrap_non_nullable_string "$PARENT_PKGORIGIN") 181 | local PARENT_FLAVORw=$(wrap_non_nullable_string "$PARENT_FLAVOR") 182 | local PARENT_PHASEw=$(wrap_nullable_string "$PARENT_PHASE") 183 | 184 | # write into DB 185 | run_SQL "INSERT OR IGNORE INTO Depends(PARENT_PKGORIGIN,PARENT_FLAVOR,PARENT_PHASE,CHILD_PKGORIGIN,CHILD_FLAVOR,KIND) VALUES($PARENT_PKGORIGINw,$PARENT_FLAVORw,$PARENT_PHASEw,$CHILD_PKGORIGINw,$CHILD_FLAVORw,'$KIND')" 186 | done 187 | } 188 | insert_github() { 189 | $STRICT 190 | run_SQL "INSERT INTO GitHub(PKGORIGIN, FLAVOR, USE_GITHUB, GH_ACCOUNT, GH_PROJECT, GH_TAGNAME) VALUES($PKGORIGINw,$FLAVORw,$USE_GITHUBw,$GH_ACCOUNTw,$GH_PROJECTw,$GH_TAGNAMEw)" 191 | } 192 | insert_gitlab() { 193 | $STRICT 194 | run_SQL "INSERT INTO GitLab(PKGORIGIN, FLAVOR, USE_GITLAB, GL_SITE, GL_ACCOUNT, GL_PROJECT, GL_COMMIT) VALUES($PKGORIGINw,$FLAVORw,$USE_GITLABw,$GL_SITEw,$GL_ACCOUNTw,$GL_PROJECTw,$GL_COMMITw)" 195 | } 196 | insert_deprecated() { 197 | $STRICT 198 | run_SQL "INSERT INTO Deprecated(PKGORIGIN, FLAVOR, DEPRECATED, EXPIRATION_DATE) VALUES($PKGORIGINw,$FLAVORw,$DEPRECATEDw,$EXPIRATION_DATEw)" 199 | } 200 | insert_broken() { 201 | $STRICT 202 | run_SQL "INSERT INTO Broken(PKGORIGIN, FLAVOR, BROKEN) VALUES($PKGORIGINw,$FLAVORw,$BROKENw)" 203 | } 204 | insert_makefile_dependencies() { 205 | $STRICT 206 | for m in $MAKEFILES; do 207 | if [ $m != "Makefile" ]; then 208 | m=$(cd $PORTSDIR/$PKGORIGIN && normalize_makefile $m) 209 | if [ -n "$m" ]; then 210 | run_SQL "INSERT INTO MakefileDependencies(PKGORIGIN, FLAVOR, MAKEFILE) VALUES($PKGORIGINw,$FLAVORw,$(wrap_non_nullable_string $m))" 211 | fi 212 | fi 213 | done 214 | } 215 | 216 | ## 217 | ## MAIN: insert records into the DB 218 | ## 219 | 220 | if [ -z "$FLAVORS" -o $(list_begins_with "$FLAVOR" "$FLAVORS") = "YES" ]; then # no flavors or default flavor 221 | log "adding Port record for $PKGORIGIN (FLAVOR=$FLAVOR FLAVORS=$FLAVORS)" 222 | insert_port 223 | insert_flavor 224 | else # subsequent flavors 225 | log "adding subsequent PortFlavor record for $PKGORIGIN: FLAVOR=$FLAVOR FLAVORS=$FLAVORS PKGBASE=$PKGBASE PKGNAME=$PKGNAME PKGNAMESUFFIX=$PKGNAMESUFFIX" 226 | insert_flavor 227 | fi 228 | 229 | # add dependency records 230 | insert_dependencies $PKGORIGIN "$FLAVOR" "$PKG_DEPENDS" G 231 | insert_dependencies $PKGORIGIN "$FLAVOR" "$FETCH_DEPENDS" F 232 | insert_dependencies $PKGORIGIN "$FLAVOR" "$EXTRACT_DEPENDS" E 233 | insert_dependencies $PKGORIGIN "$FLAVOR" "$PATCH_DEPENDS" P 234 | insert_dependencies $PKGORIGIN "$FLAVOR" "$BUILD_DEPENDS" B 235 | insert_dependencies $PKGORIGIN "$FLAVOR" "$LIB_DEPENDS" L 236 | insert_dependencies $PKGORIGIN "$FLAVOR" "$RUN_DEPENDS" R 237 | insert_dependencies $PKGORIGIN "$FLAVOR" "$TEST_DEPENDS" T 238 | 239 | # add GitHub record 240 | if [ -n "$USE_GITHUB" ]; then 241 | insert_github 242 | fi 243 | 244 | # add GitLab record 245 | if [ -n "$USE_GITLAB" ]; then 246 | insert_gitlab 247 | fi 248 | 249 | # add Deprecated record 250 | if [ -n "$DEPRECATED" ]; then 251 | insert_deprecated 252 | fi 253 | 254 | # add Broken record 255 | if [ -n "$BROKEN" ]; then 256 | insert_broken 257 | fi 258 | 259 | insert_makefile_dependencies 260 | -------------------------------------------------------------------------------- /import.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | 6 | ## 7 | ## import.sh is a script that 8 | ## creates the PortsDB SQLite database 9 | ## and adds all ports from the ports tree 10 | ## specified in the PORTSDIR environment 11 | ## variable. 12 | ## 13 | ## 14 | ## Arguments: 15 | ## - DB: (optional) database file to be created (default: ports.sqlite) 16 | ## - SQL_FILE: (optional) file to write the SQL dump from which the database can be recreated 17 | ## 18 | ## Environment variables: 19 | ## - PORTSDIR: (optional) ports tree to build the database for (default: /usr/ports) 20 | ## - SUBDIR: (optional, DEBUG) choose a subdir in the ports tree, generally produces a broken DB with foreign key violations 21 | ## 22 | 23 | ## 24 | ## set strict mode 25 | ## 26 | 27 | STRICT="set -euo pipefail" 28 | $STRICT 29 | 30 | ## 31 | ## find CODEBASE 32 | ## 33 | 34 | SCRIPT=$(readlink -f "$0") 35 | CODEBASE=$(dirname "$SCRIPT") 36 | 37 | ## 38 | ## include function libraries and read parameters 39 | ## 40 | 41 | . $CODEBASE/include/functions.sh 42 | . $CODEBASE/include/functions-import.sh 43 | . $CODEBASE/include/functions-sql.sh 44 | . $CODEBASE/params.sh 45 | 46 | ## 47 | ## read arguments and set defaults 48 | ## 49 | 50 | DB=${1-} # write the SQLite DB (do not write DB when empty) 51 | SQL_FILE=${2-} # save SQL statements into this file, if set 52 | SQL_FILE_ARG="${SQL_FILE}" 53 | 54 | ## 55 | ## read env variables 56 | ## 57 | 58 | PORTSDIR=${PORTSDIR:-/usr/ports} # default value 59 | SUBDIR=${SUBDIR-} 60 | 61 | ## 62 | ## global variables 63 | ## 64 | 65 | PORTSDIR_EFFECTIVE="" 66 | PERFORM_ACTION_WRITE_DB=no 67 | PERFORM_ACTION_WRITE_SQL=no 68 | 69 | ## 70 | ## check dependencies 71 | ## 72 | 73 | check_dependencies || fail "" 74 | 75 | ## 76 | ## usage 77 | ## 78 | 79 | usage() { 80 | $STRICT 81 | fail "Usage: $0 [{sync|async}]" 82 | } 83 | 84 | ## 85 | ## set defaults 86 | ## 87 | 88 | if [ -z "$DB" -a -z "$SQL_FILE" ]; then 89 | # no DB or SQL file is supplied, default to ports.sqlite 90 | DB="ports.sqlite" 91 | fi 92 | 93 | ## 94 | ## what do we do 95 | ## 96 | 97 | [ -n "$DB" ] && PERFORM_ACTION_WRITE_DB=yes 98 | [ -n "$SQL_FILE_ARG" ] && PERFORM_ACTION_WRITE_FILE=yes 99 | 100 | ## 101 | ## check arguments and required enviroment values 102 | ## 103 | 104 | if ! is_ports_tree_directory $PORTSDIR; then 105 | perror "error: the PORTSDIR environment variable should point to a valid ports tree" 106 | usage 107 | fi 108 | 109 | if [ -n "$SUBDIR" ] && ! [ -f "$PORTSDIR/$SUBDIR/Makefile" ]; then 110 | echo "error: the SUBDIR environment variable is expected to point to a valid subdirectory in the ports tree" 111 | usage 112 | fi 113 | 114 | 115 | # create the file for SQL statements that will be written after traversing the ports tree 116 | if [ -z "$SQL_FILE" ]; then 117 | # generate temporary SQL file if not provided by the user 118 | SQL_FILE=$(mktemp -t ports.sql) 119 | fi 120 | 121 | ## 122 | ## adjust values 123 | ## 124 | 125 | PORTSDIR=$(make_file_path_global $PORTSDIR) 126 | if [ -n "$DB" ]; then 127 | DB=$(make_file_path_global "$DB") 128 | fi 129 | if [ -n "$SQL_FILE" ]; then 130 | SQL_FILE=$(make_file_path_global "$SQL_FILE") 131 | fi 132 | 133 | ## 134 | ## save arguments and other values in environment 135 | ## 136 | 137 | export DB 138 | export SQL_FILE 139 | export CODEBASE 140 | export PORTSDIR 141 | 142 | ## 143 | ## MAIN 144 | ## 145 | 146 | # announcement 147 | announcement "starting to import" 148 | 149 | # explain 150 | explain_action 151 | 152 | # initialize 153 | initialize 154 | 155 | # traverse 156 | PORTSDIR_EFFECTIVE=$(effective_ports_tree $PORTSDIR) 157 | import $PORTSDIR_EFFECTIVE 158 | 159 | # save Git revision of the ports tree 160 | write_ports_tree_revision $PORTSDIR "imported ports tree revision $(ports_tree_get_current_revision $PORTSDIR)" 161 | 162 | # finalize 163 | finalize 164 | 165 | # remove temp file 166 | delete_temp_files 167 | 168 | # status report 169 | status_report 170 | 171 | exit 0 172 | -------------------------------------------------------------------------------- /include/functions-import.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | ## 6 | ## library of functions for the import module 7 | ## 8 | 9 | ## set strict mode 10 | STRICT="set -euo pipefail" 11 | $STRICT 12 | 13 | 14 | explain_action() { 15 | $STRICT 16 | echo "actions that will be performed:" 17 | if [ $PERFORM_ACTION_WRITE_DB = yes ]; then 18 | echo " - import the ports tree into the database $DB" 19 | fi 20 | if [ $PERFORM_ACTION_WRITE_SQL = yes ]; then 21 | echo " - import the ports tree into the SQL dump file $SQL_FILE_ARG" 22 | fi 23 | if [ -n "$SUBDIR" ]; then 24 | echo " (!) only the subdirectory '$SUBDIR' of the ports tree will be imported" 25 | fi 26 | } 27 | 28 | initialize() { 29 | $STRICT 30 | # begin the SQL file 31 | sql_file_begin "$SQL_FILE" 1 # with schema 32 | } 33 | 34 | import() { 35 | $STRICT 36 | local PD=$1 37 | local NOBUF="stdbuf -i0 -o0 -e0" 38 | 39 | ports_tree_traverse $PD "$SUBDIR" 2>&1 | 40 | $NOBUF grep "^===> " | 41 | $NOBUF sed -e 's|===> ||' | 42 | $NOBUF $CODEBASE/util/number-lines.sh | 43 | $NOBUF $CODEBASE/util/squeeze-log-into-line.sh 44 | } 45 | 46 | finalize() { 47 | $STRICT 48 | # end the SQL file 49 | cat $CODEBASE/sql/fix-default-parent-flavor.sql >> "$SQL_FILE" 50 | echo "-- end of file" >> "$SQL_FILE" 51 | 52 | # create DB from the bulk SQL 53 | if [ $PERFORM_ACTION_WRITE_DB = yes ]; then 54 | # SQL dump to DB 55 | rm -f "$DB" 56 | sqlite3 \ 57 | -cmd "PRAGMA journal_mode = MEMORY; PRAGMA synchronous = OFF;" \ 58 | "$DB" \ 59 | < "$SQL_FILE" 60 | 61 | # verify foreign keys 62 | db_check_fk_violations 63 | fi 64 | } 65 | 66 | status_report() { 67 | $STRICT 68 | echo "PortsDB has finished to import the ports tree at $(date "+%Y-%m-%d %H:%M:%S %Z (%z)") on host $(hostname)" 69 | 70 | if [ $PERFORM_ACTION_WRITE_DB = yes ]; then 71 | echo "- PortsDB created the SQLite database $DB" 72 | db_print_stats $DB 73 | fi 74 | [ $PERFORM_ACTION_WRITE_SQL = yes ] && 75 | echo "- PortsDB created the file with SQL statements to create the DB: $SQL_FILE" 76 | return 0 77 | } 78 | 79 | -------------------------------------------------------------------------------- /include/functions-sql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | 6 | ## 7 | ## functions (shared between low and high level) 8 | ## 9 | 10 | ## set strict mode 11 | STRICT="set -euo pipefail" 12 | $STRICT 13 | 14 | 15 | run_SQL() { 16 | $STRICT 17 | local SQL="$1" 18 | 19 | # save SQL statements into a file 20 | echo "$SQL;" >> "$SQL_FILE" 21 | } 22 | -------------------------------------------------------------------------------- /include/functions-update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | ## 6 | ## library of functions for the update module 7 | ## 8 | 9 | ## set strict mode 10 | STRICT="set -euo pipefail" 11 | $STRICT 12 | 13 | 14 | explain_action() { 15 | $STRICT 16 | echo "actions that will be performed:" 17 | if [ $PERFORM_ACTION_WRITE_DB = yes ]; then 18 | echo " - update database $DB with ports tree updates" 19 | fi 20 | if [ $PERFORM_ACTION_WRITE_SQL = yes ]; then 21 | echo " - import the ports tree updates into the SQL dump file $SQL_FILE_ARG" 22 | fi 23 | if [ -n "$SUBDIR" ]; then 24 | echo " (!) updated only for the subdirectory '$SUBDIR' of the ports tree will be processed" 25 | fi 26 | } 27 | 28 | initialize() { 29 | $STRICT 30 | # begin the SQL file 31 | sql_file_begin "$SQL_FILE" 0 # no schema 32 | } 33 | 34 | update() { 35 | $STRICT 36 | local PD=$1 37 | local SUBDIR=$1 38 | 39 | local old_revision new_revision pkgorigins 40 | old_revision=$(db_read_last_ports_tree_revision) 41 | new_revision=$(ports_tree_get_current_revision $PD) 42 | if [ -z "$old_revision" -o -z "$new_revision" ]; then 43 | fail "invalid revision" # this should never happen 44 | fi 45 | 46 | # any updates? 47 | if [ $new_revision = $old_revision ]; then 48 | echo "no updates were found (ports tree revision is $new_revision) - nothing to do" 49 | return 50 | fi 51 | 52 | # find all pkgorigins in delta 53 | pkgorigins="$(git_diff_revisions_to_pkgorigin $PD $old_revision $new_revision)" 54 | 55 | # limit how many pkgorigins we can do 56 | local num=1 57 | for po in $pkgorigins; do 58 | echo "pkgorigin[$num]=$po" 59 | num=$((num+1)) 60 | if [ $num = $PARAM_PORTSDB_UPDATE_LIMIT ]; then 61 | echo "TODO too many ($num) pkgorigins are updated - need to run full import" 62 | fi 63 | done 64 | num=$((num-1)) 65 | 66 | # report if there are no pkgorigin updates 67 | [ $num = 0 ] && 68 | echo "no pkgorigin updates were found" && 69 | echo "... between ports tree revisions $old_revision..$new_revision" && 70 | echo "... last revision in DB will still be updated" 71 | 72 | # update 73 | local n=1 74 | for po in $pkgorigins; do 75 | # delete previous records 76 | run_SQL "$(db_delete_pkgorigin_sql $po)" 77 | # insert new records 78 | if [ -d $PD/$po ]; then # otherwise this pkgorigin was removed 79 | echo "updating pkgorigin[$n of $num]=$po" 80 | ports_tree_traverse $PD $po > /dev/null 2>&1 || fail "failed to update of the pkgorigin[$n]=$po ... exiting" 81 | else 82 | echo "removing pkgorigin[$n of $num]=$po" 83 | fi 84 | n=$((n+1)) 85 | # count 86 | UPDATED_PKGORIGIN_COUNT=$((UPDATED_PKGORIGIN_COUNT+1)) 87 | done 88 | 89 | # set global value 90 | UPDATED=yes 91 | UPDATED_FROM_REVISION=$old_revision 92 | } 93 | 94 | finalize() { 95 | $STRICT 96 | # end the SQL file 97 | cat $CODEBASE/sql/fix-default-parent-flavor.sql >> "$SQL_FILE" 98 | echo "-- end of file" >> "$SQL_FILE" 99 | 100 | # create DB from the bulk SQL if SQL wasn't requested by the user 101 | if [ $PERFORM_ACTION_WRITE_DB = yes -a $UPDATED = yes ]; then 102 | # SQL dump to DB 103 | sqlite3 \ 104 | -cmd "PRAGMA journal_mode = MEMORY; PRAGMA synchronous = OFF;" \ 105 | "$DB" \ 106 | < "$SQL_FILE" 107 | 108 | # verify foreign keys 109 | db_check_fk_violations 110 | fi 111 | } 112 | 113 | status_report() { 114 | $STRICT 115 | [ $UPDATED = yes ] && echo "PortsDB has finished to update the ports tree at $(date "+%Y-%m-%d %H:%M:%S %Z (%z)") on host $(hostname)" 116 | 117 | if [ $PERFORM_ACTION_WRITE_DB = yes ]; then 118 | echo " - PortsDB updated $UPDATED_PKGORIGIN_COUNT $(plural_msg $UPDATED_PKGORIGIN_COUNT "pkgorigin" "pkgorigins") in the SQLite database $DB" 119 | db_print_stats $DB 120 | fi 121 | [ $PERFORM_ACTION_WRITE_SQL = yes ] && 122 | echo " - PortsDB wrote $UPDATED_PKGORIGIN_COUNT $(plural_msg $UPDATED_PKGORIGIN_COUNT "pkgorigin" "pkgorigins") updates into the SQL file $SQL_FILE_ARG" 123 | 124 | return 0 125 | } 126 | -------------------------------------------------------------------------------- /include/functions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | 6 | ## 7 | ## functions.sh module is a library of functions used by programs 8 | ## 9 | 10 | 11 | ## 12 | ## set strict mode 13 | ## 14 | 15 | STRICT="set -euo pipefail" 16 | $STRICT 17 | 18 | 19 | ## general functions 20 | 21 | fail() { 22 | $STRICT 23 | local msg="$1" 24 | [ -n "$msg" ] && echo $msg >&2 25 | exit 1 26 | } 27 | 28 | perror() { 29 | $STRICT 30 | local msg="$1" 31 | echo $msg >&2 32 | } 33 | 34 | is_ports_tree_directory() { 35 | $STRICT 36 | local PD=$1 37 | [ -f "$PD/Makefile" -a -f "$PD/Mk/bsd.port.mk" -a -d "$PD/.git" ] 38 | } 39 | 40 | plural_msg() { 41 | $STRICT 42 | local N=$1 43 | local STR_SINGLE="$2" 44 | local STR_PLURAL="$3" 45 | 46 | [ $((N%10)) = 1 -a $((N%100)) != 11 ] && echo $STR_SINGLE || echo $STR_PLURAL 47 | } 48 | 49 | check_dependencies() { 50 | $STRICT 51 | local res=0 52 | 53 | for dep in cat cp date false git grep gsed hostname make mktemp patch printf realpath rm sed sha256 sort sqlite3 sysctl true uniq wc xargs [; do 54 | if ! which -s $dep; then 55 | perror "error: $dep dependency is missing" 56 | res=1 57 | fi 58 | done 59 | 60 | return $res 61 | } 62 | 63 | announcement() { 64 | $STRICT 65 | local action="$1" 66 | echo "PortsDB is $action the ports tree at $(date "+%Y-%m-%d %H:%M:%S %Z (%z)") on host $(hostname)" 67 | } 68 | 69 | make_file_path_global() { 70 | $STRICT 71 | local path="$1" 72 | 73 | case "$path" in 74 | /*) 75 | # do nothing 76 | echo "$path" 77 | ;; 78 | *) 79 | # make path global 80 | echo "$(pwd)/$path" 81 | ;; 82 | esac 83 | } 84 | 85 | patch_ports_tree() { 86 | $STRICT 87 | local PD_ORIGINAL="$1" 88 | local PD_PATCHED 89 | 90 | # generate temp file name 91 | PD_PATCHED=$(mktemp -d -t portsdir) 92 | 93 | # copy 94 | $CODEBASE/util/copy-tree.sh $PD_ORIGINAL $PD_PATCHED || fail "failed to copy the ports tree $PD_ORIGINAL -> $PD_PATCHED" 95 | 96 | # patch 97 | (cd $PD_PATCHED && patch -p 1 --quiet < $CODEBASE/patches/Mk-portsdb.patch >&2) || fail "failed to patch the ports tree" 98 | 99 | # done 100 | echo $PD_PATCHED 101 | } 102 | 103 | effective_ports_tree() { 104 | $STRICT 105 | local PD=$1 106 | 107 | if [ $PARAM_PORTSTREE_NEEDS_PATCHING = yes ]; then 108 | patch_ports_tree $PD 109 | else 110 | echo $PD 111 | fi 112 | } 113 | 114 | describe_command() { 115 | $STRICT 116 | # build DESCRIBE_COMMAND for 'make describe' 117 | local cmd_args="" # args to supply to add-port.sh 118 | for name in \ 119 | FLAVOR PKGORIGIN PORTNAME PORTVERSION DISTVERSION DISTVERSIONPREFIX DISTVERSIONSUFFIX PORTREVISION \ 120 | PORTEPOCH CATEGORIES \ 121 | MAINTAINER WWW \ 122 | CPE_STR \ 123 | COMPLETE_OPTIONS_LIST OPTIONS_DEFAULT \ 124 | FLAVORS \ 125 | COMMENT PKGBASE PKGNAME PKGNAMESUFFIX USES \ 126 | PKG_DEPENDS FETCH_DEPENDS EXTRACT_DEPENDS PATCH_DEPENDS BUILD_DEPENDS LIB_DEPENDS RUN_DEPENDS TEST_DEPENDS \ 127 | USE_GITHUB GH_ACCOUNT GH_PROJECT GH_TAGNAME \ 128 | USE_GITLAB GL_SITE GL_ACCOUNT GL_PROJECT GL_COMMIT \ 129 | DEPRECATED EXPIRATION_DATE \ 130 | BROKEN \ 131 | .MAKE.MAKEFILES ; \ 132 | do 133 | if [ $name = "COMMENT" -o $name = "DEPRECATED" -o $name = "BROKEN" ]; then 134 | cmd_args="$cmd_args '@@@{$name:S/\\@@@/%%DOLLAR%%/g}'" 135 | else 136 | cmd_args="$cmd_args '@@@{$name}'" 137 | fi 138 | done 139 | 140 | echo "$CODEBASE/add-port.sh '$DB' $cmd_args" 141 | } 142 | 143 | ## DB-related functions 144 | 145 | db_create() { 146 | $STRICT 147 | rm -f "$DB" 148 | sqlite3 "$DB" < $CODEBASE/sql/schema.sql 149 | } 150 | 151 | db_validate() { 152 | $STRICT 153 | local DB="$1" 154 | [ -f "$DB" ] && sqlite3 "$DB" "PRAGMA integrity_check;" > /dev/null 2>&1 155 | } 156 | 157 | db_delete_pkgorigin_sql() { 158 | $STRICT 159 | local pkgorigin=$1 160 | local SQL="" 161 | SQL="${SQL}DELETE FROM MakefileDependencies WHERE PKGORIGIN='$pkgorigin';\n" 162 | SQL="${SQL}DELETE FROM Broken WHERE PKGORIGIN='$pkgorigin';\n" 163 | SQL="${SQL}DELETE FROM Deprecated WHERE PKGORIGIN='$pkgorigin';\n" 164 | SQL="${SQL}DELETE FROM GitLab WHERE PKGORIGIN='$pkgorigin';\n" 165 | SQL="${SQL}DELETE FROM GitHub WHERE PKGORIGIN='$pkgorigin';\n" 166 | SQL="${SQL}DELETE FROM Depends WHERE CHILD_PKGORIGIN='$pkgorigin';\n" 167 | SQL="${SQL}DELETE FROM PortFlavor WHERE PKGORIGIN='$pkgorigin';\n" 168 | SQL="${SQL}DELETE FROM Port WHERE PKGORIGIN='$pkgorigin';\n" 169 | printf "$SQL" 170 | } 171 | 172 | db_read_last_ports_tree_revision() { 173 | $STRICT 174 | sqlite3 "$DB" "SELECT GIT_HASH FROM RevisionLog ORDER BY UPDATE_TIMESTAMP DESC LIMIT 1;" 175 | } 176 | 177 | db_get_fk_violations_count() { 178 | $STRICT 179 | sqlite3 "$DB" "PRAGMA foreign_key_check;" | wc -l | sed -e 's| ||g' 180 | } 181 | 182 | db_check_fk_violations() { 183 | $STRICT 184 | local violations 185 | violations=$(db_get_fk_violations_count) 186 | 187 | [ $violations != 0 ] && 188 | echo "warning: database has $violations foreign key $(plural_msg $violations "violation" "violations")" && 189 | echo "info: foreign key violations are most likely due to missing flavors in some Python ports" 190 | } 191 | 192 | db_print_stats() { 193 | $STRICT 194 | local DB="$1" 195 | echo "... the database has:" 196 | echo "... - $(sqlite3 $DB 'SELECT count(*) FROM Port;') port records" 197 | echo "... - $(sqlite3 $DB 'SELECT count(*) FROM PortFlavor;') flavor records" 198 | echo "... - $(sqlite3 $DB 'SELECT count(*) FROM Depends;') dependency records" 199 | echo "... - $(sqlite3 $DB 'SELECT count(*) FROM GitHub;') GitHub records" 200 | echo "... - $(sqlite3 $DB 'SELECT count(*) FROM GitLab;') GitLab records" 201 | echo "... - $(sqlite3 $DB 'SELECT count(*) FROM Deprecated;') Deprecated records" 202 | echo "... - $(sqlite3 $DB 'SELECT count(*) FROM Broken;') Broken records" 203 | } 204 | 205 | ## ports tree revision handling functions 206 | 207 | ports_tree_get_current_revision() { 208 | $STRICT 209 | local PD=$1 210 | 211 | (cd $PD && git rev-parse HEAD) \ 212 | || fail "git failed while getting current revision" 213 | } 214 | 215 | ports_tree_traverse() { 216 | $STRICT 217 | local PD=$1 218 | local SUBDIR=$2 219 | 220 | (cd $PD && PORTSDIR=$PD make -I $PD -C $PD/$SUBDIR describe DESCRIBE_COMMAND="$(describe_command)" -j $(sysctl -n hw.ncpu)) \ 221 | || fail "ports tree traversal failed" 222 | } 223 | 224 | save_ports_tree_revision() { 225 | $STRICT 226 | local revision=$1 227 | local comment="$2" 228 | 229 | run_SQL "INSERT into RevisionLog(UPDATE_TIMESTAMP, GIT_HASH, COMMENT) VALUES(DATETIME('now'), '$revision', '$comment');" 230 | } 231 | 232 | write_ports_tree_revision() { 233 | $STRICT 234 | local PD=$1 235 | local comment="$2" 236 | local revision 237 | 238 | revision=$(ports_tree_get_current_revision $PD) 239 | save_ports_tree_revision $revision "$comment" 240 | } 241 | 242 | 243 | ## git-related 244 | 245 | git_diff_revisions_to_pkgorigin() { # returns list of changed pkgorigins between two given revisions 246 | $STRICT 247 | local PD=$1 248 | local rev1="$2" 249 | local rev2="$3" 250 | 251 | local subdir_term="" 252 | [ -n "$SUBDIR" -a $SUBDIR != . ] && subdir_term="-- $SUBDIR" 253 | 254 | (cd $PD && 255 | git diff-tree --no-commit-id --name-only -r $rev1..$rev2 $subdir_term | 256 | (grep -E "^[^/]+/[^/]+/.*" || true) | 257 | (sed -E "s|^([^/]+/[^/]+)/.*|\1|" || true) | 258 | (grep -v "^Mk/" || true) | 259 | sort | 260 | uniq 261 | ) || fail "git_diff_revisions_to_pkgorigin pipe failed" 262 | } 263 | 264 | ## SQL file handling 265 | 266 | sql_file_begin() { 267 | $STRICT 268 | local sql_file="$1" 269 | local with_schema="$2" 270 | 271 | ( 272 | echo "--" 273 | echo "-- SQL dump of the PortsDB" 274 | echo "-- - PORTSDIR=$PORTSDIR" 275 | echo "-- - SUBDIR=$SUBDIR" 276 | echo "--" 277 | echo "" 278 | 279 | if [ $with_schema = 1 ]; then 280 | echo "-- schema" 281 | cat $CODEBASE/sql/schema.sql 282 | echo "" 283 | fi 284 | 285 | echo "-- insert statements" 286 | ) > "$sql_file" 287 | } 288 | 289 | ## temporary file handling 290 | 291 | delete_temp_files() { 292 | $STRICT 293 | local tmp_files="" 294 | 295 | if [ $PORTSDIR_EFFECTIVE != $PORTSDIR ]; then 296 | tmp_files="$tmp_files $PORTSDIR_EFFECTIVE" 297 | fi 298 | if [ -z "$SQL_FILE_ARG" -a -n "$SQL_FILE" ]; then 299 | tmp_files="$tmp_files $SQL_FILE" 300 | fi 301 | 302 | for f in $tmp_files; do 303 | rm -rf $f 304 | done 305 | } 306 | -------------------------------------------------------------------------------- /params.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | set -euo pipefail 6 | 7 | 8 | PARAM_PORTSTREE_NEEDS_PATCHING=yes 9 | PARAM_PORTSDB_UPDATE_LIMIT=1000000 # number of updated pkgorigins that triggers import instead of update (TODO) 10 | -------------------------------------------------------------------------------- /patches/Mk-portsdb.patch: -------------------------------------------------------------------------------- 1 | diff --git a/Mk/bsd.port.mk b/Mk/bsd.port.mk 2 | index 1600ed176739..cf10d5e871dc 100644 3 | --- a/Mk/bsd.port.mk 4 | +++ b/Mk/bsd.port.mk 5 | @@ -4287,6 +4287,10 @@ create-manifest: 6 | ${PKG_NOTES_ENV} \ 7 | ${SH} ${SCRIPTSDIR}/create-manifest.sh 8 | 9 | +.if defined(DESCRIBE_COMMAND) 10 | +_DESCRIBE_COMMAND_EXPANDED_1:= ${DESCRIBE_COMMAND:S/'/%%%/g:S/@@@/\$/g} # shadow quotes in the supplied command, substitute dollar signs 11 | +_DESCRIBE_COMMAND_EXPANDED:= ${_DESCRIBE_COMMAND_EXPANDED_1:S/'/'\''/g:S/`/\`/g:S/(/\(/g:S/)/\)/g:S/"/\"/g:S/%%%/'/g} # escape quotes, braces in the expanded strings, restore shadowed quotes from the supplied command 12 | +.endif 13 | 14 | # Print out package names. 15 | 16 | @@ -4361,7 +4365,11 @@ INDEX_OUT=/dev/stdout 17 | 18 | . if empty(FLAVORS) || defined(_DESCRIBE_WITH_FLAVOR) 19 | describe: 20 | +. if !defined(DESCRIBE_COMMAND) 21 | @(${ECHO_CMD} "${PKGNAME}|${.CURDIR}|${PREFIX}|"${COMMENT:Q}"|${_DESCR}|${MAINTAINER}|${CATEGORIES}|${_EXTRACT_DEPENDS}|${_PATCH_DEPENDS}|${_FETCH_DEPENDS}|${_BUILD_DEPENDS:O:u}|${_RUN_DEPENDS:O:u}|${_WWW}" >> ${INDEX_OUT}) 22 | +. else 23 | + @${_DESCRIBE_COMMAND_EXPANDED} 24 | +. endif 25 | . else # empty(FLAVORS) 26 | describe: ${FLAVORS:S/^/describe-/} 27 | . for f in ${FLAVORS} 28 | -------------------------------------------------------------------------------- /periodic/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | STRICT="set -euo pipefail" 4 | $STRICT 5 | 6 | ## 7 | ## run.sh is a script that updates PortsDB that can be periodically 8 | ## run by the cron daemon. It expects that 9 | ## 1. FreeBSD ports are cloned into the 'ports' subdirectory 10 | ## 2. the user would run the command 'PORTSDB=ports import.sh' 11 | ## 3. two variables below are filled in by the user 12 | ## 13 | ## run.sh performs following actions: 14 | ## 1. updates ports.sqlite in the same folder where it is run 15 | ## 2. uploads ports.sqlite with the user-supplied command 16 | ## 3. maintains the log file portsdb.log 17 | ## 18 | 19 | ## 20 | ## user-supplied commands 21 | ## 22 | 23 | PORTDSB_UPDATE_CMD="" # please add the path to PortDB's update.sh 24 | UPLOAD_CMD="" # please add the command that uploads the DB 25 | 26 | ## 27 | ## global values 28 | ## 29 | 30 | PORTSDIR=ports 31 | 32 | ## 33 | ## checks 34 | ## 35 | 36 | if [ -z "$PORTDSB_UPDATE_CMD" -o -z "$UPLOAD_CMD" ]; then 37 | echo "error: please define PORTDSB_UPDATE_CMD and UPLOAD_CMD" 38 | exit 1 39 | 40 | fi 41 | 42 | for dep in date sha256 sqlite3 git gsed cat sysctl xz; do 43 | if ! which -s $dep; then 44 | echo "error: dependency '$dep' is missing" 45 | exit 1 46 | fi 47 | done 48 | 49 | if ! [ -d $PORTSDIR ]; then 50 | echo "error: ports tree has to be present (please check it out with 'git clone https://git.FreeBSD.org/ports.git')" 51 | exit 1 52 | fi 53 | 54 | if ! [ -f ports.sqlite ]; then 55 | echo "error: ports.sqlite has to be present (please build it with 'PORTSDIR=ports update.sh')" 56 | exit 1 57 | fi 58 | 59 | ## 60 | ## functions 61 | ## 62 | 63 | git_last_commit_hash() { 64 | $STRICT 65 | (cd $PORTSDIR && git log -n1 --format=format:"%H") 66 | } 67 | 68 | db_hash() { 69 | $STRICT 70 | sha256 -q ports.sqlite 71 | } 72 | 73 | timestamp() { 74 | $STRICT 75 | local location=$1 76 | echo "timestamp($location, pid=$$): $(date "+%Y-%m-%d %H:%M:%S")" 77 | } 78 | 79 | fail() { 80 | $STRICT 81 | local msg="$1" 82 | 83 | timestamp "error" 84 | echo "******ERROR: $msg******" >&2 85 | 86 | exit 1 87 | } 88 | 89 | ## 90 | ## MAIN 91 | ## 92 | 93 | ( 94 | # start 95 | echo "" 96 | 97 | # variables 98 | ANY_UPDATES=no 99 | 100 | # timestamp 101 | timestamp "begin" 102 | 103 | # pull (remote ports repo -> local ports repo) 104 | (cd $PORTSDIR && git pull) > git-pull.log 2>git-pull.err || fail "git pull failed: $(cat git-pull.err)" 105 | if [ "$(cat git-pull.log)" = "Already up to date." ]; then 106 | echo "no updates: nothing to import into PortsDB" 107 | elif [ -z "$(cat git-pull.log)" ]; then # no exit code and no printout isn't normal for git 108 | if [ -n "$(cat git-pull.err)" ]; then 109 | fail "git pull failed: 'git pull' exited with no error code, no stdout printout, and this error: $(cat git-pull.err)" 110 | else 111 | fail "git pull failed: 'git pull' exited with no error code, no stdout or stderr printout" 112 | fi 113 | else 114 | ANY_UPDATES=yes 115 | fi 116 | 117 | # report git log 118 | if [ $ANY_UPDATES = yes ]; then 119 | echo "---begin git log---" 120 | cat git-pull.log 121 | echo "---end git log---" 122 | fi 123 | 124 | # update the DB (local ports repo -> DB) 125 | if [ $ANY_UPDATES = yes ] || ! [ -f repo.commit.sha256 ] || [ "$(cat repo.commit.sha256)" != $(git_last_commit_hash) ]; then 126 | # save DB hash 127 | DB_HASH=$(db_hash) 128 | # actual update 129 | (PORTSDIR=$PORTSDIR $PORTDSB_UPDATE_CMD) 2>update.err || fail "update command failed: $(cat update.err)" 130 | if [ $(db_hash) = $DB_HASH ]; then 131 | echo "no updates: git commits didn't update any pkgorigins" 132 | ANY_UPDATES=no 133 | fi 134 | # save last commit's hash that's in DB 135 | git_last_commit_hash > repo.commit.sha256 136 | # compress DB 137 | xz -T 0 -9 < ports.sqlite > ports.sqlite.xz 138 | fi 139 | 140 | # upload (DB -> file hosting) 141 | if [ $ANY_UPDATES = yes ] || ! [ -f ports.sqlite.sha256 ] || [ "$(cat ports.sqlite.sha256)" != $(db_hash) ]; then 142 | echo "uploading ports.sqlite with sha256=$(db_hash) ..." 143 | # actual upload 144 | ($UPLOAD_CMD ports.sqlite.xz) > upload.log 2>upload.err || fail "upload command failed: $(cat upload.err)" 145 | # save last uploaded DB hash 146 | db_hash > ports.sqlite.sha256 147 | fi 148 | 149 | # timestamp 150 | timestamp "end" 151 | ) >> portsdb.log 2>&1 152 | -------------------------------------------------------------------------------- /run-sql: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | ## 6 | ## run-sql allows users to run prepared SQL statements against PortsDB 7 | ## 8 | 9 | ## 10 | ## set strict mode 11 | ## 12 | 13 | set -euo pipefail 14 | 15 | ## 16 | ## find CODEBASE 17 | ## 18 | 19 | SCRIPT=$(readlink -f "$0") 20 | CODEBASE=$(dirname "$SCRIPT") 21 | 22 | ## 23 | ## include functions 24 | ## 25 | 26 | . $CODEBASE/include/functions.sh 27 | 28 | ## 29 | ## find PortsDB 30 | ## 31 | 32 | PORTSDB=${PORTSDB:-./ports.sqlite} # default value 33 | [ -f "$PORTSDB" ] || PORTSDB=$CODEBASE/ports.sqlite 34 | 35 | [ -f $PORTSDB ] || echo "can't find PortsDB at '$PORTSDB', please set the PORTSDB environment variable" 36 | 37 | ## 38 | ## arguments 39 | ## 40 | 41 | SQL_FILE="$1" 42 | 43 | [ -f "$SQL_FILE" ] || fail "supplied argument '$SQL_FILE' isn't a file" 44 | 45 | ## 46 | ## substitute arguments if any 47 | ## 48 | 49 | SQL=$(cat $SQL_FILE) 50 | shift 51 | SQL="$(printf "$SQL" "$@")" 52 | 53 | ## 54 | ## execute query 55 | ## 56 | 57 | sqlite3 $PORTSDB "$SQL" 58 | -------------------------------------------------------------------------------- /sql-misc-queries/find-broken-ports-maintained-by-{user}.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | P.PKGORIGIN, 3 | B.FLAVOR, 4 | B.BROKEN 5 | FROM 6 | Port P, 7 | Broken B 8 | WHERE 9 | MAINTAINER = '%s' 10 | AND 11 | B.PKGORIGIN = P.PKGORIGIN 12 | -------------------------------------------------------------------------------- /sql-misc-queries/find-distinct-projects.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | CASE 3 | WHEN P.PKGORIGIN LIKE '%%'||PF.PKGNAMESUFFIX THEN substr(PF.PKGORIGIN, 0, length(PF.PKGORIGIN) - length(PF.PKGNAMESUFFIX) + 1) 4 | ELSE PF.PKGORIGIN 5 | END as PO 6 | FROM 7 | Port P, 8 | PortFlavor PF 9 | WHERE 10 | PF.PKGORIGIN = P.PKGORIGIN 11 | GROUP BY 12 | PO 13 | -------------------------------------------------------------------------------- /sql-misc-queries/find-duplicate-ports.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- finds duplicate ports with names that are same with case insensitive comparison 3 | -- 4 | 5 | -- 6 | -- SLOW query because it can't use the Port.PKGORIGIN index 7 | -- 8 | 9 | SELECT 10 | * 11 | FROM 12 | Port P 13 | WHERE 14 | EXISTS( 15 | SELECT 16 | * 17 | FROM 18 | Port PI 19 | WHERE -- names are same when compared w/out case 20 | PI.PKGORIGIN <> P.PKGORIGIN 21 | AND 22 | LOWER(PI.PKGORIGIN) = LOWER(P.PKGORIGIN) 23 | ) 24 | -------------------------------------------------------------------------------- /sql-misc-queries/find-ports-depending-on-{pkgorigin}.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | CHILD_PKGORIGIN, 3 | KIND 4 | FROM 5 | Depends 6 | WHERE 7 | PARENT_PKGORIGIN = '%s' 8 | -------------------------------------------------------------------------------- /sql-misc-queries/find-ports-maintained-by-{user}-with-missing-built-packages.sql: -------------------------------------------------------------------------------- 1 | ATTACH '/var/db/pkg/repo-FreeBSD.sqlite' AS REPO; 2 | 3 | SELECT 4 | PKGORIGIN 5 | FROM 6 | Port 7 | WHERE 8 | MAINTAINER = '%s' 9 | AND 10 | NOT EXISTS( 11 | SELECT * FROM REPO.packages WHERE origin = Port.PKGORIGIN 12 | ) 13 | -------------------------------------------------------------------------------- /sql-misc-queries/find-ports-maintained-by-{user}-with-{uses}.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | PKGORIGIN 3 | FROM 4 | Port P 5 | WHERE 6 | MAINTAINER = '%s' 7 | AND 8 | EXISTS( 9 | SELECT 10 | * 11 | FROM 12 | PortFlavor PF 13 | WHERE 14 | PF.PKGORIGIN = P.PKGORIGIN 15 | AND 16 | USES like '%%%s%%' 17 | ) 18 | -------------------------------------------------------------------------------- /sql-misc-queries/find-ports-maintained-by-{user}.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | PKGORIGIN 3 | FROM 4 | Port 5 | WHERE 6 | MAINTAINER = '%s' 7 | -------------------------------------------------------------------------------- /sql-misc-queries/find-ports-that-have-test-depends.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | P.PKGORIGIN 3 | FROM 4 | Port P 5 | WHERE 6 | EXISTS( 7 | SELECT 8 | * 9 | FROM 10 | Depends D 11 | WHERE 12 | D.CHILD_PKGORIGIN = P.PKGORIGIN 13 | AND 14 | D.KIND = 'T' 15 | ) 16 | -------------------------------------------------------------------------------- /sql-misc-queries/find-ports-with-missing-built-packages.sql: -------------------------------------------------------------------------------- 1 | ATTACH '/var/db/pkg/repo-FreeBSD.sqlite' AS REPO; 2 | 3 | SELECT 4 | PKGORIGIN 5 | FROM 6 | Port 7 | WHERE 8 | NOT EXISTS( 9 | SELECT * FROM REPO.packages WHERE origin = Port.PKGORIGIN 10 | ) 11 | -------------------------------------------------------------------------------- /sql-misc-queries/find-ports-with-{string}-in-comment.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | PKGORIGIN, 3 | FLAVOR, 4 | COMMENT 5 | FROM 6 | PortFlavor 7 | WHERE 8 | COMMENT like '%%%s%%' 9 | -------------------------------------------------------------------------------- /sql-misc-queries/get-ports-list.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | PKGORIGIN 3 | FROM 4 | Port 5 | -------------------------------------------------------------------------------- /sql/fix-default-parent-flavor.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Fix Depends.PARENT_FLAVOR when depending on non-empty default flavor 3 | -- 4 | 5 | UPDATE 6 | Depends 7 | SET 8 | PARENT_FLAVOR=(SELECT FLAVOR from PortFlavor WHERE PKGORIGIN=Depends.PARENT_PKGORIGIN ORDER BY RowId ASC LIMIT 1) -- default flavor 9 | WHERE 10 | PARENT_FLAVOR='' 11 | AND 12 | EXISTS (SELECT * from PortFlavor PF WHERE PKGORIGIN=Depends.PARENT_PKGORIGIN AND PF.FLAVOR!='') 13 | ; 14 | -------------------------------------------------------------------------------- /sql/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE Port( 2 | PKGORIGIN TEXT NOT NULL, 3 | PORTNAME TEXT NOT NULL, 4 | PORTVERSION TEXT NOT NULL, 5 | DISTVERSION TEXT NOT NULL, 6 | DISTVERSIONPREFIX TEXT NULL, 7 | DISTVERSIONSUFFIX TEXT NULL, 8 | PORTREVISION INTEGER NULL, 9 | PORTEPOCH INTEGER NULL, 10 | CATEGORIES TEXT NOT NULL, 11 | MAINTAINER TEXT NOT NULL, 12 | WWW TEXT NULL, 13 | CPE_STR TEXT NULL, 14 | COMPLETE_OPTIONS_LIST TEXT NULL, 15 | OPTIONS_DEFAULT TEXT NULL, 16 | FLAVORS TEXT NULL, 17 | PRIMARY KEY (PKGORIGIN) 18 | ); 19 | CREATE TABLE PortFlavor( 20 | PKGORIGIN TEXT NOT NULL, 21 | FLAVOR TEXT NOT NULL, 22 | COMMENT TEXT NOT NULL, 23 | PKGBASE TEXT NOT NULL, 24 | PKGNAME TEXT NOT NULL, 25 | PKGNAMESUFFIX TEXT NULL, 26 | USES TEXT NULL, 27 | PRIMARY KEY (PKGORIGIN, FLAVOR), 28 | FOREIGN KEY (PKGORIGIN) REFERENCES Port (PKGORIGIN) 29 | ); 30 | CREATE TABLE Depends( 31 | PARENT_PKGORIGIN TEXT NOT NULL, 32 | PARENT_FLAVOR TEXT NOT NULL, 33 | PARENT_PHASE TEXT NULL, -- there can't be multiple phases for the same combination of other fields, so it isn't included in PK 34 | CHILD_PKGORIGIN TEXT NOT NULL, 35 | CHILD_FLAVOR TEXT NOT NULL, 36 | KIND CHAR NOT NULL, 37 | PRIMARY KEY (PARENT_PKGORIGIN, PARENT_FLAVOR, CHILD_PKGORIGIN, CHILD_FLAVOR, KIND), 38 | FOREIGN KEY (PARENT_PKGORIGIN, PARENT_FLAVOR) REFERENCES PortFlavor(PKGORIGIN, FLAVOR), 39 | FOREIGN KEY (CHILD_PKGORIGIN, CHILD_FLAVOR) REFERENCES PortFlavor(PKGORIGIN, FLAVOR) 40 | ); 41 | CREATE TABLE GitHub( 42 | PKGORIGIN TEXT NOT NULL, 43 | FLAVOR TEXT NOT NULL, 44 | USE_GITHUB TEXT NOT NULL, 45 | GH_ACCOUNT TEXT NOT NULL, 46 | GH_PROJECT TEXT NOT NULL, 47 | GH_TAGNAME TEXT NOT NULL, 48 | PRIMARY KEY (PKGORIGIN, FLAVOR), 49 | FOREIGN KEY (PKGORIGIN, FLAVOR) REFERENCES PortFlavor (PKGORIGIN, FLAVOR) 50 | ); 51 | CREATE TABLE GitLab( 52 | PKGORIGIN TEXT NOT NULL, 53 | FLAVOR TEXT NOT NULL, 54 | USE_GITLAB TEXT NOT NULL, 55 | GL_SITE TEXT NOT NULL, 56 | GL_ACCOUNT TEXT NOT NULL, 57 | GL_PROJECT TEXT NOT NULL, 58 | GL_COMMIT TEXT NULL, 59 | PRIMARY KEY (PKGORIGIN, FLAVOR), 60 | FOREIGN KEY (PKGORIGIN, FLAVOR) REFERENCES PortFlavor (PKGORIGIN, FLAVOR) 61 | ); 62 | CREATE TABLE Deprecated( 63 | PKGORIGIN TEXT NOT NULL, 64 | FLAVOR TEXT NOT NULL, 65 | DEPRECATED TEXT NOT NULL, 66 | EXPIRATION_DATE TEXT NULL, 67 | PRIMARY KEY (PKGORIGIN, FLAVOR) 68 | ); 69 | CREATE TABLE Broken( 70 | PKGORIGIN TEXT NOT NULL, 71 | FLAVOR TEXT NOT NULL, 72 | BROKEN TEXT NOT NULL, 73 | PRIMARY KEY (PKGORIGIN, FLAVOR) 74 | ); 75 | CREATE TABLE MakefileDependencies( 76 | PKGORIGIN TEXT NOT NULL, 77 | FLAVOR TEXT NOT NULL, 78 | MAKEFILE TEXT NOT NULL, 79 | PRIMARY KEY (PKGORIGIN, FLAVOR, MAKEFILE), 80 | FOREIGN KEY (PKGORIGIN, FLAVOR) REFERENCES PortFlavor (PKGORIGIN, FLAVOR) 81 | ); 82 | CREATE TABLE RevisionLog( 83 | UPDATE_TIMESTAMP TEXT NOT NULL, 84 | GIT_HASH TEXT NOT NULL, 85 | COMMENT TEXT NOT NULL, 86 | PRIMARY KEY (UPDATE_TIMESTAMP) 87 | ); 88 | 89 | CREATE INDEX Port_ByPortname ON Port(PORTNAME); 90 | CREATE INDEX Port_ByMaintaner ON Port(MAINTAINER); 91 | CREATE INDEX Depends_ByChild ON Depends(CHILD_PKGORIGIN, CHILD_FLAVOR); 92 | CREATE INDEX GitHub_ByAccountProject ON GitHub(GH_ACCOUNT, GH_PROJECT); 93 | CREATE INDEX GitLab_ByAccountProject ON GitLab(GL_ACCOUNT, GL_PROJECT); 94 | CREATE INDEX Deprecated_ByExpirationDate ON Deprecated(EXPIRATION_DATE); 95 | CREATE INDEX MakefileDependencies_ByMakefile ON MakefileDependencies(MAKEFILE); 96 | -------------------------------------------------------------------------------- /update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | 6 | ## 7 | ## update.sh is a script that 8 | ## updates the PortsDB SQLite database 9 | ## and adds all ports from the ports tree 10 | ## specified in the PORTSDIR environment 11 | ## variable. 12 | ## 13 | ## 14 | ## Arguments: 15 | ## - DB: (optional) database file to be updated (default: ports.sqlite) 16 | ## - SQL_FILE: (optional) file to write the SQL dump from which the database can be recreated 17 | ## 18 | ## Environment variables: 19 | ## - PORTSDIR: (optional) ports tree to build the database for (default: /usr/ports) 20 | ## - SUBDIR: (optional, DEBUG) choose a subdir in the ports tree, generally produces a broken DB with foreign key violations 21 | ## 22 | 23 | ## 24 | ## set strict mode 25 | ## 26 | 27 | STRICT="set -euo pipefail" 28 | $STRICT 29 | 30 | ## 31 | ## find CODEBASE 32 | ## 33 | 34 | SCRIPT=$(readlink -f "$0") 35 | CODEBASE=$(dirname "$SCRIPT") 36 | 37 | ## 38 | ## include function libraries and read parameters 39 | ## 40 | 41 | . $CODEBASE/include/functions.sh 42 | . $CODEBASE/include/functions-update.sh 43 | . $CODEBASE/include/functions-sql.sh 44 | . $CODEBASE/params.sh 45 | 46 | ## 47 | ## read arguments and set defaults 48 | ## 49 | 50 | DB=${1:-ports.sqlite} # write the SQLite DB 51 | SQL_FILE=${2-} # save SQL statements into this file, if set 52 | SQL_FILE_ARG="${SQL_FILE}" 53 | 54 | ## 55 | ## read env variables 56 | ## 57 | 58 | PORTSDIR=${PORTSDIR:-/usr/ports} # default value 59 | SUBDIR=${SUBDIR-} 60 | 61 | ## 62 | ## global variables 63 | ## 64 | 65 | PORTSDIR_EFFECTIVE="" 66 | UPDATED=no 67 | UPDATED_PKGORIGIN_COUNT=0 68 | UPDATED_FROM_REVISION="" 69 | PERFORM_ACTION_WRITE_DB=no 70 | PERFORM_ACTION_WRITE_SQL=no 71 | 72 | ## 73 | ## check dependencies 74 | ## 75 | 76 | check_dependencies || fail "" 77 | 78 | ## 79 | ## usage 80 | ## 81 | 82 | usage() { 83 | $STRICT 84 | fail "Usage: $0 [{sync|async}]" 85 | } 86 | 87 | ## 88 | ## what do we do 89 | ## 90 | 91 | if [ -z "$SQL_FILE_ARG" ]; then # update.sh only writes either DB or SQL, but not both 92 | PERFORM_ACTION_WRITE_DB=yes 93 | else 94 | PERFORM_ACTION_WRITE_FILE=yes 95 | fi 96 | 97 | ## 98 | ## check arguments and required enviroment values 99 | ## 100 | 101 | if ! is_ports_tree_directory $PORTSDIR; then 102 | perror "error: the PORTSDIR environment variable should point to a valid ports tree" 103 | usage 104 | fi 105 | 106 | if [ -n "$SUBDIR" ] && ! [ -f "$PORTSDIR/$SUBDIR/Makefile" ]; then 107 | echo "error: the SUBDIR environment variable is expected to point to a valid subdirectory in the ports tree" 108 | usage 109 | fi 110 | 111 | # database will be written after traversing the ports tree 112 | if [ -z "$SQL_FILE" ]; then 113 | # generate temporary SQL file if not provided by the user 114 | SQL_FILE=$(mktemp -t ports.sql) 115 | fi 116 | 117 | ## 118 | ## adjust values 119 | ## 120 | 121 | PORTSDIR=$(make_file_path_global $PORTSDIR) 122 | if [ -n "$DB" ]; then 123 | DB=$(make_file_path_global "$DB") 124 | fi 125 | SQL_FILE=$(make_file_path_global "$SQL_FILE") 126 | 127 | ## 128 | ## save arguments and other values in environment 129 | ## 130 | 131 | export DB 132 | export SQL_FILE 133 | export CODEBASE 134 | 135 | ## 136 | ## MAIN 137 | ## 138 | 139 | # announcement 140 | announcement "starting to update" 141 | 142 | # explain 143 | explain_action 144 | 145 | # validate DB 146 | db_validate "$DB" || fail "error: DB file '$DB' doesn't exist or isn't a valid SQLite database file" 147 | 148 | # initialize 149 | initialize 150 | 151 | # update 152 | PORTSDIR_EFFECTIVE=$(effective_ports_tree $PORTSDIR) 153 | update $PORTSDIR_EFFECTIVE "$SUBDIR" 154 | 155 | # save Git revision of the ports tree 156 | [ $UPDATED = yes ] && 157 | write_ports_tree_revision \ 158 | $PORTSDIR \ 159 | "updated ports tree for revisions $UPDATED_FROM_REVISION..$(ports_tree_get_current_revision $PORTSDIR), $UPDATED_PKGORIGIN_COUNT $(plural_msg $UPDATED_PKGORIGIN_COUNT "pkgorigin was" "pkgorigins were") affected" 160 | 161 | # finalize 162 | finalize 163 | 164 | # remove temp file 165 | delete_temp_files 166 | 167 | # status report 168 | [ $UPDATED = yes ] && status_report 169 | 170 | exit 0 171 | -------------------------------------------------------------------------------- /util/copy-tree.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | set -euo pipefail 6 | 7 | 8 | PORTSDIR=$1 9 | NEW_TREE=$2 # assume that the $NEW_TREE directory exists 10 | 11 | for d in $(cd $PORTSDIR && ls); do 12 | if [ -d "$PORTSDIR/$d" -a -f "$PORTSDIR/$d/Makefile" ]; then 13 | ln -s $PORTSDIR/$d $NEW_TREE/$d 14 | fi 15 | done 16 | 17 | ln -s $PORTSDIR/Makefile $NEW_TREE/Makefile 18 | cp -r $PORTSDIR/Mk $NEW_TREE/Mk 19 | ln -s $PORTSDIR/.git $NEW_TREE/.git 20 | -------------------------------------------------------------------------------- /util/number-lines.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | set -euo pipefail 6 | 7 | num=1 8 | while read line; do 9 | # end? 10 | [ "$line" = "%%END%%" ] && exit 0 11 | 12 | # print 13 | echo "$num: $line" 14 | 15 | # inc 16 | num=$((num+1)) 17 | done 18 | -------------------------------------------------------------------------------- /util/squeeze-log-into-line.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2022-2023 by Yuri Victorovich. All rights reserved. 4 | 5 | STRICT="set -euo pipefail" 6 | $STRICT 7 | 8 | ESC=$'\e' 9 | 10 | in_terminal=no 11 | [ -t 1 ] && in_terminal=yes 12 | 13 | len=0 14 | 15 | clr() { 16 | $STRICT 17 | if [ $len -gt 0 ]; then 18 | echo -n "${ESC}[1K" # erase 19 | echo -n "${ESC}[${len}D" # backwards 20 | fi 21 | } 22 | 23 | tm_start=$(date +"%s") 24 | tm_last=$tm_start 25 | while read line; do 26 | # end? 27 | [ "$line" = "%%END%%" ] && break 28 | 29 | # display 30 | if [ $in_terminal = yes ]; then 31 | # current time 32 | tm_now=$(date +"%s") 33 | 34 | if [ $((tm_last+1)) -le $tm_now ]; then 35 | # clear the previous one if present 36 | clr 37 | 38 | # add time 39 | line="$line ($((tm_now - tm_start)) sec)" 40 | 41 | # save length 42 | len=${#line} 43 | 44 | # display 45 | echo -n "$line" 46 | 47 | # update time 48 | tm_last=$tm_now 49 | fi 50 | else 51 | echo $line 52 | fi 53 | done 54 | 55 | # clear last 56 | [ $in_terminal = yes ] && clr 57 | --------------------------------------------------------------------------------