├── releases
├── books-0.6.tar.gz
├── books-0.6.1.tar.gz
├── books-0.6.2.tar.gz
├── books-0.6.3.tar.gz
├── books-0.6.4.tar.gz
├── books-0.6.5.tar.gz
├── books-0.7.1.tar.gz
└── books-0.7.2.tar.gz
├── screenshots
├── screenshot_region-20162728012748.png
└── screenshot_region-20162828002811.png
├── books_functions
├── tm
├── import_metadata.sh
├── import_metadata
├── refresh_libgen
├── classify
├── update_libgen
├── LICENSE
├── README.md
└── books
/releases/books-0.6.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/releases/books-0.6.tar.gz
--------------------------------------------------------------------------------
/releases/books-0.6.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/releases/books-0.6.1.tar.gz
--------------------------------------------------------------------------------
/releases/books-0.6.2.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/releases/books-0.6.2.tar.gz
--------------------------------------------------------------------------------
/releases/books-0.6.3.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/releases/books-0.6.3.tar.gz
--------------------------------------------------------------------------------
/releases/books-0.6.4.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/releases/books-0.6.4.tar.gz
--------------------------------------------------------------------------------
/releases/books-0.6.5.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/releases/books-0.6.5.tar.gz
--------------------------------------------------------------------------------
/releases/books-0.7.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/releases/books-0.7.1.tar.gz
--------------------------------------------------------------------------------
/releases/books-0.7.2.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/releases/books-0.7.2.tar.gz
--------------------------------------------------------------------------------
/screenshots/screenshot_region-20162728012748.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/screenshots/screenshot_region-20162728012748.png
--------------------------------------------------------------------------------
/screenshots/screenshot_region-20162828002811.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yetangitu/books/HEAD/screenshots/screenshot_region-20162828002811.png
--------------------------------------------------------------------------------
/books_functions:
--------------------------------------------------------------------------------
1 | # shellcheck shell=bash disable=SC2154
2 |
3 | # UTITLITIES
4 |
5 | # find tool, returns the first|one|found, exit with error message if none found
6 | find_tool () {
7 | IFS='|' read -ra tools <<< "$*"
8 |
9 | found=0
10 |
11 | for tool in "${tools[@]}"; do
12 | if [[ -n $(which "$tool") ]]; then
13 | found=1
14 | break
15 | fi
16 | done
17 |
18 | if [[ "$found" -eq 0 ]]; then
19 | if [[ ${#tools[@]} -gt 1 ]]; then
20 | exit_with_error "missing programs: $*; install at least one of these: ${tools[*]} and try again"
21 | else
22 | exit_with_error "missing program: $1; please install and try again"
23 | fi
24 | fi
25 |
26 | echo "$tool"
27 | }
28 |
29 | url_available () {
30 | url="$1"
31 | dl_tool=$(find_tool "curl|wget")
32 |
33 | case "$dl_tool" in
34 | curl)
35 | ${torsocks:-} curl --output /dev/null --silent --fail -r 0-0 "$url"
36 | ;;
37 | wget)
38 | ${torsocks:-} wget -q --spider "$url"
39 | ;;
40 | *)
41 | exit_with_error "unknown download tool ${dl_tool}"
42 | ;;
43 | esac
44 | }
45 |
46 | add_cron_job () {
47 | job="$*"
48 |
49 | (crontab -l ; echo "*/1 * * * * $job") 2>/dev/null | sort | uniq | crontab -
50 | }
51 |
52 | # leave
and to enable some simple formatting tasks
53 | strip_html () {
54 | #echo "$*"|sed -e 's/
/\n/g;s/<[^>]*>//g;s/\n/
/g'
55 | echo "$*"
56 | }
57 |
58 | is_true () {
59 | val="${1,,}"
60 | if [[ "${val:0:1}" == "y" || "$val" -gt 0 ]]; then
61 | true
62 | else
63 | false
64 | fi
65 | }
66 |
67 | # dummmy cleanup function
68 | cleanup () {
69 | true
70 | }
71 |
72 | # echo error message to stderr and terminate main
73 | exit_with_error () {
74 | echo -e "$(basename "$0"): $*" >&2
75 |
76 | kill -s TERM "$TOP_PID"
77 | }
78 |
79 | trap_error () {
80 | cleanup
81 |
82 | exit 1
83 | }
84 |
85 | trap_clean () {
86 | cleanup
87 |
88 | exit
89 | }
90 |
91 | _log () {
92 | msg="$*"
93 | logdir="${XDG_STATE_HOME:-$HOME/.state}/books"
94 | logfile=$(basename "$0").log
95 | mkdir -p "$logdir"
96 | echo "$(date -Iseconds): $msg" >> "$logdir/$logfile"
97 | }
98 |
99 | log_err () {
100 | _log "E: $*"
101 | }
102 |
103 | log_warn () {
104 | _log "W: $*"
105 | }
106 |
107 | log_info () {
108 | _log "I: $*"
109 | }
110 |
111 | log_debug () {
112 | _log "D: $*"
113 | }
114 |
115 | # DATABASE
116 | dbx () {
117 | db="$1"
118 | shift
119 |
120 | mysql=$(find_tool "mysql")
121 |
122 | if [ $# -gt 0 ]; then
123 | "$mysql" -N -Bsss -h "$dbhost" -P "$dbport" -u "$dbuser" "$db" -e "$*"
124 | else
125 | "$mysql" -N -Bsss -h "$dbhost" -P "$dbport" -u "$dbuser" "$db"
126 | fi
127 | }
128 |
129 | # LOCKING
130 |
131 | exlock () {
132 | cmd="$1"
133 |
134 | lockfile="/var/lock/$(basename "$0")"
135 | lockfd=99
136 |
137 | flock=$(find_tool "flock")
138 |
139 | case "$cmd" in
140 | prepare)
141 | eval "exec $lockfd<>\"$lockfile\""
142 | trap 'exlock nolock' EXIT
143 | ;;
144 |
145 | now)
146 | $flock -xn $lockfd
147 | ;;
148 |
149 | lock)
150 | $flock -x $lockfd
151 | ;;
152 |
153 | shlock)
154 | $flock -s $lockfd
155 | ;;
156 |
157 | unlock)
158 | $flock -u $lockfd
159 | ;;
160 |
161 | nolock)
162 | $flock -u $lockfd
163 | $flock -xn $lockfd && rm -f "$lockfile"
164 | trap_clean
165 | ;;
166 |
167 | *)
168 | exit_with_error "unknown lock command: $cmd"
169 | ;;
170 | esac
171 | }
172 |
--------------------------------------------------------------------------------
/tm:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # shellcheck disable=SC2034,SC1090,SC2207
3 |
4 | shopt -s extglob
5 |
6 | export TOP_PID=$$
7 |
8 | version="0.1.2"
9 | release="20210512"
10 |
11 | main () {
12 | config=${XDG_CONFIG_HOME:-$HOME/.config}/tm.conf
13 | netrc=~/.tm-netrc
14 | tm_host="localhost:4081"
15 | # source config file if it exists
16 | [[ -f ${config} ]] && source "${config}"
17 |
18 | declare -a commands=($(declare -F|grep tm_|sed -e 's/.*tm_\(\S\+\).*/\1/'))
19 | declare -a programs=($(declare -F|grep tm_|sed -e 's/.*\(tm_\S\+\).*/\1/;s/_/-/g'))
20 |
21 |
22 | while getopts "a:chH:kln:" OPTION
23 | do
24 | case $OPTION in
25 | k)
26 | create_symlinks
27 | exit
28 | ;;
29 | a)
30 | tm_add "${OPTARG}"
31 | exit
32 | ;;
33 | l)
34 | tm_ls
35 | exit
36 | ;;
37 | n)
38 | netrc="${OPTARG}"
39 | ;;
40 | H)
41 | tm_host="${OPTARG}"
42 | ;;
43 | c)
44 | if [[ ! -f "${config}" ]]; then
45 | cat <<-EOT > "${config}"
46 | netrc="$netrc"
47 | tm_host="$tm_host"
48 | EOT
49 | else
50 | exit_with_error "-c: config file ${config} exists, either remove it or edit it directly"
51 | fi
52 | ;;
53 | h)
54 | help
55 | exit
56 | ;;
57 | *)
58 | exit_with_error "unknown option $OPTION"
59 | ;;
60 | esac
61 | done
62 |
63 | # shift out options
64 | shift $((OPTIND-1))
65 |
66 | cmd="${1//-/_}"
67 | program=$(basename "$0")
68 |
69 | IFS='|'
70 | if [[ $cmd =~ ${commands[*]} ]]; then
71 | unset IFS
72 | shift
73 | tm_"$cmd" "$@"
74 | elif [[ $program =~ ${programs[*]} ]]; then
75 | unset IFS
76 | ${program//-/_} "$@"
77 | else
78 | unset IFS
79 | exit_with_error "no such command: $cmd\navailable commands: ${commands[*]//_/-}"
80 | fi
81 | }
82 |
83 | # commands
84 |
85 | tm_cmd () {
86 | if [[ -n $(which transmission-remote) ]]; then
87 | transmission-remote "$tm_host" -N "$netrc" "$@"
88 | else
89 | exit_with_error "transmission-remote not found, please install it first (apt install transmission-cli)"
90 | fi
91 | }
92 |
93 | tm_help () {
94 | tm_cmd -h
95 | }
96 |
97 | tm_add () {
98 | tm_cmd -a "$@"
99 | }
100 |
101 | tm_add_selective () {
102 | torrent="$1"
103 | shift
104 | files="${*,,}"
105 |
106 | keep=0
107 | count=0
108 |
109 | if [[ -z "$torrent" || -z "$files" ]]; then
110 | echo 'use: tm-add-selective [,file2,file3,file4...]'
111 | exit 1
112 | fi
113 |
114 | check_torrent "$torrent"
115 |
116 | tm_cmd --start-paused
117 | btih=$(tm_torrent_hash "$torrent")
118 |
119 | # check if torrent is already downloading
120 | if tm_active "$btih"; then
121 | running=1
122 | tm_stop "$btih"
123 | else
124 | tm_add "$torrent"
125 | fi
126 |
127 | if tm_info "$btih" > /dev/null; then
128 | count=$(tm_file_count "$btih")
129 | # if the torrent only has 1 file it does not make sense to do a selective download...
130 | if [[ $count -gt 1 ]]; then
131 | if [[ $running -eq 0 ]]; then
132 | # need to keep at least 1 file active, otherwise transmission removes the torrent
133 | tm_cmd -t "$btih" -G 1-$((count-1))
134 | fi
135 | while read -r id; do
136 | [[ $id -eq 0 ]] && keep=1
137 | tm_cmd -t "$btih" -g "$id"
138 | done < <(tm_cmd -t "$btih" -f|grep -E "${files/,/|}"|cut -d ':' -f 1)
139 | [[ $keep -eq 0 && $running -eq 0 ]] && tm_cmd -t "$btih" -G 0
140 | fi
141 | else
142 | echo "error adding torrent"
143 | exit 1
144 | fi
145 | tm_cmd --no-start-paused
146 | tm_start "$btih"
147 | }
148 |
149 | tm_remove () {
150 | tm_cmd -t "$@" -r
151 | }
152 |
153 | tm_start () {
154 | if tm_active "$@"; then
155 | tm_cmd -t "$@" -s
156 | fi
157 | }
158 |
159 | tm_stop () {
160 | tm_cmd -t "$@" -S
161 | }
162 |
163 | tm_info () {
164 | tm_cmd -t "$@" -i
165 | }
166 |
167 | tm_files () {
168 | tm_cmd -t "$@" -f
169 | }
170 |
171 | tm_ls () {
172 | tm_cmd -l
173 | }
174 |
175 | tm_file_count () {
176 | tm_files "$1"|head -1|sed 's/.* (\([[:digit:]]\+\) files):/\1/'
177 | }
178 |
179 | tm_active () {
180 | tm_cmd -t "$@" -ip|grep -q Address
181 | }
182 |
183 | # torrent file related commands
184 |
185 | tm_torrent_show () {
186 | check_torrent "$@"
187 | transmission-show "$@"
188 | }
189 |
190 | tm_torrent_files () {
191 | tm_torrent_show "$@"|awk '/^FILES/ {start=1}; NF>1 && start==1 {print $0}'|sed -e 's/^\s\+\(.*\) ([0-9.]\+ .B)$/\1/'
192 | }
193 |
194 | tm_torrent_hash () {
195 | tm_torrent_show "$@"|awk ' /^\s+Hash:/ {print $2}'
196 | }
197 |
198 | # helper functions
199 |
200 | exit_with_error () {
201 | echo -e "$(basename "$0"): $*" >&2
202 |
203 | kill -s TERM $TOP_PID
204 | }
205 |
206 | check_torrent () {
207 | if ! (file "$1"|grep -i bittorrent)>/dev/null; then
208 | exit_with_error "$1 is not a torrent file"
209 | fi
210 | }
211 |
212 | create_symlinks () {
213 | basedir="$(dirname "$0")"
214 | sourcefile="$(readlink -e "$0")"
215 | prefix=$(basename "$sourcefile")
216 | for cmd in "${commands[@]}"; do
217 | name="${prefix}-${cmd//_/-}"
218 | if [[ ! -e "$basedir/$name" ]]; then
219 | ln -s "$sourcefile" "$basedir/$name"
220 | fi
221 | done
222 |
223 | exit
224 | }
225 |
226 | help () {
227 | sourcefile="$(readlink -e "$0")"
228 | prefix=$(basename "$sourcefile")
229 | echo "$(basename "$(readlink -f "$0")")" "version $version"
230 | cat <<- EOF
231 |
232 | Use: $prefix COMMAND OPTIONS [parameters]
233 | $prefix-COMMAND OPTIONS [parameters]
234 |
235 | A helper script for transmission-remote and related tools, adding some
236 | functionality like selective download etc.
237 |
238 | PROGRAMS/COMMANDS
239 |
240 | EOF
241 |
242 | for cmd in "${programs[@]}"; do
243 | echo -e " $cmd\r\t\t\t${cmd/$prefix-}"
244 | done
245 |
246 | cat <<- EOF
247 |
248 | OPTIONS
249 |
250 | -k create symbolic links
251 | creates links to all supported commands
252 | e.g. $prefix-cmd, $prefix-ls, $prefix-add, ...
253 | links are created in the directory where $prefix resides
254 |
255 | -n NETRC set netrc ($netrc)
256 |
257 | -H HOST set host ($tm_host)
258 |
259 | -c create a config file using current settings (see -n, -H)
260 |
261 | -l execute command 'ls'
262 |
263 | -a TORR execute command 'add'
264 |
265 | -h this help message
266 |
267 | EXAMPLES
268 |
269 | In all cases it is possible to replace $prefix-COMMAND with $prefix COMMAND
270 |
271 | show info about running torrents:
272 |
273 | $ $prefix-ls
274 |
275 | add a torrent or a magnet link:
276 |
277 | $prefix-add /path/to/torrent/file.torrent
278 | $prefix-add 'magnet:?xt=urn:btih:123...'
279 |
280 | add a torrent and selectivly download two files
281 | this only works with torrent files (i.e. not magnet links) for now
282 |
283 | $prefix-add-selective /path/to/torrent/file.torrent filename1,filename2
284 |
285 | show information about a running torrent, using its btih or ID:
286 |
287 | $prefix-show f0a7524fe95910da462a0d1b11919ffb7e57d34a
288 | $prefix-show 21
289 |
290 | show files for a running torrent identified by btih (can also use ID)
291 |
292 | $prefix-files f0a7524fe95910da462a0d1b11919ffb7e57d34a
293 |
294 | stop a running torrent, using its ID (can also use btih)
295 |
296 | $prefix-stop 21
297 |
298 | get btih for a torrent file
299 |
300 | $prefix-torrent-hash /path/to/torrent/file.torrent
301 |
302 | remove a torrent from transmission
303 |
304 | $prefix-remove 21
305 |
306 | execute any transmission-remote command - notice the double dash
307 | see man transmission-remote for more info on supported commands
308 |
309 |
310 | $prefix-cmd -- -h
311 | $prefix cmd -h
312 |
313 |
314 | CONFIGURATION FILES
315 |
316 | $config
317 |
318 | $prefix can be configured by editing the script itself or the configuration file:
319 |
320 | netrc=~/.tm-netrc
321 | tm_host="transmission-host.example.org:4081"
322 |
323 | values set in the configuration file override those in the script
324 |
325 | EOF
326 | }
327 |
328 | main "$@"
329 |
--------------------------------------------------------------------------------
/import_metadata.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | #shellcheck disable=SC2034,SC1090
3 | #
4 | # import_metadata - import metadata to libgen/libgen_fiction
5 | #
6 | # input: a [single line of|file containg] CSV-ordered metadata
7 |
8 | shopt -s extglob
9 | trap "trap_error" TERM
10 | trap "trap_clean" EXIT
11 | export TOP_PID=$$
12 |
13 | version="0.1.0"
14 | release="20210518"
15 |
16 | functions="$(dirname "$0")/books_functions"
17 | if [ -f "$functions" ]; then
18 | source "$functions"
19 | else
20 | echo "$functions not found"
21 | exit 1
22 | fi
23 |
24 | main () {
25 |
26 | exlock now || exit 1
27 |
28 | coproc coproc_ddc { coproc_ddc; }
29 | coproc coproc_fast { coproc_fast; }
30 |
31 | # PREFERENCES
32 | config=${XDG_CONFIG_HOME:-$HOME/.config}/books.conf
33 |
34 | dbhost="localhost"
35 | dbport="3306"
36 | db="libgen"
37 | dbuser="libgen"
38 |
39 | tmpdir=$(mktemp -d '/tmp/import_metadata.XXXXXX')
40 | update_sql="${tmpdir}/update_sql"
41 |
42 | # input field filters
43 | declare -A filter=(
44 | [md5]=filter_md5
45 | [ddc]=filter_ddc
46 | [lcc]=filter_ddc
47 | [nlm]=filter_ddc
48 | [fast]=filter_fast
49 | [author]=filter_fast
50 | [title]=filter_fast
51 | )
52 |
53 | # redirect OCLC [key] to field
54 | declare -A redirect=(
55 | [fast]="tags"
56 | )
57 |
58 | # used to get index for field / field for index
59 | keys="md5 ddc lcc nlm fast author title"
60 | declare -A headers
61 | index=0
62 | for key in $keys;do
63 | headers["$key"]=$index
64 | ((index++))
65 | done
66 |
67 | declare -A tables=(
68 | [libgen]="updated"
69 | [libgen_fiction]="fiction"
70 | )
71 |
72 | # source config file if it exists
73 | [[ -f ${config} ]] && source "${config}"
74 |
75 | declare -a csvdata
76 | declare -a csv
77 |
78 | while getopts "d:f:F:ns:vh" OPTION; do
79 | case $OPTION in
80 | d)
81 | if [ -n "${tables[$OPTARG]}" ]; then
82 | db="$OPTARG"
83 | else
84 | exit_with_error "-d $OPTARG: no such database"
85 | fi
86 | ;;
87 | f)
88 | for n in $OPTARG; do
89 | if [ -n "${headers[$n]}" ]; then
90 | fields+="${fields:+ }$n"
91 | else
92 | exit_with_error "no such field: $n"
93 | fi
94 | done
95 | ;;
96 | F)
97 | if [ -f "$OPTARG" ]; then
98 | csvfile="$OPTARG"
99 | else
100 | exit_with_error "-f $OPTARG: no such file"
101 | fi
102 | ;;
103 | s)
104 | sqlfile="$OPTARG"
105 | if ! touch "$sqlfile"; then
106 | exit_with_error "-s $OPTARG: can not write to file"
107 | fi
108 | ;;
109 | n)
110 | dry_run=1
111 | ;;
112 | v)
113 | ((verbose++))
114 | ;;
115 | h)
116 | help
117 | exit
118 | ;;
119 | *)
120 | exit_with_error "unknown option: -$OPTION"
121 | ;;
122 | esac
123 | done
124 |
125 | shift $((OPTIND-1))
126 |
127 | [[ -z "$db" ]] && exit_with_error "no database defined, use -d database"
128 | [[ -z "$fields" ]] && exit_with_error "no fields defined, use -f 'field1 field2' or -f field1 -f field2"
129 |
130 | if [ -z "$dry_run" ]; then
131 | declare -A current_fields='('$(get_current_fields "$db")')'
132 | for field in $fields; do
133 | [[ -n "${redirect[$field]}" ]] && field="${redirect[$field]}"
134 | if [[ ! "${!current_fields[*]}" =~ "${field,,}" ]]; then
135 | exit_with_error "field $field not in database $db"
136 | fi
137 | done
138 | fi
139 |
140 | if [[ -n "$csvfile" ]]; then
141 | readarray -t csvdata < <(cat "$csvfile")
142 | else
143 | readarray -t csvdata <<< "$*"
144 | fi
145 |
146 | printf "start transaction;\n" > "${update_sql}"
147 |
148 | for line in "${csvdata[@]}"; do
149 | readarray -d',' -t csv <<< "$line"
150 |
151 | if [[ "$verbose" -ge 2 ]]; then
152 | index=0
153 | for key in $keys; do
154 | echo "${key^^}: $(${filter[$key]} "$key")"
155 | ((index++))
156 | done
157 | fi
158 |
159 | sql="$(build_sql)"
160 |
161 | printf "$sql\n" >> "$update_sql"
162 |
163 | if [[ "$verbose" -ge 3 ]]; then
164 | echo "$sql"
165 | fi
166 |
167 | [[ -n "$sqlfile" ]] && printf "$sql\n" >> "$sqlfile"
168 |
169 | unset key
170 | unset sql
171 | csv=()
172 | done
173 |
174 | printf "commit;\n" >> "$update_sql"
175 | [[ -z "$dry_run" ]] && dbx "$db" < "$update_sql"
176 | }
177 |
178 | filter_md5 () {
179 | field="$1"
180 | printf "${csv[${headers[$field]}]}"
181 | }
182 |
183 | filter_ddc () {
184 | field="$1"
185 |
186 | # without coprocess
187 | # echo "${csv[${headers[$field]}]}"|sed 's/"//g;s/[[:blank:]]\+/,/g'
188 |
189 | # with coprocess (30% faster)
190 | printf "${csv[${headers[$field]}]}\n" >&${coproc_ddc[1]}
191 | IFS= read -ru ${coproc_ddc[0]} value
192 | printf "$value"
193 | }
194 |
195 | coproc_ddc () {
196 | sed -u 's/"//g;s/[[:blank:]]\+/,/g'
197 | }
198 |
199 |
200 | filter_fast () {
201 | field="$1"
202 |
203 | # without coprocess
204 | # echo "${csv[${headers[$field]}]}"|base64 -d|sed -u 's/\(["\\'\'']\)/\\\1/g;s/\r/\\r/g;s/\n/\\n/g;s/\t/\\t/g'
205 |
206 | # with coprocess (30% faster)
207 | # base64 can not be used as a coprocess due to its uncurable buffering addiction
208 | value=$(printf "${csv[${headers[$field]}]}"|base64 -d)
209 | printf "$value\n" >&${coproc_fast[1]}
210 | IFS= read -ru ${coproc_fast[0]} value
211 | printf "$value"
212 | }
213 |
214 | coproc_fast () {
215 | sed -u 's/\(["\\'\'']\)/\\\1/g;s/\r/\\r/g;s/\n/\\n/g;s/\t/\\t/g'
216 | }
217 |
218 |
219 | get_field () {
220 | field="$1"
221 | value="${csv[${headers[$field]}]}"
222 | if [ -n "${filters[$field]}" ]; then
223 | printf "$value"|eval "${filters[$field]}"
224 | else
225 | printf "$value"
226 | fi
227 | }
228 |
229 | get_current_fields () {
230 | db="$1"
231 | for table in "${tables[$db]}"; do
232 | dbx "$db" "describe $table;"|awk '{printf "[%s]=%s ",tolower($1),"'$table'"}'
233 | done
234 | }
235 |
236 | build_sql () {
237 | sql=""
238 | for field in $fields; do
239 | data=$(${filter[$field]} "$field")
240 | if [ -n "$data" ]; then
241 | [[ -n "${redirect[$field]}" ]] && field="${redirect[$field]}"
242 | sql+="${sql:+,}${field^^}='${data}'"
243 | fi
244 | done
245 |
246 | if [ -n "$sql" ]; then
247 | printf "update ${tables[$db]} set $sql where MD5='$(${filter['md5']} md5)';"
248 | fi
249 | }
250 |
251 | cleanup () {
252 | rm -rf "${tmpdir}"
253 | }
254 |
255 | # HELP
256 |
257 | help () {
258 | echo "$(basename "$(readlink -f "$0")")" "version $version"
259 | cat <<- EOHELP
260 |
261 | Use: import_metadata [OPTIONS] -d database -f "field1 field2" [-F CSVDATAFILE | single line of csv data ]
262 |
263 | Taking either a single line of CSV-formatted data or a file containing
264 | such data, this tool can be used to update a libgen / libgen_fiction
265 | database with fresh metadata. It can also be used to produce SQL (using
266 | the -s sqlfile option) which can be used to update multiple database
267 | instances.
268 |
269 | CSV data format:
270 |
271 | $(hkeys=${keys^^};echo ${hkeys// /,})
272 |
273 | CSV field names are subject to redirection to database field names,
274 | currently these redirections are active (CSV -> DB):
275 |
276 | $(for field in "${!redirect[@]}";do echo " ${field^^} -> ${redirect[$field]^^}";done)
277 |
278 | OPTIONS:
279 |
280 | -d DB define which database to use (libgen/libgen_fiction)
281 |
282 | -f 'field1 field2'
283 | -f field1 -f field2
284 |
285 | define which fields to update
286 |
287 | -F CSVFILE
288 |
289 | define CSV input file
290 |
291 | -s SQLFILE
292 |
293 | write SQL to SQLFILE
294 |
295 | -n do not update database
296 | use with -s SQLFILE to produce SQL for later use
297 | use with -vv to see data from CSVFILE
298 | use with -vvv to see SQL
299 |
300 | -v verbosity
301 | repeat to increase verbosity
302 |
303 | -h this help message
304 |
305 | Examples
306 |
307 | $ import_metadata -d libgen -F csv/update-0000 -f 'ddc lcc fast'
308 |
309 | update database 'libgen' using data from CSV file csv/update-0000,
310 | fields DDC, LCC and FAST (which is redirected to libgen.Tags)
311 |
312 | $ for f in csv/update-*;do
313 | import_metadata -d libgen -s sql/metadata.sql -n -f 'ddc lcc fast' -F "\$f"
314 | done
315 |
316 | create SQL (-s sql/metadata.sql) to update database using fields
317 | DDC, LCC and FAST from all files matching glob csv/update-*,
318 | do not update database (-n option)
319 |
320 |
321 | EOHELP
322 | }
323 |
324 | exlock prepare || exit 1
325 |
326 | main "$@"
327 |
--------------------------------------------------------------------------------
/import_metadata:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | #
3 | # import_metadata - import metadata to libgen/libgen_fiction
4 |
5 | import base64
6 | import csv
7 | import getopt
8 | import os
9 | import pymysql
10 | import re
11 | import sys
12 |
13 |
14 | version="0.1.0"
15 | release="20210521"
16 |
17 | def exit_with_error(msg):
18 | sys.exit(os.path.basename(sys.argv[0])+" "+msg)
19 |
20 | def try_file(f,p):
21 | try:
22 | fp=open(f,p)
23 | fp.close()
24 | return True
25 | except IOError as x:
26 | exit_with_error(str(x))
27 |
28 | def main():
29 |
30 | config = {
31 | 'dbhost': 'base.unternet.org',
32 | 'dbport': '3306',
33 | 'db': '',
34 | 'dbuser': 'libgen'
35 | }
36 |
37 | verbose = 0
38 | dry_run = False
39 | sqlfile = None
40 | csvfile = None
41 | use_fields=[]
42 | sql=[]
43 |
44 | re_csv=re.compile('(\s+)')
45 |
46 | # read books config file (a bash source file) and interpret it
47 | # works only for single-line static declarations (no shell code)
48 | def read_conf(conf):
49 | if 'APPDATA' in os.environ:
50 | confdir = os.environ['APPDATA']
51 | elif 'XDG_CONFIG_HOME' in os.environ:
52 | confdir = os.environ['XDG_CONFIG_HOME']
53 | else:
54 | confdir = os.path.join(os.environ['HOME'], '.config')
55 |
56 | conffile = os.path.join(confdir, 'books.conf')
57 |
58 | if try_file(conffile,'r'):
59 | line_re = re.compile('(?:export )?(?P\w+)(?:\s*\=\s*)(?P.+)')
60 | value_re = re.compile('(?P^[^#]+)(?P#.*)?$')
61 | for line in open(conffile):
62 | m = line_re.match(line)
63 | if m:
64 | name = m.group('name')
65 | value = ''
66 | if m.group('value'):
67 | value = m.group('value')
68 | m = value_re.match(value)
69 | if m:
70 | value=m.group('value')
71 |
72 | conf[name]=value.strip('\"').strip("\'")
73 |
74 | return conf
75 |
76 | config=read_conf(config)
77 |
78 | def to_itself(field):
79 | return field
80 |
81 | def to_csv(field):
82 | return re_csv.sub(',', field)
83 |
84 | def to_sqlescape(field):
85 | return pymysql.escape_string(base64.b64decode(field).decode().rstrip())
86 |
87 | fields=['md5','ddc','lcc','nlm','fast','author','title']
88 |
89 | filters = {
90 | 'md5': to_itself,
91 | 'ddc': to_csv,
92 | 'lcc': to_csv,
93 | 'nlm': to_csv,
94 | 'fast': to_sqlescape,
95 | 'author': to_sqlescape,
96 | 'title': to_sqlescape
97 | }
98 |
99 | redirects = {
100 | 'fast': 'tags'
101 | }
102 |
103 | tables = {
104 | 'libgen': 'updated',
105 | 'libgen_fiction': 'fiction'
106 | }
107 |
108 |
109 | def redirect(field):
110 | if field in redirects:
111 | return redirects[field]
112 | else:
113 | return field
114 |
115 | def usage():
116 | msg=[]
117 | def fmt_dict(lst):
118 | for key in lst:
119 | msg.append(str(key+" -> "+lst[key]).upper())
120 | return msg
121 |
122 | print(helpmsg.format(
123 | progname=os.path.basename(sys.argv[0]),
124 | version="v."+version,
125 | csvfields=','.join(fields).upper(),
126 | redirects=fmt_dict(redirects)
127 | ))
128 | sys.exit()
129 |
130 | try:
131 | opts, args = getopt.getopt(sys.argv[1:], "d:f:F:H:u:U:ns:vh")
132 | except getopt.GetoptError as err:
133 | print(str(err))
134 | usage()
135 |
136 | for o, a in opts:
137 | if o == "-v":
138 | verbose+=1
139 | elif o in ("-h"):
140 | usage()
141 | elif o in ("-d"):
142 | config['db'] = a
143 | elif o in ("-f"):
144 | for f in a.split(','):
145 | if f in fields:
146 | use_fields.append(f)
147 | else:
148 | exit_with_error("-f "+f+" : no such field")
149 | elif o in ("-F"):
150 | if try_file(a,'r'):
151 | csvfile = a
152 | elif o in ("-H"):
153 | config['dbhost'] = a
154 | elif o in ("-U"):
155 | config['dbuser'] = a
156 | elif o in ("-n"):
157 | dry_run = True
158 | elif o in ("-s"):
159 | if try_file(a,'w'):
160 | sqlfile = a
161 | else:
162 | exit_with_error("unhandled option")
163 |
164 | if len(sys.argv) <= 2:
165 | exit_with_error("needs at least 3 parameters: -d database -f field1,field2 -F csvfile")
166 |
167 | if not config['db'] or config['db'] not in tables:
168 | exit_with_error("-d "+config['db']+": no such database")
169 |
170 | if not use_fields:
171 | exit_with_error("no fields defined, use -f field1 -f field2")
172 |
173 | with open(csvfile) as cf:
174 | reader = csv.DictReader(cf, fieldnames=fields)
175 |
176 | if verbose >= 1:
177 | sys.stdout.writelines(['\n#----DATA----------------------\n\n'])
178 |
179 | for row in reader:
180 | if verbose >= 1:
181 | for field in fields:
182 | print(field.upper()+": "+filters[field](row[field]))
183 | print("")
184 |
185 | updates=""
186 | comma=""
187 | for field in use_fields:
188 | value=filters[field](row[field])
189 | if value:
190 | if updates:
191 | comma=","
192 | updates+=comma+redirect(field).upper()+"='"+value+"'"
193 |
194 | if updates:
195 | sql.append("update updated set "+updates+" where md5='"+row['md5']+"';\n")
196 | else:
197 | if verbose:
198 | print("-- fields "+str(use_fields)+" not defined for md5:"+row['md5'])
199 |
200 | if sql:
201 | if sqlfile:
202 | fp=open(sqlfile,'a')
203 | fp.writelines([
204 | '-- csvfile: '+csvfile+'\n',
205 | '-- database: '+config['db']+'\n',
206 | '-- fields: '+str(use_fields)+'\n',
207 | '-- command: '+' '.join(sys.argv)+'\n',
208 | 'start transaction;\n'
209 | ])
210 | fp.writelines(sql)
211 | fp.writelines(['commit;\n'])
212 | fp.close()
213 |
214 | if verbose >= 2:
215 | sys.stdout.writelines(['\n#----SQL-----------------------\n\n'])
216 | sys.stdout.writelines(sql)
217 |
218 | if not dry_run:
219 | conn=pymysql.connect(
220 | read_default_file='~/.my.cnf',
221 | host=config['dbhost'],
222 | port=config['dbport'],
223 | user=config['dbuser'],
224 | database=config['db']
225 | )
226 |
227 | with conn:
228 | with conn.cursor() as cursor:
229 | for line in sql:
230 | cursor.execute(line)
231 |
232 | conn.commit()
233 |
234 | helpmsg = """
235 | {progname} {version}
236 |
237 | Use: {progname} [OPTIONS] -d database -f "field1,field2" -F CSVDATAFILE
238 |
239 | Taking a file containing lines of CSV-formatted data, this tool can be
240 | used to update a libgen / libgen_fiction database with fresh metadata.
241 | It can also be used to produce SQL (using the -s sqlfile option) which
242 | can be used to update multiple database instances.
243 |
244 | CSV data format:
245 |
246 | {csvfields}
247 |
248 | Fields FAST, AUTHOR and TITLE should be base64-encoded.
249 |
250 | CSV field names are subject to redirection to database field names,
251 | currently these redirections are active (CSV -> DB):
252 |
253 | {redirects}
254 |
255 | OPTIONS:
256 |
257 | -d DB define which database to use (libgen/libgen_fiction)
258 |
259 | -f field1,field2
260 | -f field1 -f field2
261 | define which fields to update
262 |
263 | -F CSVFILE
264 | define CSV input file
265 |
266 | -s SQLFILE
267 | write SQL to SQLFILE
268 |
269 | -n do not update database
270 | use with -s SQLFILE to produce SQL for later use
271 | use with -v to see data from CSVFILE
272 | use with -vv to see SQL
273 |
274 | -v verbosity
275 | repeat to increase verbosity
276 |
277 | -h this help message
278 |
279 | Examples
280 |
281 | $ import_metadata -d libgen -F csv/update-0000 -f 'ddc lcc fast'
282 |
283 | update database 'libgen' using data from CSV file csv/update-0000,
284 | fields DDC, LCC and FAST (which is redirected to libgen.Tags)
285 |
286 | $ for f in csv/update-*;do
287 | {progname} -d libgen -s "$f.sql" -n -f 'ddc,lcc,fast' -F "$f"
288 | done
289 |
290 | create SQL (-s "$f.sql") to update database using fields
291 | DDC, LCC and FAST from all files matching glob csv/update-*,
292 | do not update database (-n option)
293 | """
294 |
295 | if __name__ == "__main__":
296 | main()
297 |
298 |
--------------------------------------------------------------------------------
/refresh_libgen:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # shellcheck disable=SC2034,SC1090
3 | #
4 | # refresh libgen databases from dump files
5 |
6 | version="0.6.2"
7 | release="20210601"
8 |
9 | trap "trap_error" TERM
10 | trap "trap_clean" EXIT
11 | export TOP_PID=$$
12 |
13 | functions="$(dirname "$0")/books_functions"
14 | if [ -f "$functions" ]; then
15 | source "$functions"
16 | else
17 | echo "$functions not found"
18 | exit 1
19 | fi
20 |
21 | main () {
22 |
23 | exlock now || exit 1
24 |
25 | # PREFERENCES
26 | config=${XDG_CONFIG_HOME:-$HOME/.config}/books.conf
27 |
28 | # maximum age (in days) of database dump file to use
29 | max_age=5
30 |
31 | # database server to use
32 | dbhost="localhost"
33 | dbport="3306"
34 | dbuser="libgen"
35 |
36 | # where to get updates. A change here probably necessitates a change in the urls array
37 | # as dump file names can be site-specific.
38 | base="http://libgen.rs/dbdumps/"
39 |
40 | # database names
41 | declare -A databases=(
42 | [libgen]=libgen
43 | [compact]=libgen_compact
44 | [fiction]=libgen_fiction
45 | )
46 |
47 | # source config file if it exists
48 | [[ -f ${config} ]] && source "${config}"
49 |
50 | # (mostly) END OF PREFERENCES
51 |
52 | # urls for dump files (minus datestamp and extension)
53 | declare -A urls=(
54 | [libgen]="${base}/libgen"
55 | [compact]="${base}/libgen_compact"
56 | [fiction]="${base}/fiction"
57 | )
58 |
59 | # sql to get time last modified for database
60 | declare -A lastmodified=(
61 | [libgen]="select max(timelastmodified) from updated;"
62 | [compact]="select max(timelastmodified) from updated;"
63 | [fiction]="select max(timelastmodified) from fiction;"
64 | )
65 |
66 | declare -A filter=(
67 | [libgen]='s/DEFINER[ ]*=[ ]*[^*]*\*/\*/;s/DEFINER[ ]*=[ ]*[^*]*PROCEDURE/PROCEDURE/;s/DEFINER[ ]*=[ ]*[^*]*FUNCTION/FUNCTION/'
68 | [compact]='s/DEFINER[ ]*=[ ]*[^*]*\*/\*/;s/DEFINER[ ]*=[ ]*[^*]*PROCEDURE/PROCEDURE/;s/DEFINER[ ]*=[ ]*[^*]*FUNCTION/FUNCTION/'
69 | )
70 |
71 | # sql to run BEFORE update
72 | declare -A before_update=(
73 | )
74 |
75 | # sql to run AFTER update
76 | declare -A after_update=(
77 | [compact]="drop trigger updated_edited;create table description (id int(11) not null auto_increment, md5 varchar(32) not null default '', descr varchar(20000) not null default '', toc mediumtext not null, TimeLastModified timestamp not null default current_timestamp on update current_timestamp, primary key (id), unique key md5_unique (md5) using btree, key time (timelastmodified) using btree, key md5_hash (md5) using hash);"
78 | )
79 |
80 | declare -A options=(
81 | [wget]="-nv"
82 | [wget_verbose]=""
83 | [unrar]="-inul"
84 | [unrar_verbose]=""
85 | )
86 |
87 |
88 | tmpdir=$(mktemp -d /var/tmp/libgen.XXXXXX)
89 |
90 | unrar=$(find_tool "unrar")
91 | wget=$(find_tool "wget")
92 | w3m=$(find_tool "w3m")
93 |
94 | while getopts "a:cd:efhH:knP:u:U:v@" OPTION
95 | do
96 | case $OPTION in
97 | n)
98 | no_action=1
99 | ;;
100 | f)
101 | force_refresh=1
102 | ;;
103 | d)
104 | max_age=${OPTARG}
105 | ;;
106 | u)
107 | if [[ -v "databases[${OPTARG}]" ]]; then
108 | dbs+=" ${OPTARG}"
109 | else
110 | exit_with_error "-u ${OPTARG}: no such database"
111 | fi
112 | ;;
113 | v)
114 | pv=$(find_tool "pv")
115 | verbose="_verbose"
116 | ;;
117 | H)
118 | dbhost="${OPTARG}"
119 | ;;
120 | P)
121 | dbport="${OPTARG}"
122 | ;;
123 | U)
124 | dbuser="${OPTARG}"
125 | ;;
126 | c)
127 | if [[ ! -f "${config}" ]]; then
128 | cat <<-EOT > "${config}"
129 | dbhost=${dbhost}
130 | dbport=${dbport}
131 | dbuser=${dbuser}
132 | base=${base}
133 | EOT
134 | else
135 | exit_with_error "-c: config file ${config} exists, either remove it or edit it directly"
136 | fi
137 | exit
138 | ;;
139 | e)
140 | if [[ -f "$config" ]]; then
141 | if [[ "$VISUAL" ]]; then "$VISUAL" "$config";
142 | elif [[ "$EDITOR" ]]; then "$EDITOR" "$config";
143 | else exit_with_error "-e: no editor configured, can not edit $config"
144 | fi
145 | else
146 | exit_with_error "-e: config file does not exist, create is first (see -c)"
147 | fi
148 | exit
149 | ;;
150 | a)
151 | if url_available "${OPTARG}"; then
152 | base="${OPTARG}"
153 | else
154 | exit_with_error "-a ${OPTARG}: repository not available"
155 | fi
156 | ;;
157 | @)
158 | torsocks=$(find_tool "torsocks")
159 | export TORSOCKS_TOR_PORT=$OPTARG
160 | ;;
161 | k)
162 | keep_downloaded_files=1
163 | ;;
164 | h)
165 | help
166 | exit
167 | ;;
168 | *)
169 | exit_with_error "unknown option: $OPTION"
170 | ;;
171 | esac
172 | done
173 |
174 | [[ -z ${dbs} ]] && dbs="${!databases[*]}"
175 |
176 | pushd "$tmpdir" >/dev/null || exit_with_error "can not change directory to $tmpdir"
177 | for db in ${dbs}; do
178 | database=${databases[$db]}
179 | if [[ $(db_exists "$database") ]]; then
180 | db_dump=$(is_available "${db}" "${max_age}")
181 | if [[ -n $db_dump ]]; then
182 | [[ -n $verbose ]] && echo "update available for ${db}: ${db_dump}"
183 | if [[ -z ${no_action} ]]; then
184 | $torsocks "$wget" "${options[$wget${verbose}]}" "${db_dump}"
185 | $unrar "${options[$unrar${verbose}]}" x "$(basename "${db_dump}")"
186 | [[ -n "${filter[$db]}" ]] && run_filter "$($unrar lb "$(basename "${db_dump}")")" "${filter[$db]}"
187 | drop_tables=$(drop_table_sql "${database}")
188 | [[ -n $drop_tables ]] && dbx "${database}" "${drop_tables}"
189 | [[ -n ${before_update[$db]} ]] && dbx "${database}" "${before_update[$db]}"
190 | [[ -n ${filter[$db]} ]] && filter_command="|sed -e '${filter[$db]}'"
191 | if [[ -n $verbose ]]; then
192 | echo "importing $(basename "${db_dump}") into ${database}"
193 | $pv "$($unrar lb "$(basename "${db_dump}")")" | dbx "${database}"
194 | else
195 | dbx "${database}" < "$($unrar lb "$(basename "${db_dump}")")"
196 | fi
197 | [[ -n ${after_update[$db]} ]] && dbx "${database}" "${after_update[$db]}"
198 | fi
199 | else
200 | [[ -n $verbose ]] && echo "no update available for ${db}"
201 | fi
202 | else
203 | echo "database '$database' does not exist, please create it before attempting to refresh" >&2
204 | fi
205 | done
206 | popd >/dev/null || exit_with_error "popd failed?"
207 | }
208 |
209 | # check whether there is a dump file which is more recent than the current database and no older
210 | # than $max_age
211 | is_available () {
212 | db="$1"
213 | max_age="$2"
214 |
215 | db_age=$(db_age "$db")
216 |
217 | age=0
218 |
219 | while [[ $age -lt $db_age && $age -lt $max_age ]]; do
220 | timestamp=$(date -d "@$(($(date +%s) - $((60*60*24*age))))" +%Y-%m-%d)
221 | result=$($w3m -dump "${base}" | awk '{ print $1 }'|grep "$(basename "${urls[$db]}_${timestamp}.rar")")
222 | [[ -n $result ]] && break
223 | ((age++))
224 | done
225 |
226 | [[ -n $result ]] && echo "$(dirname "${urls[$db]}")"/"${result}"
227 | }
228 |
229 | # drop tables to prepare database for refresh
230 | drop_table_sql () {
231 | database="$1"
232 | dbx "$database" "SELECT concat('DROP TABLE IF EXISTS ', table_name, ';') FROM information_schema.tables WHERE table_schema = '$database';"
233 | }
234 |
235 | # returns database name if it exists, nothing otherwise
236 | db_exists () {
237 | database="$1"
238 | dbx "$database" "select schema_name from information_schema.schemata where schema_name='$database';" 2>/dev/null
239 | }
240 |
241 | # return database age in days
242 | db_age () {
243 | db="$1"
244 | now=$(date +%s)
245 | age=0
246 | if [[ "$force_refresh" -gt 0 ]]; then
247 | age=$max_age
248 | else
249 | db_last_modified=$(date -d "$(dbx "$database" "${lastmodified[$db]}")" +%s)
250 | age=$(((now-db_last_modified)/60/60/24))
251 | fi
252 | echo -n $age
253 | }
254 |
255 | # run filter on dump
256 | run_filter () {
257 | dump_file="$1"
258 | flt="$2"
259 | if [[ -n $verbose ]]; then
260 | echo "running '$flt' on '$dump_file'"
261 | fi
262 | sed -i -e "$flt" "$dump_file"
263 | }
264 |
265 | check_credentials () {
266 | if [[ ! $(dbx "" "select true;" 2>/dev/null) ]]; then
267 | exit_with_error "database connection error, bad username or password?"
268 | fi
269 | }
270 |
271 | url_available () {
272 | url="$1"
273 | $torsocks "$wget" -q --spider "$url"
274 | }
275 |
276 | cleanup () {
277 | if [[ ! -v keep_downloaded_files ]]; then
278 | rm -rf "${tmpdir}"
279 | else
280 | echo "-k option active, temporary directory ${tmpdir} not removed"
281 | fi
282 | }
283 |
284 | help () {
285 | echo "$(basename "$(readlink -f "$0")")" "version $version"
286 | cat <<- EOT
287 |
288 | Usage: refresh_libgen OPTIONS
289 |
290 | Performs a refresh from a database dump file for the chosen libgen databases.
291 |
292 | Make sure the database credentials are configured (in \$HOME/.my.cnf) before
293 | using this tool.
294 |
295 | -n do not refresh database
296 | use together with '-v' to check if recent dumps are available
297 | -f force refresh, use this on first install
298 | -v be verbose about what is being updated
299 | -d DAYS only use database dump files no older than DAYS days (default: ${max_age})
300 | -u DBS refresh DBS databases (default: ${!databases[@]})
301 |
302 | -H DBHOST database host (${dbhost})
303 | -P DBPORT database port (${dbport})
304 | -U DBUSER database user (${dbuser})
305 | -a REPO dump repository (${base})
306 | -c create a config file using current settings (see -H, -P, -U, -R)
307 | -e edit config file
308 |
309 | -@ TORPORT use tor (through torsocks) to connect to libgen server
310 | -k keep downloaded files after exit
311 | -h this help message
312 |
313 | EOT
314 | }
315 |
316 | exlock prepare || exit 1
317 |
318 | main "$@"
319 |
--------------------------------------------------------------------------------
/classify:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | #shellcheck disable=SC2034,SC1090
3 | #
4 | # classify - return classification data for ISBN (etc.) or MD5 (from libgen/libgen_fiction)
5 |
6 | shopt -s extglob
7 | trap "trap_error" TERM
8 | trap "trap_clean" EXIT
9 | export TOP_PID=$$
10 |
11 | version="0.5.1"
12 | release="20210601"
13 |
14 | functions="$(dirname "$0")/books_functions"
15 | if [ -f "$functions" ]; then
16 | source "$functions"
17 | else
18 | echo "$functions not found"
19 | exit 1
20 | fi
21 |
22 | main () {
23 | # PREFERENCES
24 | config=${XDG_CONFIG_HOME:-$HOME/.config}/books.conf
25 | # OCLC classify API
26 | oclc="http://classify.oclc.org/classify2/Classify"
27 |
28 | declare -A API=(
29 | [response]='/classify/response/@code'
30 | [owi]='/classify/works/work[1]/@owi'
31 | [wi]='/classify/works/work[1]/@wi'
32 | [fast]='join(/classify/recommendations/fast/headings/heading,",")'
33 | [ddc]='join(/classify/recommendations/ddc/mostPopular/@nsfa)'
34 | [lcc]='join(/classify/recommendations/lcc/mostPopular/@nsfa)'
35 | [nlm]='join(/classify/recommendations/nlm/mostPopular/@sfa)'
36 | [author]='/classify/work/@author'
37 | [authors]='join(/classify/authors/author," | ")'
38 | [title]='/classify/work/@title'
39 | )
40 |
41 | declare -A filters=(
42 | [filename]="sed -e 's/[^-[:alnum:]:;?!.,+@#%]/_/g;s/^\([-_]\)*//'"
43 | )
44 |
45 | declare -A tables=(
46 | [libgen]="updated"
47 | [libgen_fiction]="fiction"
48 | )
49 |
50 | xidel=$(find_tool "xidel")
51 | curl=$(find_tool "curl")
52 | xq="$xidel -s"
53 |
54 | request=""
55 |
56 | TMPDIR="/tmp"
57 | xml=$(mktemp -p $TMPDIR classify.XXXXX)
58 |
59 | # source config file if it exists
60 | [[ -f ${config} ]] && source "${config}"
61 |
62 | while getopts "owdlnfatVAD:C:X:G@:h" OPTION; do
63 | case $OPTION in
64 | o)
65 | request="$request owi"
66 | ;;
67 | w)
68 | request="$request wi"
69 | ;;
70 | d)
71 | request="$request ddc"
72 | ;;
73 | l)
74 | request="$request lcc"
75 | ;;
76 | n)
77 | request="$request nlm"
78 | ;;
79 | f)
80 | request="$request fast"
81 | ;;
82 | a)
83 | request="$request author"
84 | ;;
85 | t)
86 | request="$request title"
87 | ;;
88 | V)
89 | verbose=1
90 | ;;
91 | D)
92 | db="$OPTARG"
93 | ;;
94 | C)
95 | [ -z "$db" ] && exit_with_error "use -D to define which database to use"
96 | build_csv=1
97 | md5="$OPTARG"
98 | idents=$(get_identifiers "$db" "$md5")
99 | [ -z "$idents" ] && exit_with_error "no identifier found in $db for MD5 = $md5"
100 | ;;
101 | X)
102 | save_xml="$OPTARG"
103 | [[ ! -d "$save_xml" ]] && exit_with_error "Save XML (-X $OPTARG): directory does not exist?"
104 | ;;
105 | A)
106 | request="author title fast owi wi ddc lcc nlm"
107 | verbose=1
108 | ;;
109 | G)
110 | ((debug++))
111 | ;;
112 | @)
113 | torsocks=$(find_tool "torsocks")
114 | export TORSOCKS_TOR_PORT=${OPTARG}
115 | ;;
116 | h)
117 | help
118 | exit
119 | ;;
120 |
121 | *)
122 | exit_with_error "unknown option: $OPTION"
123 | ;;
124 | esac
125 | done
126 |
127 | shift $((OPTIND-1))
128 | [ -z "$idents" ] && idents="$1"
129 |
130 | IFS=',' read -ra idarr <<< "$idents"
131 |
132 | for ident in "${idarr[@]}"; do
133 |
134 | [[ -n "$debug" ]] && echo "trying $ident..."
135 |
136 | get_xml "$xml" "stdnbr=${ident// }"
137 | response=$(get "response" "$xml")
138 |
139 | case "$response" in
140 | 0)
141 | success=1
142 | break
143 | ;;
144 | 2)
145 | success=1
146 | break
147 | ;;
148 | 4)
149 | wi=$(get "wi" "$xml")
150 | get_xml "$xml" "wi=$wi"
151 | if [[ $(get "response" "$xml") =~ 0|2 ]]; then
152 | success=1
153 | break
154 | else
155 | continue
156 | fi
157 | ;;
158 | *)
159 | continue
160 | ;;
161 | esac
162 | done
163 |
164 | [[ -z "$success" ]] && exit_with_error "no valid response for identifier(s) $idents"
165 |
166 | if [[ -n "$save_xml" ]]; then
167 | [[ -z "$md5" ]] && exit_with_error "Save XML (-X) only works with a defined MD5 (-C MD5)"
168 | cp "$xml" "$save_xml/$md5.xml"
169 | fi
170 |
171 | if [[ -n "$debug" ]]; then
172 | cat "$xml"
173 | fi
174 |
175 | if [[ -n "$build_csv" ]]; then
176 | build_csv "$db" "$md5" "$xml"
177 | else
178 | show_data "$request"
179 | fi
180 | }
181 |
182 | get_xml () {
183 | xml="$1"
184 | shift
185 | query="$*"
186 | $torsocks "$curl" -s "${oclc}?summary=false&${query}" --output "$xml"
187 | }
188 |
189 | get () {
190 | parameter="$1"
191 | xml="$2"
192 | shift 2
193 | filter="$*"
194 | [[ -z "$filter" ]] && filter='cat -'
195 | $xq "$xml" -e "${API[$parameter]}"|eval "$filter"
196 | }
197 |
198 | get_identifiers () {
199 | db="$1"
200 | md5="$2"
201 |
202 | declare -A sql_identifier=(
203 | [libgen]="select IdentifierWODash from updated where md5='${md5}';"
204 | [libgen_fiction]="select Identifier from fiction where md5='${md5}';"
205 | )
206 |
207 | sql="${sql_identifier[$db]}"
208 | dbx "$db" "$sql"
209 | }
210 |
211 | show_data () {
212 | request="$*"
213 |
214 | for parameter in $request; do
215 | data=$(get "$parameter" "$xml")
216 | [[ -n "$verbose" ]] && legend="${parameter^^}: "
217 | [[ -n "$data" ]] && echo "${legend}${data}"
218 | done
219 | }
220 |
221 | build_csv () {
222 | db="$1"
223 | md5="$2"
224 | xml="$3"
225 |
226 | updates="${md5}"
227 |
228 | for parameter in ddc lcc nlm; do
229 | data=$(get "$parameter" "$xml")
230 | updates+=",\"${data}\""
231 | done
232 |
233 | for parameter in fast author title; do
234 | data=$(get "$parameter" "$xml" "base64 -w0")
235 | updates+=",${data}"
236 | done
237 |
238 | echo "$updates"
239 | }
240 |
241 | cleanup () {
242 | base=$(basename "$xml")
243 | rm -f "$TMPDIR/$base"
244 | }
245 |
246 | help () {
247 | cat <<-EOHELP
248 | $(basename "$(readlink -f "$0")") "version $version"
249 |
250 | Use: classify [OPTIONS] identifier[,identifier...]
251 |
252 | Queries OCLC classification service for available data
253 | Supports: DDC, LCC, NLM, Author and Title
254 |
255 | Valid identifiers are ISBN, ISSN, UPC and OCLC/OWI
256 |
257 | OPTIONS:
258 |
259 | -d show DDC
260 | -l show LCC
261 | -n show NLM
262 | -f show FAST
263 | -a show Author
264 | -t show Title
265 |
266 | -o show OWI (OCLC works identifier)
267 | -w show WI (OCLC works number)
268 |
269 | -C md5 create CSV (MD5,DDC,LCC,NLM,FAST,AUTHOR,TITLE)
270 | use -D libgen/-D libgen_fiction to indicate database
271 |
272 | -X dir save OCLC XML response to \$dir/\$md5.xml
273 | only works with a defined MD5 (-C MD5)
274 |
275 | -D db define which database to use (libgen/libgen_fiction)
276 |
277 | -A show all available data for identifier
278 |
279 | -V show labels
280 |
281 | -@ PORT use torsocks to connect to the OCLC classify service.
282 | use this to avoid getting your IP blocked by OCLC
283 |
284 | -h show this help message
285 |
286 | Examples
287 |
288 | $ classify -A 0199535760
289 | AUTHOR: Plato | Jowett, Benjamin, 1817-1893 Translator; Editor; Other] ...
290 | TITLE: The republic
291 | DDC: 321.07
292 | LCC: JC71
293 |
294 | $ classify -D libgen -C 25b8ce971343e85dbdc3fa375804b538
295 | 25b8ce971343e85dbdc3fa375804b538,"321.07","JC71","",UG9saXRpY2FsI\
296 | HNjaWVuY2UsVXRvcGlhcyxKdXN0aWNlLEV0aGljcyxQb2xpdGljYWwgZXRoaWNzLFB\
297 | oaWxvc29waHksRW5nbGlzaCBsYW5ndWFnZSxUaGVzYXVyaQo=,UGxhdG8gfCBKb3dl\
298 | dHQsIEJlbmphbWluLCAxODE3LTE4OTMgW1RyYW5zbGF0b3I7IEVkaXRvcjsgT3RoZX\
299 | JdIHwgV2F0ZXJmaWVsZCwgUm9iaW4sIDE5NTItIFtUcmFuc2xhdG9yOyBXcml0ZXIg\
300 | b2YgYWRkZWQgdGV4dDsgRWRpdG9yOyBPdGhlcl0gfCBMZWUsIEguIEQuIFAuIDE5MD\
301 | gtMTk5MyBbVHJhbnNsYXRvcjsgRWRpdG9yOyBBdXRob3Igb2YgaW50cm9kdWN0aW9u\
302 | XSB8IFNob3JleSwgUGF1bCwgMTg1Ny0xOTM0IFtUcmFuc2xhdG9yOyBBdXRob3I7IE\
303 | 90aGVyXSB8IFJlZXZlLCBDLiBELiBDLiwgMTk0OC0gW1RyYW5zbGF0b3I7IEVkaXRv\
304 | cjsgT3RoZXJdCg==,VGhlIHJlcHVibGljCg==
305 |
306 |
307 | Classifying libgen/libgen_fiction
308 |
309 | This tool can be used to add classification data to libgen and
310 | libgen_fiction databases. It does not directy modify the database,
311 | instead producing CSV which can be used to apply the modifications.
312 | The best way to do this is to produce a list of md5 hashes for
313 | publications which do have Identifier values but lack values for DDC
314 | and/or LCC. Such lists can be produced by the following SQL:
315 |
316 | libgen: select md5 from updated where IdentifierWODash<>"" and DDC="";
317 | libgen_fiction: select md5 from fiction where Identifier<>"" and DDC="";
318 |
319 | Run these as batch jobs (mysql -B .... -e 'sql_code_here;' > md5_list), split
320 | the resulting file in ~1000 line sections and feed these to this tool,
321 | preferably with a random pause between requests to keep OCLC's intrusion
322 | detection systems from triggering too early. It is advisable to use
323 | this tool through Tor (using -@ TORPORT to enable torsocks, make sure it
324 | is configured correctly for your Tor instance) to avoid having too
325 | many requests from your IP to be registered, this again to avoid
326 | your IP being blocked. The OCLC classification service is not
327 | run as a production service (I asked them).
328 |
329 | Return values are stored in the following order:
330 |
331 | MD5,DDC,LCC,NLM,FAST,AUTHOR,TITLE
332 |
333 | DDC, LCC and NLM are enclosed within double quotes and can contain
334 | multiple space-separated values. FAST, AUTHOR and TITLE are base64 encoded
335 | since these fields can contain a whole host of unwholesome characters
336 | which can mess up CSV. The AUTHOR field currentlydecodes to a pipe ('|')
337 | separated list of authors in the format:
338 |
339 | LAST_NAME, NAME_OR_INITIALS, DATE_OF_BIRTH-[DATE_OF_DEATH] [[ROLE[[;ROLE]...]]]
340 |
341 | This format could change depending on what OCLC does with the
342 | (experimental) service.
343 |
344 | EOHELP
345 | }
346 |
347 | main "$@"
348 |
--------------------------------------------------------------------------------
/update_libgen:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # shellcheck disable=SC2034,SC1090,SC2155,SC2207
3 |
4 | version="0.6.1"
5 | release="20210512"
6 |
7 | trap "trap_error" TERM
8 | trap "trap_clean" EXIT
9 | export TOP_PID=$$
10 |
11 | LC_ALL=C
12 |
13 | functions="$(dirname "$0")/books_functions"
14 | if [ -f "$functions" ]; then
15 | source "$functions"
16 | else
17 | echo "$functions not found"
18 | exit 1
19 | fi
20 |
21 | main () {
22 |
23 | exlock now || exit 1
24 |
25 | # PREFERENCES
26 | config=${XDG_CONFIG_HOME:-$HOME/.config}/books.conf
27 |
28 | dbhost="localhost"
29 | dbport="3306"
30 | db="libgen"
31 | dbuser="libgen"
32 | limit=1000
33 |
34 | api="http://libgen.rs/json.php"
35 |
36 | # source config file if it exists
37 | [[ -f ${config} ]] && source "${config}"
38 |
39 | # (more or less) END OF PREFERENCES
40 |
41 | jq=$(find_tool "jq")
42 | curl=$(find_tool "curl")
43 |
44 | tmpdir=$(mktemp -d '/tmp/update_libgen.XXXXXX')
45 | updates="${tmpdir}/updates"
46 | update_count="${tmpdir}/update_count"
47 | update_sql="${tmpdir}/update_sql"
48 | update_last_modified="${tmpdir}/update_last_modified"
49 | update_last_id="${tmpdir}/update_last_id"
50 | update_newer="${tmpdir}/update_newer"
51 |
52 | verbose=0
53 | no_action=0
54 | unknown_fields=""
55 |
56 | re_type='[a-z]+'
57 | re_int='[0-9]+'
58 | re_year='[0-9]{4}'
59 | re_timestamp='[0-9]{4}-[0-9]{2}-[0-9]{2} [0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
60 |
61 | declare -a tables="(description hashes updated)"
62 | declare -A current_fields="($(get_current_fields))"
63 | declare -A field_types="($(get_field_types))"
64 | declare -A field_sizes="($(get_field_sizes))"
65 | declare -A columns=()
66 | declare -A values=()
67 | declare -A upsert=()
68 |
69 | while getopts "a:D:j:hH:i:l:nP:U:qs:ct:u:v@:" OPTION
70 | do
71 | case $OPTION in
72 | j)
73 | json_dump="${OPTARG}"
74 | ;;
75 | s)
76 | sql_dump="${OPTARG}"
77 | ;;
78 | c)
79 | classify=$(find_tool "classify")
80 | import_metadata=$(find_tool "import_metadata")
81 | classifile="${tmpdir}/classifile"
82 | ;;
83 | v)
84 | ((verbose++))
85 | ;;
86 | n)
87 | no_action=1
88 | ;;
89 | l)
90 | limit="${OPTARG}"
91 | if [[ $limit -le 1 ]]; then
92 | exit_wit_error "limit too low (-l ${limit}), minimum is 2"
93 | fi
94 | ;;
95 | t)
96 | startdatetime="${OPTARG}"
97 | ;;
98 | i)
99 | echo "${OPTARG}" > "${update_last_id}"
100 | ;;
101 | u)
102 | api="${OPTARG}"
103 | ;;
104 | H)
105 | dbhost="${OPTARG}"
106 | ;;
107 | P)
108 | dbport="${OPTARG}"
109 | ;;
110 | U)
111 | dbuser="${OPTARG}"
112 | ;;
113 | D)
114 | db="${OPTARG}"
115 | ;;
116 | a)
117 | if url_available "${OPTARG}?fields=id&ids=0"; then
118 | api="${OPTARG}"
119 | else
120 | exit_with_error "-a ${OPTARG}: API endpoint not available"
121 | fi
122 | ;;
123 | @)
124 | torsocks=$(find_tool "torsocks")
125 | export TORSOCKS_TOR_PORT=${OPTARG}
126 | ;;
127 | q)
128 | quiet=1
129 | ;;
130 | h)
131 | help
132 | exit
133 | ;;
134 | *)
135 | exit_with_error "unknown option $OPTION"
136 | ;;
137 |
138 | esac
139 | done
140 |
141 | check_fields
142 |
143 | while (
144 | if [[ -s ${update_last_modified} ]]; then
145 | last_update="$(cat "${update_last_modified}")"
146 | last_update_in_db="$(get_time_last_modified)"
147 | if [[ $last_update != "$last_update_in_db" && $no_action == 0 ]]; then
148 | exit_with_error "uh oh... something went wrong, last update in db does not equal last update from api response..."
149 | fi
150 | elif [[ -n $startdatetime ]]; then
151 | last_update="${startdatetime}"
152 | else
153 | last_update="$(get_time_last_modified)"
154 | fi
155 |
156 | last_id=$([[ -s ${update_last_id} ]] && cat "${update_last_id}" || get_max_id)
157 |
158 | get_updates "$last_id" "$limit" "$last_update"
159 |
160 | updcnt=$(get_update_count)
161 |
162 | [[ -n $json_dump ]] && cat "${updates}" >> "${json_dump}"
163 |
164 | if [[ $verbose -ge 1 ]]; then
165 | echo "database last modified: $last_update";
166 | # update counter is 0-based, humans prefer 1-based notation
167 | if [[ ${updcnt} -gt 0 ]]; then
168 | echo "$((updcnt+1)) updates";
169 | else
170 | more=$([[ -s $update_last_id ]] && echo "more " || echo "")
171 | echo "no ${more}updates"
172 | fi
173 | echo ;
174 | fi
175 |
176 | test "$updcnt" -gt 0
177 | ); do
178 |
179 | updcnt=$(get_update_count)
180 | count=0
181 | echo "start transaction;" > "${update_sql}"
182 |
183 | while [[ $count -le $updcnt ]]; do
184 | declare -A record
185 | while IFS="=" read -r key value; do
186 | # drop unknown fields
187 | if [[ ! $unknown_fields =~ ${key,,} ]]; then
188 | # limit field size to avoid choking jq on overly long strings
189 | [[ ${#value} -gt 1000 ]] && value="${value:0:997}..."
190 | record[${key,,}]="$value"
191 | fi
192 | done < <($jq -r ".[$count]"'|to_entries|map("\(.key)=\(.value|tostring|.[0:4000]|gsub("\n";"\\n"))")|.[]' "${updates}")
193 |
194 | # record current position
195 | echo "${record['id']}" > "${update_last_id}"
196 | echo "${record['timelastmodified']}" > "${update_last_modified}"
197 |
198 | if [[ $verbose -ge 2 ]]; then
199 | echo "ID: ${record['id']}";
200 | echo "Author: ${record['author']}";
201 | echo "Title: ${record['title']}";
202 | echo "Modified: ${record['timelastmodified']}";
203 | echo
204 | fi
205 |
206 | if [[ -n "$classifile" && -n "${record['identifierwodash']}" ]]; then
207 | echo "${record['md5']}" >> "$classifile"
208 | fi
209 |
210 | keys=${!record[*]}
211 |
212 | md5="${record[md5]}"
213 |
214 | # split fields between tables
215 | for key in "${!record[@]}"; do
216 | table=${current_fields[$key]}
217 | columns[$table]+="${key},"
218 | value=${record[$key]}
219 | if [ -n "$value" ]; then
220 | value=$(sanitize_field "$key" "$value")
221 | fi
222 | values[$table]+="'$value',"
223 | upsert[$table]+="${key} = values(${key}),"
224 | done
225 |
226 | # add md5 to secondary tables (all but the last)
227 | for n in $(seq 0 $((${#tables[@]}-2))); do
228 | table="${tables[$n]}"
229 | if [[ -n "${columns[$table]}" ]]; then
230 | columns[$table]+="md5,"
231 | values[$table]+="'$md5',"
232 | upsert[$table]+="md5 = values(md5),"
233 | fi
234 | done
235 |
236 | # main table (last in tables array) first
237 | for n in $(seq $((${#tables[@]}-1)) -1 0); do
238 | table="${tables[$n]}"
239 | if [[ -n "${columns[$table]}" ]]; then
240 | sql+="insert into $table (${columns[$table]%?}) values(${values[$table]%?}) on duplicate key update ${upsert[$table]%?};"
241 | fi
242 | done
243 |
244 | echo "${sql}" >> "${update_sql}"
245 | [[ -n $sql_dump ]] && echo "${sql}" >> "${sql_dump}"
246 |
247 | unset record
248 | unset keys
249 | unset key
250 | unset value
251 | unset sql
252 | columns=()
253 | values=()
254 | upsert=()
255 |
256 | ((count++))
257 | done
258 |
259 | echo "commit;" >> "${update_sql}"
260 |
261 | [[ $no_action == 0 ]] && dbx "$db" < "${update_sql}"
262 | done
263 |
264 | # optionally add classification data to new records
265 | # this will use tor and round-robin through TOR ports if these are
266 | # defined in classify_tor_ports in the config file
267 | if [[ -n "$classifile" && -f $classifile ]]; then
268 | now=$(date +%Y%m%d%H%M)
269 | csvfile="${classify_csv:+$classify_csv/}${now}.csv"
270 | IFS=',' read -ra torports <<< "$classify_tor_ports"
271 | if [[ ${#torports[*]} -gt 0 ]]; then
272 | torpc=${#torports[*]}
273 | fi
274 | upc=0
275 | while read md5;do
276 | $classify ${torpc:+-@ ${torports[$upc%$torpc]}} -D "$db" ${classify_xml:+-X $classify_xml} -C "$md5" >> "${csvfile}"
277 | ((upc++))
278 | done < <(cat "$classifile")
279 |
280 | if [[ -f ${csvfile} ]]; then
281 | $import_metadata -d "$db" -f "${classify_fields:-ddc,lcc,fast}" ${classify_sql:+-s $classify_sql/$now.sql} -F "${csvfile}"
282 | fi
283 | fi
284 |
285 | }
286 |
287 | get_current_fields () {
288 | for table in "${tables[@]}"; do
289 | dbx "$db" "describe $table;"|awk '{print "["tolower($1)"]='"$table"'"}'
290 | done
291 | }
292 |
293 | get_field_type () {
294 | field="$1"
295 | table="${current_fields[$field]}"
296 | dbx "$db" "show fields from $table where field=\"$field\";"|awk '{print $2}'
297 | }
298 |
299 | get_field_types () {
300 | for field in "${!current_fields[@]}"; do
301 | fieldtype=$(get_field_type "$field")
302 | [[ "$fieldtype" =~ $re_type ]]
303 | echo -n "[$field]=${BASH_REMATCH[0]} "
304 | done
305 | }
306 |
307 | get_field_sizes () {
308 | for field in "${!current_fields[@]}"; do
309 | fieldtype=$(get_field_type "$field")
310 | [[ "$fieldtype" =~ $re_int ]]
311 | if [[ "${BASH_REMATCH[0]}" -gt 0 ]]; then
312 | echo -n "[$field]=${BASH_REMATCH[0]} "
313 | fi
314 | done
315 | }
316 |
317 | # sanitize_field FIELD VALUE
318 | sanitize_field () {
319 | field=$1
320 | shift
321 | value="$*"
322 |
323 | # quote values for SQL
324 | value=${value//\\/\\\\}
325 | value=${value//\'/\\\'}
326 |
327 | # field-type specific filters
328 | case "${field_types[$field]}" in
329 | int|bigint)
330 | [[ "$value" =~ $re_int ]]
331 | value=${BASH_REMATCH[0]}
332 | value=${value:0:${field_sizes[$field]}}
333 | ;;
334 | char|varchar)
335 | value=${value:0:${field_sizes[$field]}}
336 | ;;
337 | timestamp)
338 | [[ "$value" =~ $re_timestamp ]]
339 | value=${BASH_REMATCH[0]}
340 | ;;
341 | esac
342 |
343 | # field-specific filters
344 | case "$field" in
345 | year)
346 | # filter out Chinese date stamps
347 | [[ "$value" =~ $re_year ]]
348 | value=${BASH_REMATCH[0]}
349 | ;;
350 | esac
351 |
352 | echo -n "$value"
353 | }
354 |
355 | # libgen_api ID LIMIT TIME_LAST_MODIFIED
356 | libgen_api () {
357 | id="$1"
358 | shift
359 | limit="$1"
360 | shift
361 | if ! newer=$(date -d "$*" +'%Y-%m-%d%%20%H:%M:%S'); then
362 | exit_with_error "date error: $* is not a valid date"
363 | fi
364 |
365 | echo "$newer" > "$update_newer"
366 |
367 | $torsocks "$curl" -s "${api}?"'fields=*&idnewer='"${id}"'&mode=newer&limit1='"${limit}"'&timenewer='"${newer}"
368 | }
369 |
370 | # get_updates ID LIMIT TIME_LAST_MODIFIED
371 | get_updates () {
372 | id="$1"
373 | shift
374 | limit="$1"
375 | shift
376 | last="$*"
377 | libgen_api "$id" "$limit" "$last" > "${updates}"
378 | $jq '.|length' "${updates}" > "${update_count}"
379 | }
380 |
381 |
382 | get_time_last_modified () {
383 | dbx "$db" 'select MAX(TimeLastModified) FROM updated;'|tail -1
384 | }
385 |
386 | get_max_id () {
387 | dbx "$db" 'select MAX(id) FROM updated;'|tail -1
388 | }
389 |
390 | get_update_count () {
391 | echo $(($(cat "${update_count}")-1))
392 | }
393 |
394 | check_fields () {
395 | updates_fields=($(libgen_api 1 2 '2000-01-01'|$jq -r '.[0]|keys|@sh'))
396 | db_fields="${!current_fields[*]}"
397 | db_fields="${db_fields,,}"
398 |
399 | # check for extra fields in api response
400 | for index in "${!updates_fields[@]}"; do
401 | field="${updates_fields[$index]%\'}"
402 | field="${field#\'}"
403 | if [[ ! $db_fields =~ ${field,,} ]]; then
404 | if [[ ! -v quiet ]]; then
405 | echo "unknown field in api response: ${field} (consider refreshing database from dump)"
406 | fi
407 | unknown_fields+="${field,,} "
408 | else
409 | :
410 | fi
411 | done
412 |
413 | # check for missing fields in api reponse
414 | [[ $verbose -ge 1 ]] && {
415 | for field in "${!current_fields[@]}"; do
416 | if [[ ! -v quiet && ! ${updates_fields[*],,} =~ ${field,,} ]]; then
417 | echo "missing field in api response: $field"
418 | fi
419 | done
420 | }
421 | }
422 |
423 | cleanup () {
424 | rm -rf "${tmpdir}"
425 | }
426 |
427 | help () {
428 | echo "$(basename "$(readlink -f "$0")")" "version $version"
429 | cat <<- 'EOT'
430 |
431 | Usage: update_libgen OPTIONS
432 |
433 | -l LIMIT get updates in blocks of LIMIT entries
434 | -v be verbose about what is being updated; repeat for more verbosity:
435 | -v: show basic info (number of updates, etc)
436 | -vv: show ID, Title and TimeLastModified for each update
437 | -n do not update database. Use together with -v or -vv to show
438 | how many (-v) and which (-vv) titles would be updated.
439 | -j FILE dump (append) json to FILE
440 | -s FILE dump (append) sql to FILE
441 | -u URL use URL to access the libgen API (overrides default)
442 | -t DATETIME get updates since DATETIME (ignoring TimeLastModified in database)
443 | use this option together with -s to create an sql update file to update
444 | non-networked machines
445 | -i ID get updates from ID
446 |
447 | -H DBHOST database host
448 | -P DBPORT database port
449 | -U DBUSER database user
450 | -D DATABASE database name
451 |
452 | -a APIHOST use APIHOST as API server
453 | -@ TORPORT use tor (through torsocks) to connect to libgen API server
454 | -c run classify over new records to get classification data
455 | -q don't warn about missing fields in database or api response
456 | -h this help message
457 |
458 | EOT
459 | }
460 |
461 | exlock prepare || exit 1
462 |
463 | main "$@"
464 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | GNU GENERAL PUBLIC LICENSE
2 | Version 3, 29 June 2007
3 | Copyright © 2007 Free Software Foundation, Inc.
4 |
5 | Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
6 |
7 | Preamble
8 |
9 | The GNU General Public License is a free, copyleft license for software and other kinds of works.
10 |
11 | The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too.
12 |
13 | When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
14 |
15 | To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others.
16 |
17 | For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
18 |
19 | Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it.
20 |
21 | For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions.
22 |
23 | Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users.
24 |
25 | Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free.
26 |
27 | The precise terms and conditions for copying, distribution and modification follow.
28 |
29 | TERMS AND CONDITIONS
30 |
31 | 0. Definitions.
32 |
33 | “This License” refers to version 3 of the GNU General Public License.
34 |
35 | “Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
36 |
37 | “The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations.
38 |
39 | To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work.
40 |
41 | A “covered work” means either the unmodified Program or a work based on the Program.
42 |
43 | To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
44 |
45 | To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
46 |
47 | An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
48 |
49 | 1. Source Code.
50 | The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work.
51 |
52 | A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
53 |
54 | The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
55 |
56 | The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.
57 |
58 | The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
59 |
60 | The Corresponding Source for a work in source code form is that same work.
61 |
62 | 2. Basic Permissions.
63 | All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
64 |
65 | You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
66 |
67 | Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
68 |
69 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
70 | No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
71 |
72 | When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
73 |
74 | 4. Conveying Verbatim Copies.
75 | You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
76 |
77 | You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
78 |
79 | 5. Conveying Modified Source Versions.
80 | You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
81 |
82 | a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
83 |
84 | b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”.
85 |
86 | c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
87 |
88 | d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
89 |
90 | A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
91 |
92 | 6. Conveying Non-Source Forms.
93 | You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
94 |
95 | a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
96 |
97 | b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
98 |
99 | c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
100 |
101 | d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
102 |
103 | e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
104 |
105 | A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
106 |
107 | A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
108 |
109 | “Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
110 |
111 | If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
112 |
113 | The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
114 |
115 | Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
116 |
117 | 7. Additional Terms.
118 | “Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
119 |
120 | When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
121 |
122 | Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
123 |
124 | a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
125 |
126 | b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
127 |
128 | c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
129 |
130 | d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
131 |
132 | e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
133 |
134 | f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
135 |
136 | All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
137 |
138 | If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
139 |
140 | Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
141 |
142 | 8. Termination.
143 | You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
144 |
145 | However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
146 |
147 | Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
148 |
149 | Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
150 |
151 | 9. Acceptance Not Required for Having Copies.
152 | You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
153 |
154 | 10. Automatic Licensing of Downstream Recipients.
155 | Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
156 |
157 | An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
158 |
159 | You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
160 |
161 | 11. Patents.
162 | A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's “contributor version”.
163 |
164 | A contributor's “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
165 |
166 | Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
167 |
168 | In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
169 |
170 | If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
171 |
172 | If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
173 |
174 | A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
175 |
176 | Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
177 |
178 | 12. No Surrender of Others' Freedom.
179 | If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
180 |
181 | 13. Use with the GNU Affero General Public License.
182 | Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such.
183 |
184 | 14. Revised Versions of this License.
185 | The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
186 |
187 | Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation.
188 |
189 | If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
190 |
191 | Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
192 |
193 | 15. Disclaimer of Warranty.
194 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
195 |
196 | 16. Limitation of Liability.
197 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
198 |
199 | 17. Interpretation of Sections 15 and 16.
200 | If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
201 |
202 | END OF TERMS AND CONDITIONS
203 |
204 | How to Apply These Terms to Your New Programs
205 |
206 | If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
207 |
208 | To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.
209 |
210 |
211 | Copyright (C)
212 |
213 | This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
214 |
215 | This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
216 |
217 | You should have received a copy of the GNU General Public License along with this program. If not, see .
218 |
219 | Also add information on how to contact you by electronic and paper mail.
220 |
221 | If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:
222 |
223 | Copyright (C)
224 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
225 | This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
226 |
227 | The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an “about box”.
228 |
229 | You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see .
230 |
231 | The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read .
232 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # books
2 |
3 | [B]ooks - which is only one of the names this program goes by - is a front-end for accessing a locally accessible libgen / libgen_fiction database instance, offering versatile search and download directly from the command line. The included `update_libgen` tool is used to keep the database up to date - if the database is older than a user-defined value it is updated before the query is executed. This generally only takes a few seconds, but it might take longer on a slow connection or after a long update interval. Updating can be temporarily disabled by using the '-x' command line option. To refresh the database(s) from a dump file use the included `refresh_libgen program`, see 'update_libgen vs refresh_libgen' below for more information on which tool to use.
4 |
5 | Books comes in three main flavours:
6 |
7 | * `books` / `books-all` / `fiction`: CLI search interface which dumps results to the terminal, download through MD5
8 | * `nbook` / `nfiction`: text-based browser offering limited preview and download
9 | * `xbook` / `xfiction`: gui-based browser offering preview and download
10 |
11 |
12 | The *book* tools are based on the *libgen* database, the *fiction* tools use the *libgen_fiction* database. Apart from the fact that the *fiction* tools do not support all the search criteria offered by the 'book' tools due to differences in the database layout, all programs share the same interface.
13 |
14 | The database can be searched in two modes, per-field (the default) and fulltext (which, of course, only searches book metadata, not the actual book contents). The current implementation for fulltext search is actually a pattern match search on a number of concatenated database columns, it does not use MySQL's native fulltext search. The advantage of this implementation is that it does not need a full-text index (which is not part of the libgen dump and would need to be generated locally), the disadvantage is that it does not offer more advanced natural language search options. Given the limited amount of 'natural language' available in the database the latter does not seem to be much of a disadvantage and the implementation performs well.
15 |
16 | In the (default) per-field search mode the database can be searched for patterns (SQL 'like' operator with leading and trailing wildcards) using lower-case options and/or exact matches using upper-case options. The fulltext search by necessity always uses pattern matching over the indicated fields ('title' and 'author' if no other fields are specified).
17 |
18 | Publications can be downloaded using IPFS, through torrents or from libgen download mirror servers by selecting them in the result list or by using the 'Download' button in the preview window, the `books` and `fiction` tools can be used to download publications based on their MD5 hash (use `-J ...`). When using the gui-based tools in combination with the 'yad' tool, double-clicking a row in the result list shows a preview, the other tools generate previews for selected publications using the '-w' command line option.
19 |
20 | See [Installation](#installation) for information on how to install *books*.
21 |
22 | ## How to use *books* et al.
23 |
24 | I'll let the programs themselves do the talking:
25 |
26 | ```txt
27 | $ books -h
28 | books version 0.7
29 |
30 | Use: books OPTIONS [like] []
31 |
32 | (...)
33 |
34 | SEARCH BY FIELD:
35 |
36 | This is the default search mode. If no field options are given this searches
37 | the Title field for the PATTERN. Capital options (-A, -T, etc) for exact match,
38 | lower-case (-a, -t, etc) for pattern match.
39 |
40 | FULLTEXT SEARCH (-f):
41 |
42 | Performs a pattern match search over all fields indicated by the options. If no
43 | field options are given, perform a pattern match search over the Author and
44 | Title fields.
45 |
46 | Depending on which name this program is executed under it behaves differently:
47 |
48 | books: query database and show results, direct download with md5
49 | books-all: query database and show results (exhaustive search over all tables, slow)
50 |
51 | nbook: select publications for download from list (terminal-based)
52 | xbook: select publications for download from list (GUI)
53 |
54 | fiction: query database and show results (using 'fiction' database), direct download with md5
55 |
56 | nfiction: select publications for download from list (terminal-based, use 'fiction' database)
57 | xfiction: select publications for download from list (GUI, use 'fiction' database)
58 |
59 | OPTIONS
60 |
61 | -z, -Z search on LOCATOR
62 | -y, -Y search on YEAR
63 | -v, -V search on VOLUMEINFO
64 | -t, -T search on TITLE
65 | -s, -S search on SERIES
66 | -r, -R search on PERIODICAL
67 | -q, -Q search on OPENLIBRARYID
68 | -p, -P search on PUBLISHER
69 | -o, -O search on TOPIC_DESCR
70 | -n, -N search on ASIN
71 | -m search on MD5
72 | -l, -L search on LANGUAGE
73 | -i, -I search on ISSN
74 | -g, -G search on TAGS
75 | -e, -E search on EXTENSION
76 | -d, -D search on EDITION
77 | -c, -C search on CITY
78 | -b, -B search on IDENTIFIERWODASH
79 | -a, -A search on AUTHOR
80 |
81 | -f fulltext search
82 | searches for the given words in the fields indicated by the other options.
83 | when no other options are given this will perform a pattern match search
84 | for the given words over the Author and Title fields.
85 |
86 | -w preview publication info before downloading (cover preview only in GUI tools)
87 | select one or more publication to preview and press enter/click OK.
88 |
89 | double-clicking a result row also shows a preview irrespective of this option,
90 | but this only works when using the yad gui tool
91 |
92 | -= DIR set download location to DIR
93 |
94 | -$ use extended path when downloading:
95 | nonfiction/[topic/]author[/series]/title
96 | fiction/language/author[/series]/title
97 |
98 | -u BOOL use bittorrent (-u 1 or -u y) or direct download (-u 0 or -u n)
99 | this parameter overrides the default download method
100 | bittorrent download depends on an external helper script
101 | to interface with a bittorrent client
102 |
103 | -I BOOL use ipfs (-I 1 or -I y) or direct download (-I 0 or -I n)
104 | this parameter overrides the default download method
105 | ipfs download depends on a functioning ipfs gateway.
106 | default gateway is hosted by Cloudfront, see https://ipfs.io/
107 | for instructions on how to run a local gateway
108 |
109 | -U MD5 print torrent path (torrent#/md5) for given MD5
110 |
111 | -j MD5 print filename for given MD5
112 |
113 | -J MD5 download file for given MD5
114 | can be combined with -u to download with bittorrent
115 |
116 | -M MD5 fast path search on md5, only works in _books_ and _fiction_
117 | can be combined with -F FIELDS to select fields to be shown
118 | output goes directly to the terminal (no pager)
119 |
120 | -F FIELDS select which fields to show in pager output
121 |
122 | -# LIMIT limit search to LIMIT hits (default: 1000)
123 |
124 | -x skip database update
125 | (currently only the 'libgen' database can be updated)
126 |
127 | -@ TORPORT use torsocks to connect to the libgen server(s). You'll need to install
128 | torsocks before using this option; try this in case your ISP
129 | (or a transit provider somewhere en-route) blocks access to libgen
130 |
131 | -k install symlinks for all program invocations
132 |
133 | -h show this help message
134 |
135 | EXAMPLES
136 |
137 | Do a pattern match search on the Title field for 'ilias' and show the results in the terminal
138 |
139 | $ books like ilias
140 |
141 |
142 | Do an exact search on the Title field for 'The Odyssey' and show the results in the terminal
143 |
144 | $ books 'the odyssey'
145 |
146 |
147 | Do an exact search on the Title field for 'The Odyssey' and the Author field for 'Homer', showing
148 | the result in the terminal
149 |
150 | $ books -T 'The Odyssey' -A 'Homer'
151 |
152 |
153 | Do the same search as above, showing the results in a list on the terminal with checkboxes to select
154 | one or more publications for download
155 |
156 | $ nbook -T 'The Odyssey' -A 'Homer'
157 |
158 |
159 | A case-insensitive pattern search using an X11-based interface; use bittorrent (-u y or -u 1) when downloading files
160 |
161 | $ xbook -u y -t 'the odyssey' -a 'homer'
162 |
163 |
164 | Do a fulltext search over the Title, Author, Series, Periodical and Publisher fields, showing the
165 | results in a terminal-based checklist for download after preview (-w)
166 |
167 | $ nbook -w -f -t -a -s -r -p 'odyssey'
168 |
169 |
170 | Walk over a directory of publications, compute md5 and use this to generate file names:
171 |
172 | $ find /path/to/publications -type f|while read f; do books -j $(md5sum "$f"|awk '{print $1}');done
173 |
174 |
175 | As above, but print torrent number and path in torrent file
176 |
177 | $ find /path/to/publications -type f|while read f; do books -U $(md5sum "$f"|awk '{print $1}');done
178 |
179 |
180 | Find publications by author 'thucydides' and show their md5,title and year in the terminal
181 |
182 | $ books -a thucydides -F md5,title,year
183 |
184 |
185 | Get data on a single publication using fast path MD5 search, show author, title and extension
186 |
187 | $ books -M 51b4ee7bc7eeb6ed7f164830d5d904ae -F author,title,extension
188 |
189 |
190 | Download a publication using its MD5 (-J MD5), using bittorrent (-u y or -u 1) to download
191 |
192 | $ books -u y -J 51b4ee7bc7eeb6ed7f164830d5d904ae
193 |
194 | ```
195 |
196 | ```txt
197 | $ update_libgen -h
198 | update_libgen version 0.6
199 |
200 | Usage: update_libgen OPTIONS
201 |
202 | -l LIMIT get updates in blocks of LIMIT entries
203 | -v be verbose about what is being updated; repeat for more verbosity:
204 | -v: show basic info (number of updates, etc)
205 | -vv: show ID, Title and TimeLastModified for each update
206 | -n do not update database. Use together with -v or -vv to show
207 | how many (-v) and which (-vv) titles would be updated.
208 | -j FILE dump (append) json to FILE
209 | -s FILE dump (append) sql to FILE
210 | -u URL use URL to access the libgen API (overrides default)
211 | -t DATETIME get updates since DATETIME (ignoring TimeLastModified in database)
212 | use this option together with -s to create an sql update file to update
213 | non-networked machines
214 | -i ID get updates from ID
215 |
216 | -H DBHOST database host
217 | -P DBPORT database port
218 | -U DBUSER database user
219 | -D DATABASE database name
220 |
221 | -a APIHOST use APIHOST as API server
222 | -@ TORPORT use tor (through torsocks) to connect to libgen API server
223 | -c run classify over new records to get classification data
224 | -q don't warn about missing fields in database or api response
225 | -h this help message
226 | ```
227 |
228 | ```txt
229 | $ refresh_libgen -h
230 | refresh_libgen version 0.6.1
231 |
232 | Usage: refresh_libgen OPTIONS
233 |
234 | Performs a refresh from a database dump file for the chosen libgen databases.
235 |
236 | -n do not refresh database
237 | use together with '-v' to check if recent dumps are available
238 | -f force refresh, use this on first install
239 | -v be verbose about what is being updated
240 | -d DAYS only use database dump files no older than DAYS days (default: 5)
241 | -u DBS refresh DBS databases (default: compact fiction libgen)
242 |
243 | -H DBHOST database host (localhost)
244 | -P DBPORT database port (3306)
245 | -U DBUSER database user (libgen)
246 | -R REPO dump repository (http://gen.lib.rus.ec/dbdumps/)
247 | -c create a config file using current settings (see -H, -P, -U, -R)
248 | -e edit config file
249 |
250 | -@ TORPORT use tor (through torsocks) to connect to libgen server
251 | -k keep downloaded files after exit
252 | -h this help message
253 | ```
254 |
255 | ## IPFS, Torrents, direct download...
256 |
257 | *Books* (et al) can download files either through IPFS (using `-I 1` or `-I y`), from torrents (using `-u y` or `-u 1`) or from one of the libgen download mirrors (default, use `-I n`/`-u n` or `-I 0`/`-u 0` in case IPFS or torrent download is set as default). To limit the load on the download servers it is best to use IPFS or torrents whenever possible. The latest publications are not yet available through IPFS or torrents since those are only created for batches of 1000 publications. The feasibility of torrent download also depends on whether the needed torrents are seeded while for IPFS download a working IPFS gateway is needed. Publications which can not be downloaded through IPFS or torrents can be downloaded directly.
258 |
259 | ### IPFS download process
260 | IPFS download makes use of an IPFS gateway, by default this is set to Cloudflare's gateway:
261 |
262 | ```
263 | # ipfs gateway
264 | ipfs_gw="https://cloudflare-ipfs.com"
265 | ```
266 |
267 | This can be changed in the config file (usually `$HOME/.config/books.conf`)
268 |
269 | The actual download works exactly the same as the direct download, only the source is changed from a direct download server to the IPFS gateway. Download speed depends on whether the gateway has the file in cache or not, in the latter case it can take a bit more time - be patient.
270 |
271 | ### Torrent download process
272 | Torrent download works by selecting individual files for download from the 'official' torrents, i.e. it is *not* necessary to download the whole torrent for a single publication. This process is automated by means of a helper script which is used to interface *books* with a torrent client. Currently the only torrent client for which a helper script is available is *transmission-daemon*, the script uses the related *transmission-remote* program to interface with the daemon. Writing a helper script should not be that hard for other torrent clients as long as these can be controlled through the command line or via an API.
273 |
274 | When downloading through torrents *books* first tries to download the related torrent file from the 'official' repository, if this fails it gives up and suggests using direct download instead. Once the torrent file has been downloaded it is checked to see whether it contains the required file. If this check passes the torrent is submitted to the torrent client with only the required file selected for download. A job script is created which can be used to control the torrent job, if the `torrent_cron_job` parameter in the PREFERENCES section or the config file is set to `1` it is submitted as a cron job. The task of this script is to copy the downloaded file from the torrent client download directory (`torrent_download_directory` in books.conf or the PREFERENCES section) to the target directory (preference `target_directory`) under the correct name. Once the torrent has finished downloading the job script will copy the file to that location and remove the cron job. If `torrent_cron_job` is not set (or is set to `0`) the job script can be called 'by hand' to copy the file, it can also be used to perform other tasks like retrying the download from a libgen download mirror server (use `-D`, this will cancel the torrent and cron job for this file) or to retry the torrent download (use `-R`). The script has the following options:
275 |
276 | ```txt
277 | $ XYZ.job -h
278 | Use: bash jobid.job [-s] -[i] [-r] [-R] [-D] [-h] [torrent_download_directory]
279 |
280 | Copies file from libgen/libgen_fiction torrent to correct location and name
281 |
282 | -S show job status
283 | -s show torrent status (short)
284 | -i show torrent info (long)
285 | -I show target file name
286 | -r remove torrent and cron jobs
287 | -R restart torrent download (does not restart cron job)
288 | -D direct download (removes torrent and cron jobs)
289 | -h show this help message
290 | ```
291 |
292 | ### The torrent helper script interface
293 | The torrent helper script (here named `ttool`) needs to support the following commands:
294 |
295 | * `ttool add-selective `
296 | download file `` from torrent ``
297 | * `ttool torrent-hash `
298 | get btih (info-hash) for ``
299 | * `ttool torrent-files `
300 | list files in ``
301 | * `ttool remove `
302 | remove active torrent with info-hash ``
303 | * `ttool ls `
304 | show download status for active torrent with info-hash ``
305 | * `ttool info `
306 | show extensive info (files, peers, etc) for torrent with info-hash ``
307 | * `ttool active `
308 | return `true` if the torrent is active, `false` otherwise
309 |
310 | Output should be the requested data without any headers or other embellishments. Here is an example using the (included) `tm` helper script for the *transmission-daemon* torrent client, showing all required commands:
311 |
312 | ```txt
313 | $ tm torrent-files r_2412000.torrent
314 | 2412000/00b3c21460499dbd80bb3a118974c879
315 | 2412000/00b64be1207c374e8719ee1186a33c4d
316 | 2412000/00c4f3a075d3af0813479754f010c491
317 | ...
318 | ... (994 files omitted for brevity)
319 | ...
320 | 2412000/ff2473a3b8ec1439cc459711fb2a4b97
321 | 2412000/ff913204c002f19ed2ee1e2bdfd236d4
322 | 2412000/ffb249ae5d148639d38f2af2dba6c681
323 |
324 | $ tm torrent-hash r_2412000.torrent
325 | e73d4bc21d0f91088c174834840f7da232330b4d
326 |
327 | $ tm add-selective r_2412000.torrent 00c4f3a075d3af0813479754f010c491
328 | ... (torrent client output omitted)
329 |
330 | $ tm ls 6934f632c06a91572b4401e5b4c96eec89d311d7
331 | ID Done Have ETA Up Down Ratio Status Name
332 | 25 0% None Unknown 0.0 0.0 None Idle 762000
333 | Sum: None 0.0 0.0
334 |
335 | (output from transmission-daemon, format is client-dependent)
336 |
337 | $ tm info 6934f632c06a91572b4401e5b4c96eec89d311d7
338 | ... (torrent client output omitted)
339 |
340 | $ tm active 6934f632c06a91572b4401e5b4c96eec89d311d7; echo "torrent is $([[ $? -gt 0 ]] && echo "not ")active"
341 | torrent is active
342 |
343 | $ if tm active 6934f632c06a91572b4401e5b4c96eec89d311d7; then echo "torrent is active"; fi
344 | torrent is active
345 |
346 | $ tm active d34db33f; echo "torrent is $([[ $? -gt 0 ]] && echo "not ")active"
347 | torrent is not active
348 | ```
349 |
350 | #### The `tm` torrent helper script
351 | The `tm` torrent helper script supports the following options:
352 | ```txt
353 | $ tm -h
354 | tm version 0.1
355 |
356 | Use: tm COMMAND OPTIONS [parameters]
357 | tm-COMMAND OPTIONS [parameters]
358 |
359 | A helper script for transmission-remote and related tools, adding some
360 | functionality like selective download etc.
361 |
362 | PROGRAMS/COMMANDS
363 |
364 | tm-active active
365 | tm-add add
366 | tm-add-selective add-selective
367 | tm-cmd cmd
368 | tm-file-count file-count
369 | tm-files files
370 | tm-help help
371 | tm-info info
372 | tm-ls ls
373 | tm-remove remove
374 | tm-start start
375 | tm-stop stop
376 | tm-torrent-files torrent-files
377 | tm-torrent-hash torrent-hash
378 | tm-torrent-show torrent-show
379 |
380 | OPTIONS
381 |
382 | -k create symbolic links
383 | creates links to all supported commands
384 | e.g. tm-cmd, tm-ls, tm-add, ...
385 | links are created in the directory where tm resides
386 |
387 | -n NETRC set netrc (/home/frank/.tm-netrc)
388 |
389 | -H HOST set host (p2p:4081)
390 |
391 | -c create a config file using current settings (see -n, -H)
392 |
393 | -l execute command 'ls'
394 |
395 | -a TORR execute command 'add'
396 |
397 | -h this help message
398 |
399 | EXAMPLES
400 |
401 | In all cases it is possible to replace tm-COMMAND with tm COMMAND
402 |
403 | show info about running torrents:
404 |
405 | $ tm-ls
406 |
407 | add a torrent or a magnet link:
408 |
409 | tm-add /path/to/torrent/file.torrent
410 | tm-add 'magnet:?xt=urn:btih:123...'
411 |
412 | add a torrent and selectivly download two files
413 | this only works with torrent files (i.e. not magnet links) for now
414 |
415 | tm-add-selective /path/to/torrent/file.torrent filename1,filename2
416 |
417 | show information about a running torrent, using its btih or ID:
418 |
419 | tm-show f0a7524fe95910da462a0d1b11919ffb7e57d34a
420 | tm-show 21
421 |
422 | show files for a running torrent identified by btih (can also use ID)
423 |
424 | tm-files f0a7524fe95910da462a0d1b11919ffb7e57d34a
425 |
426 | stop a running torrent, using its ID (can also use btih)
427 |
428 | tm-stop 21
429 |
430 | get btih for a torrent file
431 |
432 | tm-torrent-hash /path/to/torrent/file.torrent
433 |
434 | remove a torrent from transmission
435 |
436 | tm-remove 21
437 |
438 | execute any transmission-remote command - notice the double dash
439 | see man transmission-remote for more info on supported commands
440 |
441 |
442 | tm-cmd -- -h
443 | tm cmd -h
444 |
445 |
446 | CONFIGURATION FILES
447 |
448 | /home/username/.config/tm.conf
449 |
450 | tm can be configured by editing the script itself or the configuration file:
451 |
452 | netrc=~/.tm-netrc
453 | tm_host="transmission-host.example.org:4081"
454 |
455 | values set in the configuration file override those in the script
456 | ```
457 |
458 |
459 | ## Classify
460 | Classify is a tool which, when fed an *identifier* (ISBN or ISSN, it also works
461 | with UPC and OCLC OWI/WI but these are not in the database) [i]or[/i] a
462 | database name and MD5 can be used to extract classification data from the OCLC
463 | classifier. Depending on what OCLC returns it can be used to add or update the
464 | following fields:
465 |
466 | ### Always present:
467 | - Author
468 | - Title
469 |
470 | ### One or more of:
471 | - [DDC](https://en.wikipedia.org/wiki/Dewey_Decimal_Classification)
472 | - [LCC](https://en.wikipedia.org/wiki/Library_of_Congress_Classification)
473 | - [NLM](https://en.wikipedia.org/wiki/National_Library_of_Medicine_classification)
474 | - [FAST](https://www.oclc.org/research/areas/data-science/fast.html) (Faceted Application of Subject Terminology, basically a list of subject keywords derived from the Library of Congress Subject Headings (LCSH))
475 |
476 | The *classify* tool stores these fields in CSV files which can be fed to the
477 | *import_metadata* tool (see below)to update the database and/or produce SQL
478 | code. It can also store all XML data as returned by the OCLC classifier for
479 | later use, this offloads the OCLC classifier service which is marked as
480 | 'experimental' and 'not built for production use' and as such can change or
481 | disappear at any moment.
482 |
483 | The *classify* helper script supports the following options:
484 |
485 | ```
486 | $ classify -h
487 | classify "version 0.5.0"
488 |
489 | Use: classify [OPTIONS] identifier[,identifier...]
490 |
491 | Queries OCLC classification service for available data
492 | Supports: DDC, LCC, NLM, FAST, Author and Title
493 |
494 | Valid identifiers are ISBN, ISSN, UPC and OCLC/OWI
495 |
496 | OPTIONS:
497 |
498 | -d show DDC
499 | -l show LCC
500 | -n show NLM
501 | -f show FAST
502 | -a show Author
503 | -t show Title
504 |
505 | -o show OWI (OCLC works identifier)
506 | -w show WI (OCLC works number)
507 |
508 | -C md5 create CSV (MD5,DDC,LCC,NLM,FAST,AUTHOR,TITLE)
509 | use -D libgen/-D libgen_fiction to indicate database
510 |
511 | -X dir save OCLC XML response to $dir/$md5.xml
512 | only works with a defined MD5 (-C MD5)
513 |
514 | -D db define which database to use (libgen/libgen_fiction)
515 |
516 | -A show all available data for identifier
517 |
518 | -V show labels
519 |
520 | -@ PORT use torsocks to connect to the OCLC classify service.
521 | use this to avoid getting your IP blocked by OCLC
522 |
523 | -h show this help message
524 |
525 | Examples
526 |
527 | $ classify -A 0199535760
528 | AUTHOR: Plato | Jowett, Benjamin, 1817-1893 Translator; Editor; Other] ...
529 | TITLE: The republic
530 | DDC: 321.07
531 | LCC: JC71
532 |
533 | $ classify -D libgen -C 25b8ce971343e85dbdc3fa375804b538
534 | 25b8ce971343e85dbdc3fa375804b538,"321.07","JC71","",UG9saXRpY2FsI\
535 | HNjaWVuY2UsVXRvcGlhcyxKdXN0aWNlLEV0aGljcyxQb2xpdGljYWwgZXRoaWNzLFB\
536 | oaWxvc29waHksRW5nbGlzaCBsYW5ndWFnZSxUaGVzYXVyaQo=,UGxhdG8gfCBKb3dl\
537 | dHQsIEJlbmphbWluLCAxODE3LTE4OTMgW1RyYW5zbGF0b3I7IEVkaXRvcjsgT3RoZX\
538 | JdIHwgV2F0ZXJmaWVsZCwgUm9iaW4sIDE5NTItIFtUcmFuc2xhdG9yOyBXcml0ZXIg\
539 | b2YgYWRkZWQgdGV4dDsgRWRpdG9yOyBPdGhlcl0gfCBMZWUsIEguIEQuIFAuIDE5MD\
540 | gtMTk5MyBbVHJhbnNsYXRvcjsgRWRpdG9yOyBBdXRob3Igb2YgaW50cm9kdWN0aW9u\
541 | XSB8IFNob3JleSwgUGF1bCwgMTg1Ny0xOTM0IFtUcmFuc2xhdG9yOyBBdXRob3I7IE\
542 | 90aGVyXSB8IFJlZXZlLCBDLiBELiBDLiwgMTk0OC0gW1RyYW5zbGF0b3I7IEVkaXRv\
543 | cjsgT3RoZXJdCg==,VGhlIHJlcHVibGljCg==
544 |
545 |
546 | Classifying libgen/libgen_fiction
547 |
548 | This tool can be used to add classification data to libgen and
549 | libgen_fiction databases. It does not directy modify the database,
550 | instead producing CSV which can be used to apply the modifications.
551 | The best way to do this is to produce a list of md5 hashes for
552 | publications which do have Identifier values but lack values for DDC
553 | and/or LCC. Such lists can be produced by the following SQL:
554 |
555 | libgen: select md5 from updated where IdentifierWODash<>"" and DDC="";
556 | libgen_fiction: select md5 from fiction where Identifier<>"" and DDC="";
557 |
558 | Run these as batch jobs (mysql -B .... -e 'sql_code_here;' > md5_list), split
559 | the resulting file in ~1000 line sections and feed these to this tool,
560 | preferably with a random pause between requests to keep OCLC's intrusion
561 | detection systems from triggering too early. It is advisable to use
562 | this tool through Tor (using -@ TORPORT to enable torsocks, make sure it
563 | is configured correctly for your Tor instance) to avoid having too
564 | many requests from your IP to be registered, this again to avoid
565 | your IP being blocked. The OCLC classification service is not
566 | run as a production service (I asked them).
567 |
568 | Return values are stored in the following order:
569 |
570 | MD5,DDC,LCC,NLM,FAST,AUTHOR,TITLE
571 |
572 | DDC, LCC and NLM are enclosed within double quotes and can contain
573 | multiple space-separated values. FAST, AUTHOR and TITLE are base64 encoded
574 | since these fields can contain a whole host of unwholesome characters
575 | which can mess up CSV. The AUTHOR field currentlydecodes to a pipe ('|')
576 | separated list of authors in the format:
577 |
578 | LAST_NAME, NAME_OR_INITIALS, DATE_OF_BIRTH-[DATE_OF_DEATH] [[ROLE[[;ROLE]...]]]
579 |
580 | This format could change depending on what OCLC does with the
581 | (experimental) service.
582 | ```
583 |
584 | ## import_metadata
585 | Taking a file containing lines of CSV-formatted data, this tool can be used to
586 | update a libgen / libgen_fiction database with fresh metadata. It can also be
587 | used to produce SQL (using the -s sqlfile option) which can be used to update
588 | multiple database instances.
589 |
590 | In contrast to the other *books* tools *import_metadata* is a Python (version
591 | 3) script using the *pymysql* "pure python" driver (*python3-pymysql* on Debian)
592 | and as such should run on any device where Python is available. The
593 | distribution file contains a Bash script (*import_metadata.sh*) with the same
594 | interface and options which can be used where Python is not available.
595 |
596 |
597 | ```
598 | $ import_metadata -h
599 |
600 | import_metadata v.0.1.0
601 |
602 | Use: import_metadata [OPTIONS] -d database -f "field1,field2" -F CSVDATAFILE
603 |
604 | Taking a file containing lines of CSV-formatted data, this tool can be
605 | used to update a libgen / libgen_fiction database with fresh metadata.
606 | It can also be used to produce SQL (using the -s sqlfile option) which
607 | can be used to update multiple database instances.
608 |
609 | CSV data format:
610 |
611 | MD5,DDC,LCC,NLM,FAST,AUTHOR,TITLE
612 |
613 | Fields FAST, AUTHOR and TITLE should be base64-encoded.
614 |
615 | CSV field names are subject to redirection to database field names,
616 | currently these redirections are active (CSV -> DB):
617 |
618 | ['FAST -> TAGS']
619 |
620 | OPTIONS:
621 |
622 | -d DB define which database to use (libgen/libgen_fiction)
623 |
624 | -f field1,field2
625 | -f field1 -f field2
626 | define which fields to update
627 |
628 | -F CSVFILE
629 | define CSV input file
630 |
631 | -s SQLFILE
632 | write SQL to SQLFILE
633 |
634 | -n do not update database
635 | use with -s SQLFILE to produce SQL for later use
636 | use with -v to see data from CSVFILE
637 | use with -vv to see SQL
638 |
639 | -v verbosity
640 | repeat to increase verbosity
641 |
642 | -h this help message
643 |
644 | Examples
645 |
646 | $ import_metadata -d libgen -F csv/update-0000 -f 'ddc lcc fast'
647 |
648 | update database 'libgen' using data from CSV file csv/update-0000,
649 | fields DDC, LCC and FAST (which is redirected to libgen.Tags)
650 |
651 | $ for f in csv/update-*;do
652 | import_metadata -d libgen -s "$f.sql" -n -f 'ddc,lcc,fast' -F "$f"
653 | done
654 |
655 | create SQL (-s "$f.sql") to update database using fields
656 | DDC, LCC and FAST from all files matching glob csv/update-*,
657 | do not update database (-n option)
658 | ```
659 |
660 |
661 |
662 | ## Installation
663 | Download this repository (or a tarball) and copy the four scripts - `books`, `update_libgen`, `refresh_libgen` and `tm` (only needed when using the transmission-daemon torrent client) - into a directory which is somewhere on your $PATH ($HOME/bin would be a good spot). Run `books -k`to create symlinks to the various names under which the program can be run:
664 |
665 | * `books`
666 | * `books-all`
667 | * `fiction`
668 | * `nbook`
669 | * `xbook`
670 | * `nfiction`
671 | * `xfiction`
672 |
673 | Create a database on a mysql server somewhere within reach of the intended host. Either open *books* in an editor to configure the database details (look for `CONFIGURE ME` below) and anything else (eg. `target_directory` for downloaded books, `max_age` before update, `language` for topics, MD5 in filenames, tools, etc) or add these settings to the (optional) config file `books.conf` in $XDG_CONFIG_HOME (usually $HOME/.config). The easiest way to create the config file is to run `refresh_libgen` with the required options. As an example, the following command sets the database server to `base.example.org`, the database port to `3306` and the database username to `genesis`:
674 |
675 | ```bash
676 | $ refresh_libgen -H base.example.org -P 3306 -U genesis -c
677 | ```
678 |
679 | Make sure to add the `-c` option *at the end* of the command or it won't work. Once the config file has been created it can be edited
680 |
681 |
682 | ```bash
683 | main () {
684 | # PREFERENCES
685 | config=${XDG_CONFIG_HOME:-$HOME/.config}/books.conf
686 |
687 | # target directory for downloaded publications
688 | target_directory="${HOME}/Books" <<<<<< ... CONFIGURE ME ... >>>>>>
689 | # when defined, subdirectory of $target_directory) for torrents
690 | torrent_directory="torrents"
691 | # when defined, location where files downloaded with torrent client end up
692 | # torrent_download_directory="/net/p2p/incoming" <<<<<< ... ENABLE/CONFIGURE ME ... >>>>>>
693 | # when true, launch cron jobs to copy files from torrent download directory
694 | # to target directory using the correct name
695 | torrent_cron_job=1
696 | # default limit on queries
697 | limit=1000
698 | # maximum database age (in minutes) before attempting update
699 | max_age=120
700 | # topics are searched/displayed in this language ("en" or "ru")
701 | language="en" <<<<<<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
702 | # database host
703 | dbhost="localhost" <<<<<<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
704 | # database port
705 | dbport="3306" <<<<<<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
706 | # database user
707 | dbuser="libgen" <<<<<<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
708 | # default fields for fulltext search
709 | default_fields="author,title"
710 | # window/dialog heading for dialog and yad/zenity
711 | list_heading="Select publication(s) for download:"
712 |
713 | # add md5 to filename? Possibly superfluous as it can be derived from the file contents but a good guard against file corruption
714 | filename_add_md5=0
715 |
716 | # tool preferences, list preferred tool first
717 | gui_tools="yad|zenity"
718 | tui_tools="dialog|whiptail"
719 | dl_tools="curl|wget"
720 | parser_tools="xidel|hxwls"
721 | pager_tools="less|more"
722 |
723 | # torrent helper tools need to support the following commands:
724 | # ttool add-selective # downloads file from torrent
725 | # ttool torrent-hash # gets btih for
726 | # ttool torrent-files # lists files in
727 | torrent_tools="tm" <<<<<<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
728 |
729 | # database names to use:
730 | # books, books-all, nbook, xbook and xbook-all use the main libgen database
731 | # fiction, nfiction and xfiction use the 'fiction' database
732 | declare -A programs=(
733 | [books]=libgen <<<<<<<<<<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
734 | [books-all]=libgen <<<<<<<<<<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
735 | [nbook]=libgen <<<<<<<<<<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
736 | [xbook]=libgen <<<<<<<<<<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
737 | [fiction]=libgen_fiction <<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
738 | [nfiction]=libgen_fiction <<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
739 | [xfiction]=libgen_fiction <<<<<<<< ... CONFIGURE ME ..... >>>>>>>>>>>>>>>>
740 | [libgen_preview]=libgen # the actual database to use for preview is passed as a command line option
741 | )
742 | ```
743 |
744 | The same goes for the 'PREFERENCES' sections in `update_libgen` and `refresh_libgen`. In most cases the only parameters which might need change are `dbhost`, `dbuser`, `ipfs_gw` (if you don't want to use the default hosted by Cloudfront), `torrent_download_directory` and possibly `torrent_tools`. Since all programs use a common `books.conf` config file it is usually sufficient to add these parameters there:
745 |
746 | ```bash
747 | $ cat $HOME/.config/books.conf
748 | dbhost="base.example.org"
749 | dbuser="exampleuser"
750 | ipfs_gw="http://ipfs.example.org"
751 | torrent_download_directory="/net/p2p/incoming"
752 | torrent_tools="tm"
753 | ```
754 |
755 | Please note that there is no option to enter a database password as that would be rather insecure. Either use a read-only, password-free mysql user to access the database or enter your database details in $HOME/.my.cnf, like so:
756 |
757 | ```ini
758 | [mysql]
759 | user=exampleuser
760 | password=zooperzeekret
761 | ```
762 |
763 | Make sure the permissions on $HOME/.my.cnf are sane (eg. mode 640 or 600), see http://dev.mysql.com/doc/refman/5.7/en/ ... files.html for more info on this subject.
764 |
765 | Install symlinks to all tools by calling books with the -k option:
766 |
767 | ```
768 | $ books -k
769 | ```
770 |
771 | ## configuration file
772 | The configuration file is *source*d by all shell scripts, it is parsed and interpreted by import_metadata. There are some of the more useful parameters which can be set in this file:
773 |
774 | ```
775 | dbhost="base.example.org"
776 | dbport="3306"
777 | dbuser="libgen"
778 | ```
779 | Use these to set the database server hostname, port and username.
780 | ```
781 | torrent_download_directory="/net/incoming"
782 | torrent_cron_job=1
783 | torrent_tools="tm"
784 | ```
785 | Set torrent download directory (where the torrent client places downloaded files), whether a cron job should be created to copy downloaded publications to their final name and location, which torrent helper tool to use
786 | ```
787 | use_deep_path=1
788 | ```
789 | Add section, language, author and subject to path name (e.g. nonfication/German/Physics/Einstein, Albert./Die Evolution der Physik)
790 | ```
791 | use_ipfs=1
792 | ```
793 | Try to use IPFS when downloading, reverts to direct download for files which do not have a defined ipfs_cid
794 | ```
795 | gui_tools="yad|zenity"
796 | tui_tools="dialog|whiptail"
797 | parser_tools="xidel|hxwls"
798 | dl_tools="wget|curl"
799 | pager_tools="less|more|cat"
800 | ```
801 | Tools to be used, in|order|of|preference - the first available is used
802 | ```
803 | api=http://libgen.rs/json.php
804 | base=http://libgen.rs/dbdumps/
805 | ipfs_gw=https://cloudflare-ipfs.com
806 | #ipfs_gw=http://your_own_ipfs_node.example.org:8080
807 | ```
808 | Defines which resources to use for API, dumps, IPFS etc
809 | ```
810 | classify_xml="/home/username/Project/libgen_classify/xml"
811 | classify_csv="/home/username/Project/libgen_classify/csv"
812 | classify_sql="/home/username/Project/libgen_classify/sql"
813 | classify_fields="ddc,lcc,nlm,fast,title,author"
814 | classify_tor_ports="9100,9102,9104,9106,9108"
815 | ```
816 | Used by update_libgen to configure *classify* and *import_metadata*, defines whether files are saved and where they are saved, which fields to update in the database and whether Tor is used and if so on which port(s). It is advisable to use more than one port to spread the traffic over several exit nodes, this reduces the risk of OCLC blocking the Tor exit node.
817 |
818 | There are far more configurable parameters, check the script source for more possibilities.
819 |
820 | ## *update_libgen* vs. *refresh_libgen*
821 |
822 | If you regularly use books, nbook and/or xbook, the main (or compact) database should be kept up to date automatically. In that case it is only necessary to use *refresh_libgen* to refresh the database when you get a warning from *update_libgen* about unknown columns in the API response.
823 |
824 | If you have not used any of these tools for a while it can take a long time - and a lot of data transfer - to update the database through the API (which is what *update_libgen* does). Especially when using the compact database it can be quicker to use *refresh_libgen* to just pull the latest dump instead of waiting for *update_libgen* to do its job.
825 |
826 | The *fiction* database can not be updated through the API (yet), so for that databases *refresh_libgen* is currently the canonical way to get the latest version.
827 |
828 | ## Dependencies
829 |
830 | These tools have the following dependencies (apart from a locally available libgen/libgen_fiction instance on MySQL/MariaDB), sorted in order of preference:
831 |
832 | * all: bash 4.x or higher - the script relies on quite a number of bashisms
833 |
834 |
835 | * `books`/`fiction`: less | more (use less!)
836 | * `nbook`/`nfiction`: dialog | whiptail (whiptail is buggy, use dialog!)
837 | * `xbook`/`xfiction`: yad | zenity (more functionality with yad, but make sure your yad supports --html - you might have to build it yourself (use --enable-html during ./configure). If in doubt about the how and why of this, just use Zenity)
838 |
839 |
840 | Preview/Download has these dependencies:
841 |
842 | * awk (tested with mawk, nawk and gawk)
843 | * stdbuf (part of GNU coreutils)
844 | * xidel | hxwls (html parser tools, used for link extraction)
845 | * curl | wget
846 |
847 |
848 | `update_libgen` has the following dependencies:
849 |
850 | * jq (CLI json parser/mangler)
851 | * awk (tested with mawk, nawk and gawk)
852 |
853 |
854 | `refresh_libgen` has these dependencies:
855 |
856 | * w3m
857 | * wget
858 | * unrar
859 | * pv (only needed when using the verbose (-v) option
860 |
861 | `tm` has these dependencies:
862 |
863 | * transmission-remote
864 |
865 |
866 |
--------------------------------------------------------------------------------
/books:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | #
3 | # shellcheck disable=SC2034,SC1090,SC2254
4 |
5 | shopt -s extglob
6 | trap "trap_error" TERM
7 | trap "trap_clean" EXIT
8 | export TOP_PID=$$
9 |
10 | version="0.7.2"
11 | release="20210601"
12 |
13 | functions="$(dirname "$0")/books_functions"
14 | if [ -f "$functions" ]; then
15 | source "$functions"
16 | else
17 | echo "$functions not found"
18 | exit 1
19 | fi
20 |
21 | main () {
22 | # PREFERENCES
23 | config=${XDG_CONFIG_HOME:-$HOME/.config}/books.conf
24 |
25 | # target directory for downloaded publications
26 | target_directory="${HOME}/Books"
27 | # when defined, subdirectory of $target_directory) for torrents
28 | torrent_directory="torrents"
29 | # when defined, location where files downloaded with torrent client end up
30 | torrent_download_directory="/net/media/incoming"
31 | # when true, launch cron jobs to copy files from torrent download directory
32 | # to target directory using the correct name
33 | torrent_cron_job=1
34 | # default limit on queries
35 | limit=1000
36 | # maximum database age (in minutes) before attempting update
37 | max_age=120
38 | # topics are searched/displayed in this language ("en" or "ru")
39 | language="en"
40 | # database host
41 | dbhost="localhost"
42 | # database port
43 | dbport="3306"
44 | # database user
45 | dbuser="libgen"
46 | # default fields for fulltext search
47 | default_fields="author,title"
48 | # window/dialog heading for dialog and yad/zenity
49 | list_heading="Select publication(s) for download:"
50 |
51 |
52 | # add md5 to filename? Possibly superfluous as it can be derived from the file contents but a good guard against file corruption
53 | filename_add_md5=0
54 |
55 | # tool preferences, list preferred tool first
56 | gui_tools="zenity|yad"
57 | tui_tools="dialog|whiptail"
58 | dl_tools="curl|wget"
59 | parser_tools="xidel|hxwls"
60 | pager_tools="less|more|cat"
61 |
62 | # torrent helper tools need to support the following commands:
63 | # add-selective # downloads file from torrent
64 | # torrent-hash # gets btih for
65 | # torrent-files # lists files in
66 | # remove # remove active torrent with info-hash
67 | # ls # show download status for active torrent with info-hash
68 | # info # show extensive info (files, peers, etc) for torrent with info-hash
69 | # active # return `true` if the torrent is active, `false` otherwise
70 | torrent_tools="tm"
71 |
72 | # don't use a pager when not running in a tty
73 | [ ! -t 1 ] && pager_tools="cat"
74 |
75 |
76 | # database names to use:
77 | # books, books-all, nbook, xbook and xbook-all use the main libgen database
78 | # fiction, nfiction and xfiction use the 'fiction' database
79 | declare -A programs=(
80 | [books]=libgen
81 | [books-all]=libgen
82 | [nbook]=libgen
83 | [xbook]=libgen
84 | [fiction]=libgen_fiction
85 | [nfiction]=libgen_fiction
86 | [xfiction]=libgen_fiction
87 | [libgen_preview]=libgen # the actual database to use for preview is passed as a command line option
88 | )
89 |
90 | declare -A tables=(
91 | [libgen]="(updated topics description hashes)"
92 | [libgen_fiction]="(fiction fiction_description fiction_hashes)"
93 | )
94 |
95 | # searchable database fields
96 | declare -A schema=(
97 | [a]=author
98 | [t]=title
99 | [d]=edition
100 | [e]=extension
101 | [l]=language
102 | [y]=year
103 | [s]=series
104 | [p]=publisher
105 | [c]=city
106 | [o]=topic_descr
107 | [v]=volumeinfo
108 | [r]=periodical
109 | [g]=tags
110 | [z]=locator
111 | [i]=issn
112 | [n]=asin
113 | [q]=openlibraryid
114 | [b]=identifierwodash
115 | [m]=md5
116 | [D]=ddc
117 | [L]=lcc
118 | )
119 |
120 | # base url for covers
121 | declare -A coverurl=(
122 | [libgen]='http://93.174.95.29/covers'
123 | [libgen_fiction]='http://93.174.95.29/fictioncovers'
124 | )
125 |
126 | # download archives
127 | declare -A downloadurl=(
128 | [libgen]="http://93.174.95.29/main"
129 | [libgen_fiction]="http://93.174.95.29/fiction"
130 | )
131 |
132 | # torrent archives
133 | declare -A torrenturl=(
134 | [libgen]='http://libgen.rs/repository_torrent'
135 | [libgen_fiction]='http://libgen.rs/fiction/repository_torrent'
136 | )
137 |
138 | # torrent file name prefixes
139 | declare -A torrentprefix=(
140 | [libgen]='r'
141 | [libgen_fiction]='f'
142 | )
143 |
144 | # directory name prefixes (used with deep path (-$))
145 | declare -A dirprefix=(
146 | [libgen]="nonfiction"
147 | [libgen_fiction]="fiction"
148 | )
149 |
150 | # ipfs gateway
151 | ipfs_gw="https://cloudflare-ipfs.com"
152 |
153 | # source config file if it exists
154 | [[ -f ${config} ]] && source "${config}"
155 |
156 | # (mostly) END OF PREFERENCES
157 |
158 | # user agent string generator
159 | declare -a adjective=(white red blue pink green brown dark light big small tiny earth glass air space forest lake sea ground fire crunchy spicy boring zonked blasted stoned fried flattened stretched smelly ugly obnoxious irritating whiny lazy)
160 | declare -a critter=(goat frog hound fish lizard gator moose monkey whale hippo fox bird weasel owl cow pig hog donkey duck chicken dino sloth snake iguana gecko)
161 | user_agent="Mozilla/5.0 ($(uname -s)) ${adjective[$((RANDOM%${#adjective[*]}))]^}${critter[$((RANDOM%${#critter[*]}))]^}/${RANDOM:0:3}.${RANDOM:0:1}.${RANDOM:0:4}.${RANDOM:0:2}"
162 |
163 | # display columns for yad, zenity and pager
164 | declare -A zenity_columns=(
165 | [libgen]="--column Download --column MD5 --column Title --column Author --column Year --column Edition --column Publisher --column Language --column Topic --column Size --column Format --hide-column=2"
166 | [libgen_fiction]="--column Download --column MD5 --column Title --column Author --column Year --column Edition --column Publisher --column Language --column Commentary --column Size --column Format --hide-column=2"
167 | )
168 |
169 | declare -A yad_columns=(
170 | [libgen]="--column Download --column MD5:HD --column Title --column Author --column Year:NUM --column Edition --column Publisher --column Language --column Topic --column Size:NUM --column Format --column Info:HD --search-column=3 --print-column=2 --tooltip-column=12"
171 | [libgen_fiction]="--column Download --column MD5:HD --column Title --column Author --column Year:NUM --column Edition --column Publisher --column Language --column Commentary --column Size:NUM --column Format --column Info:HD --search-column=3 --print-column=2 --tooltip-column=12"
172 | )
173 |
174 | # defines which columns are shown by default in a command line (i.e. pager) search
175 | declare -A pager_columns=(
176 | [libgen]="Title,Author,Year,Edition,Publisher,Language,Topic_descr,Filesize,Extension,Series,MD5"
177 | [libgen_fiction]="Title,Author,Year,Edition,Publisher,Language,Commentary,Filesize,Extension,Series,MD5"
178 | )
179 |
180 | # used to select table prefix in command line search, contains regex patterns
181 | declare -A attribute_columns=(
182 | [topics]="topic_descr|topic_id"
183 | [hashes]="crc32|edonkey|aich|sha1|tth|torrent|btih|sha256"
184 | [description]="^descr$|toc"
185 | )
186 |
187 | # query output filter for different purposes
188 | declare -A filters=(
189 | [search]="cat"
190 | [xsearch]="sed -e 's/&/&/g'"
191 | [filename]="sed -e 's/[^-[:alnum:]:;?!.,+@#%]/_/g;s/^\([-_]\)*//'"
192 | [dirname]="sed -e 's/[^-[:alnum:]:;?!/.,+@#%]/_/g;s/^\([-_]\)*//'"
193 | [id]="cat"
194 | [ipfs_cid]="cat"
195 | [extension]="cat"
196 | [attributes]="cat"
197 | [preview_dialog]="cat"
198 | [preview_whiptail]="cat"
199 | [preview_yad]="sed -e 's/&/&/g'"
200 | [preview_zenity]="sed -e 's/&/&/g'"
201 | )
202 |
203 | # GETOPT config
204 | search_options='@('$(echo "${!schema[@]}"|tr ' ' '|')')'
205 | search_getopts="$(echo "${!schema[@]}"|tr ' ' ':'):"
206 |
207 | # X11-related config
208 | if xset q &>/dev/null; then
209 | # used to size yad/zenity windows
210 | min_screenres=$(xrandr|grep '\*'|sort -n|head -1|awk '{print $1}')
211 | x_size=$(($(echo "$min_screenres"|cut -d 'x' -f 1) - 50))
212 | y_size=$(($(echo "$min_screenres"|cut -d 'x' -f 2) - 30))
213 | fi
214 |
215 | # defines program behaviour to a large extent
216 | program=$(basename "$0")
217 | db="${programs[$program]}"
218 |
219 | # arrays
220 | declare -a query_result
221 |
222 | # refs
223 | declare -n sql_source
224 |
225 | # misc
226 | clauses=""
227 | fields=""
228 | opt_fields=""
229 | show_fields=""
230 | query=""
231 | no_update=0
232 |
233 | # this contains the columns in the current database, used to filter out unsupported search fields
234 | current_fields="$(get_current_fields)"
235 |
236 | # set ui_tool in order of preference (first found wins)
237 | case "$program" in
238 | xbook|xfiction|libgen_preview)
239 | ui_tool=$(find_tool "$gui_tools")
240 | ;;
241 | nbook|nfiction)
242 | ui_tool=$(find_tool "$tui_tools")
243 | ;;
244 | *)
245 | ui_tool="pager"
246 | ;;
247 | esac
248 |
249 | check_settings
250 |
251 | # PROCESS OPTIONS AND BUILD QUERY
252 |
253 | while getopts ":${search_getopts}fF:hkxX#:@wu:I:U:=:j:J:M:$" OPTION
254 | do
255 | case $OPTION in
256 | $search_options)
257 | add_clause "${OPTION}" "${OPTARG}"
258 | ;;
259 |
260 | h)
261 | help
262 | exit
263 | ;;
264 | k)
265 | create_symlinks
266 | exit
267 | ;;
268 | f)
269 | fulltext=1
270 | ;;
271 | F)
272 | if [[ "$program" =~ ^books|^fiction ]]; then
273 | show_fields="${OPTARG}"
274 | # test for non-existent fields
275 | get_fields > /dev/null
276 | else
277 | exit_with_error "-F FIELDS only works with _books_ and _fiction_"
278 | fi
279 | ;;
280 | x)
281 | no_update=1
282 | ;;
283 | X)
284 | ((exact_match++))
285 | ;;
286 | '#')
287 | limit="${OPTARG}"
288 | ;;
289 | w)
290 | preview=1
291 | ;;
292 | u)
293 | if is_true "$OPTARG"; then
294 | if [[ -n "$torrent_tools" ]]; then
295 | if find_tool $torrent_tools >/dev/null;then
296 | use_torrent=1
297 | unset use_ipfs
298 | else
299 | exit_with_error "-u: torrent helper script ($torrent_tools) not found"
300 | fi
301 | else
302 | exit_with_error "-u needs torrent helper script, see \$torrent_tools"
303 | fi
304 | else
305 | unset use_torrent
306 | fi
307 | ;;
308 |
309 | I)
310 | if [[ "$OPTARG" == "show" ]]; then
311 | get_ipfs_cid
312 | exit
313 | elif is_true "$OPTARG"; then
314 | use_ipfs=1
315 | unset use_torrent
316 | else
317 | unset use_ipfs
318 | fi
319 | ;;
320 |
321 | U)
322 | get_torrentpath "${OPTARG}"
323 | exit
324 | ;;
325 | j)
326 | get_filename "${OPTARG}"
327 | exit
328 | ;;
329 | J)
330 | md5_download=1
331 | md5="${OPTARG}"
332 | ;;
333 | M)
334 | md5_fast_path=1
335 | md5="${OPTARG}"
336 | ;;
337 | @)
338 | source "$(which torsocks)" on
339 | export TORSOCKS_TOR_PORT=$OPTARG
340 | ;;
341 | =)
342 | if [ -d "${OPTARG}" ]; then
343 | target_directory="${OPTARG}"
344 | else
345 | exit_with_error "Directory ${OPTARG} does not exist"
346 | fi
347 | ;;
348 | $)
349 | use_deep_path=1
350 | ;;
351 |
352 | esac
353 | done
354 |
355 | # direct download - download single publication using MD5
356 | if [[ -n "$md5_download" ]]; then
357 | ui_tool="none"
358 | db=$(get_db_for_md5 "$md5")
359 | if [[ -n "$db" ]]; then
360 | download "$db" "$md5"
361 | else
362 | exit_with_error "unknown md5, can not download"
363 | fi
364 | exit
365 | fi
366 |
367 | # direct info - get data on a single publication using MD5
368 | if [[ -n "$md5_fast_path" ]]; then
369 | case "$program" in
370 | books|fiction)
371 | pager_tools="cat"
372 | query="and md5='${md5}'"
373 | sql=$(prepare_sql "$db" "attributes")
374 | run_query "$db" "attributes" "$sql"
375 | list_${ui_tool} $preview
376 | exit
377 | ;;
378 | *)
379 | exit_with_error "-M MD5 (fast path search) only works in _books_ and _fiction_"
380 | esac
381 | fi
382 |
383 | # shift out options
384 | shift $((OPTIND-1))
385 |
386 | # process rest of command line
387 |
388 | # this enables a simple 'books [like] ...' search pattern
389 | operator=$1
390 |
391 | if [[ $operator == like ]]; then
392 | percent='%'
393 | operator=' like '
394 | shift
395 | else
396 | operator='='
397 | fi
398 |
399 | [[ $limit -gt 0 ]] && sql_limit="limit $limit"
400 |
401 | [[ -z $opt_fields ]] && fields="$default_fields" || fields="${opt_fields%?}"
402 |
403 | [[ $fulltext -gt 0 && ! $pattern =~ $opt_pattern ]] && pattern+="$opt_pattern"
404 |
405 | if [[ $fulltext -gt 0 ]]; then
406 | if [[ -z $* && -z $opt_pattern ]]; then
407 | echo "can not perform a fulltext search without search words"
408 | exit 1
409 | else
410 | unset clauses
411 | query="and concat($fields) like '%${opt_pattern}${*}%'"
412 | fi
413 | else
414 | if [[ -n $* ]]; then
415 | query="and title${operator}'${percent}${*}${percent}'"
416 | fi
417 | fi
418 |
419 | [[ -z ${query} && -z ${opt_pattern} ]] && exit_with_error "Empty query"
420 |
421 | window_title="$query $clauses"
422 | window_title=${window_title/and /}
423 |
424 | # RUN QUERY AND PROCESS RESULTS
425 |
426 | # update database before performing query
427 | case "$program" in
428 | books|books-all|nbook|xbook)
429 | # this depends on the external 'update_libgen' script
430 | # set no_update to 1 in case that script is not available, or when this script is run on a
431 | # system without internet access
432 | if [[ -z $no_update ]]; then
433 | update_database
434 | fi
435 | ;;
436 | *)
437 | # no database update for other databases (yet)
438 | ;;
439 | esac
440 |
441 | case "$program" in
442 |
443 | # this searches both the normal 'updated' table as well as the 'updated_edited' table
444 | books-all)
445 | sql=$(prepare_sql "$db" "search_all" $ui_tool)
446 | run_query "$db" "search" "$sql"
447 | list_${ui_tool} $preview
448 | ;;
449 |
450 | books|nbook|xbook|fiction|nfiction|xfiction)
451 | if [[ ${program:0:1} == "x" ]]; then
452 | filter="xsearch"
453 | else
454 | filter="search"
455 | fi
456 | sql=$(prepare_sql "$db" "search" $ui_tool)
457 | run_query "$db" "$filter" "$sql"
458 | list_${ui_tool} $preview
459 | ;;
460 |
461 | # preview info on publication
462 | libgen_preview)
463 | db="$1"
464 | shift
465 | md5="$2" # preview gets fed the whole row of data by yad/zenity; md5 is the second item in this row
466 |
467 | if [[ -n $md5 ]]; then
468 | preview "$db" "$md5"
469 | fi
470 | ;;
471 |
472 | # default
473 | *)
474 | exit_with_error "unknown program: $program"
475 | ;;
476 | esac
477 |
478 | }
479 |
480 | # DOWNLOAD
481 |
482 | # feed this a list of hashes to attempt to download the related publications
483 | download () {
484 | db="$1"
485 | shift
486 | for md5 in "$@"; do
487 |
488 | filename=$(get_filename "$md5" "$use_deep_path")
489 |
490 | if [[ -n "$filename" ]]; then
491 |
492 | if is_true "$use_torrent"; then
493 | dl_src_torrent "$db" "$md5" "$filename"
494 | elif is_true "$use_ipfs"; then
495 | dl_src_ipfs "$db" "$md5" "$filename"
496 | else
497 | dl_src_direct "$db" "$md5" "$filename"
498 | fi
499 | fi
500 |
501 | log_info "downloaded: $filename"
502 | done
503 | }
504 |
505 | # this attempts to download the actual publication, using one of several download tools
506 | # and reporting through one of several progress monitors
507 | get_file () {
508 | filename="$1"
509 | shift
510 | url="$*"
511 |
512 | # strip quotes
513 | filename=${filename%\'}
514 | filename=${filename#\'}
515 |
516 | dl_tool=$(find_tool "$dl_tools")
517 | stdbuf=$(find_tool "stdbuf")
518 |
519 | tmpdir=$(mktemp -d /tmp/libgen_dl.XXXXXX)
520 | touch "${tmpdir}/progress"
521 |
522 | case $dl_tool in
523 | curl)
524 | curl --user-agent "$user_agent" -L -o "$target_directory/$filename" "${url}" 2>"${tmpdir}/progress" &
525 | echo $! >"${tmpdir}/dl_pid"
526 | ;;
527 |
528 | wget)
529 | wget --no-use-server-timestamps --user-agent "$user_agent" -O "$target_directory/$filename" "${url}" -o "${tmpdir}/progress" --progress=bar:force &
530 | echo $! >"${tmpdir}/dl_pid"
531 | ;;
532 | *)
533 | exit
534 | ;;
535 | esac
536 |
537 | # mawk does not support pattern repetition, hence the funny patterns
538 | case $ui_tool in
539 | dialog)
540 | "$stdbuf" -oL tail -f "${tmpdir}/progress"|stdbuf -oL tr '\r' '\n'|awk -W posix_space -W interactive 'NF==12 { print "XXX\n" $(NF-11) "\nDownloading:\n'"$filename"'\n" $(NF-8) " of " $(NF-10) " at " $(NF) "B/s (" $(NF-1) " left)"; system(""); } /^.......................%\[....................\]/ { split($2,A,/\%/); print "XXX\n" A[1]"\nDownloading:\n'"$filename"'\n"$4 " at " $5 " (" $7 " left)"; system("");}' 2>/dev/null| dialog --backtitle "Download: $filename" --gauge "Starting download..." 10 120 2>/dev/null &
541 | echo $! >"${tmpdir}/pager_pid"
542 | ;;
543 | whiptail)
544 | "$stdbuf" -oL tail -f "${tmpdir}/progress"|stdbuf -oL tr '\r' '\n'|awk -W posix_space -W interactive 'NF==12 { print "XXX\n" $(NF-11) "\nDownloading:\n'"$filename"'\n" $(NF-8) " of " $(NF-10) " at " $(NF) "B/s (" $(NF-1) " left)"; system(""); } /^.......................%\[....................\]/ { split($2,A,/\%/); print "XXX\n" A[1]"\nDownloading:\n'"$filename"'\n"$4 " at " $5 " (" $7 " left)"; system("");}' 2>/dev/null| whiptail --clear --backtitle "Download: $filename" --gauge "Starting download..." 10 0 0 2>/dev/null &
545 | echo $! >"${tmpdir}/pager_pid"
546 | ;;
547 | yad)
548 | "$stdbuf" -oL tail -f "${tmpdir}/progress"|stdbuf -oL tr '\r' '\n'|awk -W posix_space -W interactive 'NF==12 { print $(NF-11) "\n#'"$filename"' (" $(NF) "B/s)"; system(""); } /^.......................%\[....................\]/ { split($2,A,/\%/); print A[1]"\n#'"$filename"' (" $5 ")"; system("");}' 2>/dev/null| yad --window-icon='gtk-save' --title='Downloading' --progress --progress-text="$filename" --auto-close 2>/dev/null &
549 | echo $! >"${tmpdir}/pager_pid"
550 | ;;
551 | zenity)
552 | "$stdbuf" -oL tail -f "${tmpdir}/progress"|stdbuf -oL tr '\r' '\n'|awk -W posix_space -W interactive 'NF=12 { print $(NF-11) "\n#'"$filename"' (" $(NF) "B/s)"; system(""); } /^.......................%\[....................\]/ { split($2,A,/\%/); print A[1]"\n#'"$filename"' (" $5 ")"; system("");}' 2>/dev/null| zenity --window-icon='gtk-save' --title='Downloading' --progress --auto-close 2>/dev/null &
553 | echo $! >"${tmpdir}/pager_pid"
554 | ;;
555 | *)
556 | "$stdbuf" -oL tail -f "${tmpdir}/progress" &
557 | echo $! >"${tmpdir}/pager_pid"
558 | ;;
559 | esac
560 |
561 | trap 'kill $(<"${tmpdir}"/dl_pid) $(<"${tmpdir}"/pager_pid) 2>/dev/null; rm -rf "${tmpdir}";' EXIT
562 |
563 | # wait for the pager to finish (or be closed by the user) and/or the download to finish
564 | # this replaces the (buggy) auto-kill functionality of yad and zenity (dialog does not have any auto-kill)
565 |
566 | while (kill -0 "$(<"${tmpdir}/pager_pid")" 2>/dev/null); do
567 | if (kill -0 "$(<"${tmpdir}/dl_pid")" 2>/dev/null); then
568 | sleep 1
569 | else
570 | break
571 | fi
572 | done
573 | }
574 |
575 |
576 | dl_src_direct () {
577 | db="$1"
578 | shift
579 | md5="$1"
580 | shift
581 | filename="$*"
582 |
583 | case "$db" in
584 | libgen)
585 | url="${downloadurl[$db]}/$(get_torrent "${db}" "${md5}")/${md5,,}/placeholder"
586 | ;;
587 | libgen_fiction)
588 | extension=$(get_attr 'extension' "${db}" "${md5}")
589 | url="${downloadurl[$db]}/$(get_torrent "${db}" "${md5}")/${md5,,}.${extension,,}/placeholder"
590 | ;;
591 | *)
592 | exit_with_error "no direct download available for $db"
593 | ;;
594 | esac
595 |
596 | if ! url_available "$url"; then
597 |
598 | url=""
599 | parser=$(find_tool "$parser_tools")
600 |
601 | if [[ $parser == "xidel" ]]; then
602 | url=$(xidel --user-agent "$user_agent" -s "${downloadurl[$db]}/${md5}" -e '//td[@id="info"]/h2/a/@href');
603 | elif [[ $parser == "hxwls" ]]; then
604 | url=$(hxwls "${downloadurl[$db]}/${md5}"|head -2|tail -1)
605 | fi
606 | fi
607 |
608 | [[ -n "$url" ]] && get_file "'${filename}'" "${url}"
609 | }
610 |
611 | dl_src_ipfs () {
612 | db="$1"
613 | shift
614 | md5="$1"
615 | shift
616 | filename="$*"
617 |
618 | ipfs_cid=$(get_attr 'ipfs_cid' "${db}" "${md5}")
619 |
620 | if [[ -z $ipfs_cid ]]; then
621 | echo "ipfs_cid not found, trying direct download..."
622 | dl_src_direct "${db}" "${md5}" "${filename}"
623 | else
624 | url="${ipfs_gw}/ipfs/${ipfs_cid}"
625 | get_file "'${filename}'" "${url}"
626 | fi
627 | }
628 |
629 | dl_src_torrent () {
630 | db="$1"
631 | shift
632 | md5="$1"
633 | shift
634 | dest_filename="$*"
635 |
636 | torrent_abspath=$(get_torrent_filename "${db}" "${md5}" 1)
637 | ttool=$(find_tool "$torrent_tools")
638 |
639 | if dl_torrent "$db" "$md5"; then
640 |
641 | torrent_filepath=$($ttool torrent-files "$torrent_abspath"|grep -i "$md5")
642 |
643 | if [[ -f "$torrent_abspath" ]]; then
644 | torrent_job=$(date +%Y%m%d-%H%M%S)-$(get_torrent "$db" "$md5")"-$md5.job"
645 | torrent_log="$target_directory/${torrent_directory:+$torrent_directory/}${torrent_job}"
646 | torrent_btih=$($ttool torrent-hash "$torrent_abspath")
647 |
648 | cat <<- EOT > "$torrent_log"
649 | #!/usr/bin/env bash
650 | # command: "$ttool add-selective $torrent_abspath $md5"
651 | tdir="$torrent_download_directory"
652 | tpath="$torrent_filepath"
653 | dest="$target_directory/$dest_filename"
654 | btih="$torrent_btih"
655 | ttool="$ttool"
656 | cronjob_remove () {
657 | if cronjob_active; then
658 | (crontab -l) 2>/dev/null | grep -v "$torrent_log" | sort | uniq | crontab -
659 | fi
660 | }
661 | torrent_restart () { $ttool add-selective "$torrent_abspath" "$md5"; }
662 | direct_download () { exec $program -u n -J "$md5"; }
663 | cronjob_active () { crontab -l|grep -q "$torrent_log"; }
664 | EOT
665 |
666 | cat <<- 'EOT' >> "$torrent_log"
667 | torrent_remove () { $ttool remove "$btih"; }
668 | torrent_status () { $ttool ls "$btih"; }
669 | torrent_info () { $ttool info "$btih"; }
670 | torrent_active () { $ttool active "$btih"; }
671 | job_status () {
672 | if [[ -f "$dest" ]]; then echo -e "job finished, file copied to destination:\n$dest";
673 | elif [[ -f "$tdir/$tpath" ]]; then echo "torrent downloaded";
674 | if cronjob_active; then echo "cronjob active, wait for file to be copied to destination";
675 | else echo "run this job without options to copy file to destination";
676 | fi
677 | elif torrent_active; then echo "torrent active:";torrent_status;
678 | else echo "job inactive, file not downloaded, -R or -D to retry";
679 | fi
680 | }
681 | check_md5 () {
682 | md5_real=$(md5sum "$1"|awk '{print $1}')
683 | md5=$(basename "$tpath")
684 | [[ "${md5_real,,}" == "${md5,,}" ]]
685 | }
686 | copy_file () {
687 | if [[ -f "$tdir/$tpath" ]]; then
688 | if check_md5 "$tdir/$tpath"; then
689 | install -D "$tdir/$tpath" "$dest"
690 | else
691 | false
692 | fi
693 | else
694 | false
695 | fi
696 | }
697 | show_help () {
698 | cat <<-EOHELP
699 | Use: bash jobid.job [-s] -[i] [-r] [-R] [-D] [-h] [torrent_download_directory]
700 |
701 | Copies file from libgen/libgen_fiction torrent to correct location and name
702 |
703 | -S show job status
704 | -s show torrent status (short)
705 | -i show torrent info (long)
706 | -I show target file name
707 | -r remove torrent and cron jobs
708 | -R restart torrent download (does not restart cron job)
709 | -D direct download (removes torrent and cron jobs)
710 | -h show this help message
711 | EOHELP
712 | }
713 | if [[ "$1" == "-r" ]]; then torrent_remove; cronjob_remove; exit; fi
714 | if [[ "$1" == "-R" ]]; then torrent_restart; exit; fi
715 | if [[ "$1" == "-D" ]]; then torrent_remove; direct_download; exit; fi
716 | if [[ "$1" == "-s" ]]; then torrent_status; exit; fi
717 | if [[ "$1" == "-S" ]]; then job_status; exit; fi
718 | if [[ "$1" == "-i" ]]; then torrent_info; exit; fi
719 | if [[ "$1" == "-I" ]]; then echo "$dest"; exit; fi
720 | if [[ "$1" == "-h" ]]; then show_help; exit; fi
721 | if [[ "$1" =~ \-.* ]]; then echo "unknown option: $1"; show_help; exit; fi
722 | if torrent_active; then echo "torrent has not finished downloading"; exit 10; fi
723 | if [[ -z "$tdir" ]]; then if [[ -d "$1" ]]; then tdir="$1"; else show_help; exit; fi; fi
724 | count=0
725 | until [[ $count == 5 ]]; do
726 | if copy_file; then break; fi
727 | sleep 5
728 | ((count++))
729 | done
730 | if [[ -f "$dest" ]]; then
731 | if ! check_md5 "$dest"; then
732 | echo "download corrupted, md5 does not match"
733 | fi
734 | else
735 | echo "download failed"
736 | fi
737 | cronjob_remove
738 | exit
739 |
740 | # torrent client output under this line
741 | :<<'END_OF_LOG'
742 | EOT
743 |
744 | (${ttool} add-selective "$torrent_abspath" "$md5";echo END_OF_LOG) >> "$torrent_log" 2>&1 &
745 |
746 | echo -e "torrent job started, job script:\n$torrent_log"
747 |
748 | # launch cron job?
749 | if [[ -n "$torrent_download_directory" && "$torrent_cron_job" -gt 0 ]]; then
750 | add_cron_job "bash $torrent_log"
751 | echo "torrent cron job started"
752 | fi
753 | fi
754 | else
755 | echo "try direct download (-u n)"
756 | fi
757 | }
758 |
759 | dl_torrent () {
760 | db="$1"
761 | md5="$2"
762 | torrent_filename="$(get_torrent_filename "${db}" "${md5}")"
763 | torrent_abspath="$(get_torrent_filename "${db}" "${md5}" 1)"
764 |
765 | if [[ ! -f "$torrent_abspath" ]]; then
766 | url="${torrenturl[$db]}/${torrent_filename}"
767 | if url_available "$url"; then
768 | get_file "'${torrent_directory:+$torrent_directory/}${torrent_filename}'" "${url}"
769 | else
770 | echo "Torrent $torrent_filename not available"
771 | false
772 | fi
773 | fi
774 | }
775 |
776 | # DATABASE
777 |
778 | # currently only the main libgen db can be updated through the api
779 | update_database () {
780 | db="${programs[books]}"
781 | last_update=$(($(date +%s)-$(date -d "$(get_time_last_modified "$db")" +%s)))
782 | if [[ $last_update -gt $((max_age*60)) ]]; then
783 | if [[ $no_update -eq 0 ]]; then
784 | update_libgen=$(find_tool "update_libgen")
785 | "$update_libgen"
786 | else
787 | echo "The database was last updated $((last_update/60)) minutes ago, consider updating"
788 | fi
789 | fi
790 | }
791 |
792 | get_current_fields () {
793 | db="${programs[$program]}"
794 | declare -a db_tables="${tables[$db]}"
795 |
796 | for table in "${db_tables[@]}"; do
797 | dbx "$db" "describe $table;"|awk '{print tolower($1)}'
798 | done
799 | }
800 |
801 | get_time_last_modified () {
802 | dbx "$db" 'select MAX(TimeLastModified) FROM updated;'
803 | }
804 |
805 | add_clause () {
806 | option="$1"
807 | shift
808 | pattern="$*"
809 |
810 | db="${programs[$program]}"
811 |
812 | if [[ ${pattern:0:1} == '-' ]]; then
813 | # option as argument, rewind OPTIND
814 | ((OPTIND-=1))
815 | unset pattern
816 | fi
817 |
818 | if [[ $current_fields =~ ${schema[${option}]} ]]; then
819 | if [[ -n $pattern ]]; then
820 | # escape ' " \ % _
821 | pattern=${pattern//\\/\\\\}
822 | pattern=${pattern//\'/\\\'}
823 | pattern=${pattern//\"/\\\"}
824 | pattern=${pattern//%/\\%}
825 | pattern=${pattern//_/\\_}
826 | [[ ! $opt_pattern =~ $pattern ]] && opt_pattern+=" $pattern"
827 | if [ -z "$exact_match" ]; then
828 | clauses+=" and ${schema[${option}]} like '%${pattern}%'"
829 | elif [ "$exact_match" -eq 1 ]; then
830 | clauses+=" and ${schema[${option}]} like '${pattern}%'"
831 | elif [ "$exact_match" -eq 2 ]; then
832 | clauses+=" and ${schema[${option}]} like '%${pattern}'"
833 | else
834 | clauses+=" and ${schema[${option}]}='${pattern}'"
835 | fi
836 | fi
837 |
838 | [[ ! $opt_fields =~ ${schema[${option}]} ]] && opt_fields+="${schema[${option}]},"
839 | else
840 | echo "warning: option -$option ignored (database $db does not contain column ${schema[${option}]})"
841 | fi
842 | }
843 |
844 | # the 'pager' gets special treatment as it likes its lines unbroken...
845 | run_query () {
846 | db="$1"
847 | shift
848 | filter="$1"
849 | shift
850 | sql="$*"
851 |
852 | declare -a line
853 | query_result=()
854 |
855 | while IFS=$'\t' read -ra line; do
856 | if [[ $ui_tool == "pager" ]]; then
857 | IFS='' query_result+=("$(printf '%s\t' "${line[@]}")")
858 | else
859 | query_result+=("${line[@]}")
860 | fi
861 | done < <(dbx "$db" "$sql"|(eval "${filters[$filter]}"))
862 | }
863 |
864 | get_attr () {
865 | role="$1"
866 | db="$2"
867 | md5="$3"
868 |
869 | # special case for filename with md5
870 | if [[ "$role" == 'filename' && $filename_add_md5 -gt 0 ]]; then
871 | role='filename_md5'
872 | fi
873 |
874 | sql=$(prepare_sql "${db}" "${role}")
875 |
876 | run_query "${db}" "${role}" "${sql}"
877 |
878 | if [[ ${#query_result[@]} -gt 0 ]]; then
879 | echo "${query_result[0]}"
880 | fi
881 | }
882 |
883 | get_torrent () {
884 | db="$1"
885 | md5="$2"
886 |
887 | id=$(get_attr 'id' "${db}" "${md5}")
888 |
889 | [[ -n "$id" ]] && echo "$((id-id%1000))"
890 | }
891 |
892 | # this creates leading directories when called with 2 parameters (value of $2 does not matter)
893 | get_filename () {
894 | md5="$1"
895 | create="$2"
896 | db="${programs[$program]}"
897 |
898 | filename=$(get_attr "filename" "$db" "$md5")
899 |
900 | # trim overly long filenames to 255 characters
901 | [[ ${#filename} -gt 255 ]] && filename="${filename:0:126}...${filename: -126}"
902 |
903 | if [[ -n $filename ]]; then
904 | if [[ -n $use_deep_path ]]; then
905 | dirname="${dirprefix[$db]}/$(get_attr 'dirname' "$db" "$md5")"
906 | if [[ -n $dirname && -n $create ]]; then
907 | mkdir -p "${target_directory}/${dirname}"
908 | filename="${dirname}/${filename}"
909 | fi
910 | fi
911 | echo "$filename"
912 | fi
913 | }
914 |
915 | get_torrentpath () {
916 | md5="$1"
917 | db="${programs[$program]}"
918 |
919 | torrent=$(get_torrent "$db" "$md5")
920 | [[ -n "$torrent" ]] && echo "$torrent/$md5"
921 | }
922 |
923 | get_attributes () {
924 | md5="$1"
925 | db="${programs[$program]}"
926 | role="attributes"
927 |
928 | get_attr "$role" "$db" "$md5"
929 | }
930 |
931 | # PREVIEWS
932 |
933 | preview () {
934 | db=$1
935 | shift
936 | for md5 in "$@"; do
937 | preview_${ui_tool} "$db" "$md5"
938 | done
939 | }
940 |
941 | # don't mess with the 'ugly formatting', the embedded newlines are part of the preview dialog
942 | preview_dialog () {
943 | db="$1"
944 | md5="$2"
945 | if [[ -n $db && -n $md5 ]]; then
946 | sql=$(prepare_sql "$db" "preview" "$ui_tool")
947 | run_query "$db" "preview_dialog" "${sql}"
948 | if [[ ${#query_result[@]} -gt 0 ]]; then
949 | filename=$(get_attr 'filename' "$db" "$md5")
950 | exec 3>&1
951 | dialog_result=$(dialog --backtitle "${program} - preview" --colors --cr-wrap --no-collapse --extra-button --ok-label "Download" --extra-label "Skip" --no-cancel --yesno \
952 | "\Z1Author\Zn: ${query_result[0]}
953 | \Z1Title\Zn: ${query_result[1]}
954 | \Z1Volume\Zn: ${query_result[2]} \Z1Series\Zn: ${query_result[3]} \Z1Edition\Zn: ${query_result[4]}
955 | \Z1Year\Zn: ${query_result[5]} \Z1Publisher\Zn: ${query_result[6]}
956 | \Z1Language\Zn: ${query_result[7]} \Z1Size\Zn: ${query_result[8]} \Z1Type\Zn: ${query_result[9]}
957 | \Z1OLID\Zn: ${query_result[10]} \Z1ISBN\Zn: ${query_result[11]} \Z1MD5\Zn: ${md5^^}
958 |
959 | \Z1Filename\Zn: ${filename}
960 |
961 | ${query_result[12]}" \
962 | 0 0 2>&1 1>&3)
963 | dialog_exit=$?
964 | exec 3>&-
965 | if [[ $dialog_exit -eq 0 ]]; then
966 | download "${db}" "${md5}"
967 | fi
968 | fi
969 | fi
970 | }
971 |
972 | preview_whiptail () {
973 | db=$1
974 | md5=$2
975 | if [[ -n $db && -n $md5 ]]; then
976 | sql=$(prepare_sql "$db" "preview" "$ui_tool")
977 | run_query "$db" "preview_whiptail" "${sql}"
978 | if [[ ${#query_result[@]} -gt 0 ]]; then
979 | filename=$(get_attr 'filename' "$db" "$md5")
980 | exec 3>&1
981 | whiptail_result=$(whiptail --backtitle "${program} - preview" --yes-button "Download" --no-button "Skip" --nocancel --yesno \
982 | "Author: ${query_result[0]}
983 | Title: ${query_result[1]}
984 | Volume: ${query_result[2]} Series: ${query_result[3]} Edition: ${query_result[4]}
985 | Year: ${query_result[5]} Publisher: ${query_result[6]}
986 | Language: ${query_result[7]} Size: ${query_result[8]} Type: ${query_result[9]}
987 | OLID: ${query_result[10]} ISBN: ${query_result[11]} MD5: ${md5^^}
988 |
989 | Filename: ${filename}
990 |
991 | ${query_result[12]}" \
992 | 0 0 2>&1 1>&3)
993 | whiptail_exit=$?
994 | exec 3>&-
995 | if [[ $whiptail_exit -eq 0 ]]; then
996 | download "${db}" "${md5}"
997 | fi
998 | fi
999 | fi
1000 | }
1001 |
1002 |
1003 | preview_zenity () {
1004 | db="$1"
1005 | md5="$2"
1006 | if [[ -n $db && -n $md5 ]]; then
1007 | sql=$(prepare_sql "$db" "preview" "$ui_tool")
1008 | run_query "$db" "preview_zenity" "${sql}"
1009 | if [[ ${#query_result[@]} -gt 0 ]]; then
1010 | filename=$(get_attr 'filename' "$db" "$md5")
1011 | info="| Author: | ${query_result[0]} |
| Title: | ${query_result[1]} |
| Volume: | ${query_result[2]} | Series: | ${query_result[3]} | Edition: | ${query_result[4]} |
| Year: | ${query_result[5]} | Publisher: | ${query_result[6]} |
| Language: | ${query_result[7]} | Size: | ${query_result[8]} | Type: | ${query_result[9]} |
| OLID: | ${query_result[10]} | ISBN: | ${query_result[11]} | MD5: | ${md5^^} |
${filename}
 | $(strip_html "${query_result[12]}") |
"
1012 | if zenity_result=$(echo "$info"|zenity --width $x_size --height $y_size --text-info --html --ok-label "Download" --cancel-label "Skip" --filename=/dev/stdin 2>/dev/null); then
1013 | download "${db}" "${md5}"
1014 | fi
1015 | fi
1016 | fi
1017 | }
1018 |
1019 | preview_yad () {
1020 | db="$1"
1021 | md5="$2"
1022 | for md5 in "$@"; do
1023 | sql=$(prepare_sql "$db" "preview" "$ui_tool")
1024 |
1025 | run_query "$db" "preview_yad" "${sql}"
1026 | if [[ ${#query_result[@]} -gt 0 ]]; then
1027 | filename=$(get_attr 'filename' "$db" "$md5")
1028 | info="| Author: | ${query_result[0]} |
| Title: | ${query_result[1]} |
| Volume: | ${query_result[2]} | Series: | ${query_result[3]} | Edition: | ${query_result[4]} |
| Year: | ${query_result[5]} | Publisher: | ${query_result[6]} |
| Language: | ${query_result[7]} | Size: | ${query_result[8]} | Type: | ${query_result[9]} |
| OLID: | ${query_result[10]} | ISBN: | ${query_result[11]} | MD5: | ${md5^^} |
${filename}
| $(strip_html "${query_result[12]}") |
"
1029 | if yad_result=$(echo "$info"|yad --width $x_size --height $y_size --html --button='gtk-cancel:1' --button='Download!filesave!Download this publication:0' --filename=/dev/stdin 2>/dev/null); then
1030 | download "${db}" "${md5}"
1031 | fi
1032 | fi
1033 | done
1034 | }
1035 |
1036 | # LIST VIEWS
1037 |
1038 | list_pager () {
1039 | show_preview="$1" # ignored, no preview possible using pager
1040 |
1041 | pager=$(find_tool "$pager_tools")
1042 |
1043 | [[ $pager == "less" ]] && pager_options="-S"
1044 |
1045 |
1046 | if [[ ${#query_result[@]} -gt 0 ]]; then
1047 | (for index in "${!query_result[@]}"; do
1048 | echo "${query_result[$index]}"
1049 | done)|column -t -n -x -s $'\t'|$pager $pager_options
1050 | fi
1051 | }
1052 |
1053 | list_dialog () {
1054 | show_preview="$1"
1055 | if [[ ${#query_result[@]} -gt 0 ]]; then
1056 | exec 3>&1
1057 | dialog_result=$(dialog --separate-output --no-tags --backtitle "$program - search" --title "$window_title" --checklist "${list_heading}" 0 0 0 -- "${query_result[@]}" 2>&1 1>&3)
1058 | dialog_exit=$?
1059 | exec 3>&-
1060 | clear
1061 | if [[ -n $dialog_result ]]; then
1062 | if [[ $show_preview -gt 0 ]]; then
1063 | # shellcheck disable=SC2086
1064 | preview "$db" $dialog_result
1065 | else
1066 | # shellcheck disable=SC2086
1067 | download "$db" $dialog_result
1068 | fi
1069 | fi
1070 | fi
1071 | }
1072 |
1073 | # current whiptail (as of the date of writing, 20160326) has a bug which makes it ignore --notags
1074 | # https://bugzilla.redhat.com/show_bug.cgi?id=1215239
1075 | list_whiptail () {
1076 | show_preview="$1"
1077 | if [[ ${#query_result[@]} -gt 0 ]]; then
1078 | exec 3>&1
1079 | whiptail_result=$(whiptail --separate-output --notags --backtitle "$program - search" --title "$window_title" --checklist "${list_heading}" 0 0 0 -- "${query_result[@]}" 2>&1 1>&3)
1080 | whiptail_exit=$?
1081 | exec 3>&-
1082 | clear
1083 | if [[ -n $whiptail_result ]]; then
1084 | if [[ $show_preview -gt 0 ]]; then
1085 | # shellcheck disable=SC2086
1086 | preview "$db" $whiptail_result
1087 | else
1088 | # shellcheck disable=SC2086
1089 | download "$db" $whiptail_result
1090 | fi
1091 | fi
1092 | fi
1093 | }
1094 |
1095 | list_yad () {
1096 | show_preview="$1"
1097 | db="${programs[$program]}"
1098 | if [[ ${#query_result[@]} -gt 0 ]]; then
1099 | # shellcheck disable=SC2086
1100 | yad_result=$(yad --width $x_size --height $y_size --separator=" " --title "$program :: ${window_title}" --text "${list_heading}" --list --checklist --dclick-action='bash -c "libgen_preview '"$db"' %s" &' ${yad_columns[$db]} -- "${query_result[@]}" 2>/dev/null)
1101 | if [[ -n $yad_result ]]; then
1102 | if [[ $show_preview -gt 0 ]]; then
1103 | # shellcheck disable=SC2086
1104 | preview "$db" $yad_result
1105 | else
1106 | # shellcheck disable=SC2086
1107 | download "$db" $yad_result
1108 | fi
1109 | fi
1110 | fi
1111 |
1112 | }
1113 |
1114 | # zenity does not support the '--' end of options convention leading to problems when the query result contains dashes,
1115 | # hence these are replaced by underscores in the query_result.
1116 | list_zenity () {
1117 | show_preview="$1"
1118 | db="${programs[$program]}"
1119 | if [[ ${#query_result[@]} -gt 0 ]]; then
1120 | # shellcheck disable=SC2086
1121 | zenity_result=$(zenity --width $x_size --height $y_size --separator=" " --title "$program :: ${window_title}" --text "${list_heading}" --list --checklist ${zenity_columns[$db]} "${query_result[@]}" 2>/dev/null)
1122 | if [[ -n $zenity_result ]];then
1123 | if [[ $show_preview -gt 0 ]]; then
1124 | # shellcheck disable=SC2086
1125 | preview "$db" $zenity_result
1126 | else
1127 | # shellcheck disable=SC2086
1128 | download "$db" $zenity_result
1129 | fi
1130 | fi
1131 | fi
1132 |
1133 | }
1134 |
1135 | # SQL
1136 |
1137 | get_fields () {
1138 | # shellcheck disable=SC2086
1139 | db="${programs[$program]}"
1140 | [[ -z "$show_fields" ]] && show_fields="${pager_columns[$db]}"
1141 | IFS=',' read -ra fields <<< "$show_fields"
1142 | result=""
1143 |
1144 | for field in "${fields[@]}"; do
1145 | [[ ! "${current_fields[*],,}" =~ ${field,,} ]] && exit_with_error "no such field: $field"
1146 | table="m"
1147 | for category in "${!attribute_columns[@]}"; do
1148 | if [[ "${field,,}" =~ ${attribute_columns[$category],,} ]]; then
1149 | table="${category:0:1}"
1150 | break
1151 | fi
1152 | done
1153 | result+="${result:+,}greatest(${table}.${field}, '-')"
1154 | done
1155 |
1156 | echo -n "$result"
1157 | }
1158 |
1159 |
1160 | prepare_sql () {
1161 | db="$1"
1162 | role="$2"
1163 | ui_tool="$3"
1164 |
1165 | # SQL to:
1166 | # build filenames...
1167 |
1168 | declare -A sql_filename=(
1169 | [libgen]="select concat_ws('.',concat_ws('-', trim(Series), trim(Author), trim(Title), trim(Year), trim(Publisher), trim(language)), trim(extension)) from updated where md5='${md5}' limit 1;"
1170 | [libgen_fiction]="select concat_ws('.',concat_ws('-', trim(Series), trim(Author), trim(Title), trim(Year), trim(Publisher), trim(language)), trim(extension)) from fiction where md5='${md5}' limit 1;"
1171 | )
1172 |
1173 | declare -A sql_filename_md5=(
1174 | [libgen]="select concat_ws('.',concat_ws('-', trim(Series), trim(Author), trim(Title), trim(Year), trim(Publisher), trim(language), trim(md5)), trim(extension)) from updated where md5='${md5}' limit 1;"
1175 | [libgen_fiction]="select concat_ws('.',concat_ws('-', trim(Series), trim(Author), trim(Title), trim(Year), trim(Publisher), trim(language), trim(md5)), trim(extension)) from fiction where md5='${md5}' limit 1;"
1176 | )
1177 |
1178 | # build directory names...
1179 | declare -A sql_dirname=(
1180 | [libgen]="select concat_ws('/', trim(language), regexp_replace(trim(topic_descr),'"'[\\\\]+'"','/'), trim(Author), trim(Series)) as dirname from updated as u left join topics as t on u.topic=t.topic_id and t.lang='${language}' where md5='${md5}' limit 1;"
1181 | [libgen_fiction]="select concat_ws('/', trim(language), trim(Author), trim(Series)) as dirname from fiction where md5='${md5}' limit 1;"
1182 | )
1183 |
1184 | # get id
1185 | declare -A sql_id=(
1186 | [libgen]="select id from updated where md5='${md5}' limit 1;"
1187 | [libgen_fiction]="select id from fiction where md5='${md5}' limit 1;"
1188 | )
1189 |
1190 | # get extension
1191 | declare -A sql_extension=(
1192 | [libgen]="select extension from updated where md5='${md5}' limit 1;"
1193 | [libgen_fiction]="select extension from fiction where md5='${md5}' limit 1;"
1194 | )
1195 |
1196 | # get attributes
1197 | declare -A sql_attributes=(
1198 | [libgen]="select $(get_fields) from updated as m left join description as d on m.md5=d.md5 left join topics as t on m.topic=t.topic_id and t.lang='${language}' left join hashes as h on m.md5=h.md5 where m.md5='${md5}' limit 1;"
1199 | [libgen_fiction]="select $(get_fields) from fiction as m left join fiction_description as d on m.md5=d.md5 left join fiction_hashes as h on m.md5=h.md5 where m.md5='${md5}' limit 1;"
1200 | )
1201 |
1202 | # get hashes
1203 | declare -A sql_sha1s=(
1204 | [libgen]="select sha1 from hashes where md5='$md5' limit 1;"
1205 | [libgen_fiction]="select sha1 from fiction_hashes where md5='$md5' limit 1;"
1206 | )
1207 |
1208 | # get ipfs content id hash
1209 | declare -A sql_ipfs_cid=(
1210 | [libgen]="select ipfs_cid from hashes where md5='${md5}' limit 1;"
1211 | [libgen_fiction]="select ipfs_cid from fiction_hashes where md5='${md5}' limit 1;"
1212 | )
1213 |
1214 | # preview publication...
1215 |
1216 | declare -A sql_preview_dialog=(
1217 | [libgen]="select greatest(Author, '-'), greatest(Title, '-'), greatest(VolumeInfo, '-'), greatest(Series, '-'), greatest(Edition, '-'), greatest(Year, '-'), greatest(Publisher, '-'), greatest(language, '-'), greatest(filesize, '-'), greatest(extension, '-'), greatest(OpenLibraryID, '-'), greatest(IdentifierWODash, '-'), greatest(ifnull(descr,'-'), '-'), Coverurl from updated left join description on updated.md5=description.md5 where updated.md5='${md5}' limit 1;"
1218 | [libgen_fiction]="select greatest(Author, '-'), greatest(Title, '-'), greatest(Issue, '-'), greatest(Series, '-'), greatest(Edition, '-'), greatest(Year, '-'), greatest(Publisher, '-'), greatest(Language, '-'), greatest(Filesize, '-'), greatest(Extension, '-'), greatest(Identifier,'-'), greatest(Commentary,'-'), greatest(ifnull(descr,'-'), '-'), Coverurl from fiction as f left join fiction_description as d on f.md5=d.md5 where f.md5='${md5}' limit 1;"
1219 | )
1220 |
1221 |
1222 | declare -n sql_preview_whiptail="sql_preview_dialog"
1223 |
1224 | declare -A sql_preview_zenity=(
1225 | [libgen]="select greatest(Author, '-'), greatest(Title, '-'), greatest(VolumeInfo, '-'), greatest(Series, '-'), greatest(Edition, '-'), greatest(Year, '-'), greatest(Publisher, '-'), greatest(language, '-'), greatest(filesize, '-'), greatest(extension, '-'), greatest(OpenLibraryID, '-'), greatest(IdentifierWODash, '-'), greatest(ifnull(descr,'-'), '-'), Coverurl from updated left join description on updated.md5=description.md5 where updated.md5='${md5}' limit 1;"
1226 | [libgen_fiction]="select greatest(Author, '-'), greatest(Title, '-'), greatest(Issue, '-'), greatest(Series, '-'), greatest(Edition, '-'), greatest(Year, '-'), greatest(Publisher, '-'), greatest(language, '-'), greatest(filesize, '-'), greatest(extension, '-'), greatest(Identifier, '-'), greatest(Commentary,'-'), greatest(ifnull(descr,'-'), '-'), Coverurl from fiction as f left join fiction_description as d on f.md5=d.md5 where f.md5='${md5}' limit 1;"
1227 | )
1228 |
1229 | declare -A sql_preview_yad=(
1230 | [libgen]="select greatest(Author, '-'), greatest(Title, '-'), greatest(VolumeInfo, '-'), greatest(Series, '-'), greatest(Edition, '-'), greatest(Year, '-'), greatest(Publisher, '-'), greatest(language, '-'), greatest(filesize, '-'), greatest(extension, '-'), greatest(OpenLibraryID, '-'), greatest(IdentifierWODash, '-'), greatest(ifnull(descr,'-'), '-'), Coverurl from updated left join description on updated.md5=description.md5 where updated.md5='${md5}' limit 1;"
1231 | [libgen_fiction]="select greatest(Author, '-'), greatest(Title, '-'), greatest(Issue, '-'), greatest(Series, '-'), greatest(Edition, '-'), greatest(Year, '-'), greatest(Publisher, '-'), greatest(language, '-'), greatest(filesize, '-'), greatest(extension, '-'), greatest(Identifier, '-'), greatest(Commentary,'-'), greatest(ifnull(descr,'-'), '-'), Coverurl from fiction as f left join fiction_description as d on f.md5=d.md5 where f.md5='${md5}' limit 1;"
1232 | )
1233 |
1234 | # search...
1235 |
1236 | declare -A sql_search_pager=(
1237 | [libgen]="select $(get_fields) from updated as m left join description as d on m.md5=d.md5 left join topics as t on m.topic=t.topic_id and t.lang='${language}' where TRUE ${query} ${clauses} ${sql_limit};"
1238 | [libgen_fiction]="select $(get_fields) from fiction as m left join fiction_description as d on m.md5=d.md5 where TRUE ${query} ${clauses} ${sql_limit};"
1239 | )
1240 |
1241 | declare -A sql_search_dialog=(
1242 | [libgen]="select u.MD5, concat_ws('|', rpad(greatest(u.Title,'-'),70,' '), rpad(greatest(u.Author,'-'), 30,' '), rpad(greatest(u.Year,'-'), 5,' '), rpad(greatest(u.Edition,'-'), 20, ' '), rpad(greatest(u.Publisher,'-'), 30,' '), rpad(greatest(u.Language,'-'), 10, ' '), rpad(greatest(ifnull(t.Topic_descr, '-'),'-'),30,' '), rpad(greatest(u.Filesize,'-'),10,' '), rpad(greatest(u.Extension,'-'),6,' ')), 'off' from updated as u left join topics as t on u.topic=t.topic_id and t.lang='${language}' where TRUE ${query} ${clauses} ${sql_limit};"
1243 | [libgen_fiction]="select f.MD5, concat_ws('|', rpad(greatest(f.Title,'-'),70,' '), rpad(greatest(f.Author,'-'), 30,' '), rpad(greatest(f.Year,'-'), 5,' '), rpad(greatest(f.Edition,'-'), 20, ' '), rpad(greatest(f.Publisher,'-'), 30,' '), rpad(greatest(f.Language,'-'), 10, ' '), rpad(greatest(f.Series,'-'),30,' '), rpad(greatest(f.Filesize,'-'),10,' '), rpad(greatest(f.Extension,'-'),6,' ')), 'off' from fiction as f where TRUE ${query} ${clauses} ${sql_limit};"
1244 | )
1245 |
1246 | declare -n sql_search_whiptail="sql_search_dialog"
1247 |
1248 | declare -A sql_search_yad=(
1249 | [libgen]="select 'FALSE', u.MD5, left(greatest(u.Title,'-'),70), left(greatest(u.Author, '-'),50),greatest(u.Year, '-'),left(greatest(u.Edition, '-'),20),left(greatest(u.Publisher, '-'),30),greatest(u.language, '-'), left(greatest(ifnull(t.Topic_descr,'-'),'-'),30), greatest(u.Filesize,'-'), greatest(u.Extension, '-'), concat('Title: ',Title, ' Author: ', Author, ' Path: ', Locator) from updated as u left join topics as t on u.topic=t.topic_id and t.lang='${language}' where TRUE ${query} ${clauses} ${sql_limit};"
1250 | [libgen_fiction]="select 'FALSE', f.MD5, left(greatest(f.Title,'-'),70), left(greatest(f.Author, '-'),50),greatest(f.Year, '-'),left(greatest(f.Edition, '-'),20),left(greatest(f.Publisher, '-'),30),greatest(f.language, '-'), left(greatest(f.Series,'-'),30), greatest(f.Filesize,'-'), greatest(f.Extension, '-'), concat('Title: ',Title, ' Author: ', Author, ' Path: ', Locator) from fiction as f where TRUE ${query} ${clauses} ${sql_limit};"
1251 | )
1252 |
1253 | declare -A sql_search_zenity=(
1254 | [libgen]="select 'FALSE', u.MD5, left(greatest(u.Title,'-'),70), left(greatest(u.Author, '-'),50),greatest(u.Year, '-'),left(greatest(u.Edition, '-'),20),left(greatest(u.Publisher, '-'),30),greatest(u.language, '-'), left(greatest(ifnull(t.Topic_descr,'-'),'-'),30), greatest(u.Filesize,'-'), greatest(u.Extension, '-') from updated as u left join topics as t on u.topic=t.topic_id and t.lang='${language}' where TRUE ${query} ${clauses} ${sql_limit};"
1255 | [libgen_fiction]="select 'FALSE', f.MD5, left(greatest(f.Title,'-'),70), left(greatest(f.Author, '-'),50),greatest(f.Year, '-'),left(greatest(f.Edition, '-'),20),left(greatest(f.Publisher, '-'),30),greatest(f.language, '-'), left(greatest(f.Series,'-'),30), greatest(f.Filesize,'-'), greatest(f.Extension, '-') from fiction as f where TRUE ${query} ${clauses} ${sql_limit};"
1256 | )
1257 |
1258 | declare -A sql_search_all_pager=(
1259 | [libgen]="(select u.Title, u.Author,u.Year,u.Edition,u.Publisher,u.language,ifnull(t.Topic_descr, '-'), u.Filesize, u.Extension, u.Locator, md5 from updated as u left join topics as t on u.topic=t.topic_id and t.lang='${language}' where TRUE ${query} ${clauses} ${sql_limit}) UNION (select u.Title, u.Author,u.Year,u.Edition,u.Publisher,u.language,t.Topic_descr,u.Filesize, u.Extension, u.Locator, u.md5 from updated_edited as u left join topics as t on u.topic=t.topic_id and t.lang='${language}' where TRUE ${query} ${clauses} ${sql_limit});"
1260 | )
1261 |
1262 | if [[ -n $ui_tool ]]; then
1263 | sql_source="sql_${role}_${ui_tool}"
1264 | else
1265 | sql_source="sql_${role}"
1266 | fi
1267 |
1268 | echo "${sql_source[$db]}"
1269 | }
1270 |
1271 | # UTILITY FUNCTIONS
1272 |
1273 | get_db_for_md5 () {
1274 | md5="$1"
1275 |
1276 | for dbs in "${!tables[@]}"; do
1277 | fmd5=$(get_attr "md5" "$db" "$md5")
1278 | if [[ "$fmd5" == "$md5" ]]; then
1279 | db=$dbs
1280 | fi
1281 | done
1282 |
1283 | if [[ -n "$db" ]]; then
1284 | echo -n "$db"
1285 | fi
1286 | }
1287 |
1288 | get_torrent_filename () {
1289 | db="$1"
1290 | md5="$2"
1291 | absolute="$3"
1292 |
1293 | echo -n "${absolute:+$target_directory/${torrent_directory:+$torrent_directory/}}${torrentprefix[$db]}_$(get_torrent "${db}" "${md5}").torrent"
1294 | }
1295 |
1296 | create_symlinks () {
1297 | basedir="$(dirname "$0")"
1298 | sourcefile="$(readlink -e "$0")"
1299 | for name in "${!programs[@]}"; do
1300 | if [[ ! -e "$basedir/$name" ]]; then
1301 | ln -s "$sourcefile" "$basedir/$name"
1302 | fi
1303 | done
1304 |
1305 | exit
1306 | }
1307 |
1308 | check_settings () {
1309 | # does target directory exist?
1310 | [[ ! -d "$target_directory" ]] && exit_with_error "target_directory $target_directory does not exist";
1311 | # when defined, does torrent download directory exist?
1312 | [[ -n "$torrent_download_directory" && ! -d "$torrent_download_directory" ]] && exit_with_error "torrent_download_directory $torrent_download_directory does not exist";
1313 | # when defined, does torrent helper script exist?
1314 | if [[ -n "$use_torrent" ]]; then
1315 | if [[ -z "$torrent_tools" ]]; then
1316 | exit_with_error "-u needs torrent helper script, see \$torrent_tools"
1317 | elif ! find_tool "$torrent_tools" >/dev/null; then
1318 | exit_with_error "-u: torrent helper script ($torrent_tools) not found"
1319 | fi
1320 | fi
1321 |
1322 | if [[ "$use_torrent" && "$use_ipfs" ]]; then
1323 | exit_with_error "can not use_torrent and use_ipfs at the same time, check $config"
1324 | fi
1325 | }
1326 |
1327 | cleanup () {
1328 | if [[ $ui_tool == "whiptail" ]]; then
1329 | reset
1330 | fi
1331 | }
1332 |
1333 | # HELP
1334 |
1335 | help () {
1336 | echo "$(basename "$(readlink -f "$0")")" "version $version"
1337 | cat <<- 'EOF'
1338 |
1339 | Use: books OPTIONS [like] []
1340 |
1341 | [B]ooks - which is only one of the names this program goes by - is a
1342 | front-end for accessing a locally accessible libgen / libgen_fiction database
1343 | instance, offering versatile search and download directly from the command
1344 | line. The included update_libgen tool is used to keep the database up to date -
1345 | if the database is older than a user-defined value it is updated before the
1346 | query is executed. This generally only takes a few seconds, but it might take
1347 | longer on a slow connection or after a long update interval. Updating can be
1348 | temporarily disabled by using the ‘-x’ command line option. To refresh the
1349 | database(s) from a dump file use the included refresh_libgen program.
1350 |
1351 | When books, nbook and/or xbook are regularly used the database should be kept
1352 | up to date automatically. In that case it is only necessary to use
1353 | refresh_libgen to refresh the database when you get a warning from
1354 | update_libgen about unknown columns in the API response.
1355 |
1356 | If the programs have not been used for a while it can take a long time - and a
1357 | lot of data transfer - to update the database through the API (which is what
1358 | update_libgen does). Especially when using the compact database it can be
1359 | quicker to use refresh_libgen to just pull the latest dump instead of waiting
1360 | for update_libgen to do its job.
1361 |
1362 | The fiction database can not be updated through the API (yet), so for
1363 | that databases refresh_libgen is currently the canonical way to get the latest
1364 | version.
1365 |
1366 | SEARCH BY FIELD:
1367 |
1368 | This is the default search mode. If no field options are given this searches
1369 | the Title field for the PATTERN. Search uses partial matching by default, use
1370 | -X for matching words starting with PATTERN, -XX to match words which end with
1371 | PATTERN and -XXX for exact matching.
1372 |
1373 | FULLTEXT SEARCH (-f):
1374 |
1375 | Performs a pattern match search over all fields indicated by the options. If no
1376 | field options are given, perform a pattern match search over the Author and
1377 | Title fields.
1378 |
1379 | Depending on which name this program is executed under it behaves differently:
1380 |
1381 | books: query database and show results, direct download with md5
1382 | books-all: query database and show results (exhaustive search over all tables, slow)
1383 |
1384 | nbook: select publications for download from list (terminal-based)
1385 | xbook: select publications for download from list (GUI)
1386 |
1387 | fiction: query database and show results (using 'fiction' database), direct download with md5
1388 |
1389 | nfiction: select publications for download from list (terminal-based, use 'fiction' database)
1390 | xfiction: select publications for download from list (GUI, use 'fiction' database)
1391 |
1392 | OPTIONS
1393 |
1394 | EOF
1395 |
1396 | for key in "${!schema[@]}"; do
1397 | echo " -${key} search on ${schema[$key]^^}"
1398 | done
1399 |
1400 | cat <<- 'EOF'
1401 |
1402 | -f fulltext search
1403 | searches for the given words in the fields indicated by the other options.
1404 | when no other options are given this will perform a pattern match search
1405 | for the given words over the Author and Title fields.
1406 |
1407 | -X search for fields starting with PATTERN
1408 | -XX search for fields ending with PATTERN
1409 | -XXX search for fields exacly matching PATTERN
1410 |
1411 | -w preview publication info before downloading (cover preview only in GUI tools)
1412 | select one or more publication to preview and press enter/click OK.
1413 |
1414 | double-clicking a result row also shows a preview irrespective of this option,
1415 | but this only works when using the yad gui tool
1416 |
1417 | -= DIR set download location to DIR
1418 |
1419 | -$ use extended path when downloading:
1420 | nonfiction/[topic/]author[/series]/title
1421 | fiction/language/author[/series]/title
1422 |
1423 | -u BOOL use bittorrent (-u 1 or -u y) or direct download (-u 0 or -u n)
1424 | this parameter overrides the default download method
1425 | bittorrent download depends on an external helper script
1426 | to interface with a bittorrent client
1427 |
1428 | -I BOOL use ipfs (-I 1 or -I y) or direct download (-I 0 or -I n)
1429 | this parameter overrides the default download method
1430 | ipfs download depends on a functioning ipfs gateway.
1431 | default gateway is hosted by Cloudfront, see https://ipfs.io/
1432 | for instructions on how to run a local gateway
1433 |
1434 | -U MD5 print torrent path (torrent#/md5) for given MD5
1435 |
1436 | -j MD5 print filename for given MD5
1437 |
1438 | -J MD5 download file for given MD5
1439 | can be combined with -u to download with bittorrent
1440 |
1441 | -M MD5 fast path search on md5, only works in _books_ and _fiction_
1442 | can be combined with -F FIELDS to select fields to be shown
1443 | output goes directly to the terminal (no pager)
1444 |
1445 | -F FIELDS select which fields to show in pager output
1446 |
1447 | -# LIMIT limit search to LIMIT hits (default: 1000)
1448 |
1449 | -x skip database update
1450 | (currently only the 'libgen' database can be updated)
1451 |
1452 | -@ TORPORT use torsocks to connect to the libgen server(s). You'll need to install
1453 | torsocks before using this option; try this in case your ISP
1454 | (or a transit provider somewhere en-route) blocks access to libgen
1455 |
1456 | -k install symlinks for all program invocations
1457 |
1458 | -h show this help message
1459 |
1460 | EXAMPLES
1461 |
1462 | Do a pattern match search on the Title field for 'ilias' and show the results in the terminal
1463 |
1464 | $ books like ilias
1465 |
1466 |
1467 | Do an exact search on the Title field for 'The Odyssey' and show the results in the terminal
1468 |
1469 | $ books 'the odyssey'
1470 |
1471 |
1472 | Do an exact search on the Title field for 'The Odyssey' and the Author field for 'Homer', showing
1473 | the result in the terminal
1474 |
1475 | $ books -X -t 'The Odyssey' -a 'Homer'
1476 |
1477 |
1478 | Do the same search as above, showing the results in a list on the terminal with checkboxes to select
1479 | one or more publications for download
1480 |
1481 | $ nbook -X -t 'The Odyssey' -a 'Homer'
1482 |
1483 |
1484 | A case-insensitive pattern search using an X11-based interface; use bittorrent (-u) when downloading files
1485 |
1486 | $ xbook -u y -t 'the odyssey' -a 'homer'
1487 |
1488 |
1489 | Do a fulltext search over the Title, Author, Series, Periodical and Publisher fields, showing the
1490 | results in a terminal-based checklist for download after preview (-w)
1491 |
1492 | $ nbook -w -f -t -a -s -r -p 'odyssey'
1493 |
1494 |
1495 | Walk over a directory of publications, compute md5 and use this to generate file names:
1496 |
1497 | $ find /path/to/publications -type f|while read f; do books -j $(md5sum "$f"|awk '{print $1}');done
1498 |
1499 |
1500 | As above, but print torrent number and path in torrent file
1501 |
1502 | $ find /path/to/publications -type f|while read f; do books -U $(md5sum "$f"|awk '{print $1}');done
1503 |
1504 |
1505 | Find publications by author 'thucydides' and show their md5,title and year in the terminal
1506 |
1507 | $ books -a thucydides -F md5,title,year
1508 |
1509 |
1510 | Get data on a single publication using fast path MD5 search, show author, title and extension
1511 |
1512 | $ books -M 51b4ee7bc7eeb6ed7f164830d5d904ae -F author,title,extension
1513 |
1514 |
1515 | Download a publication using its MD5 (-J MD5), using IPFS (-I y) to download
1516 |
1517 | $ books -I y -J 51b4ee7bc7eeb6ed7f164830d5d904ae
1518 |
1519 | EOF
1520 | }
1521 |
1522 | main "$@"
1523 |
--------------------------------------------------------------------------------