" > geodatadiv 171 | printf "
" >> $htmlOutput 191 | echo geodatadiv 192 | cat geodatadiv >> $htmlOutput 193 | echo "
" >> $htmlOutput 194 | 195 | # Put marker on google map 196 | ## NOTE!!! See google maps API key at the end of this line, you'll need to add yours in here for this to work. 197 | printf "\r\n \r\n \r\n" >> $htmlOutput 198 | 199 | # Close the div 200 | echo "> Please wait...
\r\n' $project) 285 | echo $pleasewait >> $htmlOutput 286 | startingscan=$(printf '> Progress: starting scan
\r\n' $project) 287 | sed -i -e "s#${pleasewait}#${startingscan}#g" $htmlOutput 288 | 289 | # Replace whitespace with new line character on links file to resolve multiple links on single line issue 290 | sed -i -e 's/ /\n/g' $linkfile 291 | 292 | while read line; do 293 | 294 | strippedline=$(echo $line | sed -e 's#.*http://##g' | sed -e 's#.*https://##g') 295 | 296 | if [[ "$exitloop" == 1 ]]; then break; fi 297 | 298 | #printf "\r\n========= Getting %4d of %4d =========\n" $noLinks $noLevelZeroLinks 299 | ProgressBar ${noLinks} ${_end} 300 | 301 | # Check if page has already been crawled 302 | if grep -q "$line" checkedpages; then 303 | echo "Page" $line "already checked" >> debug.txt 304 | else 305 | echo $line | grep "#" > hasHash 306 | #echo $line | grep -e '^.*\.comhttp.*' > multipleLinks 307 | #echo $line | grep -e '^.*\.nethttp.*' >> multipleLinks 308 | echo $line | grep -e '^.*(\.nethttp|\.comhttp|\.auhttp|\/htt).*' >> multipleLinks 309 | 310 | if [ -s hasHash ] || [ -s multipleLinks ]; then 311 | echo $line "has # in it, or has external link" >> debug.txt 312 | rm hasHash multipleLinks 313 | else 314 | 315 | # Remove // from any url's 316 | echo $line | sed -e 's#///#/#2' > downloadme 317 | sed -i -e 's#//#/#g' downloadme 318 | sed -i -e 's#http:#http:/#g' downloadme 319 | sed -i -e 's#https:#https:/#g' downloadme 320 | downloadme=$(cat downloadme) 321 | 322 | # Check for specific file types to exclude from downloading 323 | grep -E "^.*(\.exe|\.zip|\.pdf|\.tar|\.gz|\.doc|\.docx)" downloadme > ignore 324 | rm downloadme 325 | 326 | if [ -s ignore ]; then 327 | echo $downloadme "contains ignored filetype" >> debug.txt 328 | echo "$downloadme contains ignore filetype
" >> otherFiles 329 | rm ignore 330 | else 331 | echo "Getting" $downloadme >> debug.txt 332 | wget -nc --timeout=1 --tries=1 -o wgetOutPut $downloadme 333 | fi 334 | 335 | baseFile=$(basename "$downloadme") 336 | 337 | if [ "$baseFile" == '#' ]; then 338 | baseFile='index' 339 | fi 340 | 341 | #Output if page does not exist; ADD IN which page references this? 342 | if grep -q "404" wgetOutPut; then echo $downloadme "is 404" >> debug.txt 343 | echo "$downloadme is 404" >> 404s 344 | echo "$downloadme is 404" >> pages.txt 345 | 346 | else 347 | if [ -s "$baseFile" ]; then 348 | #echo "basefile exists" 349 | pages=1 350 | 351 | numberOfLinesBase=$(wc -l $baseFile | sed -e "s/$baseFile//g" ) 352 | 353 | while read htmlLine; do 354 | if [[ "$exitloop" == 1 ]]; then break; fi 355 | 356 | # Detect email addresses 357 | shopt -s extglob 358 | email='' 359 | email=$(echo $htmlLine | grep mailto | sed -e 's/*mailto:\(.*\)\s/\1/' | sed -e 's/^.*mailto://' | sed -e 's/".*$//' | sed -e 's/'\''.*$//') 360 | echo "$email" >> emails 361 | echo $downloadme "email" $email >> pages.txt 362 | 363 | # Detect soft 404s 364 | echo $htmlLine | grep "404" > isSoft404 365 | if [[ -s isSoft404 ]]; then echo "> $downloadme might be soft 404" >> 404s; 366 | else 367 | echo $htmlLine | grep -o '' | sed -e 's/ linkline 368 | cat linkline | grep "https" > isFullPath 369 | cat linkline | grep "http" >> isFullPath 370 | 371 | # Replace empty space \s in line with new line character \n to separate multiple links on a line 372 | sed -i -e 's#\s#\n#g' linkline 373 | 374 | if [[ -n linkline ]]; then 375 | if [[ linkline != *"#"* ]]; then 376 | if [ -s isFullPath ]; then 377 | link=$(cat linkline) 378 | echo $link >> $nextlevel 379 | echo $htmlLine | grep "$3" > containsSearchQuery 380 | echo $htmlLine | grep "$project" > isInternalLink 381 | if [[ -s containsSearchQuery && -s isInternalLink ]]; then 382 | echo $downloadme "references:" $link >> pages.txt 383 | fi 384 | rm containsSearchQuery isInternalLink 385 | # html output 386 | #echo "" $downloadme "references:" $link >> $htmlOutput 387 | echo $downloadme >> checkedpages 388 | rm isFullPath 389 | else 390 | cat linkline | grep "/" > hasslash 391 | if [ -s hasslash ]; then 392 | sed -i -e "s#^#${1}#" linkline 393 | rm hasslash 394 | else 395 | sed -i -e "s#^#${1}#" linkline 396 | fi 397 | 398 | link=$(cat linkline) 399 | echo $link >> $nextlevel 400 | # If downloaded page contains the search query, add to pages.txt 401 | # Make the graph more useful 402 | echo $htmlLine | grep "$3" > containsSearchQuery 403 | echo $htmlLine | grep "$project" > isInternalLink 404 | if [[ -s containsSearchQuery && -s isInternalLink ]]; then 405 | echo $downloadme "references:" $link >> pages.txt 406 | fi 407 | rm containsSearchQuery isInternalLink 408 | #html output 409 | #echo ' '${baseFile}' references: '${link}'' >> ${htmlOutput} 410 | echo $downloadme >> checkedpages 411 | fi 412 | sort pages.txt > upages.txt 413 | uniq upages.txt > pages.txt 414 | sort $nextlevel > sortednext 415 | uniq sortednext > $nextlevel 416 | #uniqpages=$(wc -l ../pages.txt) 417 | #printf "\r==== Checked %6d of %6d lines ====" $pages $numberOfLinesBase 418 | pages=$((pages+1)) 419 | sed -i '/^$/d' $nextlevel 420 | rm linkline 421 | fi 422 | fi 423 | fi 424 | done < $baseFile 425 | fi 426 | fi 427 | fi 428 | fi 429 | noLinks=$((noLinks+1)) 430 | #rm baseFile isFullPath startHere 431 | done < $linkfile 432 | fi 433 | 434 | # Iterate $n to move through depth levels 435 | (( n=n+1 )) 436 | 437 | # Time taken to scan depth $n 438 | endTime=`date +%s` 439 | runTime=$((endTime-startTime)) 440 | printf '\r\n' 441 | printf '> Completed scan depth %d in %d seconds
' $n $runTime >> $htmlOutput 442 | done 443 | 444 | if [[ $exitloop == 1 ]]; then printf '\r\n> Search terminated by user
' >> $htmlOutput; fi 445 | 446 | cp pages.txt /var/www/html/pages.html 447 | 448 | ######################################################### 449 | # Search query 450 | ######################################################### 451 | 452 | echo "Doing a search now..." 453 | 454 | instancesOfSearch=0 455 | 456 | #ls | grep -P ".[0-9]{1}" | xargs -d"\n" rm 457 | ls > pagestocheck 458 | 459 | sed -i 's/404s//g; s/emails//g; s/headertags//g; s/checkedpages//g; s/\s//g; s/debug.txt//g; s/errors.txt//g; s/hasHash//g; s/level0.links//g; s/level1.links//g; s/level2.links//g; s/level3.links//g; s/sortednext//g; s/index.html/Homepage/g; s/ignore//g; s/linkline//g; s/multipleLinks//g; s/pagestocheck//g; s/pages.txt//g; s/startHere//g; s/wgetlog//g; s/wgetOutPut//g;' pagestocheck 460 | 461 | if [ -s pagestocheck ]; then 462 | while read line; do 463 | if [ "$line"!='' ]; then 464 | cat "${line}" | grep "$3" > searchOut 465 | if [ -s searchOut ]; then 466 | occurences=$(grep -o -i $3 "${line}" | wc -l) 467 | 468 | # Number of heading tags 469 | h1tags=$(cat ${line} | grep "^<[Hh][1]>" | sed -e 's/" >> $htmlOutput 521 | #cat "$line" | tee --append $htmlOutput 522 | while read line; do 523 | echo $line | sed "s/[^ ][^ ]*/${line}<\/a>/g" | tee --append $htmlOutput 524 | done < pagesWithSearchQuery 525 | echo "
" >> $htmlOutput 526 | # Heading for contents from files with search query 527 | echo "