├── README.md ├── backup-mediawiki-de.sh ├── backup-mediawiki.sh ├── boot ├── bundle-ids.sh ├── cd2ogg ├── change-extension ├── clearqs.sh ├── cloc ├── convertnotes ├── create-random-mac.sh ├── diff.py ├── dmginstall.sh ├── dropbox.py ├── dtrx ├── extract-sqlite-blob.py ├── filter-files.sh ├── fing ├── fix-permissions.sh ├── flac2alac ├── flac2mp3 ├── formd ├── fzf ├── gcloghuman.sh ├── git-hooks ├── git-shorten ├── google_maps_starred_locations.py ├── grep-utf8.sh ├── html2markdown.py ├── install-jdk5-osxlion ├── ip.sh ├── known-wlans.sh ├── lein ├── list-extensions.sh ├── load-test-config.txt ├── load-test-format.txt ├── load-test-urls.txt ├── load-test.sh ├── location ├── markdown2dayone.sh ├── mateup ├── md2wiki.pl ├── mvnda.sh ├── ogg2mp3 ├── osx-cleanup-open-with.sh ├── papers2bibtex ├── pdfdiff.py ├── pickjdk ├── prettify-xml.sh ├── remove-duplicate-blank-lines.sh ├── remove-trailing-whitespace.sh ├── replace-underscore-with-space.sh ├── rip2flac ├── setos ├── ssh-copy-id.sh ├── sshagent ├── sublimeup ├── suspend_until ├── svnauthors.sh ├── svninit2git ├── swapkeys.sh ├── textile2html.rb ├── textile2latex.rb ├── textile2markdown ├── toc.rb ├── tunnel.sh ├── webm2mp3 ├── webm2ogg ├── which-shell.sh └── write-sqlite-blob.py /README.md: -------------------------------------------------------------------------------- 1 | # README # 2 | 3 | Scripts I collected over the years. Some are more useful, some are not. Take your pick. 4 | 5 | I attributed the source whenever possible. If you feel that some information is missing please contact me. -------------------------------------------------------------------------------- /backup-mediawiki-de.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | # Inspired by blogpost from http://www-public.it-sudparis.eu/~berger_o/weblog/2008/05/30/offline-backup-mediawiki-with-httrack/ 4 | 5 | # -w mirror web sites (--mirror) 6 | # -O backup directory 7 | # -%P extended parsing, attempt to parse all links, even in unknown tags or Javascript (%P0 don't use) (--extended-parsing[=N]) 8 | # -N0 Saves files like in site Site-structure (default) 9 | # -s0 follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) (--robots[=N]) 10 | # -p7 Expert options, priority mode: 7 > get html files before, then treat other files 11 | # -S Expert option, stay on the same directory 12 | # -a Expert option, stay on the same address 13 | # -K0 keep original links (e.g. http://www.adr/link) (K0 *relative link, K absolute links, K3 absolute URI links) (--keep-links[=N] 14 | # -A25000 maximum transfer rate in bytes/seconds (1000=1kb/s max) (--max-rate[=N]) 15 | # -F user-agent field (-F "user-agent name") (--user-agent ) 16 | # -%s update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..) (--updatehack) 17 | # -x Build option, replace external html links by error pages 18 | # -%x Build option, do not include any password for external password protected websites (%x0 include) (--no-passwords) 19 | 20 | site=www.host.tld 21 | topurl=http://$site 22 | backupdir=~/Downloads/mirrors/$site 23 | 24 | httrack -w $topurl/Spezial:Alle_Seiten \ 25 | -O "$backupdir" -%P -N0 -s0 -p7 -S -a -K0 -A999000 \ 26 | -F "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" \ 27 | -%s -x -%x \ 28 | "-$site/Spezial:*" \ 29 | "+$site/Spezial:Letzte_Änderungen" \ 30 | "-$site/index.php?*" \ 31 | "-$site/Diskussion:*" \ 32 | "-$site/Benutzer_*" \ 33 | "-$site/Kategorie_Diskussion_*" \ 34 | "+$site/images/*" \ 35 | "+*.css" 36 | 37 | for page in $(grep "link updated: $site/index.php/" $backupdir/hts-log.txt | sed "s,^.*link updated: $site/index.php/,," | sed ’s/ ->.*//’ | grep -v Spezial:) 38 | do 39 | wget -nv -O $backupdir/$site/index.php/${page}_raw.txt "$topurl/index.php?index=$page&action=raw" 40 | done -------------------------------------------------------------------------------- /backup-mediawiki.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | # Inspired by blogpost from http://www-public.it-sudparis.eu/~berger_o/weblog/2008/05/30/offline-backup-mediawiki-with-httrack/ 4 | 5 | # %i ??? 6 | # -w mirror web sites (--mirror) 7 | # -O backup directory 8 | # -%P extended parsing, attempt to parse all links, even in unknown tags or Javascript (%P0 don't use) (--extended-parsing[=N]) 9 | # -N0 Saves files like in site Site-structure (default) 10 | # -s0 follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) (--robots[=N]) 11 | # -p7 Expert options, priority mode: 7 > get html files before, then treat other files 12 | # -S Expert option, stay on the same directory 13 | # -a Expert option, stay on the same address 14 | # -K0 keep original links (e.g. http://www.adr/link) (K0 *relative link, K absolute links, K3 absolute URI links) (--keep-links[=N] 15 | # -%k ??? 16 | # -A25000 maximum transfer rate in bytes/seconds (1000=1kb/s max) (--max-rate[=N]) 17 | # -F user-agent field (-F "user-agent name") (--user-agent ) 18 | # -%s update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..) (--updatehack) 19 | # -x Build option, replace external html links by error pages 20 | # -%x Build option, do not include any password for external password protected websites (%x0 include) (--no-passwords) 21 | # -%u ??? 22 | 23 | site=wiki.host.tld 24 | topurl=http://$site 25 | 26 | backupdir=~/Downloads/mirrors/$site 27 | 28 | httrack -%i -w $topurl/index.php/Special:Allpages \ 29 | -O "$backupdir" -%P -N0 -s0 -p7 -S -a -K0 -%k -A25000 \ 30 | -F "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" \ 31 | -%s -x -%x -%u \ 32 | "-$site/index.php/Special:*" \ 33 | "-$site/index.php?title=Special:*" \ 34 | "+$site/index.php/Special:RecentChanges" \ 35 | "+*.css" \ 36 | "-$site/index.php?title=*&oldid=*" \ 37 | "-$site/index.php?title=*&action=edit" \ 38 | "-$site/index.php?title=*&curid=*" \ 39 | "+$site/index.php?title=*&action=history" \ 40 | "-$site/index.php?title=*&action=history&*" \ 41 | "-$site/index.php?title=*&curid=*&action=history*" \ 42 | "-$site/index.php?title=*&limit=*&action=history" 43 | 44 | for page in $(grep "link updated: $site/index.php/" $backupdir/hts-log.txt | sed "s,^.*link updated: $site/index.php/,," | sed ’s/ ->.*//’ | grep -v Special:) 45 | do 46 | wget -nv -O $backupdir/$site/index.php/${page}_raw.txt "$topurl/index.php?index=$page&action=raw" 47 | done -------------------------------------------------------------------------------- /boot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oschrenk/scripts/eb4205c7a4e7b14fa0ff2fb6a5edaddcf335cd91/boot -------------------------------------------------------------------------------- /bundle-ids.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | names=" 4 | App Store 5 | Automator 6 | Calculator 7 | Calendar 8 | Cog 9 | Contacts 10 | Dashboard 11 | Dictionary 12 | Gimp 13 | Google Chrome 14 | iPhoto 15 | iTunes 16 | LibreOffice 17 | Mail 18 | MPlayerX 19 | Preview 20 | Sublime Text 2 21 | The Unarchiver 22 | VLC 23 | " 24 | 25 | tmp=/tmp/$(uuidgen) 26 | mdfind -onlyin / 'kMDItemContentType==com.apple.application-bundle' > $tmp 27 | echo "$names" | sed 's/^/\//g;s/$/.app/g;s/ /\\ /g' | xargs -L 1 -J '{}' grep -m 1 -i -F '{}' $tmp | sed 's/ /\\ /g' | xargs mdls -name kMDItemCFBundleIdentifier | sed 's/.*= "\(.*\)"$/\1/' 28 | rm $tmp -------------------------------------------------------------------------------- /change-extension: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Filename ren_ext ( cht 13-Dec-2005 ) 3 | # Purpose : Rename all indicated files in current directory, 4 | # replacing the file extension .old with .new 5 | # 6 | # Usage: 7 | # change-extension old new file [ file ... ] 8 | # where old and new are any names used as filename extensions 9 | # Effect: 10 | # Each .old in the list of files is renamed to .new 11 | # Caution: 12 | # Any file (without extension .old) in the list of files 13 | # is renamed to .new 14 | #---------------------------------------------------------------------------- 15 | # Example: In the current directory there are files 16 | # fic.test, mic.test, ficfic ( and no other *.test ). 17 | # running 18 | # ren_ext test tzt *.test ficfic 19 | # will rename those 3 files to 20 | # fic.tzt, mic.tzt, ficfic.tzt 21 | 22 | if [ $# -lt 3 ] ; then 23 | echo "Usage:" 24 | echo " $0 old new file [ file ... ]" 25 | exit 1 26 | fi 27 | 28 | old=$1; shift 29 | new=$1; shift 30 | 31 | while [ $# -gt 0 ] ; do 32 | item=$1; shift 33 | # echo "item: $item" 34 | mv $item ${item%.$old}.$new 35 | done 36 | exit 0 -------------------------------------------------------------------------------- /clearqs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Clear Quicksilver Caches 4 | 5 | rm -rf ~/Library/Caches/com.blacktree.Quicksilver/ 6 | rm -rf ~/Library/Caches/Quicksilver/ -------------------------------------------------------------------------------- /convertnotes: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | for file in *.textile 4 | do 5 | if [ -f $file ] 6 | then 7 | name=$file 8 | mark=${name%.*}.markdown 9 | temp=${name%.*}.temp 10 | git mv $file $mark 11 | textile2html.rb $mark | html2markdown.py > $temp 12 | cat $temp > $mark 13 | rm $temp 14 | fi 15 | done 16 | 17 | -------------------------------------------------------------------------------- /create-random-mac.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | openssl rand -hex 6 | sed 's/\(..\)/\1:/g; s/.$//' -------------------------------------------------------------------------------- /diff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys 4 | import os 5 | 6 | os.system('meld "%s" "%s"' % (sys.argv[2], sys.argv[5])) 7 | -------------------------------------------------------------------------------- /dmginstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Downloads and install a .dmg from a URL 4 | # 5 | # Usage 6 | # $ dmginstall [url] 7 | # 8 | # For example, for installing alfred.app 9 | # $ dmginstall http://cachefly.alfredapp.com/alfred_1.3.1_261.dmg 10 | # 11 | # TODO 12 | # - currently only handles .dmg with .app folders, not .pkg files 13 | # - handle .zip files as well 14 | 15 | 16 | if [[ $# -lt 1 ]]; then 17 | echo "Usage: dmginstall [url]" 18 | exit 1 19 | fi 20 | 21 | url=$* 22 | 23 | # Generate a random file name 24 | tmp_file=/tmp/`openssl rand -base64 10 | tr -dc '[:alnum:]'`.dmg 25 | apps_folder='/Applications' 26 | 27 | # Download file 28 | echo "Downloading $url..." 29 | curl -# -L -o $tmp_file $url 30 | 31 | echo "Mounting image..." 32 | volume=`hdiutil mount $tmp_file | tail -n1 | perl -nle '/(\/Volumes\/[^ ]+)/; print $1'` 33 | 34 | # Locate .app folder and move to /Applications 35 | app=`find $volume/. -name *.app -maxdepth 1 -type d -print0` 36 | echo "Copying `echo $app | awk -F/ '{print $NF}'` into $apps_folder..." 37 | cp -ir $app $apps_folder 38 | 39 | # Unmount volume, delete temporal file 40 | echo "Cleaning up..." 41 | hdiutil unmount $volume -quiet 42 | rm $tmp_file 43 | 44 | echo "Done!" -------------------------------------------------------------------------------- /extract-sqlite-blob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sqlite3 4 | conn = sqlite3.connect('sp_radio_0.localstorage') 5 | cursor = conn.cursor() 6 | 7 | with open("output.json", "wb") as output_file: 8 | cursor.execute("SELECT value FROM ItemTable where key='RecentStations';") 9 | ablob = cursor.fetchone() 10 | output_file.write(ablob[0]) 11 | 12 | cursor.close() 13 | conn.close() -------------------------------------------------------------------------------- /filter-files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Usage $ filter-files phrase-to-use-as-filter 4 | 5 | for f in * 6 | do 7 | if [ -n "`grep $1 $f`" ] 8 | then 9 | cp $f temp/ 10 | fi 11 | done -------------------------------------------------------------------------------- /fing: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | nmap -sP 192.168.0.1/24 | grep report | awk '{print $5}' | xargs resolveip 2>/dev/null | awk '{print $4" "$6}' | cut -d. -f-4 -------------------------------------------------------------------------------- /fix-permissions.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oschrenk/scripts/eb4205c7a4e7b14fa0ff2fb6a5edaddcf335cd91/fix-permissions.sh -------------------------------------------------------------------------------- /flac2alac: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # flac2alac By Arlindo \"Nighto\" Pereira 4 | # (C) 2010. Licensed on GPLv3" 5 | 6 | # modified by jeffrey paul 7 | 8 | # inspiration from http://ca.ubuntuforums.org/member.php?u=6176 (MetalMusicAddict) 9 | # script at http://ca.ubuntuforums.org/showthread.php?t=889700&page=2 10 | 11 | # this requires ImageMagick (for convert), mpeg4ip (for mp4tags), 12 | # recent flac (for metaflac with --export-picture-to), and 13 | # ffmpeg with alac (apple lossless) and flac support built in 14 | 15 | # no longer requires 'convert' as through testing i've found that 16 | # mp4tags does not require a png cover file as the manpage says 17 | # also iTunes 10.x will accept either embedded png or jpeg in an alac m4a 18 | 19 | function _convert_flac2alac { 20 | 21 | NF="`basename \"$1\" .flac`.m4a" 22 | D="`dirname \"$1\"`" 23 | 24 | flac -dc "$1" > "${D}/.flacdecode.${NF}.wav" 25 | 26 | if [ $? -ne 0 ]; then 27 | rm -f "${D}/.flacdecode.${NF}.wav" 28 | echo "ERROR: corrupt or invalid flac file, exiting." > /dev/stderr 29 | exit 1 30 | fi 31 | 32 | ARTIST="`metaflac --show-tag=ARTIST \"$1\" | perl -pe 's/ARTIST=//ig'`" 33 | ALBUMARTIST="`metaflac --show-tag=ALBUMARTIST \"$1\" | perl -pe 's/ALBUMARTIST=//ig'`" 34 | TITLE="`metaflac --show-tag=TITLE \"$1\" | perl -pe 's/TITLE=//ig'`" 35 | ALBUM="`metaflac --show-tag=ALBUM \"$1\" | perl -pe 's/ALBUM=//ig'`" 36 | DATE="`metaflac --show-tag=DATE \"$1\" | perl -pe 's/DATE=//ig'`" 37 | GENRE="`metaflac --show-tag=GENRE \"$1\" | perl -pe 's/GENRE=//ig'`" 38 | TRACKNUMBER="`metaflac --show-tag=TRACKNUMBER \"$1\" | perl -pe 's/TRACKNUMBER=//ig'`" 39 | TRACKTOTAL="`metaflac --show-tag=TRACKTOTAL \"$1\" | perl -pe 's/TRACKTOTAL=//ig'`" 40 | DISCNUMBER="`metaflac --show-tag=DISCNUMBER \"$1\" | perl -pe 's/DISCNUMBER=//ig'`" 41 | DISCTOTAL="`metaflac --show-tag=DISCTOTAL \"$1\" | perl -pe 's/DISCTOTAL=//ig'`" 42 | DESCRIPTION="`metaflac --show-tag=DESCRIPTION \"$1\" | perl -pe 's/DESCRIPTION=//ig'`" 43 | COMPOSER="`metaflac --show-tag=COMPOSER \"$1\" | perl -pe 's/COMPOSER=//ig'`" 44 | ARTFILE=".arttmp.${NF}" 45 | # ARTFORMAT="`metaflac --export-picture-to=- \"$1\" | file -i -b - | awk '{split(\$1,arr,\";\"); print arr[1]}'`" 46 | # 47 | # if [ $? -eq 0 ]; then 48 | # 49 | # if [ "$ARTFORMAT" != "application/x-empty" ]; then 50 | # metaflac --export-picture-to="${D}/${ARTFILE}" "$1" 51 | # if [ "$ARTFORMAT" = "image/png" ]; then ARTEXT="png"; fi 52 | # if [ "$ARTFORMAT" = "image/jpeg" ]; then ARTEXT="jpg"; fi 53 | # if [ -z "$ARTEXT" ]; then 54 | # echo "unknown embedded album art format ${ARTFORMAT}, cannot continue." > /dev/stderr 55 | # exit 1; 56 | # fi 57 | # fi 58 | # fi 59 | 60 | ffmpeg -v -1 -i "$1" -ar 44100 -acodec alac "${D}/.tmp.${NF}" 61 | 62 | if [ $? -ne 0 ]; then 63 | echo "Problem running conversion, exiting." > /dev/stderr 64 | rm -f "${D}/.flacdecode.${NF}.wav" 65 | rm -f "${D}/.tmp.${NF}" "${D}/${ARTFILE}" 66 | exit 1 67 | fi 68 | 69 | mp4tags -s "$TITLE" "${D}/.tmp.${NF}" 70 | mp4tags -t "$TRACKNUMBER" "${D}/.tmp.${NF}" 71 | mp4tags -T "$TRACKTOTAL" "${D}/.tmp.${NF}" 72 | mp4tags -d "$DISCNUMBER" "${D}/.tmp.${NF}" 73 | mp4tags -D "$DISCTOTAL" "${D}/.tmp.${NF}" 74 | mp4tags -w "$COMPOSER" "${D}/.tmp.${NF}" 75 | mp4tags -c "$DESCRIPTION" "${D}/.tmp.${NF}" 76 | mp4tags -a "$ARTIST" "${D}/.tmp.${NF}" 77 | mp4tags -R "$ALBUMARTIST" "${D}/.tmp.${NF}" 78 | mp4tags -A "$ALBUM" "${D}/.tmp.${NF}" 79 | mp4tags -g "$GENRE" "${D}/.tmp.${NF}" 80 | mp4tags -y "$DATE" "${D}/.tmp.${NF}" 81 | 82 | if [ -f "${D}/$ARTFILE" ]; then 83 | # this embeds a png or jpeg cover into the m4a if it was 84 | # found in the original flac file 85 | mp4tags -P "${D}/$ARTFILE" "${D}/.tmp.${NF}" 86 | rm -f "${D}/${ARTFILE}" 87 | fi 88 | 89 | 90 | ffmpeg -i "${D}/.tmp.${NF}" "${D}/.alacdecode.${NF}.wav" 91 | if [ $? -ne 0 ]; then 92 | rm -f "${D}/.flacdecode.${NF}.wav" 93 | rm -f "${D}/.alacdecode.${NF}.wav" 94 | rm -f "${D}/${NF}" 95 | echo "ERROR: unable to decode new ALAC, exiting." > /dev/stderr 96 | exit 1 97 | fi 98 | ORIG="`md5 \"${D}/.flacdecode.${NF}.wav\" | awk '{print $1}'`" 99 | NEW="`md5 \"${D}/.alacdecode.${NF}.wav\" | awk '{print $1}'`" 100 | rm -f "${D}/.alacdecode.${NF}.wav" 101 | rm -f "${D}/.flacdecode.${NF}.wav" 102 | if [ "$ORIG" != "$NEW" ]; then 103 | echo "ERROR: Newly converted ALAC is not identical!" > /dev/stderr 104 | echo "Aborting!" > /dev/stderr 105 | rm -f "${D}/.tmp.${NF}" 106 | exit 1 107 | else 108 | mv "${D}/.tmp.${NF}" "${D}/${NF}" 109 | OLDSIZE="`du -s \"$1\" | awk '{print $1}'`" 110 | NEWSIZE="`du -s \"${D}/${NF}\" | awk '{print $1}'`" 111 | PCT=$(( $NEWSIZE * 100 / $OLDSIZE )) 112 | echo "Successfully converted:" 113 | echo "$1 -> ${D}/$NF" 114 | echo "STATS: ALAC is ${PCT}% the size of the input FLAC" 115 | fi 116 | } 117 | 118 | if [ $# -lt 1 ]; then 119 | echo "usage: $0 [-d] [file2.flac] [...]" > /dev/stderr 120 | exit 1 121 | fi 122 | 123 | for filename in "$@"; do 124 | if [ -f "$filename" ]; then 125 | _convert_flac2alac "$filename" 126 | fi 127 | done 128 | -------------------------------------------------------------------------------- /flac2mp3: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # flacmp3 - convert a flac file to mp3 4 | # 5 | if [ "$1" ] 6 | then 7 | for file 8 | do 9 | if [ -e "$file" ] 10 | then 11 | flac -c -d "$file" | lame -V1 - "$(basename "$file" .flac).mp3" 12 | else 13 | echo >&2 "No such file: "$1"" 14 | exit 1 15 | fi 16 | done 17 | else 18 | echo >&2 "Usage: "$(basename "$0")" INPUTFILE [...]" 19 | exit 1 20 | fi -------------------------------------------------------------------------------- /formd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding=utf8 3 | 4 | # formd—A Markdown formatting tool by Seth brown 5 | # formd is a tool for (for)matting (M)ark(d)own 6 | # that allows rapid #conversion between the two 7 | # styles of Markdown links and images—inline and referenced. 8 | 9 | # Original can be found at https://github.com/drbunsen/formd 10 | 11 | """ 12 | Seth Brown 13 | 02-24-12 14 | """ 15 | from sys import stdin, stdout 16 | import argparse 17 | import re 18 | from collections import OrderedDict 19 | 20 | class ForMd(object): 21 | """Format mardown text""" 22 | def __init__(self, text): 23 | super(ForMd, self).__init__() 24 | self.text = text 25 | self.match_links = re.compile(r'(\[[^^]*?\])\s?(\[.*?\]|\(.*?\))', 26 | re.DOTALL | re.MULTILINE) 27 | self.match_refs = re.compile(r'(?<=\n)\[[^^]*?\]:\s?.*') 28 | self.data = [] 29 | 30 | def _links(self, ): 31 | """find Markdown links""" 32 | links = re.findall(self.match_links, self.text) 33 | for link in links: 34 | # remove newline breaks from urls spanning multi-lines 35 | parsed_link = [s.replace('\n','') for s in link] 36 | yield parsed_link 37 | 38 | def _refs(self): 39 | """find Markdown references""" 40 | refs = re.findall(self.match_refs, self.text) 41 | refs.sort() 42 | refs = OrderedDict(i.split(":", 1) for i in refs) 43 | return refs 44 | 45 | def _format(self): 46 | """process text""" 47 | links = (i for i in self._links()) 48 | refs = self._refs() 49 | for n, link in enumerate(links): 50 | text, ref = link 51 | ref_num = ''.join(("[",str(n+1),"]: ")) 52 | if ref in refs.keys(): 53 | url = refs.get(ref).strip() 54 | formd_ref = ''.join((ref_num, url)) 55 | formd_text = ''.join((text, ref_num)) 56 | self.data.append([formd_text, formd_ref]) 57 | elif text in refs.keys(): 58 | url = refs.get(text).strip() 59 | formd_ref = ''.join((ref_num, url)) 60 | formd_text = ''.join((text, ref_num)) 61 | self.data.append([formd_text, formd_ref]) 62 | elif ref not in refs.keys(): 63 | parse_ref = ref.strip("()") 64 | formd_ref = ''.join((ref_num, parse_ref)) 65 | formd_text = ''.join((text,ref_num)) 66 | self.data.append([formd_text, formd_ref]) 67 | 68 | def inline_md(self): 69 | """generate inline markdown """ 70 | self._format() 71 | text_link = iter([''.join((_[0].split("][",1)[0], 72 | "](", _[1].split(":",1)[1].strip(), ")")) for _ in self.data]) 73 | formd_text = self.match_links.sub(lambda _: next(text_link), md) 74 | formd_md = self.match_refs.sub('', formd_text).strip() 75 | yield formd_md 76 | 77 | def ref_md(self): 78 | """generate referenced markdown""" 79 | self._format() 80 | ref_nums = iter([_[0].rstrip(" :") for _ in self.data]) 81 | formd_text = self.match_links.sub(lambda _: next(ref_nums), md) 82 | formd_refs = self.match_refs.sub('', formd_text).strip() 83 | references = (i[1] for i in self.data) 84 | formd_md = '\n'.join((formd_refs, '\n', '\n'.join(i for i in references))) 85 | yield formd_md 86 | 87 | def flip(self): 88 | """convert markdown to the opposite style of the first text link""" 89 | first_match = re.search(self.match_links, self.text).group(0) 90 | if '(' and ')' in first_match: 91 | formd_md = self.ref_md() 92 | else: 93 | formd_md = self.inline_md() 94 | return formd_md 95 | 96 | if __name__ == '__main__': 97 | description = 'formd: A (for)matting (M)ark(d)own tool.' 98 | p = argparse.ArgumentParser(description=description) 99 | p.add_argument('-r', '--ref', 100 | help="convert text to referenced Markdown", action='store_true', default=False) 101 | p.add_argument('-i', '--inline', 102 | help="convert text to inline Markdown", action='store_true', default=False) 103 | p.add_argument('-f', '--flip', 104 | help="convert to opposite style Markdown", action='store_true', default=True) 105 | args = p.parse_args() 106 | md = stdin.read() 107 | text = ForMd(md) 108 | if (args.inline): [stdout.write(t) for t in text.inline_md()] 109 | elif (args.ref): [stdout.write(t) for t in text.ref_md()] 110 | elif (args.flip): [stdout.write(t) for t in text.flip()] -------------------------------------------------------------------------------- /fzf: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # encoding: utf-8 3 | # 4 | # ____ ____ 5 | # / __/___ / __/ 6 | # / /_/_ / / /_ 7 | # / __/ / /_/ __/ 8 | # /_/ /___/_/ Fuzzy finder for your shell 9 | # 10 | # URL: https://github.com/junegunn/fzf 11 | # Author: Junegunn Choi 12 | # License: MIT 13 | # Last update: November 10, 2013 14 | # 15 | # Copyright (c) 2013 Junegunn Choi 16 | # 17 | # MIT License 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining 20 | # a copy of this software and associated documentation files (the 21 | # "Software"), to deal in the Software without restriction, including 22 | # without limitation the rights to use, copy, modify, merge, publish, 23 | # distribute, sublicense, and/or sell copies of the Software, and to 24 | # permit persons to whom the Software is furnished to do so, subject to 25 | # the following conditions: 26 | # 27 | # The above copyright notice and this permission notice shall be 28 | # included in all copies or substantial portions of the Software. 29 | # 30 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 31 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 32 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 33 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 34 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 35 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 36 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 37 | 38 | def usage x 39 | puts %[usage: fzf [options] 40 | 41 | -m, --multi Enable multi-select 42 | -s, --sort=MAX Maximum number of matched items to sort. Default: 500. 43 | +s, --no-sort Do not sort the result. Keep the sequence unchanged. 44 | +i Case-sensitive match 45 | +c, --no-color Disable colors] 46 | exit x 47 | end 48 | 49 | stdout = $stdout.clone 50 | $stdout.reopen($stderr) 51 | 52 | usage 0 unless (%w[--help -h] & ARGV).empty? 53 | @rxflag = ARGV.delete('+i') ? 0 : Regexp::IGNORECASE 54 | @sort = %w[+s --no-sort].map { |e| ARGV.delete e }.compact.empty? ? 500 : nil 55 | @color = %w[+c --no-color].map { |e| ARGV.delete e }.compact.empty? 56 | @multi = !%w[-m --multi].map { |e| ARGV.delete e }.compact.empty? 57 | rest = ARGV.join ' ' 58 | if sort = rest.match(/(-s|--sort=?) ?([0-9]+)/) 59 | usage 1 unless @sort 60 | @sort = sort[2].to_i 61 | rest = rest.delete sort[0] 62 | end 63 | usage 1 unless rest.empty? 64 | 65 | require 'thread' 66 | require 'curses' 67 | 68 | @mtx = Mutex.new 69 | @smtx = Mutex.new 70 | @cv = ConditionVariable.new 71 | @lists = [] 72 | @new = [] 73 | @query = '' 74 | @matches = [] 75 | @count = 0 76 | @cursor_x = 0 77 | @vcursor = 0 78 | @events = {} 79 | @selects = {} # ordered >= 1.9 80 | 81 | case RUBY_PLATFORM 82 | when /darwin/ 83 | module UConv 84 | CHOSUNG = 0x1100 85 | JUNGSUNG = 0x1161 86 | JONGSUNG = 0x11A7 87 | CHOSUNGS = 19 88 | JUNGSUNGS = 21 89 | JONGSUNGS = 28 90 | JJCOUNT = JUNGSUNGS * JONGSUNGS 91 | NFC_BEGIN = 0xAC00 92 | NFC_END = NFC_BEGIN + CHOSUNGS * JUNGSUNGS * JONGSUNGS 93 | 94 | def self.nfd str 95 | ret = '' 96 | str.split(//).each do |c| 97 | cp = c.ord 98 | if cp >= NFC_BEGIN && cp < NFC_END 99 | idx = cp - NFC_BEGIN 100 | cho = CHOSUNG + idx / JJCOUNT 101 | jung = JUNGSUNG + (idx % JJCOUNT) / JONGSUNGS 102 | jong = JONGSUNG + idx % JONGSUNGS 103 | ret << cho << jung 104 | ret << jong if jong != JONGSUNG 105 | else 106 | ret << c 107 | end 108 | end 109 | ret 110 | end 111 | 112 | def self.nfc str, b = 0, e = 0 113 | ret = '' 114 | omap = [] 115 | pend = [] 116 | str.split(//).each_with_index do |c, idx| 117 | cp = c.ord 118 | omap << ret.length 119 | unless pend.empty? 120 | if cp >= JUNGSUNG && cp < JUNGSUNG + JUNGSUNGS 121 | pend << cp - JUNGSUNG 122 | next 123 | elsif cp >= JONGSUNG && cp < JONGSUNG + JONGSUNGS 124 | pend << cp - JONGSUNG 125 | next 126 | else 127 | omap[-1] = omap[-1] + 1 128 | ret << [NFC_BEGIN + pend[0] * JJCOUNT + 129 | (pend[1] || 0) * JONGSUNGS + 130 | (pend[2] || 0)].pack('U*') 131 | pend.clear 132 | end 133 | end 134 | if cp >= CHOSUNG && cp < CHOSUNG + CHOSUNGS 135 | pend << cp - CHOSUNG 136 | else 137 | ret << c 138 | end 139 | end 140 | return [ret, omap[b] || 0, omap[e] || ((omap.last || 0) + 1)] 141 | end 142 | end 143 | 144 | def convert_query q 145 | UConv.nfd(q).split(//) 146 | end 147 | 148 | def convert_item item 149 | UConv.nfc(*item) 150 | end 151 | else 152 | def convert_query q 153 | q.split(//) 154 | end 155 | 156 | def convert_item item 157 | item 158 | end 159 | end 160 | 161 | def emit event 162 | @mtx.synchronize do 163 | @events[event] = yield 164 | @cv.broadcast 165 | end 166 | end 167 | 168 | C = Curses 169 | def max_items; C.lines - 2; end 170 | def cursor_y; C.lines - 1; end 171 | def cprint str, col 172 | C.attron(col) do 173 | C.addstr str.gsub("\0", '') 174 | end if str 175 | end 176 | 177 | def print_input 178 | C.setpos cursor_y, 0 179 | C.clrtoeol 180 | cprint '> ', color(:blue, true) 181 | C.attron(C::A_BOLD) do 182 | C.addstr @query 183 | end 184 | end 185 | 186 | def print_info selected, msg = nil 187 | @fan ||= '-\|/-\|/'.split(//) 188 | C.setpos cursor_y - 1, 0 189 | C.clrtoeol 190 | prefix = 191 | if fan = @fan.shift 192 | @fan.push fan 193 | cprint fan, color(:fan, true) 194 | ' ' 195 | else 196 | ' ' 197 | end 198 | C.attron color(:info, false) do 199 | C.addstr "#{prefix}#{@matches.length}/#{@count}" 200 | C.addstr " (#{selected})" if selected > 0 201 | C.addstr msg if msg 202 | end 203 | end 204 | 205 | def refresh 206 | C.setpos cursor_y, 2 + width(@query[0, @cursor_x]) 207 | C.refresh 208 | end 209 | 210 | def ctrl char 211 | char.to_s.ord - 'a'.ord + 1 212 | end 213 | 214 | if RUBY_VERSION.split('.').map { |e| e.rjust(3, '0') }.join > '001009' 215 | @wrx = Regexp.new '\p{Han}|\p{Katakana}|\p{Hiragana}|\p{Hangul}' 216 | def width str 217 | str.gsub(@wrx, ' ').length 218 | end 219 | 220 | def trim str, len, left 221 | width = width str 222 | diff = 0 223 | while width > len 224 | width -= (left ? str[0, 1] : str[-1, 1]) =~ @wrx ? 2 : 1 225 | str = left ? str[1..-1] : str[0...-1] 226 | diff += 1 227 | end 228 | [str, diff] 229 | end 230 | else 231 | def width str 232 | str.length 233 | end 234 | 235 | def trim str, len, left 236 | diff = str.length - len 237 | if diff > 0 238 | [left ? str[diff..-1] : str[0...-diff], diff] 239 | else 240 | [str, 0] 241 | end 242 | end 243 | 244 | class String 245 | def ord 246 | self.unpack('c').first 247 | end 248 | end 249 | 250 | class Fixnum 251 | def ord 252 | self 253 | end 254 | end 255 | end 256 | 257 | C.init_screen 258 | C.start_color 259 | dbg = 260 | if C.respond_to?(:use_default_colors) 261 | C.use_default_colors 262 | -1 263 | else 264 | C::COLOR_BLACK 265 | end 266 | C.raw 267 | C.noecho 268 | 269 | if @color 270 | if C.can_change_color? 271 | fg = ENV.fetch('FZF_FG', 252).to_i 272 | bg = ENV.fetch('FZF_BG', 236).to_i 273 | C.init_pair 1, 110, dbg 274 | C.init_pair 2, 108, dbg 275 | C.init_pair 3, fg + 2, bg 276 | C.init_pair 4, 151, bg 277 | C.init_pair 5, 148, dbg 278 | C.init_pair 6, 144, dbg 279 | C.init_pair 7, 161, bg 280 | else 281 | C.init_pair 1, C::COLOR_BLUE, dbg 282 | C.init_pair 2, C::COLOR_GREEN, dbg 283 | C.init_pair 3, C::COLOR_YELLOW, C::COLOR_BLACK 284 | C.init_pair 4, C::COLOR_GREEN, C::COLOR_BLACK 285 | C.init_pair 5, C::COLOR_GREEN, dbg 286 | C.init_pair 6, C::COLOR_WHITE, dbg 287 | C.init_pair 7, C::COLOR_RED, C::COLOR_BLACK 288 | end 289 | 290 | def color sym, bold = false 291 | C.color_pair([:blue, :match, :chosen, 292 | :match!, :fan, :info, :red].index(sym) + 1) | 293 | (bold ? C::A_BOLD : 0) 294 | end 295 | else 296 | def color sym, bold = false 297 | case sym 298 | when :chosen 299 | bold ? C::A_REVERSE : 0 300 | when :match 301 | C::A_UNDERLINE 302 | when :match! 303 | C::A_REVERSE | C::A_UNDERLINE 304 | else 305 | 0 306 | end | (bold ? C::A_BOLD : 0) 307 | end 308 | end 309 | 310 | @read = 311 | if $stdin.tty? 312 | if default_command = ENV['FZF_DEFAULT_COMMAND'] 313 | IO.popen(default_command) 314 | elsif !`which find`.empty? 315 | IO.popen("find * -path '*/\\.*' -prune -o -type f -print -o -type l -print 2> /dev/null") 316 | else 317 | exit 1 318 | end 319 | else 320 | $stdin 321 | end 322 | 323 | reader = Thread.new { 324 | while line = @read.gets 325 | emit(:new) { @new << line.chomp } 326 | end 327 | emit(:loaded) { true } 328 | @smtx.synchronize { @fan = [] } 329 | } 330 | 331 | main = Thread.current 332 | searcher = Thread.new { 333 | events = {} 334 | fcache = {} 335 | matches = [] 336 | selects = {} 337 | mcount = 0 # match count 338 | plcount = 0 # prev list count 339 | q = '' 340 | vcursor = 0 341 | delay = -5 342 | 343 | begin 344 | while true 345 | @mtx.synchronize do 346 | while true 347 | events.merge! @events 348 | 349 | if @events.empty? # No new events 350 | @cv.wait @mtx 351 | next 352 | end 353 | @events.clear 354 | break 355 | end 356 | 357 | if events[:new] 358 | @lists << [@new, {}] 359 | @count += @new.length 360 | @new = [] 361 | fcache = {} 362 | end 363 | 364 | if events[:select] 365 | selects = @selects.dup 366 | end 367 | end#mtx 368 | 369 | new_search = events[:key] || events.delete(:new) 370 | user_input = events[:key] || events[:vcursor] || events.delete(:select) 371 | progress = 0 372 | started_at = Time.now 373 | 374 | if new_search && !@lists.empty? 375 | q = events.delete(:key) || q 376 | 377 | unless q.empty? 378 | q = q.downcase if @rxflag != 0 379 | regexp = Regexp.new(convert_query(q).inject('') { |sum, e| 380 | e = Regexp.escape e 381 | sum << "#{e}[^#{e}]*?" 382 | }, @rxflag) 383 | end 384 | 385 | matches = fcache[q] ||= 386 | begin 387 | found = [] 388 | skip = false 389 | cnt = 0 390 | @lists.each do |pair| 391 | list, cache = pair 392 | cnt += list.length 393 | 394 | @mtx.synchronize { 395 | skip = @events[:key] 396 | progress = (100 * cnt / @count) 397 | } 398 | break if skip 399 | 400 | found.concat(cache[q] ||= q.empty? ? list : begin 401 | if progress < 100 && Time.now - started_at > 0.5 402 | @smtx.synchronize do 403 | print_info selects.length, " (#{progress}%)" 404 | refresh 405 | end 406 | end 407 | 408 | prefix, suffix = @query[0, @cursor_x], @query[@cursor_x..-1] || '' 409 | prefix_cache = suffix_cache = nil 410 | 411 | (prefix.length - 1).downto(1) do |len| 412 | break if prefix_cache = cache[prefix[0, len]] 413 | end 414 | 415 | 0.upto(suffix.length - 1) do |idx| 416 | break if suffix_cache = cache[suffix[idx..-1]] 417 | end unless suffix.empty? 418 | 419 | partial_cache = [prefix_cache, suffix_cache].compact.sort_by { |e| e.length }.first 420 | (partial_cache ? partial_cache.map { |e| e.first } : list).map { |line| 421 | # Ignore errors: e.g. invalid byte sequence in UTF-8 422 | md = line.match(regexp) rescue nil 423 | md && [line, *md.offset(0)] 424 | }.compact 425 | end) 426 | end 427 | next if skip 428 | @sort ? found : found.reverse 429 | end 430 | 431 | mcount = matches.length 432 | if @sort && mcount <= @sort && !q.empty? 433 | matches.replace matches.sort_by { |triple| 434 | line, b, e = triple 435 | [b ? (e - b) : 0, line.length, line] 436 | } 437 | end 438 | end#new_search 439 | 440 | # This small delay reduces the number of partial lists 441 | sleep((delay = [20, delay + 5].min) * 0.01) unless user_input 442 | 443 | if events.delete(:vcursor) || new_search 444 | @mtx.synchronize do 445 | plcount = [@matches.length, max_items].min 446 | @matches = matches 447 | vcursor = @vcursor = [0, [@vcursor, mcount - 1, max_items - 1].min].max 448 | end 449 | end 450 | 451 | # Output 452 | @smtx.synchronize do 453 | item_length = [mcount, max_items].min 454 | if item_length < plcount 455 | plcount.downto(item_length) do |idx| 456 | C.setpos cursor_y - idx - 2, 0 457 | C.clrtoeol 458 | end 459 | end 460 | 461 | maxc = C.cols - 3 462 | matches[0, max_items].each_with_index do |item, idx| 463 | next if !new_search && !((vcursor-1)..(vcursor+1)).include?(idx) 464 | 465 | line, b, e = convert_item item 466 | b ||= 0 467 | e ||= 0 468 | row = cursor_y - idx - 2 469 | chosen = idx == vcursor 470 | 471 | # Overflow 472 | if width(line) > maxc 473 | ewidth = width(line[0...e]) 474 | # Stri.. 475 | if ewidth <= maxc - 2 476 | line, _ = trim line, maxc - 2, false 477 | line << '..' 478 | # ..ring 479 | else 480 | # ..ri.. 481 | line = line[0...e] + '..' if ewidth < width(line) - 2 482 | line, diff = trim line, maxc - 2, true 483 | b += 2 - diff 484 | e += 2 - diff 485 | b = [2, b].max 486 | line = '..' + line 487 | end 488 | end 489 | 490 | C.setpos row, 0 491 | C.clrtoeol 492 | cprint chosen ? '>' : ' ', color(:red, true) 493 | selected = selects.include?([*item][0]) 494 | cprint selected ? '>' : ' ', 495 | chosen ? color(:chosen) : (selected ? color(:red, true) : 0) 496 | 497 | C.attron color(:chosen, true) if chosen 498 | 499 | if b < e 500 | C.addstr line[0, b] 501 | cprint line[b...e], color(chosen ? :match! : :match, chosen) 502 | C.attron color(:chosen, true) if chosen 503 | C.addstr line[e..-1] || '' 504 | else 505 | C.addstr line 506 | end 507 | C.attroff color(:chosen, true) if chosen 508 | end 509 | 510 | print_info selects.length if !@lists.empty? || events[:loaded] 511 | refresh 512 | end 513 | end#while 514 | rescue Exception => e 515 | main.raise e 516 | end 517 | } 518 | 519 | got = nil 520 | begin 521 | tty = IO.open(IO.sysopen('/dev/tty'), 'r') 522 | input = '' 523 | cursor = 0 524 | actions = { 525 | :nop => proc {}, 526 | ctrl(:c) => proc { exit 1 }, 527 | ctrl(:d) => proc { exit 1 if input.empty? }, 528 | ctrl(:m) => proc { 529 | @mtx.synchronize do 530 | got = [*@matches.fetch(@vcursor, [])][0] 531 | end 532 | exit 0 533 | }, 534 | ctrl(:u) => proc { input = input[cursor..-1]; cursor = 0 }, 535 | ctrl(:a) => proc { cursor = 0 }, 536 | ctrl(:e) => proc { cursor = input.length }, 537 | ctrl(:j) => proc { emit(:vcursor) { @vcursor -= 1 } }, 538 | ctrl(:k) => proc { emit(:vcursor) { @vcursor += 1 } }, 539 | ctrl(:w) => proc { 540 | ridx = (input[0...cursor - 1].rindex(/\S\s/) || -2) + 2 541 | input = input[0...ridx] + input[cursor..-1] 542 | cursor = ridx 543 | }, 544 | 127 => proc { input[cursor -= 1] = '' if cursor > 0 }, 545 | 9 => proc { 546 | emit(:select) { 547 | if sel = [*@matches.fetch(@vcursor, [])][0] 548 | if @selects.has_key? sel 549 | @selects.delete sel 550 | else 551 | @selects[sel] = 1 552 | end 553 | @vcursor = [0, @vcursor - 1].max 554 | end 555 | } if @multi 556 | }, 557 | :left => proc { cursor = [0, cursor - 1].max }, 558 | :right => proc { cursor = [input.length, cursor + 1].min }, 559 | } 560 | actions[ctrl(:b)] = actions[:left] 561 | actions[ctrl(:f)] = actions[:right] 562 | actions[ctrl(:h)] = actions[127] 563 | actions[ctrl(:n)] = actions[ctrl(:j)] 564 | actions[ctrl(:p)] = actions[ctrl(:k)] 565 | 566 | while true 567 | # Update user input 568 | @smtx.synchronize do 569 | @cursor_x = cursor 570 | print_input 571 | refresh 572 | end 573 | 574 | ord = tty.getc.ord 575 | if ord == 27 576 | ord = tty.getc.ord 577 | if ord == 91 578 | ord = case tty.getc.ord 579 | when 68 then :left 580 | when 67 then :right 581 | when 66 then ctrl(:j) 582 | when 65 then ctrl(:k) 583 | else :nop 584 | end 585 | end 586 | end 587 | 588 | actions.fetch(ord, proc { |ord| 589 | char = [ord].pack('U*') 590 | if char =~ /[[:print:]]/ 591 | input.insert cursor, char 592 | cursor += 1 593 | end 594 | }).call(ord) 595 | 596 | # Dispatch key event 597 | emit(:key) { @query = input.dup } 598 | end 599 | ensure 600 | C.close_screen 601 | if got 602 | @selects.delete got 603 | @selects.each do |sel, _| 604 | stdout.puts sel 605 | end 606 | stdout.puts got 607 | end 608 | end 609 | 610 | -------------------------------------------------------------------------------- /gcloghuman.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # see http://176.34.122.30/blog/2010/05/26/human-readable-jvm-gc-timestamps/ 4 | 5 | import sys, os, datetime 6 | 7 | # true if string is a positive float 8 | def validSeconds(str_sec): 9 | try: 10 | return 0 < float(str_sec) 11 | except ValueError: 12 | return False 13 | 14 | # show usage 15 | if len(sys.argv) < 2: 16 | print "Usage: %s " % (sys.argv[0]) 17 | sys.exit(1) 18 | 19 | file_str = sys.argv[1] 20 | lastmod_date = datetime.datetime.fromtimestamp(os.path.getmtime(file_str)) 21 | 22 | file = open(file_str, 'r') 23 | lines = file.readlines() 24 | file.close() 25 | 26 | # get last elapsed time 27 | for line in reversed(lines): 28 | parts = line.split(':') 29 | if validSeconds(parts[0]): 30 | break 31 | 32 | # calculate start time 33 | start_date = lastmod_date - datetime.timedelta(seconds=float(parts[0])) 34 | 35 | # print file prepending human readable time where appropiate 36 | for line in lines: 37 | parts = line.split(':') 38 | if not validSeconds(parts[0]): 39 | print line.rstrip() 40 | continue 41 | line_date = start_date + datetime.timedelta(seconds=float(parts[0])) 42 | print "%s: %s" % (line_date.isoformat(), line.rstrip()) 43 | -------------------------------------------------------------------------------- /git-hooks: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright (c) 2010, Benjamin C. Meyer 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 1. Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # 2. Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 3. Neither the name of the project nor the 14 | # names of its contributors may be used to endorse or promote products 15 | # derived from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER ''AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 | # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 24 | # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | # 28 | 29 | function hook_dirs 30 | { 31 | if [ ! -z "${1}" ] ; then 32 | hook="/${1}" 33 | else 34 | hook="" 35 | fi 36 | echo "${HOME}/.git_hooks${hook}" 37 | GITDIR=`git rev-parse --git-dir` 38 | cd $GITDIR/.. 39 | echo "${PWD}/git_hooks${hook}" 40 | eval echo "`git config hooks.global`"${hook} 41 | } 42 | 43 | function list_hooks_in_dir 44 | { 45 | find "${1}/" -executable -type f 2>/dev/null | grep -v "^.$" | sort 46 | } 47 | 48 | function run_hooks 49 | { 50 | dir="${1}" 51 | if [[ -z ${dir} || ! -d "${dir}" ]] ; then 52 | echo "run_hooks requires a directory name as an argument." 53 | return 1 54 | fi 55 | shift 1 56 | for hook in `list_hooks_in_dir "${dir}"` 57 | do 58 | export last_run_hook="${hook} $@" 59 | if [ ! -z ${GIT_HOOKS_VERBOSE} ] ; then 60 | echo -n "@@ Running hook: " 61 | echo -n `basename \`dirname "${hook}"\`` 62 | echo "/`basename "${hook}"`" 63 | fi 64 | ${hook} "$@" 65 | done 66 | } 67 | 68 | function run_hook 69 | { 70 | set -e 71 | hook=`basename "${1}"` 72 | if [ -z ${hook} ] ; then 73 | echo "run requires a hook argument" 74 | return 1 75 | fi 76 | shift 1 77 | for dir in `hook_dirs "${hook}"`; do 78 | if [ ! -d "${dir}" ] ; then 79 | continue 80 | fi 81 | run_hooks "${dir}" "$@" 82 | done 83 | set +e 84 | } 85 | 86 | function install_hooks 87 | { 88 | GITDIR=`git rev-parse --git-dir` 89 | if [ ! $? -eq 0 ] ; then 90 | echo "$1 must be run inside a git repository" 91 | return 1 92 | fi 93 | cd $GITDIR 94 | if [ "${1}" = "--install" ] ; then 95 | mv hooks hooks.old 96 | mkdir hooks 97 | cd hooks 98 | for file in applypatch-msg commit-msg post-applypatch post-checkout post-commit post-merge post-receive pre-applypatch pre-auto-gc pre-commit prepare-commit-msg pre-rebase pre-receive update 99 | do 100 | echo '#!/bin/bash 101 | git-hooks run "$0" "$@"' > "${file}" 102 | chmod +x "${file}" 103 | done 104 | else 105 | if [ ! -d hooks.old ] ; then 106 | echo "Error, hooks.old doesn't exists, aborting uninstall to not destroy something" 107 | return 1 108 | fi 109 | rm -rf hooks 110 | mv hooks.old hooks 111 | fi 112 | } 113 | 114 | function list_hooks 115 | { 116 | GITDIR=`git rev-parse --git-dir` 117 | cat $GITDIR/hooks/pre-commit 2> /dev/null | grep 'git-hooks' > /dev/null 2> /dev/null 118 | if [ $? = 0 ] ; then 119 | echo "Git hooks ARE installed in this repository." 120 | echo "" 121 | else 122 | echo "Git hooks are NOT installed in this repository." 123 | echo "" 124 | fi 125 | 126 | echo 'Listing User, Project, and Global hooks:' 127 | echo '---' 128 | for dir in `hook_dirs`; do 129 | echo "${dir}:" 130 | for hook in `list_hooks_in_dir "${dir}"` ; do 131 | echo -n `basename \`dirname "${hook}"\`` 132 | echo -e "/`basename "${hook}"` \t- `${hook} --about`" 133 | done 134 | echo "" 135 | done 136 | } 137 | 138 | function report_error { 139 | echo "Hook failed: $last_run_hook" 140 | exit 1 141 | } 142 | 143 | case $1 in 144 | run ) 145 | if [ ! -z "${GIT_DIR}" ] ; then 146 | unset GIT_DIR 147 | fi 148 | shift 149 | trap report_error ERR 150 | run_hook "$@" 151 | ;; 152 | --install|--uninstall ) 153 | install_hooks "$1" 154 | ;; 155 | -h|--help ) 156 | echo 'Git Hooks' 157 | echo '' 158 | echo 'Options:' 159 | echo ' --install Replace existing hooks in this repository with a call to' 160 | echo ' git hooks run [hook]. Move old hooks directory to hooks.old' 161 | echo ' --uninstall Remove existing hooks in this repository and rename hooks.old' 162 | echo ' back to hooks' 163 | echo " run [cmd] Run the hooks for cmd (such as pre-commit)" 164 | echo " (no arguments) Show currently installed hooks" 165 | ;; 166 | * ) 167 | list_hooks 168 | ;; 169 | esac 170 | -------------------------------------------------------------------------------- /git-shorten: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | curl -is http://git.io -F "url=$1" | grep "Location" | cut -d\ -f 2 -------------------------------------------------------------------------------- /google_maps_starred_locations.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Taken from: https://gist.github.com/endolith/389694 4 | 5 | Dependencies: 6 | 7 | pip install lxml 8 | pip install simplekml 9 | 10 | Usage: 11 | 12 | Go to Google Bookmarks: https://www.google.com/bookmarks/ 13 | 14 | On the bottom left, click "Export bookmarks": https://www.google.com/bookmarks/bookmarks.html?hl=en 15 | 16 | After downloading the html file, run this script on it to generate a Kml. 17 | 18 | """ 19 | 20 | from lxml.html import document_fromstring 21 | import simplekml 22 | 23 | from urllib2 import urlopen 24 | import re 25 | import time 26 | 27 | filename = r'GoogleBookmarks.html' 28 | 29 | with open(filename) as bookmarks_file: 30 | data = bookmarks_file.read() 31 | 32 | kml = simplekml.Kml() 33 | 34 | # Hacky and doesn't work for all of the stars: 35 | lat_re = re.compile('markers:[^\]]*latlng[^}]*lat:([^,]*)') 36 | lon_re = re.compile('markers:[^\]]*latlng[^}]*lng:([^}]*)') 37 | coords_in_url = re.compile('\?q=(-?\d{,3}\.\d*),\s*(-?\d{,3}\.\d*)') 38 | 39 | doc = document_fromstring(data) 40 | for element, attribute, url, pos in doc.body.iterlinks(): 41 | if 'maps.google' in url: 42 | description = element.text or '' 43 | print description.encode('UTF8') 44 | print u"URL: {0}".format(url) 45 | 46 | if coords_in_url.search(url): 47 | # Coordinates are in URL itself 48 | latitude = coords_in_url.search(url).groups()[0] 49 | longitude = coords_in_url.search(url).groups()[1] 50 | else: 51 | # Load map and find coordinates in source of page 52 | try: 53 | sock = urlopen(url.replace(' ','+').encode('UTF8')) 54 | except Exception, e: 55 | print 'Connection problem:' 56 | print repr(e) 57 | print 'Waiting 2 minutes and trying again' 58 | time.sleep(120) 59 | sock = urlopen(url.replace(' ','+').encode('UTF8')) 60 | content = sock.read() 61 | sock.close() 62 | time.sleep(3) # Don't annoy server 63 | try: 64 | latitude = lat_re.findall(content)[0] 65 | longitude = lon_re.findall(content)[0] 66 | except IndexError: 67 | print '[Coordinates not found]' 68 | print 69 | continue 70 | 71 | print latitude, longitude 72 | try: 73 | kml.newpoint(name=description, 74 | coords=[(float(longitude), float(latitude))]) 75 | except ValueError: 76 | print '[Invalid coordinates]' 77 | print 78 | 79 | kml.save("GoogleBookmarks.kml") 80 | -------------------------------------------------------------------------------- /grep-utf8.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Normally you would 4 | # grep --color='auto' -P -n "[\x80-\xFF]" $1 5 | 6 | # but OS X uses BSD grep so it doesn't support PCRE ("Perl-compatible 7 | # regular expressions") 8 | # Alternative is to `brew install pcre` which gives you 9 | pcregrep --color='auto' -n "[\x80-\xFF]" $1 -------------------------------------------------------------------------------- /html2markdown.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # html2markdown 4 | # Copyright 2005 Dale Sedivec 5 | # 6 | # This program is free software; you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation; either version 2 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 19 | # USA 20 | # 21 | # XXX 22 | # TODO: 23 | # * Change constant names to upper case. 24 | # * Test wrapping of HTML in Markdown source with long attributes that 25 | # have whitespace in their contents. 26 | # * Should probably put non-breaking spaces in the middle of a 27 | # Markdown image markup. 28 | # * Stop all the interpolation and concatenation operations and take 29 | # advantage of buffers more (use write not +) 30 | # * In code, do a consistency check WRT indentation on continued 31 | # statements. 32 | # * Look at inline HTML in indented block elements (block quote, list, 33 | # maybe code block) 34 | # * Test CLI. 35 | # * Check through for classes that are too big (refactoring) 36 | # * Write test for
  • [whitespace]

    ...

    • . I'm not sure that 37 | # Markdown will ever generate this, but it still looks likely to 38 | # happen in hand-written HTML. 39 | # * Make test with numeric entity to make sure handle_charref is 40 | # implemented. 41 | # * It's possible that (almost) everywhere we do an isinstance() 42 | # check, we should really be doing some kind of hasFeature() check, 43 | # hasFeature() being a method we implement? More flexible. 44 | 45 | from HTMLParser import HTMLParser 46 | from StringIO import StringIO 47 | import logging 48 | import textwrap 49 | import re 50 | import string 51 | import inspect 52 | import sys 53 | from itertools import repeat, chain 54 | 55 | WRAP_AT_COLUMN = 70 56 | # XXX This is kind of dumb, really, since certain types of syntax 57 | # demand certain types of indents. To parameterize this, we should 58 | # probably find all indent instances, change them to this variable, 59 | # then see what breaks with one indent or the other and hard code that 60 | # particular indent. 61 | MARKDOWN_INDENT = " " 62 | 63 | log = logging.getLogger("html2markdown") 64 | 65 | try: 66 | any 67 | except NameError: 68 | def any(items): 69 | for item in items: 70 | if item: 71 | return True 72 | return False 73 | 74 | def all(items): 75 | for item in items: 76 | if not item: 77 | return False 78 | return True 79 | 80 | # XXX TEST this is not tested? Plus it probably doesn't belong here. 81 | # At least document it. 82 | # def getMyCaller(): #pragma: no cover 83 | # try: 84 | # callerFrame = inspect.getouterframes(inspect.currentframe())[2] 85 | # return "%s:%d" % (callerFrame[3], callerFrame[2]) 86 | # finally: 87 | # del callerFrame 88 | 89 | 90 | class Box (object): 91 | def __init__(self): 92 | self.parent = None 93 | 94 | def render(self, writer): 95 | raise NotImplementedError("you must overload this") #pragma: no cover 96 | 97 | width = property(fget=lambda self: self.parent.width) 98 | 99 | class ContainerBox (Box): 100 | def __init__(self): 101 | super(ContainerBox, self).__init__() 102 | self.children = [] 103 | 104 | def addChild(self, child): 105 | self.children.append(child) 106 | child.parent = self 107 | 108 | def makeChild(self, childClass): 109 | child = childClass() 110 | self.addChild(child) 111 | return child 112 | 113 | class CompositeBox (ContainerBox): 114 | def __init__(self, addNewLines=True): 115 | super(CompositeBox, self).__init__() 116 | self.__addNewLineAfterChild = [] 117 | self.__addNewLines = addNewLines 118 | 119 | def addChild(self, child): 120 | super(CompositeBox, self).addChild(child) 121 | self.__addNewLineAfterChild.append(self.__addNewLines) 122 | 123 | def insertNewLineAfterChild(self, childIndex): 124 | assert childIndex >= 0, childIndex 125 | self.__addNewLineAfterChild[childIndex] = True 126 | 127 | def insertNewLineBeforeLastChild(self): 128 | self.__addNewLineAfterChild[-2] = True 129 | 130 | def render(self, writer): 131 | if self.children: 132 | assert len(self.__addNewLineAfterChild) == len(self.children) 133 | addNewLine = iter(self.__addNewLineAfterChild) 134 | self.children[0].render(writer) 135 | for child in self.children[1:]: 136 | if addNewLine.next(): 137 | writer("\n") 138 | child.render(writer) 139 | 140 | class RootBox (CompositeBox): 141 | # Override the property set in a superclass. (XXX Is this the 142 | # cleanest way to do this?) 143 | width = None 144 | 145 | def __init__(self, width): 146 | super(RootBox, self).__init__() 147 | self.width = width 148 | 149 | def ijoin(iterable, joinString): 150 | """Yields joinString between items from iterable. 151 | 152 | s.join(i) == "".join(ijoin(i, s)) 153 | 154 | """ 155 | iterator = iter(iterable) 156 | yield iterator.next() 157 | for item in iterator: 158 | yield joinString 159 | yield item 160 | 161 | class TextBox (Box): 162 | def __init__(self): 163 | self.__lines = [StringIO()] 164 | 165 | def addText(self, text): 166 | self.__lines[-1].write(text) 167 | 168 | def addLineBreak(self): 169 | self.__lines.append(StringIO()) 170 | 171 | def _iterLines(self): 172 | for line in self.__lines: 173 | yield line.getvalue() 174 | 175 | def render(self, writer): 176 | for string in ijoin(self._iterLines(), " \n"): 177 | writer(string) 178 | if string[-1] != "\n": 179 | writer("\n") 180 | 181 | class iterAllButLast (object): 182 | def __init__(self, iterable): 183 | self._iterator = iter(iterable) 184 | 185 | def __iter__(self): 186 | lastItem = self._iterator.next() 187 | for item in self._iterator: 188 | yield lastItem 189 | lastItem = item 190 | self.last = lastItem 191 | 192 | class WrappedTextBox (TextBox): 193 | __wordBoundaryRegexp = re.compile(r'(\s+)') 194 | 195 | def render(self, writer): 196 | def fill(line, lastLineSuffix=""): 197 | return self.__fill(line, self.width, lastLineSuffix, writer) 198 | 199 | lines = iterAllButLast(self._iterLines()) 200 | for line in lines: 201 | writer(fill(line, " ")) 202 | writer(fill(lines.last)) 203 | 204 | # XXX REFACTOR I'd say refactor this, but right now I don't see a 205 | # particularly clean way to do it. 206 | # 207 | # There should be a way, though. All this code seems so verbose, 208 | # if not needlessly complex. 209 | def __fill(self, text, width, lastLineSuffix, writer): 210 | log.debug("fill text=%r suffix=%r" % (text, lastLineSuffix)) 211 | words = self.__splitTextIntoWordsAndSpaces(text) 212 | firstSpace, firstWord = words.pop(0) 213 | linePosition = self.__writeFirstWordOnLine(firstWord, writer) 214 | for spaceBefore, word in words: 215 | spaceLen = len(spaceBefore) 216 | wordLen = len(word) 217 | if (linePosition + spaceLen + wordLen) > width: 218 | writer("\n") 219 | self.__writeFirstWordOnLine(word, writer) 220 | linePosition = wordLen 221 | else: 222 | writer(spaceBefore) 223 | writer(word) 224 | linePosition += spaceLen + wordLen 225 | writer(lastLineSuffix) 226 | writer("\n") 227 | 228 | # The second grouping prevents **strong** from tripping this 229 | # regular expression. 230 | __beginningOfLineTokens = re.compile(r"^([0-9]+\.|[*+-]([^*]|$)|#)") 231 | 232 | def __writeFirstWordOnLine(self, word, writer): 233 | """Writes the first word using writer, adding escaping if needed. 234 | 235 | Markdown assigns special meaning to certain tokens when they 236 | appear at the beginning of a line. We have to esacpe these 237 | special characters if they happen to appear at the beginning 238 | of a line after a paragraph is wrapped. This function will 239 | return the total number of characters written, which might be 240 | bigger than len(word) if an escape character is added. 241 | 242 | """ 243 | wordLen = len(word) 244 | tokenMatch = self.__beginningOfLineTokens.search(word) 245 | if tokenMatch: 246 | matchEndPosition = tokenMatch.end(1) 247 | log.debug("word=%r matchEndPosition=%r" % (word, matchEndPosition)) 248 | writer(word[0:matchEndPosition - 1]) 249 | writer("\\") 250 | writer(word[matchEndPosition - 1:]) 251 | return wordLen + 1 252 | else: 253 | log.debug("word=%r no match" % (word,)); 254 | writer(word) 255 | return wordLen 256 | 257 | def __splitTextIntoWordsAndSpaces(self, text): 258 | """ 259 | 260 | Builds and returns a list of tuples in the form (space 261 | before word, word), where the spaces and words are determined 262 | by splitting text on word boundaries. This is used primarily 263 | by the fill() method. 264 | 265 | """ 266 | log.debug("splitTextIntoWordsAndSpaces: text=%r" % (text,)) 267 | parts = self.__wordBoundaryRegexp.split(text) 268 | log.debug("splitTextIntoWordsAndSpaces: normalizing %r" % (parts,)) 269 | self.__normalizeSplitTextParts(parts) 270 | log.debug("splitTextIntoWordsAndSpaces: after normalizing %r" 271 | % (parts,)) 272 | words = [] 273 | lastWord = "" 274 | for spaceBefore, word in zip(parts[::2], parts[1::2]): 275 | spaceBefore = self.__normalizeWordSpacing(spaceBefore, lastWord) 276 | words.append((spaceBefore, word)) 277 | lastWord = word 278 | return words 279 | 280 | def __normalizeWordSpacing(self, spaceBefore, precedingWord): 281 | # If the input is "foo.\nbar" you'll end up with "foo. bar" 282 | # even if you separate your sentences with two spaces. I'm 283 | # not inclined to do anything to fix this until someone really 284 | # bitches about it. Also, two spaces are "safer" than one in 285 | # the case of (for example) "Mr.\nSmith". 286 | if spaceBefore[0:2] == " " and precedingWord[-1] in ".?!:": 287 | spaceBefore = " " 288 | else: 289 | spaceBefore = " " 290 | return spaceBefore 291 | 292 | def __normalizeSplitTextParts(self, parts): 293 | """ 294 | 295 | This method makes sure that the parts list is a list of space, 296 | word, space, word, space, word, ... The first element in the 297 | list will always be the empty string (an empty space). 298 | 299 | This method is used by the wrapping code. 300 | 301 | """ 302 | if parts[0] == "": 303 | del parts[1] 304 | else: 305 | parts.insert(0, "") 306 | if parts[-1] == "": 307 | del parts[-2:] 308 | assert (len(parts) % 2) == 0, "List normalizing failed: %r" % (parts,) 309 | 310 | class IndentedBox (ContainerBox): 311 | def __init__(self, indent, firstLineIndent=None): 312 | super(IndentedBox, self).__init__() 313 | self.__indentLength = len(indent) 314 | self.__subsequentLineIndent = indent 315 | if firstLineIndent is not None: 316 | assert len(firstLineIndent) == self.__indentLength 317 | self.__firstLineIndent = firstLineIndent 318 | else: 319 | self.__firstLineIndent = indent 320 | 321 | def render(self, writer): 322 | childRendering = StringIO() 323 | self.__renderChildren(childRendering.write) 324 | self.__rewindFile(childRendering) 325 | self.__renderLinesFromFile(childRendering, writer) 326 | 327 | def __renderLinesFromFile(self, childRendering, writer): 328 | indentGenerator = chain([self.__firstLineIndent], 329 | repeat(self.__subsequentLineIndent)) 330 | for line in childRendering: 331 | indent = indentGenerator.next() 332 | if self.__isBlankLine(line): 333 | indent = indent.rstrip() 334 | writer(indent) 335 | writer(line) 336 | 337 | def __isBlankLine(self, line): 338 | return not line.rstrip("\r\n") 339 | 340 | def __rewindFile(self, childRendering): 341 | childRendering.seek(0) 342 | 343 | def __renderChildren(self, writer): 344 | for child in self.children: 345 | child.render(writer) 346 | 347 | def _getWidth(self): 348 | return super(IndentedBox, self).width - self.__indentLength 349 | width = property(fget=_getWidth) 350 | 351 | class RawTextBox (TextBox): 352 | """A TextBox whose contents shouldn't have Markdown elements escaped.""" 353 | pass 354 | 355 | # Based on DOM. Should probably refer to this as MDDOM (Markdown 356 | # DOM). I think I used "micro-DOM" somewhere else. 357 | class Node (object): 358 | def __init__(self): 359 | self.parent = None 360 | 361 | class ContainerNode (Node): 362 | def __init__(self): 363 | super(ContainerNode, self).__init__() 364 | self.children = [] 365 | 366 | def makeChild(self, type): 367 | child = type() 368 | self.addChild(child) 369 | return child 370 | 371 | def addChild(self, child): 372 | self.children.append(child) 373 | child.parent = self 374 | 375 | # An InlineNode is a Node that does not render to a Box, but rather 376 | # modifies the Box inside which it occurs. Currently this is used to 377 | # mark Nodes whose transformation requires a Box that supports 378 | # addText(). 379 | class InlineNode (Node): 380 | pass 381 | 382 | # A TextContainer is a ContainerNode that may also hold 383 | # TextRelatedNodes. The HTML parser will ignore text that occurs 384 | # outside of a TextContainer. 385 | class TextContainer (ContainerNode): 386 | pass 387 | 388 | class InlineTextContainer (InlineNode, TextContainer): 389 | pass 390 | 391 | class Text (InlineNode): 392 | def __init__(self, text=""): 393 | super(Node, self).__init__() 394 | self.text = text 395 | 396 | class Document (ContainerNode): 397 | pass 398 | 399 | class List (ContainerNode): 400 | pass 401 | 402 | class OrderedList (List): 403 | def getChildIndex(self, child): 404 | return self.children.index(child) 405 | 406 | class UnorderedList (List): 407 | pass 408 | 409 | class ListItem (TextContainer): 410 | def getItemNumber(self): 411 | # This method is only valid when this is an item in an 412 | # OrderedList. Obviously. 413 | return self.parent.getChildIndex(self) + 1 414 | 415 | class BlockQuote (ContainerNode): 416 | pass 417 | 418 | class Paragraph (TextContainer): 419 | pass 420 | 421 | class Preformatted (TextContainer): 422 | pass 423 | 424 | class HTML (TextContainer): 425 | pass 426 | 427 | class Code (InlineTextContainer): 428 | pass 429 | 430 | class Emphasized (InlineTextContainer): 431 | pass 432 | 433 | class Strong (InlineTextContainer): 434 | pass 435 | 436 | class LineBreak (InlineNode): 437 | pass 438 | 439 | class Image (InlineNode): 440 | def __init__(self, url, alternateText=None, title=None): 441 | super(Image, self).__init__() 442 | self.url = url 443 | self.alternateText = alternateText 444 | self.title = title 445 | 446 | class Heading (TextContainer): 447 | def __init__(self, level): 448 | super(Heading, self).__init__() 449 | self.level = level 450 | 451 | class HorizontalRule (Node): 452 | pass 453 | 454 | class Anchor (InlineTextContainer): 455 | def __init__(self, url, title=None): 456 | super(Anchor, self).__init__() 457 | self.url = url 458 | self.title = title 459 | 460 | class UnknownInlineElement (InlineTextContainer): 461 | def __init__(self, tag, attributes): 462 | super(UnknownInlineElement, self).__init__() 463 | self.tag = tag 464 | self.attributes = attributes 465 | 466 | 467 | class MarkdownTransformer (object): 468 | __formattingCharactersRegexp = re.compile(r"((?<=\S)([*_])|([*_])(?=\S))") 469 | 470 | def transform(self, document): 471 | rootBox = RootBox(width=WRAP_AT_COLUMN) 472 | self.__dispatchChildren(document, rootBox) 473 | return rootBox 474 | 475 | def __dispatch(self, node, parentBox): 476 | log.debug("Dispatching node=%r parentBox=%r" % (node, parentBox)) 477 | if isinstance(node, List): 478 | nodeTypeName = "List" 479 | else: 480 | nodeTypeName = type(node).__name__ 481 | getattr(self, "_transform" + nodeTypeName)(node, parentBox) 482 | # self.__handlers[type(node)](self, node, parentBox) 483 | 484 | def __dispatchChildren(self, node, parentBox): 485 | self.__dispatchList(node.children, parentBox) 486 | 487 | def __dispatchList(self, nodeList, parentBox): 488 | for node in nodeList: 489 | self.__dispatch(node, parentBox) 490 | 491 | def _transformParagraph(self, node, parentBox): 492 | box = parentBox.makeChild(WrappedTextBox) 493 | self.__dispatchChildren(node, box) 494 | 495 | def _transformBlockQuote(self, node, parentBox): 496 | indentedBox = IndentedBox(indent="> ") 497 | parentBox.addChild(indentedBox) 498 | dividedBox = indentedBox.makeChild(CompositeBox) 499 | self.__dispatchChildren(node, dividedBox) 500 | 501 | def _transformPreformatted(self, node, parentBox): 502 | indentedBox = IndentedBox(indent=MARKDOWN_INDENT) 503 | parentBox.addChild(indentedBox) 504 | textBox = indentedBox.makeChild(TextBox) 505 | self.__dispatchChildren(node, textBox) 506 | 507 | def _transformText(self, node, parentBox): 508 | if isinstance(node.parent, (HTML, Preformatted, Code)) \ 509 | or isinstance(parentBox, RawTextBox): 510 | text = node.text 511 | else: 512 | text = self.__escapeFormattingCharacters(node.text) 513 | parentBox.addText(text) 514 | 515 | def __escapeFormattingCharacters(self, data): 516 | escapedData = data.replace("\\", "\\\\") 517 | escapedData = self.__formattingCharactersRegexp.sub(r"\\\1", 518 | escapedData) 519 | return escapedData 520 | 521 | def _transformList(self, node, parentBox): 522 | box = CompositeBox(addNewLines=False) 523 | parentBox.addChild(box) 524 | self.__dispatchChildren(node, box) 525 | self.__addExplicitParagraphsInList(node, box) 526 | 527 | # XXX REFACTOR if you dare. The list code (here and ListItem 528 | # processing) is nigh incomprehensible. Of course, I can't even 529 | # figure out how to simplify this function since the way it 530 | # figures out where to put explicit paragraphs is so arcane (and 531 | # the rules for how to generate

    are, shall we say, 532 | # "tedious"). 533 | def __addExplicitParagraphsInList(self, node, box): 534 | paragraphAnalysis = [] 535 | for listItem in node.children: 536 | isSingleParagraph = False 537 | if isinstance(listItem.children[0], Paragraph): 538 | isSingleParagraph = True 539 | for child in listItem.children[1:]: 540 | if isinstance(child, List): 541 | break 542 | elif not isinstance(child, Text): 543 | isSingleParagraph = False 544 | break 545 | paragraphAnalysis.append(isSingleParagraph) 546 | log.debug("paragraphAnalysis=%r" % (paragraphAnalysis,)) 547 | 548 | consecutiveSingleParas = 0 549 | for childIndex, isSingleParagraph in enumerate(paragraphAnalysis): 550 | if isSingleParagraph: 551 | consecutiveSingleParas += 1 552 | if consecutiveSingleParas >= 2: 553 | box.insertNewLineAfterChild(childIndex - 1) 554 | else: 555 | if consecutiveSingleParas == 1: 556 | if any([ isinstance(n, List) for n 557 | in node.children[childIndex - 1].children ]): 558 | # A List node's children can only be 559 | # ListItems, and a ListItem always generates 560 | # an outer CompositeBox, so box.children are 561 | # all CompositeBoxes. 562 | box.children[childIndex - 1].insertNewLineAfterChild(0) 563 | else: 564 | box.insertNewLineBeforeLastChild() 565 | consecutiveSingleParas = 0 566 | # XXX Near exact copy of above code. 567 | if consecutiveSingleParas == 1: 568 | if any([ isinstance(n, List) for n 569 | in node.children[childIndex].children ]): 570 | box.children[childIndex].insertNewLineAfterChild(0) 571 | else: 572 | box.insertNewLineBeforeLastChild() 573 | 574 | # XXX REFACTOR 575 | def _transformListItem(self, node, parentBox): 576 | BOX_AT_BULLET_LEVEL = 1 577 | BOX_AT_LIST_ITEM_LEVEL = 2 578 | 579 | outerBox = CompositeBox(addNewLines=False) 580 | parentBox.addChild(outerBox) 581 | # XXX This code to determine indents will have a tendency to 582 | # not work right if you want to make MARKDOWN_INDENT = "\t" 583 | # (for example). 584 | bulletIndent = " " 585 | if isinstance(node.parent, OrderedList): 586 | number = "%d. " % (node.getItemNumber(),) 587 | number = number + " " * (4 - len(number)) 588 | # XXX Should we use len(number) instead of 4 here? Are 589 | # more than four spaces on continued lines fine with 590 | # Markdown? 591 | indentedBox = IndentedBox(firstLineIndent=number, 592 | indent=bulletIndent) 593 | else: 594 | indentedBox = IndentedBox(firstLineIndent="* ", 595 | indent=bulletIndent) 596 | outerBox.addChild(indentedBox) 597 | innerBox = indentedBox.makeChild(CompositeBox) 598 | 599 | children = node.children[:] 600 | # The first child has to be in the indent box that has the 601 | # list bullet. 602 | if isinstance(children[0], InlineNode): 603 | # A ListItem that starts with text can only have text or 604 | # nested lists under it. I think. 605 | log.debug("List item dispatching text children") 606 | textBox = innerBox.makeChild(WrappedTextBox) 607 | while children and isinstance(children[0], InlineNode): 608 | self.__dispatch(children.pop(0), textBox) 609 | elif isinstance(children[0], List): 610 | # Immediate sublist. 611 | listIndentBox = IndentedBox(indent=MARKDOWN_INDENT) 612 | innerBox.addChild(listIndentBox) 613 | self.__dispatch(children.pop(0), listIndentBox) 614 | else: 615 | self.__dispatch(children.pop(0), innerBox) 616 | 617 | innerBoxType = BOX_AT_BULLET_LEVEL 618 | for child in children: 619 | if isinstance(child, Text): 620 | # Ignore whitespace that occurs between elements. 621 | continue 622 | elif isinstance(child, (Preformatted, List)): 623 | if innerBoxType != BOX_AT_LIST_ITEM_LEVEL: 624 | innerBox = IndentedBox(indent=MARKDOWN_INDENT) 625 | outerBox.addChild(innerBox) 626 | if isinstance(child, Preformatted): 627 | outerBox.insertNewLineBeforeLastChild() 628 | innerBoxType = BOX_AT_LIST_ITEM_LEVEL 629 | else: 630 | if innerBoxType != BOX_AT_BULLET_LEVEL: 631 | indentedBox = IndentedBox(indent=bulletIndent) 632 | outerBox.addChild(indentedBox) 633 | outerBox.insertNewLineBeforeLastChild() 634 | innerBox = indentedBox.makeChild(CompositeBox) 635 | innerBoxType = BOX_AT_BULLET_LEVEL 636 | self.__dispatch(child, innerBox) 637 | 638 | # XXX Might want to factor out this pattern. 639 | def _transformHTML(self, node, parentBox): 640 | box = parentBox.makeChild(TextBox) 641 | self.__dispatchChildren(node, box) 642 | 643 | __backtickRegexp = re.compile("`+") 644 | 645 | def _transformCode(self, node, parentBox): 646 | contents = self.__renderChildren(node) 647 | codeDelimiter = self.__makeCodeDelimiter(contents) 648 | parentBox.addText(codeDelimiter) 649 | if contents[0] == "`": 650 | parentBox.addText(" ") 651 | parentBox.addText(contents) 652 | if contents[-1] == "`": 653 | parentBox.addText(" ") 654 | parentBox.addText(codeDelimiter) 655 | 656 | def __makeCodeDelimiter(self, content): 657 | """Returns the correct number of backticks to set off string as code. 658 | 659 | Markdown requires you to use at least one more backtick to 660 | introduce/conclude a code span than there are backticks within 661 | the code span. For example, if contents="foo ``date`` bar", 662 | Markdown would require ``` to be used to begin/end the code 663 | span for that string. 664 | 665 | """ 666 | matches = self.__backtickRegexp.findall(content) 667 | if matches: 668 | codeDelimiterLength = max([ len(m) for m in matches ]) + 1 669 | else: 670 | codeDelimiterLength = 1 671 | return "`" * codeDelimiterLength 672 | 673 | def _transformEmphasized(self, node, parentBox): 674 | parentBox.addText("_") 675 | self.__dispatchChildren(node, parentBox) 676 | parentBox.addText("_") 677 | 678 | def _transformLineBreak(self, node, parentBox): 679 | parentBox.addLineBreak() 680 | 681 | def _transformImage(self, node, parentBox): 682 | parentBox.addText("![") 683 | parentBox.addText(node.alternateText) 684 | parentBox.addText("](") 685 | parentBox.addText(node.url) 686 | if node.title: 687 | parentBox.addText(' "') 688 | parentBox.addText(node.title) 689 | parentBox.addText('"') 690 | parentBox.addText(")") 691 | 692 | def _transformHeading(self, node, parentBox): 693 | box = parentBox.makeChild(TextBox) 694 | box.addText("#" * node.level + " ") 695 | self.__dispatchChildren(node, box) 696 | box.addText(" " + node.level * "#") 697 | 698 | def _transformHorizontalRule(self, node, parentBox): 699 | box = parentBox.makeChild(TextBox) 700 | box.addText("---") 701 | 702 | def _transformAnchor(self, node, parentBox): 703 | # Sometimes this renders the contents twice: once as "raw 704 | # text" (no escaping of formatting characters) so we can match 705 | # a URL that might have Markdown formatting characters in it 706 | # (f.e. http://example.com/foo_bar_baz), and the second time 707 | # with Markdown escaping if the contents aren't the same as 708 | # the href. 709 | linkContents = self.__renderChildren(node, boxType=RawTextBox) 710 | url = node.url 711 | isMailto = url.startswith("mailto:") 712 | if linkContents == url or (isMailto and linkContents == url[7:]): 713 | parentBox.addText("<") 714 | parentBox.addText(linkContents) 715 | parentBox.addText(">") 716 | else: 717 | parentBox.addText("[") 718 | parentBox.addText(self.__renderChildren(node)) 719 | parentBox.addText("](") 720 | parentBox.addText(url) 721 | if node.title: 722 | parentBox.addText(' "') 723 | parentBox.addText(node.title) 724 | parentBox.addText('"') 725 | parentBox.addText(")") 726 | 727 | def __renderChildren(self, node, boxType=TextBox): 728 | textBox = boxType() 729 | self.__dispatchChildren(node, textBox) 730 | contents = StringIO() 731 | textBox.render(contents.write) 732 | return contents.getvalue().strip() 733 | 734 | def _transformStrong(self, node, parentBox): 735 | parentBox.addText("**") 736 | self.__dispatchChildren(node, parentBox) 737 | parentBox.addText("**") 738 | 739 | def _transformUnknownInlineElement(self, node, parentBox): 740 | write = parentBox.addText 741 | write("<") 742 | write(node.tag) 743 | for name, value in node.attributes: 744 | if '"' in value: 745 | quotingChar = "'" 746 | else: 747 | quotingChar = '"' 748 | write(" ") 749 | write(name) 750 | write('=') 751 | write(quotingChar) 752 | write(value) 753 | write(quotingChar) 754 | if node.children: 755 | write(">") 756 | self.__dispatchChildren(node, parentBox) 757 | write("") 760 | else: 761 | write(" />") 762 | 763 | 764 | # XXX TEST Should test this? 765 | class LineNumberedBuffer (StringIO): 766 | __eolRegexp = re.compile(r"(\r?\n)") 767 | 768 | def __init__(self): 769 | StringIO.__init__(self) 770 | self.__linePositions = [0] 771 | 772 | def write(self, string): 773 | parts = self.__eolRegexp.split(string) 774 | log.debug("LineNumberedBuffer write split parts=%r" % (parts,)) 775 | for part in parts: 776 | StringIO.write(self, part) 777 | if "\n" in part: 778 | log.debug("new line at %d" % (self.tell(),)) 779 | self.__linePositions.append(self.tell()) 780 | log.debug("LineNumberedBuffer.write final pos=%d" % (self.tell(),)) 781 | 782 | def seekLinePosition(self, lineNumber, offset): 783 | """Seek to an offset from the start of line lineNumber. 784 | 785 | The first line is 1, the first character on a line is 0. This 786 | is in line with HTMLParser.getpos(). 787 | 788 | """ 789 | position = self.__linePositions[lineNumber - 1] + offset 790 | log.debug("seekLinePosition (%d,%d)=%d" % (lineNumber, offset, 791 | position)) 792 | self.seek(position, 0) 793 | log.debug("seekLinePosition tell=%d" % (self.tell(),)) 794 | assert self.tell() == position, "seekLinePosition failed" 795 | 796 | # XXX Turn this into MDDOMParser, outputs MDDOM? Then you take the 797 | # Document and ship it off to MarkdownTransformer. Should at least 798 | # give this class a better name. 799 | class MarkdownTranslator (HTMLParser): 800 | __translatedEntities = {"amp": "&", 801 | "lt": "<", 802 | "gt": ">", 803 | "quot": '"'} 804 | 805 | __unsupportedBlockElements = ("dl", "div", "noscript", "form", "table", 806 | "fieldset", "address") 807 | 808 | def reset(self): 809 | HTMLParser.reset(self) 810 | self.__shouldOutputStack = [False] 811 | self.__unsupportedElementDepth = 0 812 | self.__unsupportedBlockStart = None 813 | self.__input = LineNumberedBuffer() 814 | self.__currentNode = Document() 815 | 816 | def feed(self, text): 817 | self.__input.write(text) 818 | HTMLParser.feed(self, text) 819 | 820 | def handle_starttag(self, tag, attrs): 821 | if self.__unsupportedElementDepth: 822 | self.__unsupportedElementDepth += 1 823 | elif tag == "code" \ 824 | and isinstance(self.__currentNode, 825 | Preformatted) \ 826 | and len(self.__currentNode.children) == 0: 827 | # Special case: ignore immediately following 
    .
 828 |             # Markdown emits 
    ...
     for a
 829 |             # preformatted text block.
 830 |             #
 831 |             # XXX In the interest of moving to just a DOM HTML parser,
 832 |             # I think I support moving this logic to
 833 |             # MarkdownTransformer.
 834 |             pass
 835 |         else:
 836 |             # XXX REFACTOR
 837 |             element = None
 838 |             handler = self.__recognizedTags.get(tag)
 839 |             if handler:
 840 |                 if not isinstance(handler, type):
 841 |                     element = handler(self, tag, attrs)
 842 |                     isBlock = handler.isBlock
 843 |                 elif attrs:
 844 |                     isBlock = not issubclass(handler, InlineNode)
 845 |                 else:
 846 |                     element = self.__currentNode.makeChild(handler)
 847 |             else:
 848 |                 isBlock = tag in self.__unsupportedBlockElements
 849 |             if not element and not isBlock:
 850 |                 element = UnknownInlineElement(tag, attrs)
 851 |                 self.__currentNode.addChild(element)
 852 |             if element:
 853 |                 self.__currentNode = element
 854 |                 self.__shouldOutputStack.append(isinstance(element,
 855 |                                                            TextContainer))
 856 |             else:
 857 |                 self.__enterUnsupportedBlockElement()
 858 | 
 859 |     def handle_endtag(self, tag):
 860 |         log.debug("Leaving tag=%r" % (tag,))
 861 |         if self.__unsupportedElementDepth:
 862 |             log.debug("Leaving unsupported element")
 863 |             self.__leaveUnsupportedElement()
 864 |         elif tag == "code" and isinstance(self.__currentNode,
 865 |                                           Preformatted):
 866 |             # Special case for
    . See similar exception 867 | # in handle_starttag() for explanation. 868 | pass 869 | else: 870 | log.debug("Leaving element") 871 | self.__leaveElement() 872 | 873 | def __enterUnsupportedBlockElement(self): 874 | self.__unsupportedElementDepth = 1 875 | self.__unsupportedBlockStart = self.getpos() 876 | 877 | # XXX REFACTOR 878 | def __leaveUnsupportedElement(self): 879 | self.__unsupportedElementDepth -= 1 880 | log.debug("unsupportedBlockDepth=%r" 881 | % (self.__unsupportedElementDepth,)) 882 | if not self.__unsupportedElementDepth: 883 | log.debug("Finished with unsupported block element"); 884 | log.debug("positions begin=%r end=%r" 885 | % (self.__unsupportedBlockStart, self.getpos())) 886 | html = self.__getUnsupportedBlockElementHTML() 887 | htmlNode = self.__currentNode.makeChild(HTML) 888 | htmlNode.addChild(Text(html)) 889 | self.__positionInputBufferAtEnd() 890 | 891 | # XXX Maybe refactor -- or rename to something shorter (applies to 892 | # all methods following this naming convention). 893 | def __getUnsupportedBlockElementHTML(self): 894 | """Side effect: repositions self.__input.""" 895 | endPosition = self.__getEndOfTagPosition(self.getpos()) 896 | self.__input.seekLinePosition(*self.__unsupportedBlockStart) 897 | startPosition = self.__input.tell() 898 | htmlLength = endPosition - startPosition 899 | log.debug("endPosition=%d startPosition=%d len=%d" 900 | % (endPosition, startPosition, htmlLength)) 901 | html = StringIO() 902 | html.write(self.__input.read(htmlLength)) 903 | html.write("\n") 904 | return html.getvalue() 905 | 906 | def __getEndOfTagPosition(self, startAt): 907 | """Side effect: repositions self.__input.""" 908 | self.__input.seekLinePosition(*startAt) 909 | self.__searchInputForTagClose() 910 | return self.__input.tell() 911 | 912 | def __searchInputForTagClose(self): 913 | # XXX expensive debugging statement 914 | log.debug("searchInputForTagClose pos=%d input=%r" 915 | % (self.__input.tell(), 916 | self.__input.getvalue())) 917 | while True: 918 | nextCharacter = self.__input.read(1) 919 | if not nextCharacter: 920 | assert False, "premature tag end in input" #pragma: no cover 921 | elif nextCharacter == ">": 922 | break 923 | 924 | def __positionInputBufferAtEnd(self): 925 | self.__input.seek(0, 2) 926 | 927 | def __leaveElement(self): 928 | assert len(self.__shouldOutputStack) > 1 929 | self.__shouldOutputStack.pop() 930 | self.__currentNode = self.__currentNode.parent 931 | 932 | # XXX REFACTOR 933 | def _enterImg(self, tag, attributes): 934 | if True not in map(lambda attr: attr[0] not in ("src", "alt", "title"), 935 | attributes): 936 | attributes = dict(attributes) 937 | parameters = {"url": attributes["src"]} 938 | if "alt" in attributes: 939 | parameters["alternateText"] = attributes["alt"] 940 | if "title" in attributes: 941 | parameters["title"] = attributes["title"] 942 | image = Image(**parameters) 943 | self.__currentNode.addChild(image) 944 | return image 945 | else: 946 | return None 947 | _enterImg.isBlock = False 948 | 949 | __numericEntityRegexp = re.compile("&#(x[0-9A-F]{2}|[0-9]{2,3});") 950 | 951 | def __substituteNumericEntity(self, match): 952 | return self.__translateNumericEntity(match.group(1)) 953 | 954 | def __translateNumericEntity(self, ref): 955 | if ref[0] == "x": 956 | value = int(ref[1:], 16) 957 | else: 958 | value = int(ref) 959 | if self.__shouldDecodeNumericEntity(value): 960 | return chr(value) 961 | else: 962 | return "&#%s;" % (ref,) 963 | 964 | def __shouldDecodeNumericEntity(self, characterCode): 965 | return 32 <= characterCode <= 126 966 | 967 | def _enterA(self, tag, attributes): 968 | if all([ attr[0] in ("href", "title") for attr in attributes ]): 969 | attributes = dict(attributes) 970 | # XXX REFACTOR This indentation/wrapping is ugly and looks 971 | # unnecessary. Should think about reducing name lengths. 972 | href = self.__numericEntityRegexp.sub( 973 | self.__substituteNumericEntity, attributes["href"]) 974 | anchor = Anchor(href, title=attributes.get("title", None)) 975 | self.__currentNode.addChild(anchor) 976 | return anchor 977 | else: 978 | return None 979 | _enterA.isBlock = False 980 | 981 | # XXX TEST with attributes. 982 | def _enterHeading(self, tag, attributes): 983 | level = int(tag[1:]) 984 | heading = Heading(level) 985 | self.__currentNode.addChild(heading) 986 | return heading 987 | _enterHeading.isBlock = True 988 | 989 | def __shouldOutput(self): 990 | return self.__shouldOutputStack[-1] 991 | 992 | def handle_data(self, data): 993 | if self.__shouldOutput(): 994 | log.debug("output %r" % (data,)) 995 | self.__currentNode.addChild(Text(data)) 996 | 997 | def handle_entityref(self, name): 998 | log.debug("entity=%r" % (name,)) 999 | if not self.__unsupportedElementDepth: 1000 | if name in self.__translatedEntities: 1001 | self.handle_data(self.__translatedEntities[name]) 1002 | else: 1003 | self.handle_data("&%s;" % (name,)) 1004 | 1005 | def handle_charref(self, ref): 1006 | if not self.__unsupportedElementDepth: 1007 | self.handle_data(self.__translateNumericEntity(ref)) 1008 | 1009 | # XXX some day we should probably change this interface to write 1010 | # to a file, or to a callable 1011 | def getOutput(self): 1012 | assert isinstance(self.__currentNode, Document), `self.__currentNode` 1013 | log.debug(self.__renderTreeForDebug(self.__currentNode)) 1014 | box = MarkdownTransformer().transform(self.__currentNode) 1015 | log.debug(self.__renderTreeForDebug(box)) 1016 | result = StringIO() 1017 | box.render(result.write) 1018 | return result.getvalue() 1019 | 1020 | # XXX OPTIMIZE Could short-circuit this code when debug is off, as 1021 | # an alternative to not calling it (log.debug("%s" % 1022 | # (__renderTreeForDebug(),))). 1023 | def __renderTreeForDebug(self, node): 1024 | result = StringIO() 1025 | result.write("(%s" % (node.__class__.__name__,)) 1026 | if hasattr(node, "children"): 1027 | for child in node.children: 1028 | result.write(" ") 1029 | result.write(self.__renderTreeForDebug(child)) 1030 | result.write(")") 1031 | return result.getvalue() 1032 | 1033 | __recognizedTags = {"p": Paragraph, 1034 | "blockquote": BlockQuote, 1035 | "ol": OrderedList, 1036 | "ul": UnorderedList, 1037 | "li": ListItem, 1038 | "code": Code, 1039 | "em": Emphasized, 1040 | "pre": Preformatted, 1041 | "br": LineBreak, 1042 | "img": _enterImg, 1043 | "hr": HorizontalRule, 1044 | "a": _enterA, 1045 | "strong": Strong} 1046 | for level in range(1, 10): 1047 | __recognizedTags["h%d" % (level,)] = _enterHeading 1048 | 1049 | 1050 | def html2markdown(html): 1051 | return html2markdown_file(StringIO(html)) 1052 | 1053 | def html2markdown_file(inputFile): 1054 | translator = MarkdownTranslator() 1055 | for line in inputFile: 1056 | translator.feed(line) 1057 | translator.close() 1058 | return 0, translator.getOutput() 1059 | 1060 | if __name__ == "__main__": #pragma: no cover 1061 | logging.basicConfig() 1062 | if len(sys.argv) > 1: 1063 | inputFile = open(sys.argv[1], "r") 1064 | else: 1065 | inputFile = sys.stdin 1066 | status, output = html2markdown_file(inputFile) 1067 | if status == 0: 1068 | sys.stdout.write(output) 1069 | sys.exit(status) 1070 | -------------------------------------------------------------------------------- /install-jdk5-osxlion: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # By the way this script heavily inspired/copied from http://www.s-seven.net/java_15_lion 4 | # 5 | # This script is edited by Brice Dutheil 6 | # See there in french http://blog.arkey.fr/2011/08/22/script-pour-installer-le-jdk-5-sur-macosx-lion/ 7 | # Translate button is broken for now, please use Google to translate this website. 8 | 9 | # Make sure only root can run the script 10 | if [ $EUID -ne 0 ]; then 11 | echo "This script must be run as root" 1>&2 12 | exit 1 13 | fi 14 | 15 | # Make sure the user understand he is all alone if something goes wrong 16 | echo 'The present script has been tested on my current setup, and far from bulletproof, 17 | it might not work at all on your system. And there is *no uninstall script* for now!' 18 | echo 'Again this scripts touches system files, please be advised you are the sole 19 | responsible to run or TO NOT run this script on your machine.' 20 | echo -n 'Do you still want to proceed ? (y/n) ' 21 | read answer 22 | [ $answer != 'y' ] && echo 'JDK5 Lion Install script aborted' && exit 1 23 | 24 | 25 | # Reminder about Apple JDK updates 26 | echo 'NOTE : It seems that when applying a Java update from Apple, some important symbolic names 27 | that refer to this install are resetted to factory default values, you can just re-apply 28 | this script. 29 | ' 30 | 31 | #some variables 32 | javapkg='JavaForMacOSX10.5Update10' 33 | jvmver='1.5.0_30' 34 | 35 | if [ ! -f $javapkg.dmg ]; 36 | then 37 | echo 'The "Java for Mac OS X 10.5 Update 10" DMG ('"$javapkg.dmg"') was not found. 38 | Please download it from Apple at : http://support.apple.com/kb/DL1359' 39 | fi 40 | 41 | echo 42 | echo 'Extracting Java for Mac OS X package' 43 | mkdir /tmp/jdk5dmg 44 | hdiutil attach -quiet -nobrowse -mountpoint /tmp/jdk5dmg/ $javapkg.dmg 45 | cd /tmp/jdk5dmg/ 46 | # too bad pkgutil nor xar can stream package content 47 | pkgutil --expand $javapkg.pkg /tmp/jdk5pkg 48 | 49 | cd .. 50 | hdiutil detach -quiet -force /tmp/jdk5dmg/ 51 | rm -rf /tmp/jdk5dmg/ 52 | 53 | echo 54 | echo 'Removing previous Java 1.5 file / directory or symbolic links in :' 55 | cd /System/Library/Java/JavaVirtualMachines 56 | pwd 57 | rm -rf 1.5.0 58 | cd /System/Library/Frameworks/JavaVM.framework/Versions 59 | pwd 60 | rm 1.5/ > /dev/null 2>&1 || rm -rf 1.5 > /dev/null 2>&1 61 | rm 1.5.0/ > /dev/null 2>&1 || rm -rf 1.5.0 > /dev/null 2>&1 62 | rm -rf $jvmver 2>&1 63 | 64 | echo 65 | echo 'Preparing JavaVM framework' 66 | echo '==========================' 67 | 68 | echo 69 | echo 'Extracting JDK 1.5.0 from package payload in :' 70 | cd /System/Library/Frameworks/JavaVM.framework/Versions 71 | pwd 72 | gzip -cd /tmp/jdk5pkg/JavaForMacOSX10.5Update10.pkg/Payload | pax -r -s ',./System/Library/Frameworks/JavaVM.framework/Versions/1.5.0,./'"$jvmver"',' './System/Library/Frameworks/JavaVM.framework/Versions/1.5.0' 73 | ls -Fld 1.5* 74 | 75 | rm -rf /tmp/jdk5pkg/ 76 | 77 | echo 78 | echo 'Recreating symbolic links to ./'"$jvmver"' for 1.5 and 1.5.0 :' 79 | pwd 80 | ln -sivh ./$jvmver 1.5 81 | ln -sivh ./$jvmver 1.5.0 82 | 83 | echo 84 | echo 'Changing values in config files to make JDK work with Lion' 85 | cd $jvmver 86 | /usr/libexec/PlistBuddy -c "Set :JavaVM:JVMMaximumFrameworkVersion 14.*.*" ./Resources/Info.plist 87 | /usr/libexec/PlistBuddy -c "Set :JavaVM:JVMMaximumSystemVersion 10.7.*" ./Resources/Info.plist 88 | /usr/libexec/PlistBuddy -c "Add :CFBundleExecutable string libjava.jnilib" ./Resources/Info.plist 89 | ln -siv ./Resources/Info.plist . 90 | 91 | echo 92 | echo 'Preparing native wraping' 93 | mkdir ./MacOS 94 | ln -siv ../Libraries/libjava.jnilib ./MacOS 95 | 96 | echo 97 | echo 'Preparing Java Virtual Machines' 98 | echo '===============================' 99 | cd /System/Library/Java/JavaVirtualMachines 100 | mkdir -v 1.5.0 101 | cd 1.5.0 102 | pwd 103 | ln -sivh /System/Library/Frameworks/JavaVM.framework/Versions/$jvmver ./Contents 104 | 105 | echo 106 | echo 'And finally check that JDK 5 appears in Java Preference App' 107 | open "/Applications/Utilities/Java Preferences.app" -------------------------------------------------------------------------------- /ip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Use privoxy+tor 4 | # export http_proxy=http://127.0.0.1:8118/ 5 | 6 | while : 7 | do 8 | curl -s checkip.dyndns.org|sed -e 's/.*Current IP Address: //' -e 's/<.*$//' 9 | sleep 120 10 | done -------------------------------------------------------------------------------- /known-wlans.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | defaults read /Library/Preferences/SystemConfiguration/com.apple.airport.preferences RememberedNetworks | egrep -o '(SSID_STR|_timeStamp).+' | sed 's/^.*= \(.*\);$/\1/' | sed 's/^"\(.*\)"$/\1/' | sed 's/\([0-9]\{4\}-..-..\).*/\1/' -------------------------------------------------------------------------------- /lein: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Ensure this file is executable via `chmod a+x lein`, then place it 4 | # somewhere on your $PATH, like ~/bin. The rest of Leiningen will be 5 | # installed upon first run into the ~/.lein/self-installs directory. 6 | 7 | export LEIN_VERSION="2.5.0" 8 | 9 | case $LEIN_VERSION in 10 | *SNAPSHOT) SNAPSHOT="YES" ;; 11 | *) SNAPSHOT="NO" ;; 12 | esac 13 | 14 | if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then 15 | delimiter=";" 16 | else 17 | delimiter=":" 18 | fi 19 | 20 | if [[ "$OSTYPE" == "cygwin" ]]; then 21 | cygwin=true 22 | else 23 | cygwin=false 24 | fi 25 | 26 | function make_native_path { 27 | # ensure we have native paths 28 | if $cygwin && [[ "$1" == /* ]]; then 29 | echo -n "$(cygpath -wp "$1")" 30 | elif [[ "$OSTYPE" == "msys" && "$1" == /?/* ]]; then 31 | echo -n "$(sh -c "(cd $1 2 /dev/null 76 | download_failed_message "$LEIN_URL" "$exit_code" 77 | exit 1 78 | fi 79 | } 80 | 81 | if [ `id -u` -eq 0 ] && [ "$LEIN_ROOT" = "" ]; then 82 | echo "WARNING: You're currently running as root; probably by accident." 83 | echo "Press control-C to abort or Enter to continue as root." 84 | echo "Set LEIN_ROOT to disable this warning." 85 | read _ 86 | fi 87 | 88 | NOT_FOUND=1 89 | ORIGINAL_PWD="$PWD" 90 | while [ ! -r "$PWD/project.clj" ] && [ "$PWD" != "/" ] && [ $NOT_FOUND -ne 0 ] 91 | do 92 | cd .. 93 | if [ "$(dirname "$PWD")" = "/" ]; then 94 | NOT_FOUND=0 95 | cd "$ORIGINAL_PWD" 96 | fi 97 | done 98 | 99 | export LEIN_HOME="${LEIN_HOME:-"$HOME/.lein"}" 100 | 101 | for f in "$LEIN_HOME/leinrc" ".leinrc"; do 102 | if [ -e "$f" ]; then 103 | source "$f" 104 | fi 105 | done 106 | 107 | if $cygwin; then 108 | export LEIN_HOME=`cygpath -w "$LEIN_HOME"` 109 | fi 110 | 111 | LEIN_JAR="$LEIN_HOME/self-installs/leiningen-$LEIN_VERSION-standalone.jar" 112 | 113 | # normalize $0 on certain BSDs 114 | if [ "$(dirname "$0")" = "." ]; then 115 | SCRIPT="$(which $(basename "$0"))" 116 | else 117 | SCRIPT="$0" 118 | fi 119 | 120 | # resolve symlinks to the script itself portably 121 | while [ -h "$SCRIPT" ] ; do 122 | ls=`ls -ld "$SCRIPT"` 123 | link=`expr "$ls" : '.*-> \(.*\)$'` 124 | if expr "$link" : '/.*' > /dev/null; then 125 | SCRIPT="$link" 126 | else 127 | SCRIPT="$(dirname "$SCRIPT"$)/$link" 128 | fi 129 | done 130 | 131 | BIN_DIR="$(dirname "$SCRIPT")" 132 | 133 | export LEIN_JVM_OPTS="${LEIN_JVM_OPTS-"-XX:+TieredCompilation -XX:TieredStopAtLevel=1"}" 134 | 135 | # This needs to be defined before we call HTTP_CLIENT below 136 | if [ "$HTTP_CLIENT" = "" ]; then 137 | if type -p curl >/dev/null 2>&1; then 138 | if [ "$https_proxy" != "" ]; then 139 | CURL_PROXY="-x $https_proxy" 140 | fi 141 | HTTP_CLIENT="curl $CURL_PROXY -f -L -o" 142 | else 143 | HTTP_CLIENT="wget -O" 144 | fi 145 | fi 146 | 147 | 148 | # When :eval-in :classloader we need more memory 149 | grep -E -q '^\s*:eval-in\s+:classloader\s*$' project.clj 2> /dev/null && \ 150 | export LEIN_JVM_OPTS="$LEIN_JVM_OPTS -Xms64m -Xmx512m" 151 | 152 | if [ -r "$BIN_DIR/../src/leiningen/version.clj" ]; then 153 | # Running from source checkout 154 | LEIN_DIR="$(dirname "$BIN_DIR")" 155 | 156 | # Need to use lein release to bootstrap the leiningen-core library (for aether) 157 | if [ ! -r "$LEIN_DIR/leiningen-core/.lein-bootstrap" ]; then 158 | echo "Leiningen is missing its dependencies." 159 | echo "Please run \"lein bootstrap\" in the leiningen-core/ directory" 160 | echo "with a stable release of Leiningen. See CONTRIBUTING.md for details." 161 | exit 1 162 | fi 163 | 164 | # If project.clj for lein or leiningen-core changes, we must recalculate 165 | LAST_PROJECT_CHECKSUM=$(cat "$LEIN_DIR/.lein-project-checksum" 2> /dev/null) 166 | PROJECT_CHECKSUM=$(sum "$LEIN_DIR/project.clj" "$LEIN_DIR/leiningen-core/project.clj") 167 | if [ "$PROJECT_CHECKSUM" != "$LAST_PROJECT_CHECKSUM" ]; then 168 | if [ -r "$LEIN_DIR/.lein-classpath" ]; then 169 | rm "$LEIN_DIR/.lein-classpath" 170 | fi 171 | fi 172 | 173 | # Use bin/lein to calculate its own classpath. 174 | if [ ! -r "$LEIN_DIR/.lein-classpath" ] && [ "$1" != "classpath" ]; then 175 | echo "Recalculating Leiningen's classpath." 176 | ORIG_PWD="$PWD" 177 | cd "$LEIN_DIR" 178 | 179 | LEIN_NO_USER_PROFILES=1 $0 classpath .lein-classpath 180 | sum "$LEIN_DIR/project.clj" "$LEIN_DIR/leiningen-core/project.clj" > \ 181 | .lein-project-checksum 182 | cd "$ORIG_PWD" 183 | fi 184 | 185 | mkdir -p "$LEIN_DIR/target/classes" 186 | export LEIN_JVM_OPTS="$LEIN_JVM_OPTS -Dclojure.compile.path=$LEIN_DIR/target/classes" 187 | add_path CLASSPATH "$LEIN_DIR/leiningen-core/src/" "$LEIN_DIR/leiningen-core/resources/" \ 188 | "$LEIN_DIR/test:$LEIN_DIR/target/classes" "$LEIN_DIR/src" ":$LEIN_DIR/resources" 189 | 190 | if [ -r "$LEIN_DIR/.lein-classpath" ]; then 191 | add_path CLASSPATH "$(cat "$LEIN_DIR/.lein-classpath" 2> /dev/null)" 192 | else 193 | add_path CLASSPATH "$(cat "$LEIN_DIR/leiningen-core/.lein-bootstrap" 2> /dev/null)" 194 | fi 195 | else # Not running from a checkout 196 | add_path CLASSPATH "$LEIN_JAR" 197 | 198 | BOOTCLASSPATH="-Xbootclasspath/a:$LEIN_JAR" 199 | 200 | if [ ! -r "$LEIN_JAR" -a "$1" != "self-install" ]; then 201 | self_install 202 | fi 203 | fi 204 | 205 | # TODO: explain what to do when Java is missing 206 | export JAVA_CMD="${JAVA_CMD:-"java"}" 207 | export LEIN_JAVA_CMD="${LEIN_JAVA_CMD:-$JAVA_CMD}" 208 | 209 | if [[ -z "${DRIP_INIT+x}" && "$(basename "$LEIN_JAVA_CMD")" == *drip* ]]; then 210 | export DRIP_INIT="$(printf -- '-e\n(require (quote leiningen.repl))')" 211 | export DRIP_INIT_CLASS="clojure.main" 212 | fi 213 | 214 | # Support $JAVA_OPTS for backwards-compatibility. 215 | export JVM_OPTS="${JVM_OPTS:-"$JAVA_OPTS"}" 216 | 217 | # Handle jline issue with cygwin not propagating OSTYPE through java subprocesses: https://github.com/jline/jline2/issues/62 218 | cygterm=false 219 | if $cygwin; then 220 | case "$TERM" in 221 | rxvt* | xterm* | vt*) cygterm=true ;; 222 | esac 223 | fi 224 | 225 | if $cygterm; then 226 | LEIN_JVM_OPTS="$LEIN_JVM_OPTS -Djline.terminal=jline.UnixTerminal" 227 | stty -icanon min 1 -echo > /dev/null 2>&1 228 | fi 229 | 230 | # TODO: investigate http://skife.org/java/unix/2011/06/20/really_executable_jars.html 231 | # If you're packaging this for a package manager (.deb, homebrew, etc) 232 | # you need to remove the self-install and upgrade functionality or see lein-pkg. 233 | if [ "$1" = "self-install" ]; then 234 | if [ -r "$BIN_DIR/../src/leiningen/version.clj" ]; then 235 | echo "Running self-install from a checkout is not supported." 236 | echo "See CONTRIBUTING.md for SNAPSHOT-specific build instructions." 237 | exit 1 238 | fi 239 | echo "Manual self-install is deprecated; it will run automatically when necessary." 240 | self_install 241 | elif [ "$1" = "upgrade" ] || [ "$1" = "downgrade" ]; then 242 | if [ "$LEIN_DIR" != "" ]; then 243 | echo "The upgrade task is not meant to be run from a checkout." 244 | exit 1 245 | fi 246 | if [ $SNAPSHOT = "YES" ]; then 247 | echo "The upgrade task is only meant for stable releases." 248 | echo "See the \"Hacking\" section of the README." 249 | exit 1 250 | fi 251 | if [ ! -w "$SCRIPT" ]; then 252 | echo "You do not have permission to upgrade the installation in $SCRIPT" 253 | exit 1 254 | else 255 | TARGET_VERSION="${2:-stable}" 256 | echo "The script at $SCRIPT will be upgraded to the latest $TARGET_VERSION version." 257 | echo -n "Do you want to continue [Y/n]? " 258 | read RESP 259 | case "$RESP" in 260 | y|Y|"") 261 | echo 262 | echo "Upgrading..." 263 | TARGET="/tmp/lein-$$-upgrade" 264 | if $cygwin; then 265 | TARGET=`cygpath -w $TARGET` 266 | fi 267 | LEIN_SCRIPT_URL="https://github.com/technomancy/leiningen/raw/$TARGET_VERSION/bin/lein" 268 | $HTTP_CLIENT "$TARGET" "$LEIN_SCRIPT_URL" 269 | if [ $? == 0 ]; then 270 | cmp -s "$TARGET" "$SCRIPT" 271 | if [ $? == 0 ]; then 272 | echo "Leiningen is already up-to-date." 273 | fi 274 | mv "$TARGET" "$SCRIPT" && chmod +x "$SCRIPT" 275 | exec "$SCRIPT" version 276 | else 277 | download_failed_message "$LEIN_SCRIPT_URL" 278 | fi;; 279 | *) 280 | echo "Aborted." 281 | exit 1;; 282 | esac 283 | fi 284 | else 285 | if $cygwin; then 286 | # When running on Cygwin, use Windows-style paths for java 287 | ORIGINAL_PWD=`cygpath -w "$ORIGINAL_PWD"` 288 | fi 289 | 290 | # apply context specific CLASSPATH entries 291 | if [ -f .lein-classpath ]; then 292 | add_path CLASSPATH "$(cat .lein-classpath)" 293 | fi 294 | 295 | if [ $DEBUG ]; then 296 | echo "Leiningen's classpath: $CLASSPATH" 297 | fi 298 | 299 | if [ -r .lein-fast-trampoline ]; then 300 | export LEIN_FAST_TRAMPOLINE='y' 301 | fi 302 | 303 | if [ "$LEIN_FAST_TRAMPOLINE" != "" ] && [ -r project.clj ]; then 304 | INPUTS="$@ $(cat project.clj) $LEIN_VERSION $(test -f "$LEIN_HOME/profiles.clj" && cat "$LEIN_HOME/profiles.clj")" 305 | export INPUT_CHECKSUM=$(echo $INPUTS | shasum - | cut -f 1 -d " ") 306 | # Just don't change :target-path in project.clj, mkay? 307 | TRAMPOLINE_FILE="target/trampolines/$INPUT_CHECKSUM" 308 | else 309 | if hash mktemp 2>/dev/null; then 310 | # Check if mktemp is available before using it 311 | TRAMPOLINE_FILE="$(mktemp /tmp/lein-trampoline-XXXXXXXXXXXXX)" 312 | else 313 | TRAMPOLINE_FILE="/tmp/lein-trampoline-$$" 314 | fi 315 | trap "rm -f $TRAMPOLINE_FILE" EXIT 316 | fi 317 | 318 | if $cygwin; then 319 | TRAMPOLINE_FILE=`cygpath -w $TRAMPOLINE_FILE` 320 | fi 321 | 322 | if [ "$INPUT_CHECKSUM" != "" ] && [ -r "$TRAMPOLINE_FILE" ]; then 323 | if [ $DEBUG ]; then 324 | echo "Fast trampoline with $TRAMPOLINE_FILE." 325 | fi 326 | exec sh -c "exec $(cat $TRAMPOLINE_FILE)" 327 | else 328 | export TRAMPOLINE_FILE 329 | "$LEIN_JAVA_CMD" \ 330 | "${BOOTCLASSPATH[@]}" \ 331 | -Dfile.encoding=UTF-8 \ 332 | -Dmaven.wagon.http.ssl.easy=false \ 333 | -Dmaven.wagon.rto=10000 \ 334 | $LEIN_JVM_OPTS \ 335 | -Dleiningen.original.pwd="$ORIGINAL_PWD" \ 336 | -Dleiningen.script="$SCRIPT" \ 337 | -classpath "$CLASSPATH" \ 338 | clojure.main -m leiningen.core.main "$@" 339 | 340 | EXIT_CODE=$? 341 | 342 | if $cygterm ; then 343 | stty icanon echo > /dev/null 2>&1 344 | fi 345 | 346 | ## TODO: [ -r "$TRAMPOLINE_FILE" ] may be redundant? A trampoline file 347 | ## is always generated these days. 348 | if [ -r "$TRAMPOLINE_FILE" ] && [ "$LEIN_TRAMPOLINE_WARMUP" = "" ]; then 349 | TRAMPOLINE="$(cat $TRAMPOLINE_FILE)" 350 | if [ "$INPUT_CHECKSUM" = "" ]; then 351 | rm $TRAMPOLINE_FILE 352 | fi 353 | if [ "$TRAMPOLINE" = "" ]; then 354 | exit $EXIT_CODE 355 | else 356 | exec sh -c "exec $TRAMPOLINE" 357 | fi 358 | else 359 | exit $EXIT_CODE 360 | fi 361 | fi 362 | fi 363 | -------------------------------------------------------------------------------- /list-extensions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # on non mac-systems 'sed -rn' 4 | find . -type f | sed -En 's|.*/[^/]+\.([^/.]+)$|\1|p' | sort -u -------------------------------------------------------------------------------- /load-test-config.txt: -------------------------------------------------------------------------------- 1 | # Change the user agent 2 | #--user-agent "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7" 3 | 4 | # Follow location 5 | --location 6 | 7 | -w @load-test-format.txt 8 | #---dump-header headers.txt 9 | 10 | -H "Pragma: no-cache" 11 | -H "Cache-control: no-cache" 12 | -H "Connection: close" 13 | -------------------------------------------------------------------------------- /load-test-format.txt: -------------------------------------------------------------------------------- 1 | \n 2 | %{url_effective}\t%{http_code}\t%{content_type}\t%{time_total}\t%{time_connect}\t%{time_namelookup}\t%{time_pretransfer}\t%{time_starttransfer}\t%{size_download}\n 3 | \n -------------------------------------------------------------------------------- /load-test-urls.txt: -------------------------------------------------------------------------------- 1 | http://en.wikipedia.org/wiki/Main_Page -------------------------------------------------------------------------------- /load-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "" > output-raw.txt 4 | 5 | for url in `cat load-test-urls.txt` 6 | do 7 | curl -K load-test-config.txt $url >> output-raw.txt 8 | done 9 | 10 | echo "" > output-processed.txt 11 | 12 | sort load-test-urls.txt | uniq | while read line 13 | do 14 | grep $line output-raw.txt >> output-processed.txt 15 | done 16 | -------------------------------------------------------------------------------- /location: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | scselect `scselect | grep -i $1 | awk '{print $1}'` -------------------------------------------------------------------------------- /markdown2dayone.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | 4 | for i in * 5 | do 6 | 7 | filename=$i 8 | 9 | YEAR=${filename:0:2} 10 | MONTH=${filename:3:2} 11 | DAY=${filename:6:2} 12 | 13 | ENTRYDATE=$MONTH/$DAY/$YEAR 14 | 15 | echo $filename 16 | dayone -d="$ENTRYDATE" new < "$i" 17 | 18 | done -------------------------------------------------------------------------------- /mateup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby -w 2 | 3 | # Updates Textmate bundles using the Github repositories 4 | 5 | # Original version found on http://blog.bleything.net/pages/mateup 6 | 7 | ENV['LC_ALL'] = nil 8 | ENV['LC_CTYPE'] = 'en_US.UTF-8' 9 | 10 | github_root = 'git://github.com' 11 | bundles = { 12 | 'Ant' => 'textmate/ant.tmbundle', 13 | 'Apache' => 'textmate/apache.tmbundle', 14 | 'Copy as RTF' => 'drnic/copy-as-rtf-tmbundle', 15 | 'Git' => 'jcf/git-tmbundle', 16 | 'Html5' => 'johnmuhl/html5.tmbundle', 17 | 'LintMate' => 'rondevera/jslintmate', 18 | 'Json' => 'textmate/json.tmbundle', 19 | 'Make' => 'textmate/make.tmbundle', 20 | 'Markdown' => 'oschrenk/markdown.tmbundle', 21 | 'Maven' => 'textmate/maven.tmbundle', 22 | 'PHP' => 'textmate/php.tmbundle', 23 | 'Play' => 'oschrenk/play.tmbundle', 24 | 'RDoc' => 'joshaven/RDoc.tmbundle', 25 | 'Regexp' => 'danielvlopes/regexp-tmbundle', 26 | 'Ruby' => 'drnic/ruby-tmbundle', 27 | 'Uncrustify' => 'oschrenk/Uncrustify.tmbundle', 28 | 'Yaml' => 'textmate/yaml.tmbundle' 29 | } 30 | 31 | # escape spaces and ampersands 32 | def cleanup(str) 33 | return str.gsub(/([ &])/, '\\\\\1') 34 | end 35 | 36 | dir = "#{ENV['HOME']}/Library/Application\ Support/TextMate/Bundles" 37 | begin 38 | Dir.chdir dir 39 | rescue Errno::ENOENT 40 | puts "Bundles directory doesn't exist... creating it!" 41 | puts 42 | 43 | `mkdir -p '#{dir}'` 44 | retry 45 | end 46 | 47 | Dir.entries('.').each do |e| 48 | next if e =~ /^\./ 49 | next unless File.directory? e 50 | 51 | bundle_name = e.gsub(/.tmbundle$/, '') 52 | 53 | print "* #{bundle_name}: " 54 | 55 | if bundles.delete bundle_name 56 | puts "bundle exists, updating" 57 | `cd #{cleanup e}; git pull --quiet origin master; cd ..;` 58 | 59 | else 60 | print "don't know about this bundle. Delete it? [y/N] " 61 | 62 | while answer = gets 63 | if answer =~ /^y/ 64 | `rm -rf #{cleanup e}` 65 | puts " * deleted" 66 | break 67 | elsif answer =~ /^(n|$)/ 68 | break 69 | else 70 | print "Please enter 'y' or 'n': " 71 | end 72 | end 73 | end 74 | end 75 | 76 | bundles.each do |name, source| 77 | puts "* #{name} doesn't exist; fetching..." 78 | cmd = "git clone #{github_root}/#{source} #{cleanup name}.tmbundle" 79 | `#{cmd}`.match(/(revision \d+)/) 80 | puts " * checked out #{$1}" 81 | end 82 | 83 | `arch -i386 osascript -e 'tell app "TextMate" to reload bundles'` -------------------------------------------------------------------------------- /md2wiki.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | my $infile = shift or 7 | die "Usage: $0 \n"; 8 | 9 | open my $in, $infile or 10 | die "Cannot open $infile for reading: $!\n"; 11 | 12 | my $s = do { local $/; <$in> }; 13 | 14 | close $in; 15 | 16 | $s =~ s/^([^\s=][^\n]*?)\n=+\n/= $1 =\n/smg; 17 | $s =~ s/^([^\s=][^\n]*?)\n-+\n/== $1 ==\n/smg; 18 | $s =~ s{^\* \*\*(Syntax|Default|Context|Phase):\*\* ([^\n]+)} 19 | {my ($k, $v) = ($1, $2); $v =~ s/`//g; "'''" . lcfirst($k) . ":''' ''$v''\n"}smge; 20 | $s =~ s{^`([^`\n]*)`}{$1}smg; 21 | while ($s =~ s{^([^`\n\s].*?)`([^\`\n]*)`}{$1$2}smg) {} 22 | while ($s =~ s{^([^\*\s\n].*?)\*\*([^\n\*]*)\*\*}{$1'''$2'''}smg) {} 23 | while ($s =~ s{^([^\*\s\n].*?)\*([-\w]*)\*}{$1''$2''}smg) {} 24 | $s =~ s/^\t([^*\s])/: $1/gsm; 25 | $s =~ s/^(?:\t\t\* )/*** /gsm; 26 | $s =~ s/^(?:\t\* )/** /gsm; 27 | $s =~ s{^[ \t]+\S[^\n]*(?:\n(?:\n+|[ \t]+[^\n]*))*} 28 | {my $v = $&; $v =~ s/\n+$/\n/; qq[\n$v\n]}smge; 29 | $s =~ s{\n\n+(lua_need_request_body)}{[[#$1|$1]]}gsm; 31 | $s =~ s{((?:content|rewrite|access|set)_by_lua(?:_file|\*)?)} 32 | {my $v = $1; $v =~ s/\*$//; "[[#$v|$v]]"}gesm; 33 | $s =~ s{ngx\.(say|print|time|http_time|exit|send_header|redirect|exec|var\.VARIABLE|location\.capture(?:_multi)?)(?:\(\))?}{[[#ngx.$1|ngx.$1]]}gsm; 34 | $s =~ s/\n\n\n+/\n\n/gs; 35 | $s =~ s{ \[ ([^\n\]]*) \] \( (https?://[^\n\)]+) \)} 36 | {[$2 $1]}gmsx; 37 | $s =~ s{<(https?://[^>]+)>[ \t]*}{$1 }mgs; 38 | $s =~ s/^\d+\.\s/# /gms; 39 | 40 | print $s; 41 | 42 | 43 | -------------------------------------------------------------------------------- /mvnda.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | mvn archetype:generate -DarchetypeGroupId=com.oschrenk.maven -DarchetypeArtifactId=default-archetype -DarchetypeVersion=1.0-SNAPSHOT -------------------------------------------------------------------------------- /ogg2mp3: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | ogg2mp3 (Transcode OGG to MP3) 5 | 6 | Requirements: 7 | * python 8 | * oggdec 9 | * ogginfo 10 | * lame 11 | 12 | History: 13 | -------- 14 | 2005-07-06 Darren Stone Created. http://bitmason.com 15 | 2007-02-25 Darren Stone Fixed shell quote bug 16 | 2007-10-14 Darren Stone More decode + encode error trapping. Detect unrecognized genres (LAME limitation). 17 | 2007-12-08 Darren Stone Case-insensitive match on ogginfo keys. (Thanx to Owen Emmerson for the catch.) 18 | 19 | Run with --help for usage. 20 | 21 | Distribute, use, and modify freely, but please keep the 22 | history above and usage below up to date! 23 | """ 24 | 25 | 26 | from sys import argv, exit, stdout 27 | from os import system, stat, getpid, unlink 28 | from os.path import basename 29 | from signal import signal, SIGINT 30 | from commands import getoutput 31 | 32 | 33 | # base LAME command. 34 | # 1. ID3 tags, etc. will be automatically appended if available 35 | # 2. user-supplied command line options will also be appended 36 | lame_cmd_base = 'lame --quiet --ignore-tag-errors' 37 | 38 | # Describes mapping from ogginfo keys to LAME ID3 tag commands. 39 | # Match will be case-insensitive on the OGG side. 40 | ogg_to_id3 = { 41 | 'TITLE': '--tt', 42 | 'ARTIST': '--ta', 43 | 'ALBUM': '--tl', 44 | 'GENRE': '--tg', 45 | 'COMMENT': '--tc', 46 | 'DATE': '--ty', 47 | 'TRACKNUMBER': '--tn', 48 | } 49 | 50 | # supported ID3 genres 51 | LAME_GENRES = [] 52 | 53 | # temporary WAV file will be created (and later removed) in current working directory 54 | WAV_FILENAME_PREFIX = "_tmp_ogg2mp3_" 55 | 56 | def init_lame_genres(): 57 | """ 58 | LAME supports only a limited set of textual genres (despite ID3V2 allowing for 59 | user defined strings). This helps us detect and report this limitation. 60 | """ 61 | for line in getoutput("lame --genre-list").splitlines(): 62 | try: 63 | LAME_GENRES.append(line.strip().split(' ', 1)[1].lower()) 64 | except: 65 | pass 66 | 67 | def ogg_info_dict(oggfilename): 68 | """ 69 | Return dictionary of ogginfo, containing at least the keys in 70 | ogg_to_id3 -if- they are present in the ogg file. 71 | """ 72 | d = {} 73 | out = getoutput("ogginfo %s" % shell_quote(oggfilename)) 74 | out = out.splitlines() 75 | for line in out: 76 | for k in ogg_to_id3.keys(): 77 | i = line.lower().find(k.lower()+'=') 78 | if i != -1: 79 | d[k] = line[i+len(k)+1:].strip() 80 | return d 81 | 82 | 83 | def file_size(filename): 84 | """ 85 | Return size of file, in bytes. 86 | """ 87 | return stat(filename).st_size 88 | 89 | 90 | def size_to_human(bytes): 91 | """ 92 | Return string representation of the byte count, human-readable 93 | (i.e. in B, KB, MB, or GB) 94 | """ 95 | if bytes >= 1024*1024*1024: 96 | return "%0.1f GB" % (float(bytes)/1024.0/1024.0/1024.0) 97 | elif bytes >= 1024*1024: 98 | return "%0.1f MB" % (float(bytes)/1024.0/1024.0) 99 | elif bytes >= 1024: 100 | return "%0.1f KB" % (float(bytes)/1024.0) 101 | else: 102 | return "%d B" % bytes 103 | 104 | 105 | def file_size_human(filename): 106 | """ 107 | Return string representation of the filename, human-readable 108 | (i.e. in B, KB, MB, or GB) 109 | """ 110 | return size_to_human(stat(filename).st_size) 111 | 112 | 113 | def shell_quote(s): 114 | """ 115 | Quote and escape the given string (if necessary) for inclusion in 116 | a shell command 117 | """ 118 | return "\"%s\"" % s.replace('"', '\"') 119 | 120 | 121 | def transcode(oggfilename): 122 | """ 123 | Transcode given OGG to MP3 in current directory, with .mp3 extension, 124 | transferring meta info where possible. 125 | Return (oggsize, mp3size). 126 | """ 127 | try: 128 | wavfilename = "%s%d.wav" % (WAV_FILENAME_PREFIX, getpid()) 129 | mp3filename = "%s.mp3" % basename(oggfilename)[:-4] 130 | oggsize = file_size_human(oggfilename) 131 | stdout.write("%s (%s)\n" % (oggfilename, oggsize)) 132 | oggdict = ogg_info_dict(oggfilename) 133 | encode_cmd = lame_cmd_base 134 | for k in oggdict.keys(): 135 | k = k.upper() 136 | knote = '' 137 | if k in ogg_to_id3.keys(): 138 | if k == 'GENRE' and oggdict[k].lower() not in LAME_GENRES: 139 | knote = "[WARNING: Unrecognized by LAME so MP3 genre will be 'Other']" 140 | encode_cmd = "%s %s %s" % (encode_cmd, ogg_to_id3[k], shell_quote(oggdict[k])) 141 | stdout.write(" %s: %s %s\n" % (str(k), str(oggdict[k]), knote)) 142 | stdout.write("%s " % mp3filename) 143 | stdout.flush() 144 | decode_cmd = "oggdec --quiet -o %s %s 2>/dev/null" % (shell_quote(wavfilename), shell_quote(oggfilename)) 145 | system(decode_cmd) 146 | wavsize = 0 147 | try: 148 | wavsize = file_size(wavfilename) 149 | except: 150 | pass 151 | if wavsize <= 0: 152 | stdout.write("[FAILED] OGG did not decode to intermediate WAV\n\n") 153 | return (file_size(oggfilename), 0) 154 | encode_cmd = "%s %s %s 2>/dev/null" % (encode_cmd, wavfilename, shell_quote(mp3filename)) 155 | system(encode_cmd) 156 | try: 157 | mp3size = file_size_human(mp3filename) 158 | except: 159 | stdout.write("[FAILED] OGG decoded but MP3 encoding and/or tagging failed\n\n") 160 | return (file_size(oggfilename), 0) 161 | stdout.write("(%s)\n\n" % mp3size) 162 | except Exception, e: 163 | stdout.write(str(e)) 164 | try: 165 | unlink(wavfilename) 166 | except: 167 | pass 168 | return (file_size(oggfilename), file_size(mp3filename)) 169 | 170 | 171 | def sig_int_handler(p0, p1): 172 | """ Make CTRL-C less catasrophic """ 173 | pass 174 | 175 | 176 | if __name__ == '__main__': 177 | 178 | # TODO: ensure oggdec, ogginfo, lame are available 179 | 180 | signal(SIGINT, sig_int_handler) 181 | failure = False # True iff one or more files fail 182 | 183 | if len(argv) < 2 or (len(argv) >= 2 and argv[1] in ('-h', '--help', '-?')): 184 | progname = basename(argv[0]) 185 | print "Usage: %s [LAME_OPTIONS] FILE1 [FILE2 [FILE3 ...]]" % progname 186 | print "\nTranscode FILE(s) from OGG to MP3." 187 | print "MP3s with same basename and .mp3 extension will be written to current working" 188 | print "directory and meta info will be transferred to ID3 tags where possible." 189 | print "\nExamples:" 190 | print "%s -B 256 --vbr-new -V 0 *.ogg (decent quality VBR)" % progname 191 | print "%s -m m -s 22.05 -b 56 -q 9 --lowpass 8 *.ogg (lo-fi, fast, mono CBR)" % progname 192 | exit(1) 193 | 194 | # append user-supplied cmd line options (for LAME) 195 | argv.pop(0) 196 | while not argv[0].lower().endswith('.ogg'): 197 | lame_cmd_base = "%s %s" % (lame_cmd_base, argv[0]) 198 | argv.pop(0) 199 | 200 | init_lame_genres() 201 | 202 | fcount = 0 203 | oggsize = 0 204 | mp3size = 0 205 | for f in argv: 206 | (s1, s2) = transcode(f) 207 | fcount += 1 208 | oggsize += s1 209 | mp3size += s2 210 | if s1 == 0 or s2 == 0: failure = True 211 | 212 | # summary 213 | if fcount > 1: 214 | stdout.write("%s OGGs (%s) transcoded to MP3 (%s)\n" % ( 215 | fcount, 216 | size_to_human(oggsize), 217 | size_to_human(mp3size))) 218 | if failure: 219 | stdout.write("One or more files failed to transcode. Review output above.\n") 220 | 221 | exit(int(failure)) 222 | -------------------------------------------------------------------------------- /osx-cleanup-open-with.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | /System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister -kill -r -domain local -domain system -domain user 3 | killall Finder -------------------------------------------------------------------------------- /papers2bibtex: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2011 Steve Lianoglou, all rights reserved. 4 | # License: GPL 5 | # 6 | # Taken from https://github.com/lianos/papers-bin 7 | # 8 | # The papers.py script drives functionality against Papers2 9 | # 10 | # Currently this script was written to auto-generate a bibtex file for the 11 | # citations used in 1+ latex files. To generate one bibtex file to stdout 12 | # for all of the *.tex files in the current directory, put papers.py in 13 | # your PATH and: 14 | # 15 | # $ papers.py bibtex *.tex > refs.bib 16 | # 17 | # Requires Python >= 2.5 and Papers >= 2.0.8 18 | # 19 | # Create a `~/.papersrc` to override default settings. This file should be 20 | # formatted such that Python's ConfigParser can read it. 21 | # 22 | # For instance, I store my Papers2 folder in my Dropbox, so my `~/.papersrc` 23 | # file looks like so: 24 | # 25 | # [appinfo] 26 | # dbpath = /Users/stavros/Dropbox/Papers2/Library.papers2/Database.papersdb 27 | # 28 | # Feel free to submit bugs/feature request through the tickets for this project. If you fork and extend this project, I'll gladly accept pull requests. 29 | # 30 | # DISCLAIMER 31 | # ---------- 32 | # 33 | # This script is not supported or endorsed by the Papers2 team. I'm making queries directly against the Papers database, the structure of which may change whenever they seem it fit to do so. 34 | # 35 | # The functionality in the scripts provided will perform READ-ONLY queries 36 | # against your Papers.app sqlite database. I cannot guarantee that it will not 37 | # corrupt your database, so use it at your own risk. 38 | # 39 | # That having been said, I use this script too, so it really shouldn't corrupt 40 | # your database. Murphy's law being what it is and all, I feel compelled 41 | # to write this blurb in case it wasn't obvious. 42 | 43 | 44 | 45 | 46 | """ 47 | Drives functionality that interacts with the Papers2 database. 48 | 49 | Requires Python >= 2.5 and Papers >= 2.0.8 50 | 51 | Copyright 2011 Steve Lianoglou, all rights reserved 52 | 53 | License: GPL 54 | """ 55 | 56 | import re, os, time, sys, glob, itertools, sqlite3 57 | from optparse import OptionParser 58 | from ConfigParser import ConfigParser, NoOptionError 59 | 60 | ## You can overide these values in ~/.papersc 61 | DEFAULTS = { 62 | 'dbpath' : "~/Documents/Papers2/Library.papers2/Database.papersdb", 63 | } 64 | 65 | def filter_files(filelist): 66 | """Returns a list of files that can be 'found'""" 67 | found = [] 68 | if len(filelist) == 0: 69 | return found 70 | for infile in filelist: 71 | if os.path.isfile(infile): 72 | found.append(infile) 73 | return found 74 | 75 | def dict_factory(cursor, row): 76 | """Used to extract results from a sqlite3 row by name""" 77 | d = {} 78 | for idx, col in enumerate(cursor.description): 79 | d[col[0]] = row[idx] 80 | return d 81 | 82 | ############################################################################### 83 | ## Interface to Papers 84 | class Papers(object): 85 | """Interface to Papers2.app""" 86 | 87 | _xlate_month = { 88 | '01' : 'Jan', '02' : 'Feb', '03' : 'Mar', '04' : 'Apr', 89 | '05' : 'May', '06' : 'Jun', '07' : 'Jul', '08' : 'Aug', 90 | '09' : 'Sep', '10' : 'Oct', '11' : 'Nov', '12' : 'Dec' 91 | } 92 | 93 | def __init__(self, dbpath): 94 | self.dbpath = dbpath 95 | self.dbconn = sqlite3.connect(dbpath) 96 | 97 | ## Checks to see if this is a valid db connection 98 | c = self.dbconn.cursor() 99 | try: 100 | c.execute("SELECT * FROM metadata LIMIT 1;") 101 | except sqlite3.OperationalError: 102 | raise ValueError("Invalid Papers database") 103 | self.dbconn.row_factory = dict_factory 104 | 105 | def parse_publication_date(self, pub_date, translate_month=True, month=(6,7), 106 | year=(2,5)): 107 | """99200406011200000000222000 == Jun 2004 108 | returns (month, year) as strings 109 | """ 110 | try: 111 | cmonth = pub_date[month[0]:month[1]+1] 112 | if translate_month: 113 | cmonth = Papers._xlate_month[cmonth] 114 | except: 115 | cmonth = '' 116 | try: 117 | cyear = pub_date[year[0]:year[1]+1] 118 | except: 119 | cyear = '' 120 | return {'month' : cmonth, 'year' : cyear} 121 | 122 | def query_papers_by_citekey(self, citekeys, n=100): 123 | """Returns summary information for each paper matched to citekey(s). 124 | 125 | The returned object is a `dict` keyed on the citekey for each paper, 126 | the values are dicts with the following minimal paper info: 127 | 128 | - title : 129 | - authors : Firstname Lastname, First Last, and First Last 130 | - journal : Journal name (as listed in Publications db), this can 131 | be done better by JOINing against NameVariant, but we 132 | are not doing that for now 133 | - citekey : The citekey 134 | 135 | And optionally, if these are found in the Publication record: 136 | 137 | - volume : Journal volume 138 | - number : Journal number 139 | - pages : start--end pages 140 | - month : Month of publication date (as 3 letter name) 141 | - year : 4 digit (character) year of publication 142 | """ 143 | query = """SELECT publication_date, full_author_string, 144 | attributed_title, bundle_string, volume, number, 145 | startpage, endpage, citekey 146 | FROM Publication WHERE citekey IN (%s)""" 147 | results = {} 148 | c = self.dbconn.cursor() 149 | while len(citekeys) > 0: 150 | take = min(len(citekeys), n) 151 | cites = ['"%s"' % x for x in citekeys[0:take]] 152 | cites = ','.join(cites) 153 | citekeys = citekeys[take:] 154 | c.execute(query % cites) 155 | for row in c: 156 | date = self.parse_publication_date(row['publication_date']) 157 | citekey = row['citekey'] 158 | entry = { 159 | 'title' : row['attributed_title'], 160 | 'author' : row['full_author_string'], 161 | 'journal' : row['bundle_string'], 162 | 'citekey' : citekey 163 | } 164 | if date['month'] is not None: 165 | entry['month'] = date['month'] 166 | if date['year'] is not None: 167 | entry['year'] = date['year'] 168 | if row['number'] is not None: 169 | entry['number'] = row['number'] 170 | if row['volume'] is not None: 171 | entry['volume'] = row['volume'] 172 | if row['startpage'] is not None and row['endpage'] is not None: 173 | entry['pages'] = "%s--%s" % (row['startpage'], row['endpage']) 174 | results[citekey] = entry 175 | return results 176 | 177 | # END : Class Papers 178 | 179 | class PapersOptionParser(OptionParser, object): 180 | """Documentation for PapersOptionParser""" 181 | 182 | def __init__(self, usage=None): 183 | super(PapersOptionParser, self).__init__(usage=usage) 184 | self.add_option('-o', '--out', dest="out", default=None, 185 | help="The file to save the output to, defaults " \ 186 | "to STDOUT") 187 | self.add_option('-d', '--dbpath', dest="dbpath", default=None, 188 | help="The path to the Papers2 sqlite database, " \ 189 | "defaults to [%s]. If this is set, it will " \ 190 | "override the value set in your ~/.papersrc" \ 191 | "file." % DEFAULTS['dbpath']) 192 | self.add_option('-v', '--verbose', action='store_true', default=False, 193 | help='Make some noise') 194 | self.add_option('-c', '--config', default="~/.papersrc", 195 | help="The path to the papers utility config file") 196 | self.add_option('-f', '--force', dest='force', default=False, 197 | action='store_true', 198 | help="Set to force overwrite of existing output file") 199 | self.out = None 200 | self.report = None 201 | self.to_stdout = None 202 | 203 | 204 | def parse_args(self, args=None, values=None): 205 | """Parses the arguments. 206 | 207 | The precedence of arguments that get stuffed into `options` are 208 | (from highest to lowest): 209 | 210 | - values passed in through command line args/flags 211 | - values set in ~/.papersrc 212 | - DEFAULTS 213 | """ 214 | (options, args) = super(PapersOptionParser, self).parse_args(args, values) 215 | 216 | if options.out is None: 217 | self.to_stdout = True 218 | self.out = sys.stdout 219 | self.report = sys.stderr 220 | else: 221 | if os.path.isfile(options.out) and not options.force: 222 | self.error("Outfile already exists. Use --force to override") 223 | self.to_stdout = False 224 | self.out = open(options.out, 'w') 225 | self.report = sys.stdout 226 | 227 | ## override options with values in ~/.papersrc 228 | config_file = os.path.expanduser(options.config) 229 | if os.path.isfile(config_file): 230 | cparser = ConfigParser() 231 | cparser.read(config_file) 232 | if options.dbpath is None: 233 | try: 234 | options.dbpath = cparser.get('appinfo', 'dbpath') 235 | except NoOptionError: 236 | pass 237 | 238 | if options.dbpath is None: 239 | options.dbpath = DEFAULTS['dbpath'] 240 | 241 | return (options, args) 242 | 243 | def cleanup(self): 244 | if not self.to_stdout: 245 | self.out.close() 246 | 247 | # END : Class PapersOptionParser 248 | 249 | 250 | ############################################################################### 251 | ## BibTex Generator 252 | ## Generates rudimentary bibtex file by parsing \cite*{} 253 | ## references in a document(s), and crossreferences the citekeys 254 | ## with the ones in your Papers2 databse. 255 | ## 256 | ## 257 | ## Minimal BibTex entry looks like so: 258 | ## 259 | ## @article{Proudfoot:2004gs, citekey 260 | ## author = {Proudfoot, Nick}, author_string 261 | ## title = {{New perspectives on connecting ...}}, attributed_title 262 | ## journal = {Current opinion in cell biology}, bundle_string 263 | ## year = {2004}, publication_date (99200406011200000000222000) 264 | ## month = {jun}, 265 | ## volume = {16}, volume 266 | ## number = {3}, number 267 | ## pages = {272--278} startpage -- endpage 268 | ## } 269 | ## 270 | ## To get the journal name, use the `bundle` column in Pulblication and join it 271 | ## to NameVariant.object_id 272 | ## 273 | ## select 274 | ## p.publication_date, p.author_string, p.attributed_title, 275 | ## p.bundle, p.bundle_string, p.volume, p.number, p.startpage, 276 | ## p.endpage, n.name 277 | ## from 278 | ## Publication as p 279 | ## inner join NameVariant as n on p.bundle=n.object_id 280 | ## where 281 | ## p.citekey="Sandberg:2008ks"; 282 | ## 283 | ## Forget the complex query, just use bundle_string for journal name 284 | class BibtexOptionParser(PapersOptionParser): 285 | """OptionParser for the bibtex command""" 286 | 287 | usage = """usage: %prog bibtex [OPTIONS] FILE1 [FILES ...] 288 | 289 | Parses the file(s) identified by the unix blob-like matching patterns 290 | provided in the positional arguments for cite*{...} commands in 291 | them and generates a minimal bibtex file for them by looking up 292 | the citekeys in your Papers2 database. 293 | 294 | If a -o/--out BIBFILE.tex option is not provided, the bibtex file will 295 | be streamed to STDOUT.""" 296 | 297 | def __init__(self): 298 | super(BibtexOptionParser, self).__init__(usage=BibtexOptionParser.usage) 299 | self.infiles = [] 300 | 301 | def parse_args(self, args=sys.argv[2:], values=None): 302 | (options, args) = super(BibtexOptionParser, self).parse_args(args, values) 303 | 304 | ## OptionParser already matches and expands unix globs for us! 305 | ## match input files and flatten + uniqify potentiall nested list 306 | ## infiles = [glob.glob(fn) for fn in args] 307 | ## infiles = set(itertools.chain(*infiles)) 308 | self.infiles = filter_files(args) 309 | if len(self.infiles) == 0: 310 | self.error("Valid input file list required (no files found)") 311 | return (options, args) 312 | 313 | 314 | # END : Class BibtexOptionParser 315 | 316 | class BibtexGenerator(object): 317 | """Generats bibtex file from input""" 318 | 319 | citekey_regex = re.compile(r"""\\cite(?:t|p)?\{(.*?)\}""", re.MULTILINE) 320 | 321 | def __init__(self, app, infiles, author_style="default"): 322 | self.app = app 323 | self.infiles = filter_files(infiles) 324 | self.author_style = author_style 325 | self.citekeys = {} 326 | 327 | def extract_citekeys_from_line(self, line, store=True, regex=None): 328 | if regex is None: 329 | regex = BibtexGenerator.citekey_regex 330 | citations = regex.findall(line) 331 | citekeys = [] 332 | if len(citations) > 0: 333 | for citation in citations: 334 | for citekey in citation.split(','): 335 | citekey = citekey.strip() 336 | citekeys.append(citekey) 337 | if store: 338 | try: 339 | self.citekeys[citekey] += 1 340 | except KeyError: 341 | self.citekeys[citekey] = 1 342 | return citekeys 343 | 344 | def extract_citekeys_from_file(self, infile, store=True, regex=None): 345 | if regex is None: 346 | regex = BibtexGenerator.citekey_regex 347 | allkeys = [] 348 | fh = open(infile, 'r') 349 | for line in fh: 350 | allkeys.append(self.extract_citekeys_from_line(line, store=store)) 351 | fh.close() 352 | return allkeys 353 | 354 | def extract_citekeys(self, infiles=None): 355 | """Extracts the citekeys for `infiles`""" 356 | if infiles is None: 357 | infiles = self.infiles 358 | else: 359 | infiles = filter_files(infiles) 360 | if len(self.infiles) == 0: 361 | raise ValueError("No input files found") 362 | for infile in infiles: 363 | self.extract_citekeys_from_file(infile, store=True) 364 | 365 | def convert_author_style(self, author_string, style=None): 366 | if style is None: 367 | style = self.author_style 368 | if style == "default": 369 | authors = re.sub(r"\Wand\W", " ", author_string).split(',') 370 | mangled = list() 371 | for author in authors: 372 | pieces = author.strip().split() 373 | lastname = pieces[-1] 374 | rest = ' '.join(pieces[:-1]) 375 | mangled.append("%s, %s" % (lastname, rest)) 376 | author_string = ' and '.join(mangled) 377 | return author_string 378 | 379 | def as_bibtex(self, info): 380 | result = [] 381 | header = '@article{%s,\n' % info['citekey'].encode('utf-8') 382 | info['author'] = self.convert_author_style(info['author']) 383 | for key in info: 384 | if key == 'citekey': 385 | continue 386 | if key == 'title': 387 | add = 'title = {{%s}}' % info['title'].encode('utf-8') 388 | else: 389 | add = '%s = {%s}' % (key, info[key].encode('utf-8')) 390 | result.append(add) 391 | meta = ",\n".join(result) 392 | result = header + meta + "\n}\n" 393 | return result 394 | 395 | def generate_bibtex(self, fhandle): 396 | """Dumps the generated bibtex file into fhandle""" 397 | citations = self.app.query_papers_by_citekey(self.citekeys.keys()) 398 | for citation in citations: 399 | fhandle.write(self.as_bibtex(citations[citation])) 400 | fhandle.write("\n") 401 | 402 | # END : Class BibtexGenerator 403 | 404 | ## Drivers -- all these functions must accept a Papers (app) object as 405 | ## their single parameter 406 | 407 | def do_bibtex(app): 408 | """Run the bibtex command""" 409 | parser = BibtexOptionParser() 410 | (options, args) = parser.parse_args() 411 | 412 | report = parser.report 413 | outfile = parser.out 414 | 415 | # try: 416 | # app = Papers(options.dbpath) 417 | # except sqlite3.OperationalError: 418 | # parser.error("Problem connecting to database, is the following " \ 419 | # "path to your Database.papersdb database correct?\n" \ 420 | # "\t%s\n" % options.dbpath) 421 | 422 | if options.verbose: 423 | report.write("Parsing files: " + ','.join(parser.infiles) + "\n") 424 | 425 | bibtex = BibtexGenerator(app, parser.infiles) 426 | bibtex.extract_citekeys() 427 | bibtex.generate_bibtex(outfile) 428 | 429 | if options.verbose: 430 | report.write("=== Citekeys Used ===\n") 431 | for citation, count in bibtex.citekeys.iteritems(): 432 | report.write("%s : %d\n" % (citation, count)) 433 | 434 | parser.cleanup() 435 | 436 | 437 | if __name__ == '__main__': 438 | usage = """usage: %prog COMMAND [OPTIONS] ARGS 439 | 440 | This tool is a wrapper for (eventually) several COMMANDs that query your 441 | Papers2 database. Try `%prog COMMAND --help` for help for the specific 442 | COMMANDs that are listed below. 443 | 444 | Commands 445 | -------- 446 | - bibtex : Generates a bibtex file by parsing the references in the 447 | files provided in ARGS 448 | """ 449 | 450 | commands = {'bibtex' : do_bibtex} 451 | 452 | usage = usage.replace("%prog", os.path.basename(sys.argv[0])) 453 | parser = PapersOptionParser(usage=usage) 454 | (options, args) = parser.parse_args() 455 | 456 | if len(args) == 0: 457 | user_cmd = '' 458 | else: 459 | user_cmd = args[0] 460 | 461 | if user_cmd not in commands: 462 | if len(user_cmd) > 0: 463 | user_cmd = "'%s'" % user_cmd 464 | parser.error("Unknown command %s\n" % user_cmd) 465 | 466 | try: 467 | app = Papers(options.dbpath) 468 | except ValueError: 469 | parser.error("Problem connecting to database, is the following " \ 470 | "path to your Database.papersdb database correct?\n" \ 471 | "\t%s\n" % options.dbpath) 472 | 473 | commands[user_cmd](app) -------------------------------------------------------------------------------- /pdfdiff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # This Python file uses the following encoding: latin-1 3 | """ 4 | pdfdiff.py : inspect the difference between two PDF files. 5 | 6 | Copyright (C) 2007 Cas Cremers 7 | 8 | This program is free software; you can redistribute it and/or 9 | modify it under the terms of the GNU General Public License 10 | as published by the Free Software Foundation; either version 2 11 | of the License, or (at your option) any later version. 12 | 13 | This program is distributed in the hope that it will be useful, 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | GNU General Public License for more details. 17 | 18 | You should have received a copy of the GNU General Public License 19 | along with this program; if not, write to the Free Software 20 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 21 | 02110-1301, USA. 22 | """ 23 | 24 | """ 25 | Module dependencies 26 | """ 27 | import sys 28 | import string 29 | import commands 30 | import os.path 31 | import tempfile 32 | 33 | """ 34 | Global declarations 35 | """ 36 | # Preference order of diff viewers (top most is most preferred) 37 | # Note that e.g.: 38 | # 39 | # kdiff3 works well with unicode things and can nicely do things like 40 | # '\phi'. 41 | # 42 | # Meld shows unicode well but I couldn't get it to wrap as I wanted from 43 | # the command line (you can use preferences though). 44 | # 45 | diffViewers = [ \ 46 | "kdiff3 --cs WordWrap=1 --cs ShowWhiteSpaceCharacters=0", \ 47 | "meld", \ 48 | "tkdiff", \ 49 | "xxdiff", \ 50 | "gvimdiff", \ 51 | "vimdiff", \ 52 | "diff", \ 53 | "opendiff", \ 54 | ] 55 | 56 | # pdftotext program with switches 57 | pdftotextProgram = "pdftotext" 58 | pdftotextOptions = "-nopgbrk" 59 | 60 | # Myname 61 | progName = "pdfdiff.py" 62 | progVersion = "0.92" 63 | 64 | # Define what a long sentence is. 65 | # When a sentence is longer than this, any punctuations count as sentence 66 | # ends. 67 | longSentenceLength = 50 68 | 69 | 70 | """ 71 | Code overview. 72 | 73 | The procedure is fairly trivial. We exploit pdftotext, which converts 74 | pdf files to text. However, this does not work very well because in 75 | general, semantical sentences are distributed randomly over file lines. 76 | We use a very crude form of normalization that attempts to output (file) 77 | lines that somewhat correspond to sentences. In practice, this turns out 78 | to be sufficient for diff programs to work. 79 | 80 | With respect to the diff programs, pdftotext handles formulas amazingly 81 | well, and turns most symbols into useful unicode. Thus, it is worthwile 82 | to have a diff viewer (kdiff3, meld) that can display these. Also, make 83 | sure to turn on word wrap for full effect. 84 | 85 | I'm sure it can be done better/faster/cleaner/..., as this is just a 86 | hack, so feel free to improve it. Please send me an e-mail with the 87 | result if you do. I also bet there is somebody that can do it in one 88 | line using sed. 89 | 90 | The code is split into five sections: 91 | 92 | 1. Basics 93 | 2. Text normalization 94 | 3. Conversions from format A to B 95 | 4. High-level commands 96 | 5. Main code 97 | 98 | """ 99 | 100 | 101 | #------------------------------------------------------------------------- 102 | # 1. Basics 103 | #------------------------------------------------------------------------- 104 | 105 | def get_viewer_list(): 106 | """ 107 | Return the list of viewers 108 | """ 109 | global diffViewers 110 | 111 | return map(lambda s:(s.split())[0], diffViewers) 112 | 113 | 114 | def is_command_available(prg): 115 | """ 116 | Detect whether prg exists. Note that it may have switches, i.e. 117 | it will find "kdiff3 -a" 118 | """ 119 | cmd = "which %s" % ((prg.split())[0]) 120 | (status,out) = commands.getstatusoutput(cmd) 121 | return (status == 0) 122 | 123 | 124 | def find_first(plist): 125 | """ 126 | Find the first program from the list that exists. 127 | """ 128 | for prg in plist: 129 | if is_command_available(prg): 130 | return prg 131 | return None 132 | 133 | 134 | def apply_command_temp(prg,options,notfound,filename,prefix="",suffix=""): 135 | """ 136 | Execute 'prg options filename tempout' if prg exists. 137 | Report 'notfound' if prg is not there. 138 | 139 | Returns (tempfileFilehandle,output) tuple. 140 | """ 141 | fout = tempfile.NamedTemporaryFile(suffix=suffix,prefix=prefix) 142 | 143 | if not is_command_available(prg): 144 | print "Error: %s" % (notfound) 145 | sys.exit(1) 146 | 147 | cmd = "%s %s \"%s\" \"%s\"" % (prg,options,filename,fout.name) 148 | output = commands.getoutput(cmd) 149 | return (fout,output) 150 | 151 | 152 | def make_prefix(fname): 153 | """ 154 | Turn file name into a prefix we can use. 155 | """ 156 | (head,tail) = os.path.split(fname) 157 | (root,ext) = os.path.splitext(tail) 158 | return root + "_" 159 | 160 | 161 | def get_filetype(filename): 162 | """ 163 | Hack to determine the filetype. 164 | """ 165 | (head,tail) = os.path.split(filename) 166 | (root,ext) = os.path.splitext(tail) 167 | lowerext = ext.lower() 168 | if lowerext in ['.pdf','.fdf']: 169 | return "pdf" 170 | elif lowerext in ['.ps']: 171 | return "ps" 172 | else: 173 | return "txt" 174 | 175 | 176 | def fix_ff_problem(sentence): 177 | """ 178 | Hack to fix an often occurring latex problem with 'ff' combinations. 179 | This is ultimately a font problem (with Times New Roman), and not our 180 | problem (probably latex, alternatively pdftotext ought to fix it). 181 | For now, we just stupidly revert the weird character combos. 182 | """ 183 | sentence = sentence.replace("ffi","ffi") 184 | sentence = sentence.replace("ffl","ffl") 185 | sentence = sentence.replace("ff","ff") 186 | return sentence 187 | 188 | 189 | #------------------------------------------------------------------------- 190 | # 2. Text normalization 191 | #------------------------------------------------------------------------- 192 | 193 | def is_sentence_end(c): 194 | """ 195 | The following characters are considered to be sentence endings for our 196 | normalization. 197 | """ 198 | return c in ".!?" 199 | 200 | 201 | def is_sentence_break(c): 202 | """ 203 | The following characters are considered to be sentence breaks for our 204 | normalization of long sentences. 205 | """ 206 | return c in string.punctuation 207 | 208 | 209 | def is_sentence_done(sentence): 210 | """ 211 | Detect whether the sentence is done 212 | """ 213 | global longSentenceLength 214 | 215 | if len(sentence) > 0: 216 | if is_sentence_end(sentence[-1]): 217 | return True 218 | else: 219 | if len(sentence) >= longSentenceLength: 220 | if is_sentence_break(sentence[-1]): 221 | return True 222 | return False 223 | 224 | 225 | def flush_sentence(fout,forceNewLine = False): 226 | """ 227 | Flush the sentence buffer. 228 | """ 229 | global sentenceBuf 230 | global lastWordLength 231 | 232 | lastWordLength = 0 233 | l = sentenceBuf.lstrip() 234 | l = fix_ff_problem(l) 235 | fout.write(l) 236 | if forceNewLine or (sentenceBuf != ""): 237 | fout.write("\n") 238 | sentenceBuf = "" 239 | 240 | 241 | def normalize_text(fin,fout): 242 | """ 243 | Normalize the lines read from fin, and output to fout, which 244 | are file handles. 245 | """ 246 | global sentenceBuf 247 | global lastWordLength 248 | 249 | sentenceBuf = "" # stores unfinished sentences 250 | wordLength = 0 251 | lastWordLength = 0 252 | skipEnds = False 253 | 254 | # Alternatively, we could use xreadlines, if the files are really 255 | # really huge. 256 | for l in fin.readlines(): 257 | # Cut of spacing from both ends 258 | ls = l.strip() 259 | 260 | # Empty line or not? 261 | if ls == "": 262 | # This occurs when there is an empty line. 263 | # We flush the sentence, and force a newline. 264 | # 265 | # Any further additional empty lines have no effect, 266 | # which is enforced by skipEnds. 267 | if not skipEnds: 268 | flush_sentence(fout) 269 | flush_sentence(fout,True) 270 | skipEnds = True 271 | else: 272 | # The file line is not empty, so this is some sort of 273 | # paragraph 274 | skipEnds = False 275 | if sentenceBuf != "": 276 | if not sentenceBuf[-1] in string.whitespace: 277 | sentenceBuf += " " 278 | 279 | for c in ls: 280 | # Append the character to the current buffer. 281 | sentenceBuf += c 282 | 283 | # Some admin to know how long the last word was. 284 | if c in string.ascii_letters: 285 | wordLength += 1 286 | lastWordLength = wordLength 287 | else: 288 | wordLength = 0 289 | 290 | if is_sentence_done(sentenceBuf): 291 | # If the last word is only a single character, 292 | # it's assumed that the punctuation does not 293 | # refer to a sentence end. 294 | if lastWordLength != 1: 295 | # Sentence has ended, so flush it. 296 | # We should skip any spacing directly after 297 | # the sentence end mark. 298 | flush_sentence(fout) 299 | 300 | flush_sentence(fout) 301 | fout.flush() 302 | 303 | 304 | #------------------------------------------------------------------------- 305 | # 3. Conversions from format A to B 306 | #------------------------------------------------------------------------- 307 | 308 | def ps_to_pdf(filename,prefix=""): 309 | """ 310 | ps to pdf conversion 311 | """ 312 | prg = "ps2pdf" 313 | notfound = "Could not find 'ps2pdf', which is needed for ps to pdf conversion." 314 | (fout,output) = apply_command_temp(prg,"",notfound,filename,prefix,".pdf") 315 | return fout 316 | 317 | 318 | def pdf_to_text(filename,prefix=""): 319 | """ 320 | pdf to text conversion 321 | """ 322 | global pdftotextProgram,pdftotextOptions 323 | 324 | notfound = """\ 325 | Could not find '%s', which is needed for pdf to text conversion. 326 | %s is part of the 'xPdf' suite of programs, obtainable at: 327 | http://www.foolabs.com/xpdf/ 328 | """ % (pdftotextProgram,pdftotextProgram) 329 | (fout,output) = apply_command_temp(pdftotextProgram,pdftotextOptions,notfound,filename,prefix,".txt") 330 | return fout 331 | 332 | 333 | def normalize_anything(filename,fout=sys.stdout): 334 | """ 335 | This function takes any file type and tries to apply converters 336 | until we can finall churn out normalized text. 337 | """ 338 | prefix = make_prefix(filename) 339 | filetype = get_filetype(filename) 340 | 341 | # Iterate until we have text 342 | temphandle = None 343 | fhandle = None 344 | while filetype != "txt": 345 | if filetype == "pdf": 346 | fhandle = pdf_to_text(filename,prefix=prefix) 347 | elif filetype == "ps": 348 | fhandle = ps_to_pdf(filename,prefix=prefix) 349 | else: 350 | print "Error: Don't know how to handle file type '%s'" % (filetype) 351 | sys.exit(1) 352 | if temphandle: 353 | temphandle.close() 354 | 355 | filename = fhandle.name 356 | filetype = get_filetype(filename) 357 | # Store for destruction of intermediate objects later 358 | temphandle = fhandle 359 | 360 | if not fhandle: 361 | fhandle = open(filename,'r') 362 | 363 | # Now fhandle is considered text 364 | normalize_text(fhandle,fout) 365 | 366 | 367 | def normalize_anything_tempfile(filename): 368 | """ 369 | Normalize anything with a wrapper for tempfile generation. 370 | """ 371 | prefix = make_prefix(filename) 372 | fout = tempfile.NamedTemporaryFile(suffix=".txt",prefix=prefix) 373 | normalize_anything(filename,fout) 374 | return fout 375 | 376 | 377 | #------------------------------------------------------------------------- 378 | # 4. High-level commands 379 | #------------------------------------------------------------------------- 380 | 381 | def view_diff(fnleft,fnright): 382 | """ 383 | Show the diff between two files using the first program that is 384 | found. 385 | """ 386 | global diffViewers 387 | global diffViewerPrefix 388 | 389 | fleft = normalize_anything_tempfile(fnleft) 390 | fright = normalize_anything_tempfile(fnright) 391 | 392 | viewers = [] 393 | if diffViewerPrefix != "": 394 | # Attempt to use the prefix as a program (overrides defaults) 395 | viewers = [diffViewerPrefix] 396 | # Also add filtered known ones 397 | viewers += filter(lambda s:s.startswith(diffViewerPrefix),diffViewers) 398 | # Add known ones 399 | viewers += diffViewers 400 | 401 | prg = find_first(viewers) 402 | 403 | if prg == None: 404 | estr = "Error: Could not find a suitable diff viewer from the list %s" % (diffViewers) 405 | print estr 406 | sys.exit(1) 407 | 408 | cmd = "%s \"%s\" \"%s\"" % (prg,fleft.name,fright.name) 409 | out = commands.getoutput(cmd) 410 | # Also print the result (e.g. for programs like diff that send 411 | # output to stdout) 412 | print out 413 | 414 | fleft.close() 415 | fright.close() 416 | 417 | 418 | def display_help(): 419 | """ 420 | Program manual 421 | """ 422 | global progName, progVersion 423 | global diffViewers 424 | 425 | helpstr = """\ 426 | PRG version %s 427 | Copyright 2007 Cas Cremers 428 | 429 | Usage: PRG [switches] [] 430 | 431 | View the difference between two files, or output a normalized version 432 | of the text in a single file. 433 | Supported file types are: pdf,ps,txt. 434 | 435 | Switches: 436 | -d , --diffviewer | 437 | Try to use the diff viewer of the given name, or try to select 438 | the first available diffviewer from the list: 439 | %s 440 | that starts with . 441 | """ % (progVersion, ", ".join(get_viewer_list())) 442 | print helpstr.replace("PRG",progName) 443 | 444 | 445 | #------------------------------------------------------------------------- 446 | # 5. Main code 447 | #------------------------------------------------------------------------- 448 | 449 | if __name__ == "__main__": 450 | """ 451 | Main code 452 | """ 453 | global diffViewerPrefix 454 | 455 | args = sys.argv[1:] 456 | diffViewerPrefix = "" 457 | 458 | # No arguments, show help 459 | if len(args) == 0: 460 | display_help() 461 | sys.exit(0) 462 | 463 | # Check for special commands 464 | while len(args) > 0: 465 | optcmd = args[0] 466 | if optcmd in ["-?","-h","--help"]: 467 | # Help 468 | display_help() 469 | sys.exit(0) 470 | 471 | elif optcmd in ["-d","--diffviewer"]: 472 | # Selecting diff viewer prefix 473 | if len(args) < 2: 474 | print "Error: Diff viewer preference requires a string prefix argument" 475 | sys.exit(1) 476 | diffViewerPrefix = args[1] 477 | if len(filter(lambda s:s.startswith(diffViewerPrefix),get_viewer_list())) == 0: 478 | print "Error: No viewer from the list %s starts with '%s'" % (get_viewer_list(),diffViewerPrefix) 479 | sys.exit(1) 480 | args = args[2:] 481 | 482 | else: 483 | # Default mode: 1 argument is normalize, 2 is diff 484 | if len(args) == 1: 485 | normalize_anything(args[0]) 486 | sys.exit(0) 487 | elif len(args) == 2: 488 | view_diff(args[0],args[1]) 489 | sys.exit(0) 490 | else: 491 | print "Error: I don't know what to do with more than two files" 492 | sys.exit(1) 493 | 494 | # vim: set ts=4 sw=4 et fileencoding=latin1 list lcs=tab\:>-: 495 | -------------------------------------------------------------------------------- /pickjdk: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Provides a function that allows you to choose a JDK. Just set the environment 4 | # variable JDKS_ROOT to the directory containing multiple versions of the JDK 5 | # and the function will prompt you to select one. JAVA_HOME and PATH will be cleaned 6 | # up and set appropriately. 7 | 8 | _macosx() 9 | { 10 | if [ $(uname -s) = Darwin ]; then 11 | return 0 12 | else 13 | return 1 14 | fi 15 | } 16 | 17 | JDKS_ROOT= 18 | if _macosx; then 19 | JDKS_ROOT=/Library/Java/JavaVirtualMachines 20 | fi 21 | 22 | pickjdk() 23 | { 24 | if [ -z "$JDKS_ROOT" ]; then 25 | return 1 26 | fi 27 | 28 | declare -a JDKS 29 | local n=1 jdk total_jdks choice=0 currjdk=$JAVA_HOME explicit_jdk 30 | for jdk in $JDKS_ROOT/*; do 31 | if [ -d $jdk ]; then 32 | JDKNAMES[$n]="$(basename $jdk)" 33 | if _macosx; then 34 | jdk=$jdk/Contents/Home 35 | fi 36 | if [ -z "$1" ]; then 37 | echo -n " $n) ${JDKNAMES[$n]}" 38 | if [ $jdk = "$currjdk" ]; then 39 | echo " < CURRENT" 40 | else 41 | echo 42 | fi 43 | fi 44 | JDKS[$n]=$jdk 45 | total_jdks=$n 46 | n=$[ $n + 1 ] 47 | fi 48 | done 49 | if [ -z "$1" ]; then 50 | echo " $n) None" 51 | fi 52 | JDKS[$n]=None 53 | total_jdks=$n 54 | 55 | if [ $total_jdks -gt 1 ]; then 56 | if [ -z "$1" ]; then 57 | while [ -z "${JDKS[$choice]}" ]; do 58 | echo -n "Choose one of the above [1-$total_jdks]: " 59 | read choice 60 | done 61 | else 62 | choice=$1 63 | fi 64 | fi 65 | 66 | if [ -z "$currjdk" ]; then 67 | currjdk=$(dirname $(dirname $(type -path java))) 68 | fi 69 | 70 | if [ ${JDKS[$choice]} != None ]; then 71 | export JAVA_HOME=${JDKS[$choice]} 72 | else 73 | unset JAVA_HOME 74 | fi 75 | 76 | explicit_jdk= 77 | for jdk in ${JDKS[*]}; do 78 | if [ "$currjdk" = "$jdk" ]; then 79 | explicit_jdk=$jdk 80 | break 81 | fi 82 | done 83 | 84 | if [ "$explicit_jdk" ]; then 85 | if [ -z "$JAVA_HOME" ]; then 86 | PATH=$(echo $PATH | sed "s|$explicit_jdk/bin:*||g") 87 | else 88 | PATH=$(echo $PATH | sed "s|$explicit_jdk|$JAVA_HOME|g") 89 | fi 90 | elif [ "$JAVA_HOME" ]; then 91 | PATH="$JAVA_HOME/bin:$PATH" 92 | fi 93 | 94 | echo "New JDK: ${JDKNAMES[$choice]}" 95 | 96 | hash -r 97 | unset JDKS 98 | } -------------------------------------------------------------------------------- /prettify-xml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for file in $(ls *.xml) 4 | do 5 | echo "Formatting $file" 6 | xmllint --format $file -o $file 7 | done 8 | -------------------------------------------------------------------------------- /remove-duplicate-blank-lines.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Reduces two or more consecutive blank lines to a single blank line 4 | 5 | awk '/^$/{ if (! blank++) print; next } { blank=0; print }' "$1" > "$1.old"; rm "$1"; mv "$1.old" "$1"; -------------------------------------------------------------------------------- /remove-trailing-whitespace.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Remove trailing whitespace of text files in directory 4 | find . -type f -name "*.txt" -exec sh -c 'for i;do sed 's/[[:space:]]*$//' "$i">/tmp/.$$ && mv /tmp/.$$ "$i";done' arg0 {} + -------------------------------------------------------------------------------- /replace-underscore-with-space.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ls *_* | while read -r FILE 4 | do 5 | mv "$FILE" "`echo "$FILE" | tr '_' ' '`" 6 | done -------------------------------------------------------------------------------- /rip2flac: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Taken from http://planet.jboss.org/post/how_to_rip_audio_cd_as_flac_with_embedded_cue_sheet_in_ubuntu_linux 3 | 4 | DEVICE="/dev/cdrom" 5 | COPIES="3" 6 | 7 | DISCID="`cd-discid $DEVICE 2>&1 | egrep -o '^[0-9a-z]{8}'`" 8 | if [ "$?" != "0" ]; then 9 | echo "Failed to retrieve disc ID." 10 | exit 1 11 | fi 12 | 13 | echo -e "\033[1;32mDisc ID: $DISCID\033[0;0m" 14 | rm -f "$DISCID".* 15 | 16 | echo -e "\033[1;32mExtracting a cue sheet:\033[0;0m" 17 | 18 | cdrdao read-toc --device "$DEVICE" --datafile "$DISCID.wav" "$DISCID.toc" || exit 2 19 | cueconvert -i toc -o cue "$DISCID.toc" | grep -vP '^ISRC "' > "$DISCID.cue" || exit 2 20 | 21 | CHECKSUM='' 22 | I=0 23 | for ((I=0; I < $COPIES; I++)); do 24 | echo 25 | echo -e "\033[1;32mPass $((I+1)) of $COPIES\033[0;0m:" 26 | 27 | if [[ $I -eq 0 ]]; then 28 | OUT="$DISCID.wav" 29 | else 30 | OUT="$DISCID.new.wav" 31 | fi 32 | rm -f "$OUT" 33 | 34 | cdparanoia -zX '1-' "$OUT" 35 | if [ "$?" != "0" ]; then 36 | rm -f "$DISCID".* 37 | echo "Failed to rip a disc." 38 | exit 3 39 | fi 40 | 41 | C="`sha1sum "$OUT" | cut -f1 -d' '`" 42 | if [[ "x$CHECKSUM" = 'x' ]]; then 43 | echo "Checksum: $C" 44 | CHECKSUM=$C 45 | else 46 | rm -f "$OUT" 47 | if [[ "$CHECKSUM" != "$C" ]]; then 48 | echo "Mismatching checksum: $C" 49 | exit 4 50 | else 51 | echo "Matching checksum: $C" 52 | fi 53 | fi \ 54 | done 55 | 56 | eject "$DEVICE" & 57 | 58 | echo 59 | echo -en "\033[1;32mCompressing...\033[0;0m" 60 | flac -f -V --replay-gain --best --cuesheet="$DISCID.cue" "$DISCID.wav" 61 | if [ "$?" != "0" ]; then 62 | echo "Failed to encode the ripped tracks." 63 | exit 5 64 | fi 65 | 66 | rm -f "$DISCID.wav" "$DISCID.toc" 67 | 68 | echo 69 | echo -e "\033[1;32mAll done: $DISCID.flac\033[0;0m" 70 | -------------------------------------------------------------------------------- /setos: -------------------------------------------------------------------------------- 1 | # OS Environment Variables for tests 2 | # source: http://stackoverflow.com/questions/394230 3 | 4 | lowercase(){ 5 | echo "$1" | sed "y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/" 6 | } 7 | 8 | OS=`uname | tr '[A-Z]' '[a-z]'` 9 | KERNEL=`uname -r` 10 | MACH=`uname -m` 11 | 12 | if [ "${OS}" == "windowsnt" ]; then 13 | OS=windows 14 | elif [ "${OS}" == "darwin" ]; then 15 | OS=mac 16 | else 17 | OS=`uname` 18 | if [ "${OS}" = "SunOS" ] ; then 19 | OS=Solaris 20 | ARCH=`uname -p` 21 | OSSTR="${OS} ${REV}(${ARCH} `uname -v`)" 22 | elif [ "${OS}" = "AIX" ] ; then 23 | OSSTR="${OS} `oslevel` (`oslevel -r`)" 24 | elif [ "${OS}" = "Linux" ] ; then 25 | if [ -f /etc/redhat-release ] ; then 26 | DistroBasedOn='RedHat' 27 | DIST=`cat /etc/redhat-release |sed s/\ release.*//` 28 | PSEUDONAME=`cat /etc/redhat-release | sed s/.*\(// | sed s/\)//` 29 | REV=`cat /etc/redhat-release | sed s/.*release\ // | sed s/\ .*//` 30 | elif [ -f /etc/SuSE-release ] ; then 31 | DistroBasedOn='SuSe' 32 | PSEUDONAME=`cat /etc/SuSE-release | tr "\n" ' '| sed s/VERSION.*//` 33 | REV=`cat /etc/SuSE-release | tr "\n" ' ' | sed s/.*=\ //` 34 | elif [ -f /etc/mandrake-release ] ; then 35 | DistroBasedOn='Mandrake' 36 | PSEUDONAME=`cat /etc/mandrake-release | sed s/.*\(// | sed s/\)//` 37 | REV=`cat /etc/mandrake-release | sed s/.*release\ // | sed s/\ .*//` 38 | elif [ -f /etc/debian_version ] ; then 39 | DistroBasedOn='Debian' 40 | DIST=`cat /etc/lsb-release | grep '^DISTRIB_ID' | awk -F= '{ print $2 }'` 41 | PSEUDONAME=`cat /etc/lsb-release | grep '^DISTRIB_CODENAME' | awk -F= '{ print $2 }'` 42 | REV=`cat /etc/lsb-release | grep '^DISTRIB_RELEASE' | awk -F= '{ print $2 }'` 43 | fi 44 | if [ -f /etc/UnitedLinux-release ] ; then 45 | DIST="${DIST}[`cat /etc/UnitedLinux-release | tr "\n" ' ' | sed s/VERSION.*//`]" 46 | fi 47 | OS=`echo $OS | tr '[A-Z]' '[a-z]'` 48 | DistroBasedOn=`lowercase $DistroBasedOn` 49 | fi 50 | fi 51 | 52 | readonly OS 53 | readonly DIST 54 | readonly DistroBasedOn 55 | readonly PSEUDONAME 56 | readonly REV 57 | readonly KERNEL 58 | readonly MACH 59 | 60 | -------------------------------------------------------------------------------- /ssh-copy-id.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | KEY="$HOME/.ssh/id_rsa.pub" 4 | 5 | if [ ! -f ~/.ssh/id_rsa.pub ];then 6 | echo "private key not found at $KEY" 7 | echo "* please create it with "ssh-keygen -t rsa" *" 8 | echo "* to login to the remote host without a password, don't give the key you create with ssh-keygen a password! *" 9 | exit 10 | fi 11 | 12 | if [ -z $1 ];then 13 | echo "Please specify user@host.tld as the first switch to this script" 14 | exit 15 | fi 16 | 17 | echo "Putting your key on $1... " 18 | 19 | KEYCODE=`cat $KEY` 20 | ssh -q $1 "mkdir ~/.ssh 2>/dev/null; chmod 700 ~/.ssh; echo "$KEYCODE" >> ~/.ssh/authorized_keys; chmod 644 ~/.ssh/authorized_keys" 21 | 22 | echo "done!" -------------------------------------------------------------------------------- /sshagent: -------------------------------------------------------------------------------- 1 | # Setup SSH-agent found at http://www.cygwin.com/ml/cygwin/2001-06/msg00537.html 2 | SSH_ENV="$HOME/.ssh/environment" 3 | 4 | function start_agent { 5 | echo "Initialising new SSH agent..." 6 | /usr/bin/ssh-agent | sed 's/^echo/#echo/' > "${SSH_ENV}" 7 | echo succeeded 8 | chmod 600 "${SSH_ENV}" 9 | . "${SSH_ENV}" > /dev/null 10 | /usr/bin/ssh-add; 11 | } 12 | 13 | # Source SSH settings, if applicable 14 | 15 | if [ -f "${SSH_ENV}" ]; then 16 | . "${SSH_ENV}" > /dev/null 17 | #ps ${SSH_AGENT_PID} doesn't work under cywgin 18 | ps -ef | grep ${SSH_AGENT_PID} | grep ssh-agent$ > /dev/null || { 19 | start_agent; 20 | } 21 | else 22 | start_agent; 23 | fi 24 | -------------------------------------------------------------------------------- /sublimeup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # Original version for Textmate found on 3 | # http://blog.bleything.net/pages/mateup 4 | 5 | # Updates Sublime 2/3 packages using their respective Github repositories 6 | 7 | # Manage your packages in 8 | # touch ~/.sublime.packages 9 | # For each package create a new line like this 10 | # Git => kemayo/sublime-text-2-git#python3 11 | # 12 | # Use ' => ' as a separator between package name and github repository 13 | # Use '#branch' to pull from a specific branch 14 | require 'uri' 15 | require 'tempfile' 16 | 17 | ENV['LC_ALL'] = nil 18 | ENV['LC_CTYPE'] = 'en_US.UTF-8' 19 | 20 | github_root = 'git://github.com' 21 | 22 | packages = Hash[File.read(ENV['HOME']+'/.sublime.packages').split("\n").map{|i|i.split(' => ')}] 23 | 24 | # packages provided by default installation 25 | providedPackages = [ 26 | 'ActionScript', 27 | 'AppleScript', 28 | 'ASP',# 29 | 'Batch File', 30 | 'C#', 31 | 'C++', 32 | 'Clojure', 33 | 'Color Scheme - Default', 34 | 'CSS', 35 | 'D', 36 | 'Default', 37 | 'Diff', 38 | 'Erlang', 39 | 'Go', 40 | 'Graphviz', 41 | 'Groovy', 42 | 'Haskell', 43 | 'HTML', 44 | 'Java', 45 | 'JavaScript', 46 | 'Language - English', 47 | 'LaTeX', 48 | 'Lisp', 49 | 'Lua', 50 | 'Makefile', 51 | 'Markdown', 52 | 'Matlab', 53 | 'Objective-C', 54 | 'OCaml', 55 | 'Pascal', 56 | 'Python', 57 | 'Perl', 58 | 'PHP', 59 | 'R', 60 | 'Rails', 61 | 'Regular Expressions', 62 | 'RestructuredText', 63 | 'Ruby', 64 | 'Scala', 65 | 'ShellScript', 66 | 'SQL', 67 | 'TCL', 68 | 'Text', 69 | 'Textile', 70 | 'Theme - Default', 71 | 'User', 72 | 'Vintage', 73 | 'XML', 74 | 'YAML' 75 | ] 76 | 77 | # escape spaces and ampersands 78 | def cleanup(str) 79 | return str.gsub(/([ &])/, '\\\\\1') 80 | end 81 | 82 | 83 | def repo(source) 84 | if (source.nil? || source.split(/#/).length != 2) 85 | return source 86 | end 87 | 88 | return source.split(/#/)[0] 89 | end 90 | 91 | def branch(source) 92 | if (source.nil? || source.split(/#/).length != 2) 93 | return "master" 94 | end 95 | 96 | return source.split(/#/)[1] 97 | end 98 | 99 | def packageDir() 100 | if RUBY_PLATFORM.downcase.include?("darwin") 101 | return "#{ENV['HOME']}/Library/Application\ Support/Sublime\ Text\ 3/Packages" 102 | elsif RUBY_PLATFORM.downcase.include?("linux") 103 | return "#{ENV['HOME']}/.config/sublime-text-2/Packages" 104 | end 105 | 106 | return "" 107 | end 108 | 109 | dir = packageDir() 110 | 111 | begin 112 | Dir.chdir dir 113 | rescue Errno::ENOENT 114 | puts "Packages directory doesn't exist... creating it!" 115 | puts 116 | 117 | `mkdir -p '#{dir}'` 118 | retry 119 | end 120 | 121 | puts "Packages directory: #{dir}" 122 | 123 | # Update existing and delete unknown 124 | Dir.entries('.').each do |package_name| 125 | next if package_name =~ /^\./ 126 | next unless File.directory? package_name 127 | 128 | source = packages[package_name] 129 | branch = branch(source) 130 | 131 | print "* #{package_name} #{source}: " 132 | 133 | if packages.delete package_name 134 | print "package exists, ..." 135 | if File.directory?("#{dir}/#{package_name}/.git") 136 | puts "automatic update via Git" 137 | `cd #{cleanup package_name}; git pull; git checkout #{branch} &> /dev/null; cd ..;` 138 | else 139 | puts "not a Git repository" 140 | end 141 | else 142 | if providedPackages.index(package_name) 143 | print "Ignoring provided package\n" 144 | else 145 | print "don't know about this package. Delete it? [y/n] " 146 | 147 | while answer = gets 148 | if answer =~ /^y/ 149 | `rm -rf #{cleanup package_name}` 150 | puts " * deleted" 151 | break 152 | elsif answer =~ /^(n|$)/ 153 | break 154 | else 155 | print "Please enter 'y' or 'n': " 156 | end 157 | end 158 | end 159 | end 160 | end 161 | 162 | ## Install new 163 | packages.each do |name, source| 164 | puts "* #{name} doesn't exist; " 165 | 166 | uri = URI(source) 167 | 168 | if (uri.scheme == 'http' || uri.scheme == 'https') 169 | puts "fetching... from #{uri.to_s}" 170 | 171 | # download package to tmp file 172 | # extract to package name 173 | fileName = uri.path.gsub(/^\//, "") 174 | tmp = Tempfile.new('#{fileName}') 175 | `wget -O #{tmp.path} #{uri.to_s}` 176 | 177 | path = "#{dir}/#{name}" 178 | `unzip "#{tmp.path}" -d "#{path}"` 179 | puts " * unzipped." 180 | else 181 | repo = repo(source) 182 | branch = branch(source) 183 | 184 | puts "fetching... from #{github_root}/#{repo}" 185 | 186 | `git clone #{github_root}/#{repo} #{cleanup name}` 187 | branch = branch(source) 188 | 189 | if branch != "master" 190 | `cd #{cleanup name}; git checkout #{branch} &> /dev/null; cd ..;` 191 | end 192 | puts " * checked out #{branch}" 193 | end 194 | 195 | end 196 | -------------------------------------------------------------------------------- /suspend_until: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Auto suspend and wake-up script, taken from 4 | # 5 | # http://askubuntu.com/questions/61708/automatically-sleep-and-wake-up-at-specific-times 6 | # 7 | # Puts the computer on standby and automatically wakes it up at specified time 8 | # 9 | # Written by Romke van der Meulen 10 | # Minor mods fossfreedom for AskUbuntu 11 | # 12 | # Takes a 24hour time HH:MM as its argument 13 | # Example: 14 | # 15 | # suspend_until 9:30 16 | # suspend_until 18:45 17 | # 18 | # You can create a root cron job that calls this script to execute at a 19 | # specific time in the evening and then awake in the morning: 20 | # 21 | # sudo crontab -e 22 | # 23 | # Now enter something like to run the suspend script at 23:30: 24 | # 25 | # 30 23 * * * /home/myhomefolder/suspend_until 07:30 26 | 27 | # ------------------------------------------------------ 28 | # Argument check 29 | if [ $# -lt 1 ]; then 30 | echo "Usage: suspend_until HH:MM" 31 | exit 32 | fi 33 | 34 | # Check whether specified time today or tomorrow 35 | DESIRED=$((`date +%s -d "$1"`)) 36 | NOW=$((`date +%s`)) 37 | if [ $DESIRED -lt $NOW ]; then 38 | DESIRED=$((`date +%s -d "$1"` + 24*60*60)) 39 | fi 40 | 41 | # Kill rtcwake if already running 42 | sudo killall rtcwake 43 | 44 | # Set RTC wakeup time 45 | # N.B. change "mem" for the suspend option 46 | # find this by "man rtcwake" 47 | sudo rtcwake -l -m mem -t $DESIRED & 48 | 49 | # feedback 50 | echo "Suspending..." 51 | 52 | # give rtcwake some time to make its stuff 53 | sleep 2 54 | 55 | # then suspend 56 | # N.B. dont usually require this bit 57 | #sudo pm-suspend 58 | 59 | # Any commands you want to launch after wakeup can be placed here 60 | # Remember: sudo may have expired by now 61 | 62 | # Wake up with monitor enabled N.B. change "on" for "off" if 63 | # you want the monitor to be disabled on wake 64 | xset dpms force on 65 | 66 | # and a fresh console 67 | clear 68 | echo "Good morning!" 69 | -------------------------------------------------------------------------------- /svnauthors.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # taken from http://technicalpickles.com/posts/creating-a-svn-authorsfile-when-migrating-from-subversion-to-git/ 3 | # Run this inside an subversion checkout. It outputs a template for the 4 | # svn.authorsfile to the console, so you just need to paste it into a 5 | # document, and fill in the names and email address for your authors. 6 | # 7 | # Usage example: svnauthors.sh http://company.com/repo 8 | authors=$(svn log -q $@ | grep -e '^r[0-9]' | awk 'BEGIN { FS = "|" } ; { print $2 } $1 ~ /r([0-9]+000)/ { print "fetched revision " substr($1, 2) > "/dev/stderr" }' | sort | uniq) 9 | for author in ${authors}; do 10 | echo "${author} = NAME "; 11 | done -------------------------------------------------------------------------------- /svninit2git: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | # prefix to the svn repository 4 | repo=$1 5 | 6 | # name of the final project 7 | project=$2 8 | 9 | # use local instead of global svn authorsfile 10 | authorsfile=~/.git-svn.authorsfile 11 | 12 | rm -rf tmp-$project 13 | mkdir tmp-$project 14 | cd tmp-$project 15 | 16 | echo $repo 17 | 18 | git svn init $repo --no-metadata 19 | git config svn.authorsfile $authorsfile 20 | git svn fetch 21 | 22 | 23 | cd .. 24 | git clone tmp-$project $project 25 | 26 | rm -rf $project-tmp 27 | -------------------------------------------------------------------------------- /swapkeys.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # The script is intended to automatically switch the keybindings of 3 | # special keys on OSX which is normally done through 4 | # System Preferences -> Keybopard > Modifier keys 5 | 6 | # This Script does NOT work, see 7 | # http://forums.macrumors.com/showthread.php?t=949280 8 | # http://apple.stackexchange.com/questions/13598/updating-modifier-key-mappings-through-defaults-command-tool 9 | 10 | mappingplist=com.apple.keyboard.modifiermapping.1452-545-0 11 | 12 | if [ $1 == "emacs" ]; then 13 | echo "Switching to emacs modifiers" 14 | defaults -currentHost write -g $mappingplist '( 15 | { 16 | HIDKeyboardModifierMappingDst = 4; 17 | HIDKeyboardModifierMappingSrc = 2; }, 18 | { 19 | HIDKeyboardModifierMappingDst = 12; 20 | HIDKeyboardModifierMappingSrc = 10; 21 | }, 22 | { 23 | HIDKeyboardModifierMappingDst = 2; 24 | HIDKeyboardModifierMappingSrc = 4; 25 | }, 26 | { 27 | HIDKeyboardModifierMappingDst = 10; 28 | HIDKeyboardModifierMappingSrc = 12; 29 | })' 30 | 31 | 32 | else 33 | echo "Switching to default modifiers" 34 | defaults -currentHost delete -g $mappingplist 35 | fi -------------------------------------------------------------------------------- /textile2html.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Usage textile2html.rb file.textile > file.html 4 | 5 | require "rubygems" 6 | require "redcloth" 7 | 8 | if ARGV[0] == nil 9 | puts "No filename." 10 | exit 1 11 | end 12 | 13 | file_name = ARGV[0] 14 | document = IO.read(file_name) 15 | html = RedCloth.new(document).to_html 16 | 17 | $stdout.write "#{html}" -------------------------------------------------------------------------------- /textile2latex.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Usage textile2latex .rb file.textile > file.html 4 | 5 | require "rubygems" 6 | require "redcloth" 7 | 8 | if ARGV[0] == nil 9 | puts "No filename." 10 | exit 1 11 | end 12 | 13 | file_name = ARGV[0] 14 | document = IO.read(file_name) 15 | latex = RedCloth.new(document).to_latex 16 | 17 | $stdout.write "#{latex}" -------------------------------------------------------------------------------- /textile2markdown: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | # You need in your PATH 4 | # - html2markdown.py (http://www.codefu.org/wiki/Main/Html2markdown) 5 | # - textile2html (http://github.com/oschrenk/scripts/blob/master/textile2html.rb) 6 | 7 | # Usage textile2markdown > output.markdown 8 | 9 | textile2html.rb "$1" | html2markdown.py -------------------------------------------------------------------------------- /toc.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # usage: ruby ./toc.rb doc.textile > doc.html 4 | 5 | # author: marko dot haapala at aktagon dot com 6 | # idea taken from here: http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/134005 7 | 8 | # copied from http://snippets.aktagon.com/snippets/48-Generate-a-TOC-for-a-textile-file 9 | # under CC http://creativecommons.org/licenses/by/3.0/ 10 | 11 | require "rubygems" 12 | require "redcloth" 13 | 14 | def generate_toc (file_name, headreg) 15 | document = IO.read(file_name) 16 | toc = "" 17 | document.gsub(headreg) do |match| 18 | number = $1 19 | name = $2 20 | header = name.gsub(/\s/,"+") 21 | toc << '#' * number.to_i + ' "' + name + '":#' + header + "\n" 22 | end 23 | RedCloth.new(toc).to_html 24 | end 25 | 26 | def manipulate_body(file_name, headreg) 27 | document = IO.read(file_name) 28 | document.gsub!(headreg) do |match| 29 | number = $1 30 | name = $2 31 | header = name.gsub(/\s/,"+") 32 | "\nh" + number + '. ' + name + '' 33 | end 34 | RedCloth.new(document).to_html 35 | end 36 | 37 | if ARGV[0] == nil 38 | puts "Oh no! You didn't give me a filename :(" 39 | exit 1 40 | end 41 | 42 | file_name = ARGV[0] 43 | headreg = /^\s*h([1-6])\.\s+(.*)/ 44 | toc = generate_toc(file_name, headreg) 45 | body = manipulate_body(file_name, headreg) 46 | template = <<-'EOF' 47 | 48 | 49 | 50 | #{file_name} 51 | 52 | 53 | 54 | 55 | #{toc} 56 | #{body} 57 | 58 | 59 | EOF 60 | puts eval("\"" + template + "\"") -------------------------------------------------------------------------------- /tunnel.sh: -------------------------------------------------------------------------------- 1 | # !/bin/sh 2 | # SSH SOCKS proxy script for Mac OS X 3 | 4 | # Set PROXY_USER and PROXY_HOST 5 | # 6 | # If choosing not to set credentials as Environmental Variables, replace the values below for $PROXY_USER and $PROXY_HOST with genuine credentials. 7 | # ie 8 | # remoteuser='user' 9 | # remoteproxy='example.com' 10 | 11 | trap ctrl_c INT 12 | 13 | # Get the min and max system-available ports. 14 | lowerport=`sysctl net.inet.ip.portrange.first | cut -d " " -f 2` 15 | upperport=`sysctl net.inet.ip.portrange.last | cut -d " " -f 2` 16 | localport=`jot -r 1 $lowerport $upperport` 17 | 18 | remoteuser=$PROXY_USER 19 | remoteproxy=$PROXY_HOST 20 | remoteport="22" 21 | 22 | usage(){ 23 | echo "" 24 | echo "Usage: tunnel.sh [on|off|killall|shutdown|no_args]" 25 | echo "tunnel.sh is a proxy settings toggle script for OSX" 26 | echo "tunnel.sh initiates an SSH tunnel and then enables a Socks proxy" 27 | } 28 | 29 | function ctrl_c() { 30 | echo "SIGINT. Exiting." 31 | exit $? 32 | } 33 | 34 | unknown_input(){ 35 | echo "Unknown input, try again with different term" 36 | } 37 | 38 | get_proxy_state(){ 39 | result=`networksetup -getsocksfirewallproxy Wi-Fi | head -1 | cut -d ' ' -f2` 40 | echo $result 41 | } 42 | 43 | proxy_on(){ 44 | remote_ip_before=`curl -s http://curlmyip.com/` 45 | 46 | echo "Listening on localhost:$localport." 47 | echo "Modifying network settings..." 48 | 49 | # Ask for the administrator password upfront 50 | sudo -v 51 | 52 | IFS=$(echo -en "\n\b") 53 | for device in $(networksetup -listallnetworkservices | sed '1d' | grep -v "Bluetooth") 54 | do 55 | echo " - enabling proxy for $device" 56 | sudo networksetup -setsocksfirewallproxy "$device" 127.0.0.1 $localport off 57 | done 58 | echo "...done" 59 | echo "Starting SSH session. Will run in background for 1 day." 60 | ssh -f tunnel -N -D localhost:$localport sleep 1d 61 | 62 | remote_ip_after=`curl -s -S --socks5 127.0.0.1:$localport http://curlmyip.com/` 63 | echo "Your remote ip before connecting through the proxy is $remote_ip_before" 64 | echo "Your remote ip after connecting through the proxy is $remote_ip_after" 65 | echo "The http_proxy for the terminal has NOT been set." 66 | } 67 | 68 | proxy_off(){ 69 | echo "Modifying network settings..." 70 | 71 | # Ask for the administrator password upfront 72 | sudo -v 73 | 74 | IFS=$(echo -en "\n\b") 75 | for device in $(networksetup -listallnetworkservices | sed '1d' | grep -v "Bluetooth") 76 | do 77 | echo " - disabling proxy for $device" 78 | sudo networksetup -setsocksfirewallproxystate "$device" off 79 | done 80 | echo "... done!" 81 | } 82 | 83 | kill_all(){ 84 | pids=`ps -A | grep "ssh -f" | grep "sleep" | awk '{print $1}'` 85 | echo "Killing all running proxy connections." 86 | for pid in $pids 87 | do 88 | `kill -9 $pid` 89 | done 90 | } 91 | 92 | shutdown(){ 93 | proxy_off 94 | kill_all 95 | echo "Proxy shutdown complete!" 96 | } 97 | 98 | toggle_state(){ 99 | # result=`get_proxy_state` 100 | echo $(get_proxy_state) | grep -i 'y' >> /dev/null 101 | if [[ $? =~ 0 ]]; then 102 | shutdown 103 | echo "OFF" 104 | else 105 | proxy_on 106 | echo "ON" 107 | fi 108 | } 109 | 110 | case $1 in 111 | "on") 112 | proxy_on 113 | ;; 114 | "off") 115 | proxy_off 116 | ;; 117 | "killall") 118 | kill_all 119 | ;; 120 | "shutdown") 121 | shutdown 122 | ;; 123 | "usage") 124 | usage 125 | ;; 126 | "") 127 | toggle_state 128 | ;; 129 | *) 130 | unknown_input 131 | usage 132 | ;; 133 | esac 134 | -------------------------------------------------------------------------------- /webm2mp3: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # webm2mp3 - extract mp3 audio from web2m 4 | # 5 | if [ "$1" ] 6 | then 7 | for file 8 | do 9 | if [ -e "$file" ] 10 | then 11 | ffmpeg -i "$file" -vn -acodec copy "${file%.*}.mp3" 12 | else 13 | echo >&2 "No such file: "$1"" 14 | exit 1 15 | fi 16 | done 17 | else 18 | echo >&2 "Usage: "$(basename "$0")" INPUTFILE [...]" 19 | exit 1 20 | fi -------------------------------------------------------------------------------- /webm2ogg: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # webm2mp3 - extract mp3 audio from web2m 4 | # 5 | if [ "$1" ] 6 | then 7 | for file 8 | do 9 | if [ -e "$file" ] 10 | then 11 | ffmpeg -i "$file" -vn -acodec copy "${file%.*}.ogg" 12 | else 13 | echo >&2 "No such file: "$1"" 14 | exit 1 15 | fi 16 | done 17 | else 18 | echo >&2 "Usage: "$(basename "$0")" INPUTFILE [...]" 19 | exit 1 20 | fi -------------------------------------------------------------------------------- /which-shell.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Tell you which shell you are in by printing the current process 3 | ps -p $$ -------------------------------------------------------------------------------- /write-sqlite-blob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sqlite3 4 | conn = sqlite3.connect('sp_radio_0.localstorage') 5 | cursor = conn.cursor() 6 | 7 | with open("output.json", "rb") as input_file: 8 | ablob = input_file.read() 9 | cursor.execute("UPDATE ItemTable SET value=? WHERE key='RecentStations'; ", [sqlite3.Binary(ablob)]) 10 | conn.commit() 11 | 12 | cursor.close() 13 | conn.close() --------------------------------------------------------------------------------