├── .gitignore ├── .travis.yml ├── COPYING ├── MANIFEST.in ├── Makefile ├── README.rst ├── doc ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── commands.rst │ ├── conf.py │ ├── config.rst │ ├── index.rst │ ├── rash │ └── tips.rst ├── rash ├── __init__.py ├── cli.py ├── config.py ├── daemon.py ├── database.py ├── ext │ ├── rash.bash │ └── rash.zsh ├── functional_tests │ ├── __init__.py │ └── test_cli.py ├── index.py ├── indexer.py ├── init.py ├── interactive_search.py ├── isearch.py ├── log.py ├── model.py ├── query.py ├── record.py ├── schema.sql ├── search.py ├── show.py ├── tests │ ├── __init__.py │ ├── test_database.py │ ├── test_indexer.py │ ├── test_query.py │ └── utils.py ├── utils │ ├── __init__.py │ ├── confutils.py │ ├── iterutils.py │ ├── pathutils.py │ ├── py3compat.py │ ├── sqlconstructor.py │ ├── termdetection.py │ ├── tests │ │ ├── __init__.py │ │ └── test_termdetection.py │ └── timeutils.py └── watchrecord.py ├── rash_cli.py ├── setup.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | .tox 4 | doc/build 5 | MANIFEST 6 | /dist 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | before_install: 4 | 5 | - pip install --quiet --use-mirrors tox 6 | 7 | # Install zsh: 8 | - sudo apt-get update -qq 9 | - sudo apt-get install -qq zsh 10 | 11 | script: 12 | - tox 13 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include rash/ext/*sh 2 | include rash/schema.sql 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # E.g., RASH_TOX_OPTS=-e py27 2 | RASH_TOX_OPTS ?= 3 | 4 | .PHONY: test tox-sdist clean cog upload 5 | 6 | ## Testing 7 | test: 8 | tox $(RASH_TOX_OPTS) 9 | 10 | tox-sdist: 11 | rm -f MANIFEST 12 | tox $(RASH_TOX_OPTS) --sdistonly 13 | 14 | clean: 15 | rm -rf *.egg-info .tox MANIFEST 16 | 17 | ## Update files using cog.py 18 | cog: rash/__init__.py 19 | rash/__init__.py: README.rst 20 | cd rash && cog.py -r __init__.py 21 | 22 | ## Upload to PyPI 23 | upload: cog 24 | python setup.py register sdist upload 25 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | Rash Advances Shell History 3 | ============================= 4 | 5 | 6 | .. sidebar:: Links: 7 | 8 | * `Documentation `_ (at Read the Docs) 9 | 10 | * `Commands `_ 11 | * `Configuration `_ 12 | * `Tips `_ 13 | 14 | * `Repository `_ (at GitHub) 15 | * `Issue tracker `_ (at GitHub) 16 | * `PyPI `_ 17 | * `Travis CI `_ |build-status| 18 | 19 | 20 | What is this? 21 | ============= 22 | 23 | Shell history is useful. But it can be more useful if it logs more 24 | data points. For example, if you forget which `make` target to run 25 | for certain project, you'd want to search shell commands that are 26 | run in particular directory. Wouldn't it be nice if you can do this?:: 27 | 28 | rash search --cwd . "make*" 29 | 30 | RASH records many data points and they are stored in SQLite database. 31 | Here is a list of recorded information [#]_. 32 | 33 | #. Current directory (``$PWD``). 34 | #. Exit code (``$?``) 35 | #. Exit code of pipes (``$PIPESTATUS`` / ``$pipestatus``) 36 | #. The time command is started and terminated. 37 | #. Environment variable (``$PATH``, ``$SHELL``, ``$TERM``, ``$HOST``, etc.) 38 | #. Real terminal. ``$TERM`` is used to fake programs. 39 | RASH can detect if you are in tmux, byobu, screen, gnome-terminal, etc. 40 | #. Session information. If you go back and forth in some terminals, 41 | RASH does not loose in which sequence you ran the commands in which 42 | terminal. 43 | 44 | .. [#] If you are curious, checkout ``rash record --help``. 45 | 46 | 47 | RASH also has interactive search interface. You can see the search 48 | result as you type. If you are using zsh, you can execute the result 49 | instantaneously. 50 | 51 | .. image:: https://raw.github.com/tkf/rash/data/rash-isearch.gif 52 | :alt: RASH interactive search interface 53 | 54 | 55 | Install 56 | ======= 57 | 58 | RASH is written in Python. The easiest way to install is to use `pip` 59 | (or `easy_install`, if you wish). You may need `sudo` for installing 60 | it in a system directory.:: 61 | 62 | pip install rash 63 | pip install percol # if you want interactive search feature 64 | 65 | If you use virtualenv to install RASH, you may have trouble when 66 | switching environment. In that case, it is safe to make an alias 67 | to full path of the rash executable.:: 68 | 69 | alias rash="PATH/TO/VIRTUALENV/bin/rash" 70 | 71 | If you want to use developmental version, just clone the git repository 72 | and add the following in your RC file.:: 73 | 74 | alias rash="PATH/TO/RASH/rash_cli.py" 75 | 76 | 77 | Setup 78 | ===== 79 | Add this to your `.zshrc` or `.bashrc`. That's all.:: 80 | 81 | eval "$(rash init)" 82 | 83 | For more information, see ``rash init --help``. 84 | 85 | 86 | Usage 87 | ===== 88 | 89 | Searching history -- ``rash search`` 90 | ------------------------------------ 91 | 92 | After your shell history is accumulated by RASH, it's the time to 93 | make use of the history! See ``rash search --help`` for detailed 94 | information. Here is some examples. 95 | 96 | Forget how to run automated test for the current project?:: 97 | 98 | rash search --cwd . --include-pattern "*test*" --include-pattern "tox*" 99 | 100 | All git commands you ran in one week.:: 101 | 102 | rash search --time-after "1 week ago" "git*" 103 | 104 | Some intensive task you ran in the current project that succeeded and 105 | took longer than 30 minutes.:: 106 | 107 | rash search --cwd-under . --include-exit-code 0 --duration-longer-than 30m 108 | 109 | What did I do after `cd`-ing to some directory?:: 110 | 111 | rash search --after-context 5 "cd SOME-DIRECTORY" 112 | 113 | All failed commands you ran at this directory.:: 114 | 115 | rash search --cwd . --exclude-exit-code 0 116 | 117 | Count number of commands you ran in one day:: 118 | 119 | rash search --limit -1 --no-unique --time-after "1 day ago" | wc -l 120 | 121 | 122 | Showing detailed information -- ``rash show`` 123 | --------------------------------------------- 124 | 125 | If you give ``--with-command-id`` to ``rash search`` command, it prints out 126 | ID number for each command history.:: 127 | 128 | % rash search --with-command-id --limit 5 "*git*" 129 | 359 git log 130 | 1253 git help clone 131 | 1677 git help diff 132 | 1678 git diff --word-diff 133 | 1780 git merge 134 | 135 | You can see all information associated with a command with 136 | ``rash show`` command:: 137 | 138 | rash show 1677 139 | 140 | 141 | Interactive search -- ``rash isearch`` 142 | -------------------------------------- 143 | 144 | Searching history using command line is not fast. 145 | You can use ``rash isearch`` command to interactively search 146 | history and see the result immediately as you type. 147 | 148 | You need percol_ to use this feature. 149 | 150 | Zsh user can setup a keybind like this:: 151 | 152 | # Type `Ctrl-x r` to start isearch 153 | bindkey "^Xr" rash-zle-isearch 154 | 155 | Defining this function in your rc file can be handy and it is 156 | usable for bash users.:: 157 | 158 | rash-isearch(){ 159 | eval "$(rash isearch)" 160 | } 161 | 162 | .. _percol: https://github.com/mooz/percol 163 | 164 | 165 | Dependency 166 | ========== 167 | 168 | RASH tested against Python 2.6, 2.7 and 3.2. However, as some 169 | dependencies are not Python 3 compatible, some functionality is 170 | missing when used with Python 3. 171 | 172 | Python modules: 173 | 174 | * watchdog_ [#nopy3k]_ 175 | * parsedatetime_ [#nopy3k]_ 176 | 177 | .. _watchdog: http://pypi.python.org/pypi/watchdog/ 178 | .. _parsedatetime: http://pypi.python.org/pypi/parsedatetime/ 179 | 180 | .. [#nopy3k] These modules do not support Python 3. 181 | They are not installed in if you use Python 3 182 | and related functionality is disabled. 183 | 184 | Platforms 185 | --------- 186 | 187 | UNIX-like systems 188 | RASH is tested in Linux and I am using in Linux. 189 | It should work in other UNIX-like systems like BSD. 190 | 191 | Mac OS 192 | I guess it works. Not tested. 193 | 194 | MS Windows 195 | Probably no one wants to use a shell tool in windows, but I 196 | try to avoid stuff that is platform specific. Only the 197 | daemon launcher will not work on Windows but there is several 198 | ways to avoid using it. See ``rash init --help``. 199 | 200 | Shells 201 | ------ 202 | 203 | RASH currently supports zsh and bash. 204 | 205 | 206 | Design principle 207 | ================ 208 | 209 | RASH's design is focused on sparseness. There are several stages 210 | of data transformation until you see the search result, and they 211 | are done by separated processes. 212 | 213 | First, ``rash record`` command dumps shell history in raw JSON record. 214 | This part of program does not touches to DB to make process very fast. 215 | As there is no complex transformation in this command, probably in the 216 | future version it is better to rewrite it entirely in shell function. 217 | 218 | Second, ``rash daemon`` runs in background and watches the directory to 219 | store JSON record. When JSON record arrives, it insert the data into 220 | database. 221 | 222 | ``rash record`` and ``rash daemon`` are setup by simple shell snippet 223 | ``eval $(rash init)``. 224 | 225 | Finally, you can search through command history using search interface 226 | such as `rash search`. This search is very fast as you don't read 227 | all JSON records in separated files. 228 | 229 | :: 230 | 231 | +-------+ +--------+ +--------+ +--------+ 232 | | Shell | | Raw | | SQLite | | Search | 233 | | hooks |-------->| JSON |-------->| DB |-------->| result | 234 | +-------+ | record | +--------+ +--------+ 235 | +--------+ 236 | 237 | `rash record` `rash daemon` `rash search` 238 | `rash show` 239 | 240 | \------------------------------/ \------------/ 241 | `rash init` setups them search interface 242 | 243 | License 244 | ======= 245 | 246 | RASH is licensed under GPL v3. 247 | See COPYING for details. 248 | 249 | 250 | .. Travis CI build status badge 251 | .. |build-status| 252 | image:: https://secure.travis-ci.org/tkf/rash.png?branch=master 253 | :target: http://travis-ci.org/tkf/rash 254 | :alt: Build Status 255 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/RASH.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/RASH.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/RASH" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/RASH" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. linkcheck to check all external links for integrity 37 | echo. doctest to run all doctests embedded in the documentation if enabled 38 | goto end 39 | ) 40 | 41 | if "%1" == "clean" ( 42 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 43 | del /q /s %BUILDDIR%\* 44 | goto end 45 | ) 46 | 47 | if "%1" == "html" ( 48 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 49 | if errorlevel 1 exit /b 1 50 | echo. 51 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 52 | goto end 53 | ) 54 | 55 | if "%1" == "dirhtml" ( 56 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 57 | if errorlevel 1 exit /b 1 58 | echo. 59 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 60 | goto end 61 | ) 62 | 63 | if "%1" == "singlehtml" ( 64 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 68 | goto end 69 | ) 70 | 71 | if "%1" == "pickle" ( 72 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished; now you can process the pickle files. 76 | goto end 77 | ) 78 | 79 | if "%1" == "json" ( 80 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished; now you can process the JSON files. 84 | goto end 85 | ) 86 | 87 | if "%1" == "htmlhelp" ( 88 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can run HTML Help Workshop with the ^ 92 | .hhp project file in %BUILDDIR%/htmlhelp. 93 | goto end 94 | ) 95 | 96 | if "%1" == "qthelp" ( 97 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 98 | if errorlevel 1 exit /b 1 99 | echo. 100 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 101 | .qhcp project file in %BUILDDIR%/qthelp, like this: 102 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\RASH.qhcp 103 | echo.To view the help file: 104 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\RASH.ghc 105 | goto end 106 | ) 107 | 108 | if "%1" == "devhelp" ( 109 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 110 | if errorlevel 1 exit /b 1 111 | echo. 112 | echo.Build finished. 113 | goto end 114 | ) 115 | 116 | if "%1" == "epub" ( 117 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 118 | if errorlevel 1 exit /b 1 119 | echo. 120 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 121 | goto end 122 | ) 123 | 124 | if "%1" == "latex" ( 125 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 129 | goto end 130 | ) 131 | 132 | if "%1" == "text" ( 133 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The text files are in %BUILDDIR%/text. 137 | goto end 138 | ) 139 | 140 | if "%1" == "man" ( 141 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 145 | goto end 146 | ) 147 | 148 | if "%1" == "texinfo" ( 149 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 150 | if errorlevel 1 exit /b 1 151 | echo. 152 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 153 | goto end 154 | ) 155 | 156 | if "%1" == "gettext" ( 157 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 158 | if errorlevel 1 exit /b 1 159 | echo. 160 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 161 | goto end 162 | ) 163 | 164 | if "%1" == "changes" ( 165 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 166 | if errorlevel 1 exit /b 1 167 | echo. 168 | echo.The overview file is in %BUILDDIR%/changes. 169 | goto end 170 | ) 171 | 172 | if "%1" == "linkcheck" ( 173 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 174 | if errorlevel 1 exit /b 1 175 | echo. 176 | echo.Link check complete; look for any errors in the above output ^ 177 | or in %BUILDDIR%/linkcheck/output.txt. 178 | goto end 179 | ) 180 | 181 | if "%1" == "doctest" ( 182 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 183 | if errorlevel 1 exit /b 1 184 | echo. 185 | echo.Testing of doctests in the sources finished, look at the ^ 186 | results in %BUILDDIR%/doctest/output.txt. 187 | goto end 188 | ) 189 | 190 | :end 191 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinxcontrib-programoutput 2 | -------------------------------------------------------------------------------- /doc/source/commands.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | RASH command line interface 3 | ============================= 4 | 5 | 6 | Search interface 7 | ================ 8 | 9 | .. _rash search: 10 | 11 | :program:`rash search` 12 | ---------------------- 13 | .. program-output:: rash search --help 14 | 15 | 16 | .. _rash show: 17 | 18 | :program:`rash show` 19 | -------------------- 20 | .. program-output:: rash show --help 21 | 22 | 23 | .. _rash isearch: 24 | 25 | :program:`rash isearch` 26 | ----------------------- 27 | .. program-output:: rash isearch --help 28 | 29 | 30 | System setup interface 31 | ====================== 32 | 33 | .. _rash init: 34 | 35 | :program:`rash init` 36 | -------------------- 37 | .. program-output:: rash init --help 38 | 39 | 40 | .. _rash daemon: 41 | 42 | :program:`rash daemon` 43 | ---------------------- 44 | .. program-output:: rash daemon --help 45 | 46 | 47 | .. _rash locate: 48 | 49 | :program:`rash locate` 50 | ---------------------- 51 | .. program-output:: rash locate --help 52 | 53 | 54 | .. _rash version: 55 | 56 | :program:`rash version` 57 | ----------------------- 58 | .. program-output:: rash version --help 59 | 60 | 61 | Low level commands 62 | ================== 63 | 64 | .. _rash record: 65 | 66 | :program:`rash record` 67 | ---------------------- 68 | 69 | .. program-output:: rash record --help 70 | 71 | 72 | .. _rash index: 73 | 74 | :program:`rash index` 75 | --------------------- 76 | 77 | .. program-output:: rash index --help 78 | 79 | 80 | ZSH functions 81 | ============= 82 | 83 | .. _rash-zle-isearch: 84 | 85 | :program:`rash-zle-isearch` 86 | --------------------------- 87 | 88 | To setup :kbd:`Ctrl-x r` to start :ref:`rash isearch`, add this to 89 | your :file:`.zshrc`: 90 | 91 | .. sourcecode:: sh 92 | 93 | bindkey "^Xr" rash-zle-isearch 94 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | from os.path import dirname 5 | import sys 6 | 7 | sys.path.insert(0, dirname(dirname(dirname(__file__)))) 8 | 9 | # To make command "rash" runnable 10 | os.environ['PATH'] = dirname(__file__) + ':' + os.environ['PATH'] 11 | 12 | 13 | # -- General configuration ------------------------------------------------ 14 | extensions = [ 15 | 'sphinx.ext.autodoc', 16 | 'sphinx.ext.doctest', 17 | 'sphinx.ext.todo', 18 | 'sphinxcontrib.programoutput', 19 | ] 20 | 21 | templates_path = [] # ['_templates'] 22 | source_suffix = '.rst' 23 | master_doc = 'index' 24 | 25 | # General information about the project. 26 | project = u'RASH' 27 | copyright = u'2013, Takafumi Arakaki' 28 | 29 | # The short X.Y version. 30 | version = '0.1.3' 31 | # The full version, including alpha/beta/rc tags. 32 | release = '0.1.3' 33 | 34 | exclude_patterns = [] 35 | 36 | pygments_style = 'sphinx' 37 | 38 | 39 | # -- Options for HTML output ---------------------------------------------- 40 | html_theme = 'default' 41 | html_static_path = [] # ['_static'] 42 | 43 | # Output file base name for HTML help builder. 44 | htmlhelp_basename = 'RASHdoc' 45 | 46 | 47 | # -- Options for LaTeX output --------------------------------------------- 48 | latex_elements = { 49 | # The paper size ('letterpaper' or 'a4paper'). 50 | #'papersize': 'letterpaper', 51 | 52 | # The font size ('10pt', '11pt' or '12pt'). 53 | #'pointsize': '10pt', 54 | 55 | # Additional stuff for the LaTeX preamble. 56 | #'preamble': '', 57 | } 58 | 59 | # Grouping the document tree into LaTeX files. List of tuples 60 | # (source start file, target name, title, 61 | # author, documentclass [howto/manual]). 62 | latex_documents = [ 63 | ('index', 'RASH.tex', u'RASH Documentation', 64 | u'Takafumi Arakaki', 'manual'), 65 | ] 66 | 67 | 68 | # -- Options for manual page output --------------------------------------- 69 | # One entry per manual page. List of tuples 70 | # (source start file, name, description, authors, manual section). 71 | man_pages = [ 72 | ('index', 'rash', u'RASH Documentation', 73 | [u'Takafumi Arakaki'], 1) 74 | ] 75 | 76 | # If true, show URL addresses after external links. 77 | #man_show_urls = False 78 | 79 | 80 | # -- Options for Texinfo output ------------------------------------------- 81 | # Grouping the document tree into Texinfo files. List of tuples 82 | # (source start file, target name, title, author, 83 | # dir menu entry, description, category) 84 | texinfo_documents = [ 85 | ('index', 'RASH', u'RASH Documentation', 86 | u'Takafumi Arakaki', 'RASH', 'One line description of project.', 87 | 'Miscellaneous'), 88 | ] 89 | 90 | 91 | # -- Options for extensions ----------------------------------------------- 92 | 93 | autodoc_member_order = 'bysource' 94 | -------------------------------------------------------------------------------- /doc/source/config.rst: -------------------------------------------------------------------------------- 1 | .. automodule:: rash.config 2 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. automodule:: rash 2 | 3 | More resources 4 | ============== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | commands 10 | config 11 | tips 12 | 13 | * :ref:`genindex` 14 | * :ref:`modindex` 15 | * :ref:`search` 16 | -------------------------------------------------------------------------------- /doc/source/rash: -------------------------------------------------------------------------------- 1 | ../../rash_cli.py -------------------------------------------------------------------------------- /doc/source/tips.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | Tips 3 | ====== 4 | 5 | Define Zsh ZLE widget 6 | ===================== 7 | 8 | You can use the ZLE widget :ref:`rash-zle-isearch` loaded by 9 | :ref:`rash init` to define your own modified widget. It takes 10 | arguments and passes them to :ref:`rash isearch` directly. Here 11 | is a recipe for "Do What I Mean" search: 12 | 13 | .. sourcecode:: sh 14 | 15 | rash-zle-dwim(){ 16 | rash-zle-isearch --query-template "-x 0 -d . @ {0} " 17 | } 18 | zle -N rash-zle-dwim 19 | bindkey "^Xs" rash-zle-dwim 20 | 21 | 22 | In the :term:`configuration file`, you should define an alias 23 | called ``@`` like this (see also |search.alias|):: 24 | 25 | config.search.alias['@'] = [...] # some complex query 26 | 27 | .. |search.alias| replace:: 28 | :attr:`config.search.alias ` 29 | 30 | 31 | Using RASH in old version of zsh 32 | ================================ 33 | 34 | RASH depends on ``precmd_functions`` / ``preexec_functions`` hooks in 35 | zsh. In old version zsh doesn't have it. However, you can use RASH 36 | by adding this in your :file:`.zshrc`. 37 | 38 | .. sourcecode:: sh 39 | 40 | precmd(){ 41 | for f in $precmd_functions 42 | do 43 | "$f" 44 | done 45 | } 46 | 47 | preexec(){ 48 | for f in $preexec_functions 49 | do 50 | "$f" 51 | done 52 | } 53 | -------------------------------------------------------------------------------- /rash/__init__.py: -------------------------------------------------------------------------------- 1 | # [[[cog import cog; cog.outl('"""\n%s\n"""' % file('../README.rst').read())]]] 2 | """ 3 | ============================= 4 | Rash Advances Shell History 5 | ============================= 6 | 7 | 8 | .. sidebar:: Links: 9 | 10 | * `Documentation `_ (at Read the Docs) 11 | 12 | * `Commands `_ 13 | * `Configuration `_ 14 | * `Tips `_ 15 | 16 | * `Repository `_ (at GitHub) 17 | * `Issue tracker `_ (at GitHub) 18 | * `PyPI `_ 19 | * `Travis CI `_ |build-status| 20 | 21 | 22 | What is this? 23 | ============= 24 | 25 | Shell history is useful. But it can be more useful if it logs more 26 | data points. For example, if you forget which `make` target to run 27 | for certain project, you'd want to search shell commands that are 28 | run in particular directory. Wouldn't it be nice if you can do this?:: 29 | 30 | rash search --cwd . "make*" 31 | 32 | RASH records many data points and they are stored in SQLite database. 33 | Here is a list of recorded information [#]_. 34 | 35 | #. Current directory (``$PWD``). 36 | #. Exit code (``$?``) 37 | #. Exit code of pipes (``$PIPESTATUS`` / ``$pipestatus``) 38 | #. The time command is started and terminated. 39 | #. Environment variable (``$PATH``, ``$SHELL``, ``$TERM``, ``$HOST``, etc.) 40 | #. Real terminal. ``$TERM`` is used to fake programs. 41 | RASH can detect if you are in tmux, byobu, screen, gnome-terminal, etc. 42 | #. Session information. If you go back and forth in some terminals, 43 | RASH does not loose in which sequence you ran the commands in which 44 | terminal. 45 | 46 | .. [#] If you are curious, checkout ``rash record --help``. 47 | 48 | 49 | RASH also has interactive search interface. You can see the search 50 | result as you type. If you are using zsh, you can execute the result 51 | instantaneously. 52 | 53 | .. image:: https://raw.github.com/tkf/rash/data/rash-isearch.gif 54 | :alt: RASH interactive search interface 55 | 56 | 57 | Install 58 | ======= 59 | 60 | RASH is written in Python. The easiest way to install is to use `pip` 61 | (or `easy_install`, if you wish). You may need `sudo` for installing 62 | it in a system directory.:: 63 | 64 | pip install rash 65 | pip install percol # if you want interactive search feature 66 | 67 | If you use virtualenv to install RASH, you may have trouble when 68 | switching environment. In that case, it is safe to make an alias 69 | to full path of the rash executable.:: 70 | 71 | alias rash="PATH/TO/VIRTUALENV/bin/rash" 72 | 73 | If you want to use developmental version, just clone the git repository 74 | and add the following in your RC file.:: 75 | 76 | alias rash="PATH/TO/RASH/rash_cli.py" 77 | 78 | 79 | Setup 80 | ===== 81 | Add this to your `.zshrc` or `.bashrc`. That's all.:: 82 | 83 | eval "$(rash init)" 84 | 85 | For more information, see ``rash init --help``. 86 | 87 | 88 | Usage 89 | ===== 90 | 91 | Searching history -- ``rash search`` 92 | ------------------------------------ 93 | 94 | After your shell history is accumulated by RASH, it's the time to 95 | make use of the history! See ``rash search --help`` for detailed 96 | information. Here is some examples. 97 | 98 | Forget how to run automated test for the current project?:: 99 | 100 | rash search --cwd . --include-pattern "*test*" --include-pattern "tox*" 101 | 102 | All git commands you ran in one week.:: 103 | 104 | rash search --time-after "1 week ago" "git*" 105 | 106 | Some intensive task you ran in the current project that succeeded and 107 | took longer than 30 minutes.:: 108 | 109 | rash search --cwd-under . --include-exit-code 0 --duration-longer-than 30m 110 | 111 | What did I do after `cd`-ing to some directory?:: 112 | 113 | rash search --after-context 5 "cd SOME-DIRECTORY" 114 | 115 | All failed commands you ran at this directory.:: 116 | 117 | rash search --cwd . --exclude-exit-code 0 118 | 119 | Count number of commands you ran in one day:: 120 | 121 | rash search --limit -1 --no-unique --time-after "1 day ago" | wc -l 122 | 123 | 124 | Showing detailed information -- ``rash show`` 125 | --------------------------------------------- 126 | 127 | If you give ``--with-command-id`` to ``rash search`` command, it prints out 128 | ID number for each command history.:: 129 | 130 | % rash search --with-command-id --limit 5 "*git*" 131 | 359 git log 132 | 1253 git help clone 133 | 1677 git help diff 134 | 1678 git diff --word-diff 135 | 1780 git merge 136 | 137 | You can see all information associated with a command with 138 | ``rash show`` command:: 139 | 140 | rash show 1677 141 | 142 | 143 | Interactive search -- ``rash isearch`` 144 | -------------------------------------- 145 | 146 | Searching history using command line is not fast. 147 | You can use ``rash isearch`` command to interactively search 148 | history and see the result immediately as you type. 149 | 150 | You need percol_ to use this feature. 151 | 152 | Zsh user can setup a keybind like this:: 153 | 154 | # Type `Ctrl-x r` to start isearch 155 | bindkey "^Xr" rash-zle-isearch 156 | 157 | Defining this function in your rc file can be handy and it is 158 | usable for bash users.:: 159 | 160 | rash-isearch(){ 161 | eval "$(rash isearch)" 162 | } 163 | 164 | .. _percol: https://github.com/mooz/percol 165 | 166 | 167 | Dependency 168 | ========== 169 | 170 | RASH tested against Python 2.6, 2.7 and 3.2. However, as some 171 | dependencies are not Python 3 compatible, some functionality is 172 | missing when used with Python 3. 173 | 174 | Python modules: 175 | 176 | * watchdog_ [#nopy3k]_ 177 | * parsedatetime_ [#nopy3k]_ 178 | 179 | .. _watchdog: http://pypi.python.org/pypi/watchdog/ 180 | .. _parsedatetime: http://pypi.python.org/pypi/parsedatetime/ 181 | 182 | .. [#nopy3k] These modules do not support Python 3. 183 | They are not installed in if you use Python 3 184 | and related functionality is disabled. 185 | 186 | Platforms 187 | --------- 188 | 189 | UNIX-like systems 190 | RASH is tested in Linux and I am using in Linux. 191 | It should work in other UNIX-like systems like BSD. 192 | 193 | Mac OS 194 | I guess it works. Not tested. 195 | 196 | MS Windows 197 | Probably no one wants to use a shell tool in windows, but I 198 | try to avoid stuff that is platform specific. Only the 199 | daemon launcher will not work on Windows but there is several 200 | ways to avoid using it. See ``rash init --help``. 201 | 202 | Shells 203 | ------ 204 | 205 | RASH currently supports zsh and bash. 206 | 207 | 208 | Design principle 209 | ================ 210 | 211 | RASH's design is focused on sparseness. There are several stages 212 | of data transformation until you see the search result, and they 213 | are done by separated processes. 214 | 215 | First, ``rash record`` command dumps shell history in raw JSON record. 216 | This part of program does not touches to DB to make process very fast. 217 | As there is no complex transformation in this command, probably in the 218 | future version it is better to rewrite it entirely in shell function. 219 | 220 | Second, ``rash daemon`` runs in background and watches the directory to 221 | store JSON record. When JSON record arrives, it insert the data into 222 | database. 223 | 224 | ``rash record`` and ``rash daemon`` are setup by simple shell snippet 225 | ``eval $(rash init)``. 226 | 227 | Finally, you can search through command history using search interface 228 | such as `rash search`. This search is very fast as you don't read 229 | all JSON records in separated files. 230 | 231 | :: 232 | 233 | +-------+ +--------+ +--------+ +--------+ 234 | | Shell | | Raw | | SQLite | | Search | 235 | | hooks |-------->| JSON |-------->| DB |-------->| result | 236 | +-------+ | record | +--------+ +--------+ 237 | +--------+ 238 | 239 | `rash record` `rash daemon` `rash search` 240 | `rash show` 241 | 242 | \------------------------------/ \------------/ 243 | `rash init` setups them search interface 244 | 245 | License 246 | ======= 247 | 248 | RASH is licensed under GPL v3. 249 | See COPYING for details. 250 | 251 | 252 | .. Travis CI build status badge 253 | .. |build-status| 254 | image:: https://secure.travis-ci.org/tkf/rash.png?branch=master 255 | :target: http://travis-ci.org/tkf/rash 256 | :alt: Build Status 257 | 258 | """ 259 | # [[[end]]] 260 | 261 | # Copyright (C) 2013- Takafumi Arakaki 262 | 263 | # This program is free software: you can redistribute it and/or modify 264 | # it under the terms of the GNU General Public License as published by 265 | # the Free Software Foundation, either version 3 of the License, or 266 | # (at your option) any later version. 267 | 268 | # This program is distributed in the hope that it will be useful, 269 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 270 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 271 | # GNU General Public License for more details. 272 | 273 | # You should have received a copy of the GNU General Public License 274 | # along with this program. If not, see . 275 | 276 | 277 | __version__ = '0.1.3' 278 | __author__ = 'Takafumi Arakaki' 279 | __license__ = 'GNU General Public License v3 (GPLv3)' 280 | -------------------------------------------------------------------------------- /rash/cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | Command line interface. 3 | """ 4 | 5 | import argparse 6 | import textwrap 7 | 8 | EPILOG = """ 9 | # Copyright (C) 2013- Takafumi Arakaki 10 | 11 | # This program is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU General Public License as published by 13 | # the Free Software Foundation, either version 3 of the License, or 14 | # (at your option) any later version. 15 | 16 | # This program is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program. If not, see . 23 | """ 24 | 25 | 26 | class Formatter(argparse.RawDescriptionHelpFormatter, 27 | argparse.ArgumentDefaultsHelpFormatter): 28 | pass 29 | 30 | 31 | def get_parser(commands): 32 | """ 33 | Generate argument parser given a list of subcommand specifications. 34 | 35 | :type commands: list of (str, function, function) 36 | :arg commands: 37 | Each element must be a tuple ``(name, adder, runner)``. 38 | 39 | :param name: subcommand 40 | :param adder: a function takes one object which is an instance 41 | of :class:`argparse.ArgumentParser` and add 42 | arguments to it 43 | :param runner: a function takes keyword arguments which must be 44 | specified by the arguments parsed by the parser 45 | defined by `adder`. Docstring of this function 46 | will be the description of the subcommand. 47 | 48 | """ 49 | parser = argparse.ArgumentParser( 50 | formatter_class=Formatter, 51 | description=__doc__, 52 | epilog=EPILOG, 53 | ) 54 | subparsers = parser.add_subparsers() 55 | 56 | for (name, adder, runner) in commands: 57 | subp = subparsers.add_parser( 58 | name, 59 | formatter_class=Formatter, 60 | description=runner.__doc__ and textwrap.dedent(runner.__doc__)) 61 | adder(subp) 62 | subp.set_defaults(func=runner) 63 | 64 | return parser 65 | 66 | 67 | def main(args=None): 68 | from . import init 69 | from . import record 70 | from . import daemon 71 | from . import search 72 | from . import show 73 | from . import index 74 | from . import isearch 75 | # from . import MODULE 76 | parser = get_parser( 77 | init.commands 78 | + record.commands 79 | + daemon.commands 80 | + search.commands 81 | + show.commands 82 | + index.commands 83 | + isearch.commands 84 | # + MODULE.commands 85 | + misc_commands 86 | ) 87 | ns = parser.parse_args(args=args) 88 | applyargs = lambda func, **kwds: func(**kwds) 89 | applyargs(**vars(ns)) 90 | 91 | 92 | def version_run(): 93 | """Print version number.""" 94 | from .__init__ import __version__ 95 | print(__version__) 96 | 97 | 98 | def version_add_arguments(parser): 99 | pass 100 | 101 | 102 | def locate_run(output, target, no_newline): 103 | """ 104 | Print location of RASH related file. 105 | """ 106 | from .config import ConfigStore 107 | cfstore = ConfigStore() 108 | path = getattr(cfstore, "{0}_path".format(target)) 109 | output.write(path) 110 | if not no_newline: 111 | output.write("\n") 112 | 113 | 114 | def locate_add_arguments(parser): 115 | parser.add_argument( 116 | 'target', 117 | choices=['base', 'config', 'db', 'daemon_pid', 'daemon_log'], 118 | help='Name of file to show the path (e.g., config).') 119 | parser.add_argument( 120 | '--no-newline', '-n', action='store_true', 121 | help='do not output the trailing newline.') 122 | parser.add_argument( 123 | '--output', default='-', type=argparse.FileType('w'), 124 | help=""" 125 | Output file to write the results in. Default is stdout. 126 | """) 127 | 128 | 129 | misc_commands = [ 130 | ('version', version_add_arguments, version_run), 131 | ('locate', locate_add_arguments, locate_run), 132 | ] 133 | 134 | 135 | if __name__ == '__main__': 136 | main() 137 | -------------------------------------------------------------------------------- /rash/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | RASH configuration 3 | ================== 4 | 5 | .. autoclass:: Configuration 6 | :members: 7 | .. autoclass:: RecordConfig 8 | :members: 9 | .. autoclass:: SearchConfig 10 | :members: 11 | .. autoclass:: ISearchConfig 12 | :members: 13 | 14 | """ 15 | 16 | # FIXME: Remove ConfigStore then use ``autodoc_default_flags = ['members']`` 17 | # in ../doc/source/conf.py, so that I don't need to write `autoclass` 18 | # explicitly like above. To do so, add `get_config` function and another 19 | # sub-configurable called PathConfig. 20 | 21 | # Copyright (C) 2013- Takafumi Arakaki 22 | 23 | # This program is free software: you can redistribute it and/or modify 24 | # it under the terms of the GNU General Public License as published by 25 | # the Free Software Foundation, either version 3 of the License, or 26 | # (at your option) any later version. 27 | 28 | # This program is distributed in the hope that it will be useful, 29 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | # GNU General Public License for more details. 32 | 33 | # You should have received a copy of the GNU General Public License 34 | # along with this program. If not, see . 35 | 36 | 37 | import os 38 | 39 | from .utils.confutils import get_config_directory 40 | from .utils.pathutils import mkdirp 41 | 42 | 43 | class ConfigStore(object): 44 | 45 | """ 46 | Configuration and data file store. 47 | 48 | RASH stores data in the following directory in Linux:: 49 | 50 | * ~/.config/ # $XDG_CONFIG_HOME 51 | `--* rash/ # base_path 52 | |--* daemon.pid # PID of daemon process 53 | |--* daemon.log # Log file for daemon 54 | `--* data/ # data_path 55 | |--* db.sqlite # db_path ("indexed" record) 56 | `--* record/ # record_path ("raw" record) 57 | |--* command/ # command log 58 | `--* init/ # initialization log 59 | 60 | In Mac OS and Windows, :attr:`base_path` may be different but 61 | structure in the directory is the same. 62 | 63 | """ 64 | 65 | def __init__(self, base_path=None): 66 | self.base_path = base_path or get_config_directory('RASH') 67 | """ 68 | Root directory for any RASH related data files (``~/.config/rash``). 69 | """ 70 | 71 | self.config_path = os.path.join(self.base_path, 'config.py') 72 | """ 73 | File to store user configuration (``~/.config/rash/config.py``). 74 | """ 75 | 76 | self.data_path = os.path.join(self.base_path, 'data') 77 | """ 78 | Directory to store data collected by RASH (``~/.config/rash/data``). 79 | """ 80 | 81 | self.record_path = os.path.join(self.data_path, 'record') 82 | """ 83 | Shell history is stored in this directory at the first stage. 84 | """ 85 | 86 | self.db_path = os.path.join(self.data_path, 'db.sqlite') 87 | """ 88 | Shell history is stored in the DB at this path. 89 | """ 90 | 91 | self.daemon_pid_path = os.path.join(self.base_path, 'daemon.pid') 92 | """ 93 | A file to store daemon PID (``~/.config/rash/daemon.pid``). 94 | """ 95 | 96 | self.daemon_log_path = os.path.join(self.base_path, 'daemon.log') 97 | """ 98 | Daemon log file (``~/.config/rash/daemon.log``). 99 | """ 100 | 101 | self.daemon_log_level = 'INFO' # FIXME: make this configurable 102 | """ 103 | Daemon log level. 104 | """ 105 | 106 | mkdirp(self.record_path) 107 | 108 | def get_config(self): 109 | """ 110 | Load user configuration or return default when not found. 111 | 112 | :rtype: :class:`Configuration` 113 | 114 | """ 115 | if not self._config: 116 | namespace = {} 117 | if os.path.exists(self.config_path): 118 | execfile(self.config_path, namespace) 119 | self._config = namespace.get('config') or Configuration() 120 | return self._config 121 | _config = None 122 | 123 | 124 | class Configuration(object): 125 | 126 | """ 127 | RASH configuration interface. 128 | 129 | If you define an object named :data:`config` in the 130 | :term:`configuration file`, it is going to be loaded by RASH. 131 | :data:`config` must be an instance of :class:`Configuration`. 132 | 133 | .. glossary:: 134 | 135 | configuration file 136 | In unix-like systems, it's :file:`~/.config/rash/config.py` or 137 | different place if you set :envvar:`XDG_CONFIG_HOME`. In Mac 138 | OS, it's :file:`~/Library/Application Support/RASH/config.py`. 139 | Use ``rash locate config`` to locate the exact place. 140 | 141 | Example: 142 | 143 | >>> from rash.config import Configuration 144 | >>> config = Configuration() 145 | >>> config.isearch.query = '-u .' 146 | 147 | Here is a list of configuration variables you can set: 148 | 149 | =========================== =========================================== 150 | Configuration variables 151 | =========================== =========================================== 152 | |record.environ| Environment variables to record. 153 | |search.alias| Search query alias. 154 | |search.kwds_adapter| Transform keyword arguments. 155 | |isearch.query| Default isearch query. 156 | |isearch.query_template| Transform default query. 157 | |isearch.base_query| Default isearch base query. 158 | =========================== =========================================== 159 | 160 | .. |record.environ| replace:: 161 | :attr:`config.record.environ ` 162 | .. |search.alias| replace:: 163 | :attr:`config.search.alias ` 164 | .. |search.kwds_adapter| replace:: 165 | :attr:`config.search.kwds_adapter ` 166 | .. |isearch.query| replace:: 167 | :attr:`config.isearch.query ` 168 | .. |isearch.query_template| replace:: 169 | :attr:`config.isearch.query_template ` 170 | .. |isearch.base_query| replace:: 171 | :attr:`config.isearch.base_query ` 172 | 173 | """ 174 | 175 | def __init__(self): 176 | self.record = RecordConfig() 177 | self.search = SearchConfig() 178 | self.isearch = ISearchConfig() 179 | 180 | 181 | class RecordConfig(object): 182 | 183 | """ 184 | Recording configuration. 185 | """ 186 | 187 | def __init__(self): 188 | 189 | self.environ = { 190 | 'init': [ 191 | 'SHELL', 'TERM', 'HOST', 'TTY', 'USER', 'DISPLAY', 192 | # SOMEDAY: Reevaluate if "RASH_SPENV_TERMINAL" is the 193 | # right choice. Here, I am using `environ` dict as a 194 | # generic key value store. Using 'RASH_SPENV_' as a 195 | # prefix key, it is very easy to add new variable to 196 | # track. 197 | 'RASH_SPENV_TERMINAL', 198 | ], 199 | 'exit': [], 200 | 'command': ['PATH'], 201 | } 202 | """ 203 | Environment variables to record. 204 | 205 | Each key (str) represent record type (init/exit/command). 206 | Each value (list of str) is a list of environment variables to 207 | record. 208 | 209 | Example usage: 210 | 211 | >>> config = Configuration() 212 | >>> config.record.environ['command'] += ['VIRTUAL_ENV', 'PYTHONPATH'] 213 | 214 | """ 215 | 216 | 217 | class SearchConfig(object): 218 | 219 | """ 220 | Search configuration. 221 | """ 222 | 223 | def __init__(self): 224 | 225 | self.alias = {} 226 | r""" 227 | Search query alias. 228 | 229 | It must be a dict-like object that maps a str to a list of str 230 | when "expanding" search query. 231 | 232 | Example: 233 | 234 | >>> config = Configuration() 235 | >>> config.search.alias['test'] = \ 236 | ... ["--exclude-pattern", "*rash *", "--include-pattern", "*test*"] 237 | 238 | then,:: 239 | 240 | rash search test 241 | 242 | is equivalent to:: 243 | 244 | rash search --exclude-pattern "*rash *" --include-pattern "*test*" 245 | 246 | """ 247 | 248 | self.kwds_adapter = lambda x: x 249 | """ 250 | A function to transform keyword arguments. 251 | 252 | This function takes a dictionary from command line argument 253 | parser and can modify the dictionary to do whatever you want 254 | to do with it. It is much more lower-level and powerful than 255 | :attr:`alias`. This function must return the modified, 256 | or possibly new dictionary. 257 | 258 | Example definition that does the same effect as the example in 259 | :attr:`alias`: 260 | 261 | >>> def adapter(kwds): 262 | ... if 'test' in kwds.get('pattern', []): 263 | ... kwds['pattern'] = [p for p in kwds['pattern'] 264 | ... if p != 'test'] 265 | ... kwds['exclude_pattern'].append("*rash *") 266 | ... kwds['include_pattern'].append("*test*") 267 | ... return kwds 268 | ... 269 | >>> config = Configuration() 270 | >>> config.search.kwds_adapter = adapter 271 | 272 | """ 273 | 274 | 275 | class ISearchConfig(object): 276 | 277 | """ 278 | Configure how ``rash isearch`` is started. 279 | 280 | See also :class:`SearchConfig`. Once isearch UI is started, 281 | :class:`SearchConfig` controls how search query is interpreted. 282 | For example, aliases defined in :class:`SearchConfig` can be used 283 | in isearch. 284 | 285 | """ 286 | 287 | def __init__(self): 288 | 289 | self.query = '' 290 | """ 291 | Set default value (str) for ``--query`` option. 292 | 293 | If you want to start isearch with the query ``-d .`` (only 294 | list the command executed at this directory), use the 295 | following configuration: 296 | 297 | >>> config = Configuration() 298 | >>> config.isearch.query = '-d . ' 299 | 300 | As ``rash-zle-isearch`` passes the current line content to 301 | ``--query`` which override this setting, you need to use 302 | :attr:`query_template` instead if you want to configure the 303 | default query. 304 | 305 | """ 306 | 307 | self.query_template = '{0}' 308 | """ 309 | Transform default query using Python string format. 310 | 311 | The string format should have only one field ``{0}``. 312 | The query given by ``-query`` or the one specified by 313 | :attr:`query` fills that filed. Default value is 314 | do-nothing template ``'{0}'``. 315 | 316 | >>> config = Configuration() 317 | >>> config.isearch.query_template = '-d . {0}' 318 | 319 | """ 320 | 321 | self.base_query = [] 322 | """ 323 | Set default value (list of str) for ``--base-query`` option. 324 | """ 325 | -------------------------------------------------------------------------------- /rash/daemon.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | 19 | 20 | def daemon_run(no_error, restart, record_path, keep_json, check_duplicate, 21 | use_polling, log_level): 22 | """ 23 | Run RASH index daemon. 24 | 25 | This daemon watches the directory ``~/.config/rash/data/record`` 26 | and translate the JSON files dumped by ``record`` command into 27 | sqlite3 DB at ``~/.config/rash/data/db.sqlite``. 28 | 29 | ``rash init`` will start RASH automatically by default. 30 | But there are alternative ways to start daemon. 31 | 32 | If you want to organize background process in one place such 33 | as supervisord_, it is good to add `--restart` option to force 34 | stop other daemon process if you accidentally started it in 35 | other place. Here is an example of supervisord_ setup:: 36 | 37 | [program:rash-daemon] 38 | command=rash daemon --restart 39 | 40 | .. _supervisord: http://supervisord.org/ 41 | 42 | Alternatively, you can call ``rash index`` in cron job to 43 | avoid using daemon. It is useful if you want to use RASH 44 | on NFS, as it looks like watchdog does not work on NFS.:: 45 | 46 | # Refresh RASH DB every 10 minutes 47 | */10 * * * * rash index 48 | 49 | """ 50 | # Probably it makes sense to use this daemon to provide search 51 | # API, so that this daemon is going to be the only process that 52 | # is connected to the DB? 53 | from .config import ConfigStore 54 | from .indexer import Indexer 55 | from .log import setup_daemon_log_file, LogForTheFuture 56 | from .watchrecord import watch_record, install_sigterm_handler 57 | 58 | install_sigterm_handler() 59 | cfstore = ConfigStore() 60 | if log_level: 61 | cfstore.daemon_log_level = log_level 62 | flogger = LogForTheFuture() 63 | 64 | # SOMEDAY: make PID checking/writing atomic if possible 65 | flogger.debug('Checking old PID file %r.', cfstore.daemon_pid_path) 66 | if os.path.exists(cfstore.daemon_pid_path): 67 | flogger.debug('Old PID file exists. Reading from it.') 68 | with open(cfstore.daemon_pid_path, 'rt') as f: 69 | pid = int(f.read().strip()) 70 | flogger.debug('Checking if old process with PID=%d is alive', pid) 71 | try: 72 | os.kill(pid, 0) # check if `pid` is alive 73 | except OSError: 74 | flogger.info( 75 | 'Process with PID=%d is already dead. ' 76 | 'So just go on and use this daemon.', pid) 77 | else: 78 | if restart: 79 | flogger.info('Stopping old daemon with PID=%d.', pid) 80 | stop_running_daemon(cfstore, pid) 81 | else: 82 | message = ('There is already a running daemon (PID={0})!' 83 | .format(pid)) 84 | if no_error: 85 | flogger.debug(message) 86 | # FIXME: Setup log handler and flogger.dump(). 87 | # Note that using the default log file is not safe 88 | # since it has already been used. 89 | return 90 | else: 91 | raise RuntimeError(message) 92 | else: 93 | flogger.debug('Daemon PID file %r does not exists. ' 94 | 'So just go on and use this daemon.', 95 | cfstore.daemon_pid_path) 96 | 97 | with open(cfstore.daemon_pid_path, 'w') as f: 98 | f.write(str(os.getpid())) 99 | 100 | try: 101 | setup_daemon_log_file(cfstore) 102 | flogger.dump() 103 | indexer = Indexer(cfstore, check_duplicate, keep_json, record_path) 104 | indexer.index_all() 105 | watch_record(indexer, use_polling) 106 | finally: 107 | os.remove(cfstore.daemon_pid_path) 108 | 109 | 110 | def stop_running_daemon(cfstore, pid): 111 | import time 112 | import signal 113 | os.kill(pid, signal.SIGTERM) 114 | for _ in range(30): 115 | time.sleep(0.1) 116 | if not os.path.exists(cfstore.daemon_pid_path): 117 | break 118 | else: 119 | raise RuntimeError( 120 | 'Failed to stop running daemon process (PID={0})' 121 | .format(pid)) 122 | 123 | 124 | def start_daemon_in_subprocess(options, outpath=os.devnull): 125 | """ 126 | Run `rash daemon --no-error` in background. 127 | 128 | :type options: list of str 129 | :arg options: options for "rash daemon" command 130 | :type outpath: str 131 | :arg outpath: path to redirect daemon output 132 | 133 | """ 134 | import subprocess 135 | import sys 136 | from .utils.py3compat import nested 137 | from .utils.pathutils import mkdirp 138 | if outpath != os.devnull: 139 | mkdirp(os.path.dirname(outpath)) 140 | with nested(open(os.devnull), 141 | open(outpath, 'w')) as (stdin, stdout): 142 | subprocess.Popen( 143 | [os.path.abspath(sys.executable), '-m', 'rash.cli', 144 | 'daemon', '--no-error'] + options, 145 | preexec_fn=os.setsid, 146 | stdin=stdin, stdout=stdout, stderr=subprocess.STDOUT) 147 | 148 | 149 | def daemon_add_arguments(parser): 150 | parser.add_argument( 151 | '--no-error', action='store_true', default=False, 152 | help=""" 153 | Do nothing if a daemon is already running. 154 | """) 155 | parser.add_argument( 156 | '--restart', action='store_true', default=False, 157 | help=""" 158 | Kill already running daemon process if exist. 159 | """) 160 | parser.add_argument( 161 | '--record-path', 162 | help=""" 163 | specify the directory that has JSON records. 164 | """) 165 | parser.add_argument( 166 | '--keep-json', default=False, action='store_true', 167 | help=""" 168 | Do not remove old JSON files. It turns on --check-duplicate. 169 | """) 170 | parser.add_argument( 171 | '--check-duplicate', default=False, action='store_true', 172 | help='do not store already existing history in DB.') 173 | parser.add_argument( 174 | '--use-polling', default=False, action='store_true', 175 | help=""" 176 | Use polling instead of system specific notification. 177 | This is useful, for example, when your $HOME is on NFS where 178 | inotify does not work. 179 | """) 180 | parser.add_argument( 181 | '--log-level', 182 | choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'], 183 | help='logging level.') 184 | 185 | 186 | commands = [ 187 | ('daemon', daemon_add_arguments, daemon_run), 188 | ] 189 | -------------------------------------------------------------------------------- /rash/database.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | import re 19 | import sqlite3 20 | from contextlib import closing, contextmanager 21 | import datetime 22 | import warnings 23 | import itertools 24 | 25 | from .utils.py3compat import zip_longest 26 | from .utils.iterutils import nonempty, include_before, include_after, \ 27 | include_context 28 | from .utils.sqlconstructor import SQLConstructor 29 | from .model import CommandRecord, SessionRecord, VersionRecord, EnvironRecord 30 | 31 | schema_version = '0.1' 32 | 33 | 34 | def convert_ts(ts): 35 | """ 36 | Convert timestamp (ts) 37 | 38 | :type ts: int or str or None 39 | :arg ts: Unix timestamp 40 | :rtype: datetime.datetime or str or None 41 | 42 | """ 43 | if ts is None: 44 | return None 45 | try: 46 | return datetime.datetime.utcfromtimestamp(ts) 47 | except TypeError: 48 | pass 49 | return ts 50 | 51 | 52 | def normalize_directory(path): 53 | """ 54 | Append "/" to `path` if needed. 55 | """ 56 | if path is None: 57 | return None 58 | if path.endswith(os.path.sep): 59 | return path 60 | else: 61 | return path + os.path.sep 62 | 63 | 64 | def sql_regexp_func(expr, item): 65 | return re.match(expr, item) is not None 66 | 67 | 68 | def sql_program_name_func(command): 69 | """ 70 | Extract program name from `command`. 71 | 72 | >>> sql_program_name_func('ls') 73 | 'ls' 74 | >>> sql_program_name_func('git status') 75 | 'git' 76 | >>> sql_program_name_func('EMACS=emacs make') 77 | 'make' 78 | 79 | :type command: str 80 | 81 | """ 82 | args = command.split(' ') 83 | for prog in args: 84 | if '=' not in prog: 85 | return prog 86 | return args[0] 87 | 88 | 89 | def sql_pathdist_func(path1, path2, sep=os.path.sep): 90 | """ 91 | Return a distance between `path1` and `path2`. 92 | 93 | >>> sql_pathdist_func('a/b/', 'a/b/', sep='/') 94 | 0 95 | >>> sql_pathdist_func('a/', 'a/b/', sep='/') 96 | 1 97 | >>> sql_pathdist_func('a', 'a/', sep='/') 98 | 0 99 | 100 | """ 101 | seq1 = path1.rstrip(sep).split(sep) 102 | seq2 = path2.rstrip(sep).split(sep) 103 | return sum(1 for (p1, p2) in zip_longest(seq1, seq2) if p1 != p2) 104 | 105 | 106 | class DataBase(object): 107 | 108 | schemapath = os.path.join( 109 | os.path.dirname(os.path.abspath(__file__)), 'schema.sql') 110 | 111 | def __init__(self, dbpath): 112 | self.dbpath = dbpath 113 | if not os.path.exists(dbpath): 114 | self._init_db() 115 | self.update_version_records() 116 | 117 | def _get_db(self): 118 | """Returns a new connection to the database.""" 119 | return closing(sqlite3.connect(self.dbpath)) 120 | 121 | def _init_db(self): 122 | """Creates the database tables.""" 123 | with self._get_db() as db: 124 | with open(self.schemapath) as f: 125 | db.cursor().executescript(f.read()) 126 | db.commit() 127 | 128 | @contextmanager 129 | def connection(self, commit=False): 130 | """ 131 | Context manager to keep around DB connection. 132 | 133 | :rtype: sqlite3.Connection 134 | 135 | SOMEDAY: Get rid of this function. Keeping connection around as 136 | an argument to the method using this context manager is 137 | probably better as it is more explicit. 138 | Also, holding "global state" as instance attribute is bad for 139 | supporting threaded search, which is required for more fluent 140 | percol integration. 141 | 142 | """ 143 | if commit: 144 | self._need_commit = True 145 | if self._db: 146 | yield self._db 147 | else: 148 | try: 149 | with self._get_db() as db: 150 | self._db = db 151 | db.create_function("REGEXP", 2, sql_regexp_func) 152 | db.create_function("PROGRAM_NAME", 1, 153 | sql_program_name_func) 154 | db.create_function("PATHDIST", 2, sql_pathdist_func) 155 | yield self._db 156 | if self._need_commit: 157 | db.commit() 158 | finally: 159 | self._db = None 160 | self._need_commit = False 161 | _db = None 162 | _need_commit = False 163 | 164 | def close_connection(self): 165 | """ 166 | Close connection kept by :meth:`connection`. 167 | 168 | If commit is needed, :meth:`sqlite3.Connection.commit` 169 | is called first and then :meth:`sqlite3.Connection.interrupt` 170 | is called. 171 | 172 | A few methods/generators support :meth:`close_connection`: 173 | 174 | - :meth:`search_command_record` 175 | - :meth:`select_by_command_record` 176 | 177 | """ 178 | if self._db: 179 | db = self._db 180 | try: 181 | if self._need_commit: 182 | db.commit() 183 | finally: 184 | db.interrupt() 185 | self._db = None 186 | self._need_commit = False 187 | 188 | def _executing(self, sql, params=[]): 189 | """ 190 | Execute and yield rows in a way to support :meth:`close_connection`. 191 | """ 192 | with self.connection() as connection: 193 | for row in connection.execute(sql, params): 194 | yield row 195 | if not self._db: 196 | return 197 | 198 | def _select_rows(self, rowclass, keys, sql, params): 199 | return (rowclass(**dict(zip(keys, row))) 200 | for row in self._executing(sql, params)) 201 | 202 | def get_version_records(self): 203 | """ 204 | Yield RASH version information stored in DB. Latest first. 205 | 206 | :rtype: [VersionRecord] 207 | 208 | """ 209 | keys = ['id', 'rash_version', 'schema_version', 'updated'] 210 | sql = """ 211 | SELECT id, rash_version, schema_version, updated 212 | FROM rash_info 213 | ORDER BY id DESC 214 | """ 215 | with self.connection() as connection: 216 | for row in connection.execute(sql): 217 | yield VersionRecord(**dict(zip(keys, row))) 218 | 219 | def update_version_records(self): 220 | """ 221 | Update rash_info table if necessary. 222 | """ 223 | from .__init__ import __version__ as version 224 | with self.connection(commit=True) as connection: 225 | for vrec in self.get_version_records(): 226 | if (vrec.rash_version == version and 227 | vrec.schema_version == schema_version): 228 | return # no need to insert the new one! 229 | connection.execute( 230 | 'INSERT INTO rash_info (rash_version, schema_version) ' 231 | 'VALUES (?, ?)', 232 | [version, schema_version]) 233 | 234 | def import_json(self, json_path, **kwds): 235 | import json 236 | with open(json_path) as fp: 237 | try: 238 | dct = json.load(fp) 239 | except ValueError: 240 | warnings.warn( 241 | 'Ignoring invalid JSON file at: {0}'.format(json_path)) 242 | return 243 | self.import_dict(dct, **kwds) 244 | 245 | def import_dict(self, dct, check_duplicate=True): 246 | crec = CommandRecord(**dct) 247 | if check_duplicate and nonempty(self.select_by_command_record(crec)): 248 | return 249 | with self.connection(commit=True) as connection: 250 | db = connection.cursor() 251 | ch_id = self._insert_command_history(db, crec) 252 | self._isnert_command_environment(db, ch_id, crec.environ) 253 | self._insert_pipe_status(db, ch_id, crec.pipestatus) 254 | 255 | def _insert_command_history(self, db, crec): 256 | command_id = self._get_maybe_new_command_id(db, crec.command) 257 | session_id = self._get_maybe_new_session_id(db, crec.session_id) 258 | directory_id = self._get_maybe_new_directory_id(db, crec.cwd) 259 | terminal_id = self._get_maybe_new_terminal_id(db, crec.terminal) 260 | db.execute( 261 | ''' 262 | INSERT INTO command_history 263 | (command_id, session_id, directory_id, terminal_id, 264 | start_time, stop_time, exit_code) 265 | VALUES (?, ?, ?, ?, ?, ?, ?) 266 | ''', 267 | [command_id, session_id, directory_id, terminal_id, 268 | convert_ts(crec.start), convert_ts(crec.stop), crec.exit_code]) 269 | return db.lastrowid 270 | 271 | def _isnert_command_environment(self, db, ch_id, environ): 272 | self._insert_environ(db, 'command_environment_map', 'ch_id', ch_id, 273 | environ) 274 | 275 | def _insert_environ(self, db, table, id_name, ch_id, environ): 276 | if not environ: 277 | return 278 | for (name, value) in environ.items(): 279 | if name is None or value is None: 280 | continue 281 | ev_id = self._get_maybe_new_id( 282 | db, 'environment_variable', 283 | {'variable_name': name, 'variable_value': value}) 284 | db.execute( 285 | ''' 286 | INSERT INTO {0} 287 | ({1}, ev_id) 288 | VALUES (?, ?) 289 | '''.format(table, id_name), 290 | [ch_id, ev_id]) 291 | 292 | def _insert_pipe_status(self, db, ch_id, pipe_status): 293 | if not pipe_status: 294 | return 295 | for (i, code) in enumerate(pipe_status): 296 | db.execute( 297 | ''' 298 | INSERT INTO pipe_status_map 299 | (ch_id, program_position, exit_code) 300 | VALUES (?, ?, ?) 301 | ''', 302 | [ch_id, i, code]) 303 | 304 | def _get_maybe_new_command_id(self, db, command): 305 | if command is None: 306 | return None 307 | return self._get_maybe_new_id( 308 | db, 'command_list', {'command': command}) 309 | 310 | def _get_maybe_new_session_id(self, db, session_long_id): 311 | if session_long_id is None: 312 | return None 313 | return self._get_maybe_new_id( 314 | db, 'session_history', {'session_long_id': session_long_id}) 315 | 316 | def _get_maybe_new_directory_id(self, db, directory): 317 | if directory is None: 318 | return None 319 | directory = normalize_directory(directory) 320 | return self._get_maybe_new_id( 321 | db, 'directory_list', {'directory': directory}) 322 | 323 | def _get_maybe_new_terminal_id(self, db, terminal): 324 | if terminal is None: 325 | return None 326 | return self._get_maybe_new_id( 327 | db, 'terminal_list', {'terminal': terminal}) 328 | 329 | def _get_maybe_new_id(self, db, table, columns): 330 | kvlist = list(columns.items()) 331 | values = [v for (_, v) in kvlist] 332 | sql_select = 'SELECT id FROM "{0}" WHERE {1}'.format( 333 | table, 334 | ' AND '.join(map('"{0[0]}" = ?'.format, kvlist)), 335 | ) 336 | for (id_val,) in db.execute(sql_select, values): 337 | return id_val 338 | sql_insert = 'INSERT INTO "{0}" ({1}) VALUES ({2})'.format( 339 | table, 340 | ', '.join(map('"{0[0]}"'.format, kvlist)), 341 | ', '.join('?' for _ in kvlist), 342 | ) 343 | db.execute(sql_insert, values) 344 | return db.lastrowid 345 | 346 | def select_by_command_record(self, crec): 347 | """ 348 | Yield records that matches to `crec`. 349 | 350 | All attributes of `crec` except for `environ` are concerned. 351 | 352 | """ 353 | keys = ['command_history_id', 'command', 'session_history_id', 354 | 'cwd', 'terminal', 355 | 'start', 'stop', 'exit_code'] 356 | sql = """ 357 | SELECT 358 | command_history.id, CL.command, session_id, 359 | DL.directory, TL.terminal, 360 | start_time, stop_time, exit_code 361 | FROM command_history 362 | LEFT JOIN command_list AS CL ON command_id = CL.id 363 | LEFT JOIN directory_list AS DL ON directory_id = DL.id 364 | LEFT JOIN terminal_list AS TL ON terminal_id = TL.id 365 | WHERE 366 | (CL.command = ? OR (CL.command IS NULL AND ? IS NULL)) AND 367 | (DL.directory = ? OR (DL.directory IS NULL AND ? IS NULL)) AND 368 | (TL.terminal = ? OR (TL.terminal IS NULL AND ? IS NULL)) AND 369 | (start_time = ? OR (start_time IS NULL AND ? IS NULL)) AND 370 | (stop_time = ? OR (stop_time IS NULL AND ? IS NULL)) AND 371 | (exit_code = ? OR (exit_code IS NULL AND ? IS NULL)) 372 | """ 373 | desired_row = [ 374 | crec.command, normalize_directory(crec.cwd), crec.terminal, 375 | convert_ts(crec.start), convert_ts(crec.stop), crec.exit_code] 376 | params = list(itertools.chain(*zip(desired_row, desired_row))) 377 | return self._select_rows(CommandRecord, keys, sql, params) 378 | 379 | def search_command_record( 380 | self, 381 | after_context, before_context, context, context_type, 382 | **kwds): 383 | """ 384 | Search command history. 385 | 386 | :rtype: [CommandRecord] 387 | 388 | """ 389 | if after_context or before_context or context: 390 | kwds['condition_as_column'] = True 391 | limit = kwds['limit'] 392 | kwds['limit'] = -1 393 | kwds['unique'] = False 394 | kwds['sort_by'] = { 395 | 'session': ['session_start_time', 'start_time'], 396 | 'time': ['start_time'], 397 | }[context_type] 398 | if not kwds['reverse']: 399 | # Default (reverse=False) means latest history comes first. 400 | after_context, before_context = before_context, after_context 401 | 402 | (sql, params, keys) = self._compile_sql_search_command_record(**kwds) 403 | records = self._select_rows(CommandRecord, keys, sql, params) 404 | 405 | # SOMEDAY: optimize context search; do not create CommandRecord 406 | # object for all (including non-matching) records. 407 | predicate = lambda r: r.condition 408 | if context: 409 | records = include_context(predicate, context, records) 410 | elif before_context: 411 | records = include_before(predicate, before_context, records) 412 | elif after_context: 413 | records = include_after(predicate, after_context, records) 414 | if after_context or before_context or context and limit >= 0: 415 | records = itertools.islice(records, limit) 416 | # NOTE: as SQLite does not support row_number function, let's 417 | # do the filtering at Python side when context modifier 418 | # is given. This is *very* inefficient but at least it 419 | # works.. 420 | 421 | return records 422 | 423 | @classmethod 424 | def _compile_sql_search_command_record( 425 | cls, limit, unique, 426 | match_pattern, include_pattern, exclude_pattern, 427 | match_regexp, include_regexp, exclude_regexp, 428 | cwd, cwd_glob, cwd_under, 429 | time_after, time_before, duration_longer_than, duration_less_than, 430 | include_exit_code, exclude_exit_code, 431 | include_session_history_id, exclude_session_history_id, 432 | match_environ_pattern, include_environ_pattern, 433 | exclude_environ_pattern, 434 | match_environ_regexp, include_environ_regexp, 435 | exclude_environ_regexp, 436 | reverse, sort_by, sort_by_cwd_distance, 437 | ignore_case, 438 | additional_columns=[], condition_as_column=False, 439 | ): 440 | keys = ['command_history_id', 'command', 'session_history_id', 441 | 'cwd', 'terminal', 442 | 'start', 'stop', 'exit_code'] 443 | columns = ['command_history.id', 'CL.command', 'session_id', 444 | 'DL.directory', 'TL.terminal', 445 | 'start_time', 'stop_time', 'exit_code'] 446 | source = ( 447 | 'command_history ' 448 | 'LEFT JOIN command_list AS CL ON command_id = CL.id ' 449 | 'LEFT JOIN directory_list AS DL ON directory_id = DL.id ' 450 | 'LEFT JOIN terminal_list AS TL ON terminal_id = TL.id') 451 | 452 | if cwd_under: 453 | cwd_glob.extend(os.path.join(os.path.abspath(p), "*") 454 | for p in cwd_under) 455 | 456 | if ignore_case: 457 | glob = "glob(lower({1}), lower({0}))".format 458 | else: 459 | glob = "glob({1}, {0})".format 460 | regexp = "regexp({1}, {0})" 461 | eq = '{0} = {1}' 462 | 463 | if not unique and 'command_count' in sort_by: 464 | # When not using "GROUP BY", `COUNT(*)` yields just one 465 | # row. As unique is True by default, `unique=False` 466 | # should mean to ignore ``sort_by='command_count'``. 467 | sort_by = [k for k in sort_by if k != 'command_count'] 468 | 469 | sc = SQLConstructor(source, columns, keys, limit=limit) 470 | if sort_by_cwd_distance: 471 | col_cwd_dist = 'PATHDIST(DL.directory, ?)' 472 | if unique: 473 | col_cwd_dist = 'MIN({0})'.format(col_cwd_dist) 474 | col_cwd_dist += ' AS cwd_distance' 475 | path0 = normalize_directory(os.path.abspath(sort_by_cwd_distance)) 476 | sc.add_column(col_cwd_dist, 'cwd_distance', params=[path0]) 477 | sc.order_by('cwd_distance', 'DESC' if reverse else 'ASC') 478 | for k in sort_by: 479 | sc.order_by(k, 'ASC' if reverse else 'DESC') 480 | sc.add_matches(glob, 'CL.command', 481 | match_pattern, include_pattern, exclude_pattern) 482 | sc.add_matches(regexp, 'CL.command', 483 | match_regexp, include_regexp, exclude_regexp) 484 | sc.add_or_matches(glob, 'DL.directory', cwd_glob) 485 | sc.add_or_matches( 486 | eq, 'DL.directory', 487 | [normalize_directory(os.path.abspath(p)) for p in cwd]) 488 | sc.add_and_matches('DATETIME({0}) >= {1}', 'start_time', time_after) 489 | sc.add_and_matches('DATETIME({0}) <= {1}', 'start_time', time_before) 490 | comdura = ( 491 | '(JULIANDAY(stop_time) - JULIANDAY(start_time)) * 60 * 60 * 24') 492 | sc.add_and_matches('({0} >= {1})', comdura, duration_longer_than) 493 | sc.add_and_matches('({0} <= {1})', comdura, duration_less_than) 494 | sc.add_matches(eq, 'exit_code', 495 | [], include_exit_code, exclude_exit_code) 496 | sc.add_matches(eq, 'session_id', [], 497 | include_session_history_id, exclude_session_history_id) 498 | cls._add_environ_searches( 499 | sc, 500 | match_environ_pattern, include_environ_pattern, 501 | exclude_environ_pattern, 502 | match_environ_regexp, include_environ_regexp, 503 | exclude_environ_regexp) 504 | 505 | if unique: 506 | sc.uniquify_by('CL.command', 'start_time') 507 | 508 | additional_column_set = set(sort_by) | set(additional_columns) 509 | need = lambda *x: set(x) & additional_column_set 510 | if need('command_count'): 511 | sc.add_column('COUNT(*) as command_count', 'command_count') 512 | if need('success_count', 'success_ratio'): 513 | sc.join(cls._sc_success_count(), 514 | on='command_id = success_command.id') 515 | sc.add_column('success_count') 516 | sc.add_column('(success_count * 1.0 / COUNT(*)) AS success_ratio', 517 | 'success_ratio') 518 | if need('program_count'): 519 | sc.join(cls._sc_program_count(), 520 | on='PROGRAM_NAME(CL.command) = command_program.program') 521 | sc.add_column('program_count') 522 | if need('session_start_time', 'session_stop_time'): 523 | sc_sh = SQLConstructor( 524 | 'session_history', 525 | ['id', 526 | 'start_time AS session_start_time', 527 | 'stop_time AS session_stop_time'], 528 | table_alias='session_history') 529 | sc.join(sc_sh, on='session_id = session_history.id') 530 | sc.add_column('session_start_time', 'session_start') 531 | sc.add_column('session_stop_time', 'session_stop') 532 | 533 | if condition_as_column: 534 | sc.move_where_clause_to_column() 535 | 536 | return sc.compile() 537 | 538 | @classmethod 539 | def _add_environ_searches( 540 | cls, sc, 541 | match_pattern=[], include_pattern=[], exclude_pattern=[], 542 | match_regexp=[], include_regexp=[], exclude_regexp=[], 543 | **kwds): 544 | if not (match_pattern or include_pattern or exclude_pattern or 545 | match_regexp or include_regexp or exclude_regexp): 546 | return 547 | glob = "({0[0]} = {1} AND glob({2}, {0[1]}))".format 548 | notglob = "({0[0]} = {1} AND NOT glob({2}, {0[1]}))".format 549 | regexp = "({0[0]} = {1} AND regexp({2}, {0[1]}))".format 550 | notregexp = "({0[0]} = {1} AND NOT regexp({2}, {0[1]}))".format 551 | lhs = ['variable_name', 'variable_value'] 552 | addes = lambda *a: cls._add_environ_search_2(*a, **kwds) 553 | addes(sc, glob, notglob, lhs, 554 | match_pattern, include_pattern, exclude_pattern, '_glob') 555 | addes(sc, regexp, notregexp, lhs, 556 | match_regexp, include_regexp, exclude_regexp, '_regexp') 557 | sc.add_group_by('command_history.id') 558 | 559 | @classmethod 560 | def _add_environ_search_2( 561 | cls, sc, matcher, notmatcher, lhs, 562 | match_params=[], include_params=[], exclude_params=[], 563 | suffix='', **kwds): 564 | addes = lambda *a: cls._add_environ_search_1(*a, **kwds) 565 | addes(sc, matcher, lhs, match_params, '_match' + suffix) 566 | addes(sc, matcher, lhs, include_params, '_include' + suffix, False) 567 | addes(sc, notmatcher, lhs, exclude_params, '_exclude' + suffix) 568 | 569 | @classmethod 570 | def _add_environ_search_1( 571 | cls, sc, matcher, lhs, match_params, 572 | suffix='', and_match=True, **kwds): 573 | if not match_params: 574 | return 575 | command_table_alias = 'CEnv{0}'.format(suffix) 576 | session_table_alias = 'SEnv{0}'.format(suffix) 577 | sc_ce = cls._sc_history_environ( 578 | 'command_environment_map', 'ch_id', matcher, lhs, match_params, 579 | table_alias=command_table_alias, **kwds) 580 | sc_se = cls._sc_history_environ( 581 | 'session_environment_map', 'sh_id', matcher, lhs, match_params, 582 | table_alias=session_table_alias, **kwds) 583 | sc.join(sc_ce, op='LEFT JOIN', on='command_history.id = {r}.ch_id') 584 | sc.join(sc_se, op='LEFT JOIN', on='session_id = {r}.sh_id') 585 | if and_match: 586 | # When doing AND match, there should be at least matches 587 | # as many as ``match_params``. 588 | # SOMEDAY: add more tests for _add_environ_search_1. 589 | # Can one matcher match many rows? In that case, 590 | # this wouldn't work, no? 591 | sc.add_having( 592 | 'COUNT(DISTINCT {0}.ev_id) + ' 593 | 'COUNT(DISTINCT {1}.ev_id) >= {2}' 594 | .format(command_table_alias, 595 | session_table_alias, 596 | len(match_params))) 597 | else: 598 | # Alternative way is to use 599 | # len(match_params) if and_match else 1 600 | # instead of ``len(match_params)`` above. 601 | # But this way is probably faster. 602 | sc.add_having( 603 | '({0}.ev_id IS NOT NULL OR {1}.ev_id IS NOT NULL)' 604 | .format(command_table_alias, session_table_alias)) 605 | 606 | @staticmethod 607 | def _sc_history_environ( 608 | map_table, map_id, 609 | matcher, lhs, match_params, 610 | table_alias=None, numq=2, **kwds): 611 | sc_ev = SQLConstructor( 612 | map_table, [map_id, 'ev_id'], table_alias=table_alias) 613 | sc_ev.join('environment_variable AS EV', 614 | op='JOIN', on='ev_id = EV.id') 615 | sc_ev.add_or_matches(matcher, lhs, match_params, numq=numq, **kwds) 616 | return sc_ev 617 | 618 | @staticmethod 619 | def _sc_success_count(table_alias='success_command'): 620 | count = ('COUNT(CASE WHEN exit_code = 0 THEN 1 ELSE NULL END)' 621 | ' AS success_count') 622 | return SQLConstructor( 623 | 'command_history', 624 | ['command_id AS id', count], 625 | ['command_id', 'success_count'], 626 | group_by=['command_id'], table_alias=table_alias) 627 | 628 | @staticmethod 629 | def _sc_program_count(table_alias='command_program'): 630 | return SQLConstructor( 631 | 'command_history ' 632 | 'LEFT JOIN command_list AS CL ON command_id = CL.id', 633 | ['PROGRAM_NAME(CL.command) AS program', 634 | 'COUNT(*) AS program_count'], 635 | ['program', 'program_count'], 636 | group_by=['program'], table_alias=table_alias) 637 | 638 | @staticmethod 639 | def _sc_matched_environment_variable( 640 | match_pattern=[], include_pattern=[], exclude_pattern=[], 641 | match_regexp=[], include_regexp=[], exclude_regexp=[], 642 | table_alias='matched_environment_variable'): 643 | glob = "({0[0]} = {1} AND glob({2}, {0[1]}))".format 644 | regexp = "({0[0]} = {1} AND regexp({2}, {0[1]}))".format 645 | sc = SQLConstructor( 646 | 'environment_variable', 647 | ['id', 'variable_name', 'variable_value'], 648 | ['environment_variable_id', 'variable_name', 'variable_value'], 649 | table_alias=table_alias) 650 | sc.add_matches(glob, ['variable_name', 'variable_value'], 651 | match_pattern, include_pattern, exclude_pattern, numq=2) 652 | sc.add_matches(regexp, ['variable_name', 'variable_value'], 653 | match_regexp, include_regexp, exclude_regexp, numq=2) 654 | return sc 655 | 656 | def search_environ_record(self, **kwds): 657 | sc = self._sc_matched_environment_variable(**kwds) 658 | (sql, params, keys) = sc.compile() 659 | return self._select_rows(EnvironRecord, keys, sql, params) 660 | 661 | def import_init_dict(self, dct, overwrite=True): 662 | long_id = dct['session_id'] 663 | srec = SessionRecord(**dct) 664 | with self.connection(commit=True) as connection: 665 | db = connection.cursor() 666 | records = list(self.select_session_by_long_id(long_id)) 667 | if records: 668 | assert len(records) == 1 669 | oldrec = records[0] 670 | if oldrec.start is not None and not overwrite: 671 | return 672 | oldrec.start = srec.start 673 | sh_id = self._update_session_history(db, oldrec) 674 | else: 675 | sh_id = self._insert_session_history(db, srec) 676 | self._update_session_environ(db, sh_id, srec.environ) 677 | 678 | def import_exit_dict(self, dct, overwrite=True): 679 | long_id = dct['session_id'] 680 | srec = SessionRecord(**dct) 681 | with self.connection(commit=True) as connection: 682 | db = connection.cursor() 683 | records = list(self.select_session_by_long_id(long_id)) 684 | if records: 685 | assert len(records) == 1 686 | oldrec = records[0] 687 | if oldrec.stop is not None and not overwrite: 688 | return 689 | oldrec.stop = srec.stop 690 | self._update_session_history(db, oldrec) 691 | else: 692 | self._insert_session_history(db, srec) 693 | 694 | def _insert_session_history(self, db, srec): 695 | db.execute( 696 | ''' 697 | INSERT INTO session_history 698 | (session_long_id, start_time, stop_time) 699 | VALUES (?, ?, ?) 700 | ''', 701 | [srec.session_id, convert_ts(srec.start), convert_ts(srec.stop)]) 702 | return db.lastrowid 703 | 704 | def _update_session_history(self, db, srec): 705 | assert srec.session_history_id is not None 706 | db.execute( 707 | ''' 708 | UPDATE session_history 709 | SET session_long_id=?, start_time=?, stop_time=? 710 | WHERE id=? 711 | ''', 712 | [srec.session_id, convert_ts(srec.start), convert_ts(srec.stop), 713 | srec.session_history_id]) 714 | return srec.session_history_id 715 | 716 | def _update_session_environ(self, db, sh_id, environ): 717 | if not environ: 718 | return 719 | db.execute('DELETE FROM session_environment_map WHERE sh_id=?', 720 | [sh_id]) 721 | self._insert_session_environ(db, sh_id, environ) 722 | 723 | def _insert_session_environ(self, db, sh_id, environ): 724 | self._insert_environ(db, 'session_environment_map', 'sh_id', sh_id, 725 | environ) 726 | 727 | def select_session_by_long_id(self, long_id): 728 | keys = ['session_history_id', 'session_id', 'start', 'stop'] 729 | sql = """ 730 | SELECT id, session_long_id, start_time, stop_time 731 | FROM session_history 732 | WHERE session_long_id = ? 733 | """ 734 | params = [long_id] 735 | with self.connection() as connection: 736 | for row in connection.execute(sql, params): 737 | yield SessionRecord(**dict(zip(keys, row))) 738 | 739 | def search_session_record(self, session_id): 740 | return self.select_session_by_long_id(session_id) 741 | 742 | def get_full_command_record(self, command_history_id, 743 | merge_session_environ=True): 744 | """ 745 | Get fully retrieved :class:`CommandRecord` instance by ID. 746 | 747 | By "fully", it means that complex slots such as `environ` and 748 | `pipestatus` are available. 749 | 750 | :type command_history_id: int 751 | :type merge_session_environ: bool 752 | 753 | """ 754 | with self.connection() as db: 755 | crec = self._select_command_record(db, command_history_id) 756 | crec.pipestatus = self._get_pipestatus(db, command_history_id) 757 | # Set environment variables 758 | cenv = self._select_environ(db, 'command', command_history_id) 759 | crec.environ.update(cenv) 760 | if merge_session_environ: 761 | senv = self._select_environ( 762 | db, 'session', crec.session_history_id) 763 | crec.environ.update(senv) 764 | return crec 765 | 766 | def _select_command_record(self, db, command_history_id): 767 | keys = ['session_history_id', 'command', 'cwd', 'terminal', 768 | 'start', 'stop', 'exit_code'] 769 | sql = """ 770 | SELECT 771 | session_id, CL.command, DL.directory, TL.terminal, 772 | start_time, stop_time, exit_code 773 | FROM command_history 774 | LEFT JOIN command_list AS CL ON command_id = CL.id 775 | LEFT JOIN directory_list AS DL ON directory_id = DL.id 776 | LEFT JOIN terminal_list AS TL ON terminal_id = TL.id 777 | WHERE command_history.id = ? 778 | """ 779 | params = [command_history_id] 780 | for row in db.execute(sql, params): 781 | crec = CommandRecord(**dict(zip(keys, row))) 782 | crec.command_history_id = command_history_id 783 | return crec 784 | raise ValueError("Command record of id={0} is not found" 785 | .format(command_history_id)) 786 | 787 | def _get_pipestatus(self, db, command_history_id): 788 | sql = """ 789 | SELECT program_position, exit_code 790 | FROM pipe_status_map 791 | WHERE ch_id = ? 792 | """ 793 | params = [command_history_id] 794 | records = list(db.execute(sql, params)) 795 | length = max(r[0] for r in records) + 1 796 | pipestatus = [None] * length 797 | for (i, s) in records: 798 | pipestatus[i] = s 799 | return pipestatus 800 | 801 | def _select_environ(self, db, recname, recid): 802 | sql = """ 803 | SELECT 804 | EVar.variable_name, EVar.variable_value 805 | FROM {recname}_environment_map as EMap 806 | LEFT JOIN environment_variable AS EVar ON EMap.ev_id = EVar.id 807 | WHERE EMap.{hist_id} = ? 808 | """.format( 809 | recname=recname, 810 | hist_id='ch_id' if recname == 'command' else 'sh_id', 811 | ) 812 | params = [recid] 813 | return db.execute(sql, params) 814 | -------------------------------------------------------------------------------- /rash/ext/rash.bash: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | ### Record commands 18 | _rash-postexec(){ 19 | test -d "$PWD" && \ 20 | rash record \ 21 | --record-type command \ 22 | --session-id "$_RASH_SESSION_ID" \ 23 | --command "$_RASH_COMMAND" \ 24 | --cwd "$_RASH_PWD" \ 25 | --exit-code "$_RASH_EXIT_CODE" \ 26 | "${_RASH_OPTS[@]}" \ 27 | --pipestatus "${_RASH_PIPESTATUS[@]}" 28 | } 29 | 30 | _RASH_EXECUTING="" 31 | 32 | _rash-preexec(){ 33 | _RASH_EXECUTING=t 34 | _RASH_PWD="$PWD" 35 | } 36 | 37 | _rash-precmd(){ 38 | _RASH_EXIT_CODE="$1" 39 | shift 40 | _RASH_PIPESTATUS=("$@") 41 | _RASH_OPTS=() 42 | 43 | if [ -n "$_RASH_EXECUTING" ] 44 | then 45 | local num start command 46 | local hist="$(HISTTIMEFORMAT="%s " builtin history 1)" 47 | read -r num start command <<< "$hist" 48 | if [ -n "$start" ] 49 | then 50 | _RASH_OPTS=(--start "$start" "${_RASH_OPTS[@]}") 51 | fi 52 | _RASH_COMMAND="$command" 53 | _rash-postexec 54 | _RASH_EXECUTING="" 55 | fi 56 | _rash-preexec 57 | } 58 | 59 | export PROMPT_COMMAND="_rash-precmd \${?} \${PIPESTATUS[@]}" 60 | 61 | 62 | ### Record session initialization 63 | if [ -z "$_RASH_SESSION_ID" ] 64 | then 65 | _RASH_SESSION_ID=$(rash record --record-type init --print-session-id) 66 | fi 67 | 68 | 69 | ### Record session exit 70 | _rash-before-exit(){ 71 | rash record --record-type exit --session-id "$_RASH_SESSION_ID" 72 | } 73 | 74 | trap "_rash-before-exit" EXIT TERM 75 | -------------------------------------------------------------------------------- /rash/ext/rash.zsh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | ### Record commands 18 | _rash-postexec(){ 19 | test -d "$PWD" && \ 20 | rash record \ 21 | --record-type command \ 22 | --session-id "$_RASH_SESSION_ID" \ 23 | --command "$_RASH_COMMAND" \ 24 | --cwd "$_RASH_PWD" \ 25 | --exit-code "$_RASH_EXIT_CODE" \ 26 | "${_RASH_OPTS[@]}" \ 27 | --pipestatus "${_RASH_PIPESTATUS[@]}" 28 | } 29 | 30 | _RASH_EXECUTING="" 31 | 32 | _rash-preexec(){ 33 | _RASH_START=$(date "+%s") 34 | _RASH_EXECUTING=t 35 | _RASH_PWD="$PWD" 36 | } 37 | 38 | _rash-zshaddhistory(){ 39 | _RASH_COMMAND="${1%%$'\n'}" 40 | _RASH_DEBUG_COMMAND_LOG_TYPE=zshaddhistory # SOMEDAY: remove this 41 | } 42 | 43 | _rash-history-fallback(){ 44 | # reading "$(builtin history -n -1)" directly does not work when 45 | # command contains newlines. 46 | local num command 47 | read -r num command <<< "$(builtin history -1)" 48 | _RASH_COMMAND="$history[$num]" 49 | _RASH_DEBUG_COMMAND_LOG_TYPE=history-array 50 | 51 | # `$history' may not work as it is not documented. In that case, 52 | # fallback to $(builtin history -n -1): 53 | if [ -z "$_RASH_COMMAND" -a -n "$command" ] 54 | then 55 | _RASH_COMMAND="$(builtin history -n -1)" 56 | _RASH_DEBUG_COMMAND_LOG_TYPE=history-builtin 57 | fi 58 | } 59 | 60 | _rash-precmd(){ 61 | # Make sure to copy these variable at very first stage. 62 | # Otherwise, I will loose these information. 63 | _RASH_EXIT_CODE="$?" 64 | _RASH_PIPESTATUS=("${pipestatus[@]}") 65 | _RASH_OPTS=(--start "$_RASH_START") 66 | 67 | if [ -z "$_RASH_COMMAND" ] 68 | then 69 | # Some old zsh (< 4.3?) does not support zshaddhistory. 70 | _rash-history-fallback 71 | fi 72 | 73 | if [ -n "$_RASH_EXECUTING" ] 74 | then 75 | _rash-postexec 76 | _RASH_EXECUTING="" 77 | _RASH_COMMAND="" 78 | fi 79 | } 80 | 81 | preexec_functions+=(_rash-preexec) 82 | precmd_functions+=(_rash-precmd) 83 | zshaddhistory_functions+=(_rash-zshaddhistory) 84 | 85 | 86 | ### Record session initialization 87 | if [ -z "$_RASH_SESSION_ID" ] 88 | then 89 | _RASH_SESSION_ID=$(rash record --record-type init --print-session-id) 90 | fi 91 | 92 | 93 | ### Record session exit 94 | _rash-before-exit(){ 95 | rash record --record-type exit --session-id "$_RASH_SESSION_ID" 96 | } 97 | 98 | trap "_rash-before-exit" EXIT TERM 99 | 100 | 101 | ### zle isearch widget 102 | rash-zle-isearch(){ 103 | local result="$(rash isearch --query "$BUFFER" "$@")" 104 | if [ -n "$result" ] 105 | then 106 | BUFFER="$result" 107 | CURSOR=$#BUFFER 108 | fi 109 | zle -R -c 110 | } 111 | zle -N rash-zle-isearch 112 | -------------------------------------------------------------------------------- /rash/functional_tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests that requires shells and CLI installed. 3 | """ 4 | 5 | # Copyright (C) 2013- Takafumi Arakaki 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /rash/functional_tests/test_cli.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | import sys 19 | import subprocess 20 | import unittest 21 | import tempfile 22 | import shutil 23 | import textwrap 24 | import json 25 | import time 26 | 27 | from ..utils.py3compat import PY3 28 | from ..config import ConfigStore 29 | from ..tests.utils import BaseTestCase, skipIf 30 | 31 | BASE_COMMAND = 'rash' 32 | 33 | try: 34 | run_command = subprocess.check_output 35 | except AttributeError: 36 | 37 | def run_command(*args, **kwds): 38 | assert 'stdout' not in kwds 39 | with open(os.devnull, 'w') as devnull: 40 | kwds['stdout'] = devnull 41 | subprocess.check_call(*args, **kwds) 42 | 43 | 44 | def run_cli(command, *args, **kwds): 45 | run_command([BASE_COMMAND] + command, *args, **kwds) 46 | 47 | 48 | class TestCLI(unittest.TestCase): 49 | 50 | def test_command_init_known_shell(self): 51 | run_cli(['init', '--shell', 'zsh']) 52 | 53 | def test_command_init_unknown_shell(self): 54 | self.assertRaises( 55 | subprocess.CalledProcessError, 56 | run_cli, 57 | ['init', '--shell', 'UNKNOWN_SHELL'], stderr=subprocess.PIPE) 58 | 59 | 60 | class FunctionalTestMixIn(object): 61 | 62 | """ 63 | MixIn class for isolating functional test environment. 64 | 65 | SOMEDAY: Make FunctionalTestMixIn work in non-*nix systems. 66 | (I think) This isolation does not work in Mac OS in Windows! 67 | I can workaround this by adding --config-dir global 68 | option to specify configuration directory from 69 | command line, rather than using $HOME. 70 | 71 | """ 72 | 73 | def setUp(self): 74 | self.home_dir = tempfile.mkdtemp(prefix='rash-test-') 75 | self.config_dir = os.path.join(self.home_dir, '.config') 76 | self.conf_base_path = os.path.join(self.config_dir, 'rash') 77 | 78 | self.environ = os.environ.copy() 79 | self.environ['HOME'] = self.home_dir 80 | # FIXME: run the test w/o $TERM 81 | self.environ['TERM'] = 'xterm-256color' 82 | # Make sure that $XDG_CONFIG_HOME does not confuse sub processes 83 | if 'XDG_CONFIG_HOME' in self.environ: 84 | del self.environ['XDG_CONFIG_HOME'] 85 | 86 | self.cfstore = ConfigStore(self.conf_base_path) 87 | 88 | def tearDown(self): 89 | # Kill daemon if exists 90 | try: 91 | if os.path.exists(self.cfstore.daemon_pid_path): 92 | with open(self.cfstore.daemon_pid_path) as f: 93 | pid = f.read().strip() 94 | print("Daemon (PID={0}) may be left alive. Killing it..." 95 | .format(pid)) 96 | subprocess.call(['kill', pid]) 97 | except Exception as e: 98 | print("Got error while trying to kill daemon: {0}" 99 | .format(e)) 100 | 101 | try: 102 | shutil.rmtree(self.home_dir) 103 | except OSError: 104 | print("Failed to remove self.home_dir={0}. " 105 | "Can be timing issue. Trying again..." 106 | .format(self.home_dir)) 107 | time.sleep(0.1) 108 | shutil.rmtree(self.home_dir) 109 | 110 | def popen(self, *args, **kwds): 111 | if 'env' in kwds: 112 | raise RuntimeError('Do not use env!') 113 | if 'cwd' in kwds: 114 | raise RuntimeError('Do not use cwd!') 115 | kwds['env'] = self.environ 116 | kwds['cwd'] = self.home_dir 117 | return subprocess.Popen(*args, **kwds) 118 | 119 | 120 | class TestIsolation(FunctionalTestMixIn, BaseTestCase): 121 | 122 | """ 123 | Make sure that test environment is isolated from the real one. 124 | """ 125 | 126 | def test_config_isolation(self): 127 | proc = self.popen( 128 | [os.path.abspath(sys.executable)], 129 | stdin=subprocess.PIPE, 130 | stdout=subprocess.PIPE, 131 | stderr=subprocess.PIPE) 132 | (stdout, stderr) = proc.communicate(textwrap.dedent(""" 133 | from rash.config import ConfigStore 134 | cfstore = ConfigStore() 135 | print(repr(cfstore.base_path)) 136 | """).encode()) 137 | stderr = stderr.decode() 138 | stdout = stdout.decode() 139 | base_path = eval(stdout) 140 | self.assertEqual(base_path, self.conf_base_path) 141 | self.assertFalse(stderr) 142 | self.assertNotEqual(base_path, ConfigStore().base_path) 143 | 144 | 145 | class ShellTestMixIn(FunctionalTestMixIn): 146 | 147 | shell = 'sh' 148 | eval_command = 'eval' 149 | 150 | def run_shell(self, script): 151 | proc = self.popen( 152 | [self.shell], 153 | stdin=subprocess.PIPE, 154 | stdout=subprocess.PIPE, 155 | stderr=subprocess.PIPE) 156 | (stdout, stderr) = proc.communicate(script.encode()) 157 | return (stdout.decode(), stderr.decode()) 158 | 159 | def get_record_data(self, record_type): 160 | top = os.path.join(self.cfstore.record_path, record_type) 161 | for (root, _, files) in os.walk(top): 162 | for f in files: 163 | path = os.path.join(root, f) 164 | with open(path) as f: 165 | data = json.load(f) 166 | yield dict(path=path, data=data) 167 | 168 | def get_all_record_data(self): 169 | return dict( 170 | init=list(self.get_record_data('init')), 171 | exit=list(self.get_record_data('exit')), 172 | command=list(self.get_record_data('command')), 173 | ) 174 | 175 | def _get_init_script(self, no_daemon=True, daemon_options=[], 176 | daemon_outfile=None): 177 | options = [] 178 | if no_daemon: 179 | options.append('--no-daemon') 180 | options.extend(map('--daemon-opt={0}'.format, daemon_options)) 181 | if daemon_outfile: 182 | options.extend(['--daemon-outfile', daemon_outfile]) 183 | optstr = ' '.join(options) 184 | return "{0} $({1} init --shell {2} {3})".format( 185 | self.eval_command, BASE_COMMAND, self.shell, optstr) 186 | 187 | def get_script(self, script='', **kwds): 188 | init_script = self._get_init_script(**kwds) 189 | return '\n'.join([init_script, textwrap.dedent(script)]) 190 | 191 | def test_init(self): 192 | script = self.get_script(""" 193 | test -n "$_RASH_SESSION_ID" && echo "_RASH_SESSION_ID is defined" 194 | """) 195 | (stdout, stderr) = self.run_shell(script) 196 | self.assertFalse(stderr) 197 | self.assertIn('_RASH_SESSION_ID is defined', stdout) 198 | 199 | assert os.path.isdir(self.cfstore.record_path) 200 | records = self.get_all_record_data() 201 | self.assertEqual(len(records['init']), 1) 202 | self.assertEqual(len(records['exit']), 1) 203 | self.assertEqual(len(records['command']), 0) 204 | 205 | from ..record import get_environ 206 | subenv = get_environ(['HOST']) 207 | 208 | data = records['init'][0]['data'] 209 | assert 'start' in data 210 | assert 'stop' not in data 211 | self.assertEqual(data['environ']['HOST'], subenv['HOST']) 212 | init_id = data['session_id'] 213 | 214 | data = records['exit'][0]['data'] 215 | assert 'start' not in data 216 | assert 'stop' in data 217 | assert not data['environ'] 218 | exit_id = data['session_id'] 219 | 220 | self.assertEqual(init_id, exit_id) 221 | 222 | def test_postexec(self): 223 | script = self.get_script(self.test_postexec_script) 224 | (stdout, stderr) = self.run_shell(script) 225 | 226 | # stderr may have some errors in it 227 | if stderr: 228 | print("Got STDERR from {0} (but it's OK to ignore it)" 229 | .format(self.shell)) 230 | print(stderr) 231 | 232 | records = self.get_all_record_data() 233 | self.assertEqual(len(records['init']), 1) 234 | self.assertEqual(len(records['exit']), 1) 235 | self.assertEqual(len(records['command']), 1) 236 | 237 | init_data = records['init'][0]['data'] 238 | command_data = records['command'][0]['data'] 239 | assert command_data['session_id'] == init_data['session_id'] 240 | assert command_data['environ']['PATH'] 241 | assert isinstance(command_data['stop'], int) 242 | if self.shell.endswith('zsh'): 243 | assert isinstance(command_data['start'], int) 244 | else: 245 | assert 'start' not in command_data 246 | 247 | test_postexec_script = None 248 | """Set this to a shell script for :meth:`test_postexc`.""" 249 | 250 | def test_exit_code(self): 251 | script = self.get_script(self.test_exit_code_script) 252 | (stdout, stderr) = self.run_shell(script) 253 | 254 | # stderr may have some errors in it 255 | if stderr: 256 | print("Got STDERR from {0} (but it's OK to ignore it)" 257 | .format(self.shell)) 258 | print(stderr) 259 | 260 | records = self.get_all_record_data() 261 | self.assertEqual(len(records['init']), 1) 262 | self.assertEqual(len(records['exit']), 1) 263 | self.assertEqual(len(records['command']), 1) 264 | 265 | command_data = [d['data'] for d in records['command']] 266 | self.assertEqual(command_data[0]['exit_code'], 1) 267 | 268 | test_exit_code_script = None 269 | """Set this to a shell script for :meth:`test_exit_code`.""" 270 | 271 | def test_pipe_status(self): 272 | script = self.get_script(self.test_pipe_status_script) 273 | (stdout, stderr) = self.run_shell(script) 274 | 275 | # stderr may have some errors in it 276 | if stderr: 277 | print("Got STDERR from {0} (but it's OK to ignore it)" 278 | .format(self.shell)) 279 | print(stderr) 280 | 281 | records = self.get_all_record_data() 282 | self.assertEqual(len(records['init']), 1) 283 | self.assertEqual(len(records['exit']), 1) 284 | self.assertEqual(len(records['command']), 1) 285 | 286 | command_data = [d['data'] for d in records['command']] 287 | self.assertEqual(command_data[0]['pipestatus'], [1, 0]) 288 | 289 | test_pipe_status_script = None 290 | """Set this to a shell script for :meth:`test_pipe_status`.""" 291 | 292 | def test_non_existing_directory(self): 293 | main_script = """ 294 | _rash-precmd 295 | mkdir non_existing_directory 296 | 297 | _rash-precmd 298 | cd non_existing_directory 299 | 300 | _rash-precmd 301 | rmdir ../non_existing_directory 302 | 303 | _rash-precmd 304 | : 305 | 306 | _rash-precmd 307 | cd .. 308 | """ 309 | script = self.get_script(main_script) 310 | (stdout, stderr) = self.run_shell(script) 311 | self.assertNotIn('Traceback', stderr) 312 | 313 | @skipIf(PY3, "watchdog does not support Python 3") 314 | def test_daemon(self): 315 | daemon_outfile = os.path.join(self.cfstore.base_path, 'daemon.out') 316 | script = self.get_script( 317 | no_daemon=False, daemon_outfile=daemon_outfile, 318 | daemon_options=['--keep-json', '--log-level=DEBUG']) 319 | (stdout, stderr) = self.run_shell(script) 320 | 321 | # These are useful when debugging, so let's leave them: 322 | print(stderr) 323 | print(stdout) 324 | print(self.cfstore.daemon_pid_path) 325 | 326 | # Print daemon process output for debugging 327 | with open(daemon_outfile) as f: 328 | daemon_output = f.read().strip() 329 | if daemon_output: 330 | print("Daemon process output ({0})".format(daemon_outfile)) 331 | print(daemon_output.decode()) 332 | 333 | # The daemon process should create a PID file containing a number 334 | @self.assert_poll_do( 335 | "Daemon did not produce PID file at: {0}" 336 | .format(self.cfstore.daemon_pid_path)) 337 | def pid_file_contains_a_number(): 338 | try: 339 | with open(self.cfstore.daemon_pid_path) as f: 340 | return f.read().strip().isdigit() 341 | except IOError: 342 | return False 343 | 344 | # Read the PID file 345 | with open(self.cfstore.daemon_pid_path) as f: 346 | pid = int(f.read().strip()) 347 | 348 | # The daemon process should be alive 349 | ps_pid_cmd = ['ps', '--pid', str(pid)] 350 | try: 351 | run_command(ps_pid_cmd) 352 | except subprocess.CalledProcessError: 353 | raise AssertionError( 354 | 'At this point, daemon process should be live ' 355 | '("ps --pid {0}" failed).'.format(pid)) 356 | 357 | # The daemon should create a log file 358 | self.assert_poll(lambda: os.path.exists(self.cfstore.daemon_log_path), 359 | "daemon_log_path={0!r} is not created on time" 360 | .format(self.cfstore.daemon_log_path)) 361 | 362 | # The daemon should write some debug message to the log file 363 | # (Note: --log-level=DEBUG is given by $RASH_INIT_DAEMON_OPTIONS) 364 | with open(self.cfstore.daemon_log_path) as f: 365 | @self.assert_poll_do("Nothing written in log file.") 366 | def log_file_written(): 367 | return f.read().strip() 368 | 369 | # Kill command should succeeds 370 | run_command(['kill', '-TERM', str(pid)]) 371 | 372 | # The daemon should be killed by the TERM signal 373 | @self.assert_poll_do( 374 | "Daemon process {0} failed to exit.".format(pid)) 375 | def terminated(): 376 | try: 377 | run_command(ps_pid_cmd) 378 | return False 379 | except subprocess.CalledProcessError: 380 | return True 381 | 382 | # The daemon should remove the PID file on exit 383 | self.assert_poll( 384 | lambda: not os.path.exists(self.cfstore.daemon_pid_path), 385 | "Daemon did not remove PID file at: {0}".format( 386 | self.cfstore.daemon_pid_path)) 387 | 388 | @staticmethod 389 | def assert_poll(assertion, message, num=100, tick=0.1): 390 | """ 391 | Run `assersion` every `tick` second `num` times. 392 | 393 | If none of `assersion` call returns true, it raise 394 | an assertion error with `message`. 395 | 396 | """ 397 | for i in range(num): 398 | if assertion(): 399 | break 400 | time.sleep(tick) 401 | else: 402 | raise AssertionError(message) 403 | 404 | @classmethod 405 | def assert_poll_do(cls, message, *args, **kwds): 406 | """ 407 | Decorator to run :meth:`assert_poll` right after the definition. 408 | """ 409 | def decorator(assertion): 410 | cls.assert_poll(assertion, message, *args, **kwds) 411 | return assertion 412 | return decorator 413 | 414 | 415 | class TestZsh(ShellTestMixIn, BaseTestCase): 416 | shell = 'zsh' 417 | test_postexec_script = """ 418 | _rash-precmd 419 | """ 420 | test_exit_code_script = """ 421 | false 422 | _rash-precmd 423 | """ 424 | test_pipe_status_script = """ 425 | false | true 426 | _rash-precmd 427 | """ 428 | 429 | def test_zsh_executes_preexec(self): 430 | script = self.get_script('echo _RASH_EXECUTING=$_RASH_EXECUTING') 431 | (stdout, stderr) = self.run_shell(script) 432 | self.assertFalse(stderr) 433 | self.assertIn('_RASH_EXECUTING=t', stdout) 434 | 435 | def test_hook_installation(self): 436 | script = self.get_script(""" 437 | echo $precmd_functions 438 | echo $preexec_functions 439 | """) 440 | (stdout, stderr) = self.run_shell(script) 441 | self.assertIn('_rash-precmd', stdout) 442 | self.assertIn('_rash-preexec', stdout) 443 | 444 | 445 | class TestBash(ShellTestMixIn, BaseTestCase): 446 | shell = 'bash' 447 | test_postexec_script = """ 448 | eval "$PROMPT_COMMAND" 449 | eval "$PROMPT_COMMAND" 450 | """ 451 | test_exit_code_script = """ 452 | eval "$PROMPT_COMMAND" 453 | false 454 | eval "$PROMPT_COMMAND" 455 | """ 456 | test_pipe_status_script = """ 457 | eval "$PROMPT_COMMAND" 458 | false | true 459 | eval "$PROMPT_COMMAND" 460 | """ 461 | 462 | def test_hook_installation(self): 463 | script = self.get_script('echo $PROMPT_COMMAND') 464 | (stdout, stderr) = self.run_shell(script) 465 | self.assertIn('_rash-precmd', stdout) 466 | -------------------------------------------------------------------------------- /rash/index.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | def index_run(record_path, keep_json, check_duplicate): 18 | """ 19 | Convert raw JSON records into sqlite3 DB. 20 | 21 | Normally RASH launches a daemon that takes care of indexing. 22 | See ``rash daemon --help``. 23 | 24 | """ 25 | from .config import ConfigStore 26 | from .indexer import Indexer 27 | cfstore = ConfigStore() 28 | indexer = Indexer(cfstore, check_duplicate, keep_json, record_path) 29 | indexer.index_all() 30 | 31 | 32 | def index_add_arguments(parser): 33 | parser.add_argument( 34 | 'record_path', nargs='?', 35 | help=""" 36 | specify the directory that has JSON records. 37 | """) 38 | parser.add_argument( 39 | '--keep-json', default=False, action='store_true', 40 | help=""" 41 | Do not remove old JSON files. It turns on --check-duplicate. 42 | """) 43 | parser.add_argument( 44 | '--check-duplicate', default=False, action='store_true', 45 | help='do not store already existing history in DB.') 46 | 47 | 48 | commands = [ 49 | ('index', index_add_arguments, index_run), 50 | ] 51 | -------------------------------------------------------------------------------- /rash/indexer.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | import json 19 | import warnings 20 | 21 | from .database import DataBase 22 | 23 | 24 | class Indexer(object): 25 | 26 | """ 27 | Translate JSON files into SQLite DB. 28 | """ 29 | 30 | def __init__(self, cfstore, check_duplicate, keep_json, record_path=None): 31 | """ 32 | Create an indexer. 33 | 34 | :type cfstore: rash.config.ConfigStore 35 | :arg cfstore: 36 | :type check_duplicate: bool 37 | :arg check_duplicate: See :meth:`DataBase.import_dict`. 38 | :type keep_json: bool 39 | :arg keep_json: Do not remove JSON files. 40 | Imply ``check_duplicate=True``. 41 | :type record_path: str or None 42 | :arg record_path: Default to `cfstore.record_path`. 43 | 44 | """ 45 | from .log import logger 46 | self.logger = logger 47 | if keep_json: 48 | check_duplicate = True 49 | self.cfstore = cfstore 50 | self.check_duplicate = check_duplicate 51 | self.keep_json = keep_json 52 | self.record_path = record_path or cfstore.record_path 53 | self.db = DataBase(cfstore.db_path) 54 | if record_path: 55 | self.check_path(record_path, '`record_path`') 56 | 57 | self.logger.debug('Indexer initialized') 58 | self.logger.debug('check_duplicate = %r', self.check_duplicate) 59 | self.logger.debug('keep_json = %r', self.keep_json) 60 | self.logger.debug('record_path = %r', self.record_path) 61 | 62 | def get_record_type(self, path): 63 | relpath = os.path.relpath(path, self.cfstore.record_path) 64 | dirs = relpath.split(os.path.sep, 1) 65 | return dirs[0] if dirs else None 66 | 67 | def check_path(self, path, name='path'): 68 | if self.get_record_type(path) not in ['command', 'init', 'exit']: 69 | raise RuntimeError( 70 | '{0} must be under {1}'.format( 71 | name, 72 | os.path.join(self.cfstore.record_path, 73 | '{command,init,exit}', 74 | ''))) 75 | 76 | def index_record(self, json_path): 77 | """ 78 | Import `json_path` and remove it if :attr:`keep_json` is false. 79 | """ 80 | self.logger.debug('Indexing record: %s', json_path) 81 | json_path = os.path.abspath(json_path) 82 | self.check_path(json_path, '`json_path`') 83 | 84 | with open(json_path) as fp: 85 | try: 86 | dct = json.load(fp) 87 | except ValueError: 88 | warnings.warn( 89 | 'Ignoring invalid JSON file at: {0}'.format(json_path)) 90 | return 91 | 92 | record_type = self.get_record_type(json_path) 93 | kwds = {} 94 | if record_type == 'command': 95 | importer = self.db.import_dict 96 | kwds.update(check_duplicate=self.check_duplicate) 97 | elif record_type == 'init': 98 | importer = self.db.import_init_dict 99 | elif record_type == 'exit': 100 | importer = self.db.import_exit_dict 101 | else: 102 | raise ValueError("Unknown record type: {0}".format(record_type)) 103 | importer(dct, **kwds) 104 | 105 | if not self.keep_json: 106 | self.logger.info('Removing JSON record: %s', json_path) 107 | os.remove(json_path) 108 | 109 | def find_record_files(self): 110 | """ 111 | Yield paths to record files. 112 | """ 113 | for (root, _, files) in os.walk(self.record_path): 114 | for f in (f for f in files if f.endswith('.json')): 115 | yield os.path.join(root, f) 116 | 117 | def index_all(self): 118 | """ 119 | Index all records under :attr:`record_path`. 120 | """ 121 | self.logger.debug('Start indexing all records under: %s', 122 | self.record_path) 123 | with self.db.connection(): 124 | for json_path in sorted(self.find_record_files()): 125 | self.index_record(json_path) 126 | -------------------------------------------------------------------------------- /rash/init.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | 19 | 20 | def shell_name(shell): 21 | return shell.rsplit(os.path.sep, 1)[-1] 22 | 23 | 24 | def find_init(shell): 25 | rash_dir = os.path.dirname(__file__) 26 | return os.path.join(rash_dir, 'ext', 'rash.{0}'.format(shell_name(shell))) 27 | 28 | 29 | INIT_TEMPLATE = """\ 30 | source '{file}' 31 | _RASH_VERSION='{version}' 32 | """ 33 | # Currently `_RASH_VERSION` is not used anywhere, but it is useful to 34 | # see when RASH for a long lasting shell session is initialized. 35 | 36 | 37 | def init_run(shell, no_daemon, daemon_options, daemon_outfile): 38 | """ 39 | Configure your shell. 40 | 41 | Add the following line in your shell RC file and then you are 42 | ready to go:: 43 | 44 | eval $(%(prog)s) 45 | 46 | To check if your shell is supported, simply run:: 47 | 48 | %(prog)s --no-daemon 49 | 50 | If you want to specify shell other than $SHELL, you can give 51 | --shell option:: 52 | 53 | eval $(%(prog)s --shell zsh) 54 | 55 | By default, this command also starts daemon in background to 56 | automatically index shell history records. To not start daemon, 57 | use --no-daemon option like this:: 58 | 59 | eval $(%(prog)s --no-daemon) 60 | 61 | To see the other methods to launch the daemon process, see 62 | ``rash daemon --help``. 63 | 64 | """ 65 | import sys 66 | from .__init__ import __version__ 67 | init_file = find_init(shell) 68 | if os.path.exists(init_file): 69 | sys.stdout.write(INIT_TEMPLATE.format( 70 | file=init_file, version=__version__)) 71 | else: 72 | raise RuntimeError( 73 | "Shell '{0}' is not supported.".format(shell_name(shell))) 74 | 75 | if not no_daemon: 76 | from .daemon import start_daemon_in_subprocess 77 | start_daemon_in_subprocess(daemon_options, daemon_outfile) 78 | 79 | 80 | def init_add_arguments(parser): 81 | parser.add_argument( 82 | '--shell', default=os.environ.get('SHELL'), 83 | help=""" 84 | name of shell you are using. directory before the last / 85 | is discarded. It defaults to $SHELL. 86 | """) 87 | parser.add_argument( 88 | '--no-daemon', action='store_true', default=False, 89 | help=""" 90 | Do not start daemon. By default, daemon is started if 91 | there is no already running daemon. 92 | """) 93 | parser.add_argument( 94 | '--daemon-opt', dest='daemon_options', action='append', default=[], 95 | help=""" 96 | Add options given to daemon. See "rash daemon --help" for 97 | available options. It can be specified many times. 98 | Note that --no-error is always passed to the daemon command. 99 | """) 100 | parser.add_argument( 101 | '--daemon-outfile', default=os.devnull, 102 | help=""" 103 | Path to redirect STDOUT and STDERR of daemon process. 104 | This is mostly for debugging. 105 | """) 106 | 107 | 108 | commands = [ 109 | ('init', init_add_arguments, init_run), 110 | ] 111 | -------------------------------------------------------------------------------- /rash/interactive_search.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import re 18 | import shlex 19 | 20 | try: 21 | from percol.finder import FinderMultiQueryString 22 | assert FinderMultiQueryString # fool pyflakes 23 | except ImportError: 24 | # Dummy class for making this module importable: 25 | FinderMultiQueryString = object 26 | 27 | from .search import search_add_arguments 28 | from .query import SafeArgumentParser, expand_query, preprocess_kwds 29 | 30 | 31 | def strip_glob(string, split_str=' '): 32 | """ 33 | Strip glob portion in `string`. 34 | 35 | >>> strip_glob('*glob*like') 36 | 'glob like' 37 | >>> strip_glob('glob?') 38 | 'glo' 39 | >>> strip_glob('glob[seq]') 40 | 'glob' 41 | >>> strip_glob('glob[!seq]') 42 | 'glob' 43 | 44 | :type string: str 45 | :rtype: str 46 | 47 | """ 48 | string = _GLOB_PORTION_RE.sub(split_str, string) 49 | return string.strip() 50 | 51 | _GLOB_PORTION_RE = re.compile(r'\*|.\?|\[[^\]]+\]') 52 | 53 | 54 | class RashFinder(FinderMultiQueryString): 55 | 56 | base_query = [] 57 | 58 | def __init__(self, *args, **kwds): 59 | super(RashFinder, self).__init__(*args, **kwds) 60 | 61 | self.__parser = parser = SafeArgumentParser() 62 | search_add_arguments(parser) 63 | 64 | # Generator should be terminated in order to close connection to 65 | # sqlite. Otherwise, sqlite3 modules raise an error saying that 66 | # it doesn't support multi-threading access. 67 | lazy_finding = False 68 | 69 | and_search = False 70 | 71 | def find(self, query, collection=None): 72 | try: 73 | # shlex < 2.7.3 does not work with unicode: 74 | args = self.base_query + shlex.split(query.encode()) 75 | ns = self.__parser.parse_args(args) 76 | kwds = preprocess_kwds(expand_query(self.rashconfig, vars(ns))) 77 | except (ValueError, SyntaxError): 78 | return super(RashFinder, self).find(query, collection) 79 | 80 | # SOMEDAY: get rid of this hard-coded search limit by making 81 | # `search_command_record` thread-safe and setting 82 | # `lazy_finding = True`. 83 | kwds['limit'] = 50 if kwds['match_pattern'] else 1000 84 | 85 | records = self.db.search_command_record(**kwds) 86 | self.collection = collection = (r.command for r in records) 87 | 88 | # There will be no filtering in the super class. 89 | # I am using it for highlighting matches. 90 | queries = kwds['match_pattern'] + kwds['include_pattern'] 91 | split_str = self.split_str 92 | subquery = split_str.join(strip_glob(q, split_str) for q in queries) 93 | return super(RashFinder, self).find(subquery, collection) 94 | 95 | 96 | def load_rc(percol, path=None, encoding=None): 97 | import os 98 | from percol import debug 99 | if path is None: 100 | path = os.path.expanduser("~/.percol.d/rc.py") 101 | try: 102 | with open(path, 'r') as rc: 103 | exec(rc.read().decode(encoding or 'utf-8'), locals()) 104 | except Exception as e: 105 | debug.log("exception", e) 106 | 107 | 108 | def launch_isearch(cfstore, rcfile=None, input_encoding=None, 109 | base_query=None, query=None, query_template=None, **kwds): 110 | from percol import Percol 111 | from percol import tty 112 | import percol.actions as actions 113 | 114 | from .database import DataBase 115 | 116 | config = cfstore.get_config() 117 | default = lambda val, defv: defv if val is None else val 118 | 119 | # Pass db instance to finder. Not clean but works and no harm. 120 | RashFinder.db = DataBase(cfstore.db_path) 121 | RashFinder.base_query = default(base_query, config.isearch.base_query) 122 | RashFinder.rashconfig = config 123 | 124 | template = default(query_template, config.isearch.query_template) 125 | default_query = default(query, config.isearch.query) 126 | 127 | ttyname = tty.get_ttyname() 128 | with open(ttyname, "r+w") as tty_f: 129 | with Percol(descriptors=tty.reconnect_descriptors(tty_f), 130 | finder=RashFinder, 131 | actions=(actions.output_to_stdout,), 132 | # This will be used if the first call for RashFinder.find 133 | # fails to fetch collections from DB. 134 | candidates=[], 135 | query=template.format(default_query), 136 | **kwds) as percol: 137 | load_rc(percol, rcfile, input_encoding) 138 | exit_code = percol.loop() 139 | 140 | exit(exit_code) 141 | -------------------------------------------------------------------------------- /rash/isearch.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | def isearch_run(**kwds): 18 | r""" 19 | Interactive history search that updated as you type. 20 | 21 | The query for this program is the same as the one for 22 | ``rash search`` command. 23 | 24 | You need percol_ to use this command. 25 | 26 | _percol: https://github.com/mooz/percol 27 | 28 | If you use zsh, you can setup a keybind like this to quickly 29 | launch iserch and execute the result.:: 30 | 31 | # Type `Ctrl-x r` to start isearch 32 | bindkey "^Xr" rash-zle-isearch 33 | 34 | If you like command or you are not using zsh, you can add 35 | something like the following in your rc file to start and 36 | execute the chosen command. 37 | 38 | rash-isearch(){ 39 | eval "$(rash isearch)" 40 | } 41 | 42 | To pass long and complex query, give them after "--", 43 | like this.:: 44 | 45 | rash isearch -- \ 46 | --cwd . \ 47 | --exclude-pattern "*rash *" \ 48 | --include-pattern "*test*" \ 49 | --include-pattern "tox*" \ 50 | --include-pattern "make *test*" 51 | 52 | """ 53 | from .config import ConfigStore 54 | from .interactive_search import launch_isearch 55 | launch_isearch(ConfigStore(), **kwds) 56 | 57 | 58 | def isearch_add_arguments(parser): 59 | parser.add_argument( 60 | '--query', '-q', default=None, 61 | help='default query') 62 | parser.add_argument( 63 | '--query-template', default=None, 64 | help='Transform default query using Python string format.') 65 | parser.add_argument( 66 | 'base_query', nargs='*', default=None, 67 | help=""" 68 | The part of query that is not shown in UI and is impossible 69 | to rewrite in this session. Useful for putting long and 70 | complex query. 71 | """) 72 | parser.add_argument( 73 | '--caret', default=None, type=int, 74 | help='caret position') 75 | 76 | 77 | commands = [ 78 | ('isearch', isearch_add_arguments, isearch_run), 79 | ] 80 | -------------------------------------------------------------------------------- /rash/log.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import logging 18 | 19 | 20 | def loglevel(level): 21 | """ 22 | Convert any representation of `level` to an int appropriately. 23 | 24 | :type level: int or str 25 | :rtype: int 26 | 27 | >>> loglevel('DEBUG') == logging.DEBUG 28 | True 29 | >>> loglevel(10) 30 | 10 31 | >>> loglevel(None) 32 | Traceback (most recent call last): 33 | ... 34 | ValueError: None is not a proper log level. 35 | 36 | """ 37 | if isinstance(level, str): 38 | level = getattr(logging, level.upper()) 39 | elif isinstance(level, int): 40 | pass 41 | else: 42 | raise ValueError('{0!r} is not a proper log level.'.format(level)) 43 | return level 44 | 45 | 46 | logger = logging.getLogger('rash') 47 | 48 | 49 | def setup_daemon_log_file(cfstore): 50 | """ 51 | Attach file handler to RASH logger. 52 | 53 | :type cfstore: rash.config.ConfigStore 54 | 55 | """ 56 | level = loglevel(cfstore.daemon_log_level) 57 | handler = logging.FileHandler(filename=cfstore.daemon_log_path) 58 | handler.setLevel(level) 59 | logger.setLevel(level) 60 | logger.addHandler(handler) 61 | 62 | 63 | class LogForTheFuture(object): 64 | 65 | """ 66 | Logger that works even handler is not ready. 67 | """ 68 | 69 | def __init__(self): 70 | self._messages = [] 71 | 72 | def __getattr__(self, name): 73 | try: 74 | return super(LogForTheFuture, self).__getattr__(name) 75 | except AttributeError: 76 | assert name in ('critical', 'fatal', 'error', 'warning', 77 | 'warn', 'info', 'debug') 78 | return lambda *args, **kwds: \ 79 | self._messages.append((name, args, kwds)) 80 | 81 | def dump(self): 82 | for (name, args, kwds) in self._messages: 83 | getattr(logger, name)(*args, **kwds) 84 | -------------------------------------------------------------------------------- /rash/model.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | class CommandRecord(object): 18 | 19 | """ 20 | Command record. 21 | 22 | >>> CommandRecord() 23 | 24 | >>> CommandRecord( 25 | ... command='DUMMY-COMMAND', 26 | ... command_history_id=222, 27 | ... session_history_id=111) 28 | 29 | 30 | """ 31 | 32 | def __init__(self, **kwds): 33 | self.command_history_id = None 34 | self.command = None 35 | self.session_history_id = None 36 | self.session_id = None 37 | self.exit_code = None 38 | self.pipestatus = [] 39 | self.start = None 40 | self.stop = None 41 | self.terminal = None 42 | self.cwd = None 43 | self.environ = {} 44 | 45 | # Non-record attributes (metadata): 46 | self.command_count = None 47 | 48 | self.__dict__.update(kwds) 49 | 50 | def __repr__(self): 51 | ch_id = self.command_history_id 52 | sh_id = self.session_history_id 53 | return '<{0}: {1}({2}.{3})>'.format( 54 | self.__class__.__name__, 55 | self.command, 56 | sh_id if sh_id is not None else '?', 57 | ch_id if ch_id is not None else '?', 58 | ) 59 | 60 | 61 | class SessionRecord(object): 62 | 63 | """ 64 | Session record. 65 | 66 | >>> SessionRecord() 67 | 68 | >>> SessionRecord(command='DUMMY-COMMAND', session_history_id=111) 69 | 70 | 71 | """ 72 | 73 | def __init__(self, **kwds): 74 | self.session_history_id = None 75 | self.session_id = None 76 | self.start = None 77 | self.stop = None 78 | self.environ = {} 79 | self.__dict__.update(kwds) 80 | 81 | def __repr__(self): 82 | sh_id = self.session_history_id 83 | return '<{0}: {1}>'.format( 84 | self.__class__.__name__, 85 | sh_id if sh_id is not None else '?', 86 | ) 87 | 88 | 89 | class EnvironRecord(object): 90 | 91 | """ 92 | Environment variable record. 93 | 94 | >>> EnvironRecord(variable_name='PWD', variable_value='DUMMY/PATH') 95 | 96 | 97 | """ 98 | 99 | def __init__(self, **kwds): 100 | self.environment_variable_id = None 101 | self.variable_name = None 102 | self.variable_value = None 103 | self.__dict__.update(kwds) 104 | 105 | def __repr__(self): 106 | return '<{0}: {1}={2}>'.format( 107 | self.__class__.__name__, 108 | self.variable_name, self.variable_value, 109 | ) 110 | 111 | 112 | class VersionRecord(object): 113 | 114 | """ 115 | Version record. 116 | 117 | >>> VersionRecord(rash_version='0.2.0', schema_version='0.1') 118 | 119 | 120 | """ 121 | 122 | def __init__(self, **kwds): 123 | self.id = None 124 | self.rash_version = None 125 | self.schema_version = None 126 | self.updated = None 127 | self.__dict__.update(kwds) 128 | 129 | def __repr__(self): 130 | return '<{0}: schema={1}, rash={2}>'.format( 131 | self.__class__.__name__, 132 | self.schema_version, 133 | self.rash_version, 134 | ) 135 | -------------------------------------------------------------------------------- /rash/query.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | from argparse import ArgumentParser 17 | 18 | from .search import SORT_KEY_SYNONYMS, search_add_arguments 19 | 20 | 21 | class SafeArgumentParser(ArgumentParser): 22 | 23 | def exit(self, *_, **__): 24 | raise ValueError 25 | 26 | def print_usage(self, *_): 27 | pass 28 | 29 | print_help = print_version = print_usage 30 | 31 | 32 | def expand_query(config, kwds): 33 | """ 34 | Expand `kwds` based on `config.search.query_expander`. 35 | 36 | :type config: .config.Configuration 37 | :type kwds: dict 38 | :rtype: dict 39 | :return: Return `kwds`, modified in place. 40 | 41 | """ 42 | pattern = [] 43 | for query in kwds.pop('pattern', []): 44 | expansion = config.search.alias.get(query) 45 | if expansion is None: 46 | pattern.append(query) 47 | else: 48 | parser = SafeArgumentParser() 49 | search_add_arguments(parser) 50 | ns = parser.parse_args(expansion) 51 | for (key, value) in vars(ns).items(): 52 | if isinstance(value, (list, tuple)): 53 | if not kwds.get(key): 54 | kwds[key] = value 55 | else: 56 | kwds[key].extend(value) 57 | else: 58 | kwds[key] = value 59 | kwds['pattern'] = pattern 60 | return config.search.kwds_adapter(kwds) 61 | 62 | 63 | def preprocess_kwds(kwds): 64 | """ 65 | Preprocess keyword arguments for `DataBase.search_command_record`. 66 | """ 67 | from .utils.timeutils import parse_datetime, parse_duration 68 | 69 | for key in ['output', 'format', 'format_level', 70 | 'with_command_id', 'with_session_id']: 71 | kwds.pop(key, None) 72 | 73 | for key in ['time_after', 'time_before']: 74 | val = kwds[key] 75 | if val: 76 | dt = parse_datetime(val) 77 | if dt: 78 | kwds[key] = dt 79 | 80 | for key in ['duration_longer_than', 'duration_less_than']: 81 | val = kwds[key] 82 | if val: 83 | dt = parse_duration(val) 84 | if dt: 85 | kwds[key] = dt 86 | 87 | # interpret "pattern" (currently just copying to --include-pattern) 88 | less_strict_pattern = list(map("*{0}*".format, kwds.pop('pattern', []))) 89 | kwds['match_pattern'] = kwds['match_pattern'] + less_strict_pattern 90 | 91 | if not kwds['sort_by']: 92 | kwds['sort_by'] = ['count'] 93 | kwds['sort_by'] = [SORT_KEY_SYNONYMS[k] for k in kwds['sort_by']] 94 | return kwds 95 | -------------------------------------------------------------------------------- /rash/record.py: -------------------------------------------------------------------------------- 1 | """ 2 | Record shell history. 3 | 4 | This is a command to be called from shell-specific hooks. 5 | This Python implementation is a reference implementation. 6 | Probably it makes sense to write this command naively in 7 | shells to make it faster. 8 | 9 | The dumped data goes under the ``~/.config/rash/data/record`` 10 | directory. 11 | 12 | """ 13 | 14 | # Copyright (C) 2013- Takafumi Arakaki 15 | 16 | # This program is free software: you can redistribute it and/or modify 17 | # it under the terms of the GNU General Public License as published by 18 | # the Free Software Foundation, either version 3 of the License, or 19 | # (at your option) any later version. 20 | 21 | # This program is distributed in the hope that it will be useful, 22 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 23 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 24 | # GNU General Public License for more details. 25 | 26 | # You should have received a copy of the GNU General Public License 27 | # along with this program. If not, see . 28 | 29 | 30 | import os 31 | import time 32 | import json 33 | 34 | from .utils.pathutils import mkdirp 35 | from .utils.py3compat import getcwd 36 | from .config import ConfigStore 37 | 38 | 39 | def get_tty(): 40 | """ 41 | Return \"os.ttyname(0 or 1 or 2)\". 42 | """ 43 | for i in range(3): 44 | try: 45 | return os.ttyname(i) 46 | break 47 | except OSError: 48 | pass 49 | 50 | 51 | def get_environ(keys): 52 | """ 53 | Get environment variables from :data:`os.environ`. 54 | 55 | :type keys: [str] 56 | :rtype: dict 57 | 58 | Some additional features. 59 | 60 | * If 'HOST' is not in :data:`os.environ`, this function 61 | automatically fetch it using :meth:`platform.node`. 62 | * If 'TTY' is not in :data:`os.environ`, this function 63 | automatically fetch it using :meth:`os.ttyname`. 64 | * Set 'RASH_SPENV_TERMINAL' if needed. 65 | 66 | """ 67 | items = ((k, os.environ.get(k)) for k in keys) 68 | subenv = dict((k, v) for (k, v) in items if v is not None) 69 | needset = lambda k: k in keys and not subenv.get(k) 70 | 71 | def setifnonempty(key, value): 72 | if value: 73 | subenv[key] = value 74 | 75 | if needset('HOST'): 76 | import platform 77 | subenv['HOST'] = platform.node() 78 | if needset('TTY'): 79 | setifnonempty('TTY', get_tty()) 80 | if needset('RASH_SPENV_TERMINAL'): 81 | from .utils.termdetection import detect_terminal 82 | setifnonempty('RASH_SPENV_TERMINAL', detect_terminal()) 83 | return subenv 84 | 85 | 86 | def generate_session_id(data): 87 | """ 88 | Generate session ID based on HOST, TTY, PID [#]_ and start time. 89 | 90 | :type data: dict 91 | :rtype: str 92 | 93 | .. [#] PID of the shell, i.e., PPID of this Python process. 94 | 95 | """ 96 | host = data['environ']['HOST'] 97 | tty = data['environ'].get('TTY') or 'NO_TTY' 98 | return ':'.join(map(str, [ 99 | host, tty, os.getppid(), data['start']])) 100 | 101 | 102 | def record_run(record_type, print_session_id, **kwds): 103 | """ 104 | Record shell history. 105 | """ 106 | if print_session_id and record_type != 'init': 107 | raise RuntimeError( 108 | '--print-session-id should be used with --record-type=init') 109 | 110 | cfstore = ConfigStore() 111 | # SOMEDAY: Pass a list of environment variables to shell by "rash 112 | # init" and don't read configuration in "rash record" command. It 113 | # is faster. 114 | config = cfstore.get_config() 115 | envkeys = config.record.environ[record_type] 116 | json_path = os.path.join(cfstore.record_path, 117 | record_type, 118 | time.strftime('%Y-%m-%d-%H%M%S.json')) 119 | mkdirp(os.path.dirname(json_path)) 120 | 121 | # Command line options directly map to record keys 122 | data = dict((k, v) for (k, v) in kwds.items() if v is not None) 123 | data.update( 124 | environ=get_environ(envkeys), 125 | ) 126 | 127 | # Automatically set some missing variables: 128 | data.setdefault('cwd', getcwd()) 129 | if record_type in ['command', 'exit']: 130 | data.setdefault('stop', int(time.time())) 131 | elif record_type in ['init']: 132 | data.setdefault('start', int(time.time())) 133 | 134 | if print_session_id: 135 | data['session_id'] = generate_session_id(data) 136 | print(data['session_id']) 137 | 138 | with open(json_path, 'w') as fp: 139 | json.dump(data, fp) 140 | 141 | 142 | def record_add_arguments(parser): 143 | parser.add_argument( 144 | '--record-type', default='command', 145 | choices=['command', 'init', 'exit'], 146 | help='type of record to store.') 147 | parser.add_argument( 148 | '--command', 149 | help="command that was ran.") 150 | parser.add_argument( 151 | '--cwd', 152 | help=''' 153 | Like $PWD, but callee can set it to consider command that 154 | changes directory (e.g., cd). 155 | ''') 156 | parser.add_argument( 157 | '--exit-code', type=int, 158 | help="exit code $? of the command.") 159 | parser.add_argument( 160 | '--pipestatus', type=int, nargs='+', 161 | help="$pipestatus (zsh) / $PIPESTATUS (bash)") 162 | parser.add_argument( 163 | '--start', type=int, 164 | help='the time COMMAND is started.') 165 | parser.add_argument( 166 | '--stop', type=int, 167 | help='the time COMMAND is finished.') 168 | parser.add_argument( 169 | '--session-id', 170 | help=''' 171 | RASH session ID generated by --print-session-id. 172 | This option should be used with `command` or `exit` RECORD_TYPE. 173 | ''') 174 | parser.add_argument( 175 | '--print-session-id', default=False, action='store_true', 176 | help=''' 177 | print generated session ID to stdout. 178 | This option should be used with `init` RECORD_TYPE. 179 | ''') 180 | 181 | 182 | commands = [ 183 | ('record', record_add_arguments, record_run), 184 | ] 185 | -------------------------------------------------------------------------------- /rash/schema.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2013- Takafumi Arakaki 2 | 3 | -- This program is free software: you can redistribute it and/or modify 4 | -- it under the terms of the GNU General Public License as published by 5 | -- the Free Software Foundation, either version 3 of the License, or 6 | -- (at your option) any later version. 7 | 8 | -- This program is distributed in the hope that it will be useful, 9 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | -- GNU General Public License for more details. 12 | 13 | -- You should have received a copy of the GNU General Public License 14 | -- along with this program. If not, see . 15 | 16 | 17 | DROP TABLE IF EXISTS command_history; 18 | CREATE TABLE command_history ( 19 | id INTEGER PRIMARY KEY AUTOINCREMENT, 20 | command_id INTEGER, 21 | session_id INTEGER, 22 | directory_id INTEGER, 23 | -- SOMEDAY: Remove terminal_id or move this to session_history table. 24 | -- See the comment in record_run (./record.py). 25 | terminal_id INTEGER, 26 | start_time TIMESTAMP, 27 | stop_time TIMESTAMP, 28 | exit_code INTEGER, 29 | FOREIGN KEY(command_id) REFERENCES command_list(id), 30 | FOREIGN KEY(session_id) REFERENCES session_history(id), 31 | FOREIGN KEY(directory_id) REFERENCES directory_list(id), 32 | FOREIGN KEY(terminal_id) REFERENCES terminal_list(id) 33 | ); 34 | 35 | DROP TABLE IF EXISTS session_history; 36 | CREATE TABLE session_history ( 37 | id INTEGER PRIMARY KEY AUTOINCREMENT, 38 | session_long_id TEXT NOT NULL UNIQUE, 39 | start_time TIMESTAMP, 40 | stop_time TIMESTAMP 41 | ); 42 | 43 | DROP TABLE IF EXISTS command_list; 44 | CREATE TABLE command_list ( 45 | id INTEGER PRIMARY KEY AUTOINCREMENT, 46 | command TEXT NOT NULL UNIQUE 47 | ); 48 | 49 | DROP TABLE IF EXISTS terminal_list; 50 | CREATE TABLE terminal_list ( 51 | id INTEGER PRIMARY KEY AUTOINCREMENT, 52 | terminal TEXT NOT NULL UNIQUE 53 | ); 54 | 55 | DROP TABLE IF EXISTS environment_variable; 56 | CREATE TABLE environment_variable ( 57 | id INTEGER PRIMARY KEY AUTOINCREMENT, 58 | variable_name TEXT NOT NULL, 59 | variable_value TEXT NOT NULL 60 | ); 61 | 62 | DROP TABLE IF EXISTS directory_list; 63 | CREATE TABLE directory_list ( 64 | id INTEGER PRIMARY KEY autoincrement, 65 | directory TEXT NOT NULL UNIQUE 66 | ); 67 | 68 | DROP TABLE IF EXISTS command_environment_map; 69 | CREATE TABLE command_environment_map ( 70 | ch_id INTEGER NOT NULL, 71 | ev_id INTEGER NOT NULL, 72 | FOREIGN KEY(ch_id) REFERENCES command_history(id), 73 | FOREIGN KEY(ev_id) REFERENCES environment_variable(id) 74 | ); 75 | 76 | DROP TABLE IF EXISTS session_environment_map; 77 | CREATE TABLE session_environment_map ( 78 | sh_id INTEGER NOT NULL, 79 | ev_id INTEGER NOT NULL, 80 | FOREIGN KEY(sh_id) REFERENCES session_history(id), 81 | FOREIGN KEY(ev_id) REFERENCES environment_variable(id) 82 | ); 83 | 84 | DROP TABLE IF EXISTS pipe_status_map; 85 | CREATE TABLE pipe_status_map ( 86 | ch_id INTEGER NOT NULL, 87 | program_position INTEGER NOT NULL, 88 | exit_code INTEGER, 89 | FOREIGN KEY(ch_id) REFERENCES command_history(id) 90 | ); 91 | 92 | DROP TABLE IF EXISTS rash_info; 93 | CREATE TABLE rash_info ( 94 | id INTEGER PRIMARY KEY AUTOINCREMENT, 95 | rash_version TEXT NOT NULL, 96 | schema_version TEXT NOT NULL, 97 | updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP 98 | ); 99 | -------------------------------------------------------------------------------- /rash/search.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | SORT_KEY_SYNONYMS = { 18 | 'count': 'command_count', 19 | 'success_count': 'success_count', 20 | 'success_ratio': 'success_ratio', 21 | 'program_count': 'program_count', 22 | 'time': 'start_time', 23 | 'start': 'start_time', 24 | 'stop': 'stop_time', 25 | 'code': 'exit_code', 26 | } 27 | 28 | 29 | def search_run(output, **kwds): 30 | """ 31 | Search command history. 32 | 33 | """ 34 | from .config import ConfigStore 35 | from .database import DataBase 36 | from .query import expand_query, preprocess_kwds 37 | 38 | cfstore = ConfigStore() 39 | kwds = expand_query(cfstore.get_config(), kwds) 40 | format = get_formatter(**kwds) 41 | fmtkeys = formatter_keys(format) 42 | candidates = set([ 43 | 'command_count', 'success_count', 'success_ratio', 'program_count']) 44 | kwds['additional_columns'] = candidates & set(fmtkeys) 45 | 46 | db = DataBase(cfstore.db_path) 47 | for crec in db.search_command_record(**preprocess_kwds(kwds)): 48 | output.write(format.format(**crec.__dict__)) 49 | 50 | 51 | def get_formatter( 52 | format, with_command_id, with_session_id, format_level, **_): 53 | if format_level >= 3: 54 | format = ("{session_history_id:>5} " 55 | "{command_history_id:>5} " 56 | "{command_count:>4} {command}\n") 57 | elif format_level == 2 or with_command_id and with_session_id: 58 | format = ("{session_history_id:>5} " 59 | "{command_history_id:>5} {command}\n") 60 | elif format_level == 1 or with_command_id: 61 | format = "{command_history_id:>5} {command}\n" 62 | elif with_session_id: 63 | format = "{session_history_id:>5} {command}\n" 64 | else: 65 | format = format.decode('string_escape') 66 | return format 67 | 68 | 69 | def formatter_keys(format_string): 70 | """ 71 | Return required fields in `format_string`. 72 | 73 | >>> sorted(formatter_keys('{1} {key}')) 74 | ['1', 'key'] 75 | 76 | """ 77 | from string import Formatter 78 | return (tp[1] for tp in Formatter().parse(format_string)) 79 | 80 | 81 | def search_add_arguments(parent_parser): 82 | import argparse 83 | # Filter 84 | parser = parent_parser.add_argument_group('Filter') 85 | parser.add_argument( 86 | 'pattern', nargs='*', 87 | help=""" 88 | Glob pattern to match substring of command. It is as same as 89 | --match-pattern/-m except that the pattern is going to be 90 | wrapped by `*`s. If you want to use strict glob pattern 91 | that matches to entire command, use --match-pattern/-m. 92 | """) 93 | parser.add_argument( 94 | '--match-pattern', '-m', metavar='GLOB', action='append', default=[], 95 | help=""" 96 | Only commands that match to this glob pattern are listed. 97 | Unlike --include-pattern/-g, applying this option multiple 98 | times does AND match. 99 | """) 100 | parser.add_argument( 101 | '--include-pattern', '-g', metavar='GLOB', action='append', default=[], 102 | help='glob patterns that matches to commands to include.') 103 | parser.add_argument( 104 | '--exclude-pattern', '-G', metavar='GLOB', action='append', default=[], 105 | help='glob patterns that matches to commands to exclude.') 106 | parser.add_argument( 107 | '--match-regexp', '-M', 108 | metavar='REGEXP', action='append', default=[], 109 | help=""" 110 | Only commands that matches to this grep pattern are listed. 111 | Unlike --include-regexp/-e, applying this option multiple 112 | times does AND match. 113 | """) 114 | parser.add_argument( 115 | '--include-regexp', '-e', 116 | metavar='REGEXP', action='append', default=[], 117 | help=""" 118 | Regular expression patterns that matches to commands to include. 119 | """) 120 | parser.add_argument( 121 | '--exclude-regexp', '-E', 122 | metavar='REGEXP', action='append', default=[], 123 | help=""" 124 | Regular expression patterns that matches to commands to exclude. 125 | """) 126 | parser.add_argument( 127 | '--cwd', '-d', metavar='DIR', action='append', default=[], 128 | help=""" 129 | The working directory at the time when the command was run. 130 | When given several times, items that match to one of the 131 | directory are included in the result. 132 | """) 133 | parser.add_argument( 134 | '--cwd-glob', '-D', metavar='GLOB', action='append', default=[], 135 | help=""" 136 | Same as --cwd but it accepts glob expression. 137 | """) 138 | parser.add_argument( 139 | '--cwd-under', '-u', metavar='DIR', action='append', default=[], 140 | help=""" 141 | Same as --cwd but include all subdirectories. 142 | """) 143 | parser.add_argument( 144 | '--time-after', '-t', metavar='TIME', 145 | help='commands run after the given time') 146 | parser.add_argument( 147 | '--time-before', '-T', metavar='TIME', 148 | help='commands run before the given time') 149 | parser.add_argument( 150 | '--duration-longer-than', '-S', metavar='DURATION', 151 | help='commands that takes longer than the given time') 152 | parser.add_argument( 153 | '--duration-less-than', '-s', metavar='DURATION', 154 | help='commands that takes less than the given time') 155 | parser.add_argument( 156 | '--include-exit-code', '-x', 157 | metavar='CODE', action='append', default=[], type=int, 158 | help='include command which finished with given exit code.') 159 | parser.add_argument( 160 | '--exclude-exit-code', '-X', 161 | metavar='CODE', action='append', default=[], type=int, 162 | help='exclude command which finished with given exit code.') 163 | parser.add_argument( 164 | '--include-session', '-n', dest='include_session_history_id', 165 | metavar='ID', action='append', default=[], type=int, 166 | help=""" 167 | include command which is issued in given session. 168 | """) 169 | parser.add_argument( 170 | '--exclude-session', '-N', dest='exclude_session_history_id', 171 | metavar='ID', action='append', default=[], type=int, 172 | help=""" 173 | exclude command which is issued in given session. 174 | """) 175 | parser.add_argument( 176 | '--match-environ-pattern', 177 | metavar='ENV', action='append', default=[], nargs=2, 178 | help=""" 179 | select command which associated with environment variable 180 | that matches to given glob pattern.""") 181 | parser.add_argument( 182 | '--include-environ-pattern', '-v', 183 | metavar='ENV', action='append', default=[], nargs=2, 184 | help=""" 185 | include command which associated with environment variable 186 | that matches to given glob pattern.""") 187 | parser.add_argument( 188 | '--exclude-environ-pattern', '-V', 189 | metavar='ENV', action='append', default=[], nargs=2, 190 | help=""" 191 | exclude command which associated with environment variable 192 | that matches to given glob pattern.""") 193 | parser.add_argument( 194 | '--match-environ-regexp', 195 | metavar='ENV', action='append', default=[], nargs=2, 196 | help=""" 197 | select command which associated with environment variable 198 | that matches to given glob pattern.""") 199 | parser.add_argument( 200 | '--include-environ-regexp', '-w', 201 | metavar='ENV', action='append', default=[], nargs=2, 202 | help=""" 203 | include command which associated with environment variable 204 | that matches to given glob pattern.""") 205 | parser.add_argument( 206 | '--exclude-environ-regexp', '-W', 207 | metavar='ENV', action='append', default=[], nargs=2, 208 | help=""" 209 | exclude command which associated with environment variable 210 | that matches to given glob pattern.""") 211 | # "global" filters 212 | parser.add_argument( 213 | '--limit', '-l', metavar='NUM', type=int, default=10, 214 | help='maximum number of history to show. -1 means no limit.') 215 | parser.add_argument( 216 | '--no-unique', '-a', dest='unique', action='store_false', default=True, 217 | help=""" 218 | Include all duplicates. 219 | """) 220 | parser.add_argument( 221 | '--ignore-case', '-i', action='store_true', default=False, 222 | help=""" 223 | Do case insensitive search. 224 | """) 225 | 226 | # Sorter 227 | parser = parent_parser.add_argument_group('Sorter') 228 | parser.add_argument( 229 | '--reverse', '-r', action='store_true', default=False, 230 | help=""" 231 | Reverse order of the result. 232 | By default, most recent commands are shown. 233 | """) 234 | parser.add_argument( 235 | '--sort-by', default=[], action='append', 236 | choices=sorted(SORT_KEY_SYNONYMS), 237 | help=""" 238 | Sort keys 239 | `count`: number of the time command is executed; 240 | `success_count`: number of the time command is succeeded; 241 | `program_count`: number of the time *program* is used; 242 | `start`(=`time`): the time command is executed; 243 | `stop`: the time command is finished; 244 | `code`: exit code of the command; 245 | Note that --sort-by=count cannot be used with --no-unique. 246 | If you don't give anything, it defaults to `count`. 247 | However, if you give this option at least once, the default 248 | is ignored (i.e., the result is *not* sorted by `count` 249 | unless you give it explicitly.). 250 | """) 251 | # SOMEDAY: order of --sort-by and --sort-by-cwd-distance should be 252 | # taken into account 253 | parser.add_argument( 254 | '--sort-by-cwd-distance', '-y', metavar='DIR', 255 | help=""" 256 | Sort by distance of recorded cwd fron DIR. 257 | Commands run at DIR are listed first, then commands 258 | run at one level down or one level up directories, 259 | and then two level down/up, and so on. 260 | """) 261 | 262 | # Modifier 263 | parser = parent_parser.add_argument_group('Modifier') 264 | parser.add_argument( 265 | '--after-context', '-A', type=int, metavar='NUM', 266 | help=""" 267 | Print NUM commands executed after matching commands. 268 | See also --context option. 269 | """) 270 | parser.add_argument( 271 | '--before-context', '-B', type=int, metavar='NUM', 272 | help=""" 273 | Print NUM commands executed before matching commands. 274 | See also --context option. 275 | """) 276 | parser.add_argument( 277 | '--context', '-C', type=int, metavar='NUM', 278 | help=""" 279 | Print NUM commands executed before and after matching commands. 280 | When this option is given --no-unique is implied and --sort-by 281 | is ignored. 282 | """) 283 | # SOMEDAY: remove restrictions from --context. 284 | parser.add_argument( 285 | '--context-type', default='time', choices=['time', 'session'], 286 | help=""" 287 | `session`: commands executed in the same shell session; 288 | `time`: commands executed around the same time; 289 | """) 290 | 291 | # Formatter 292 | parser = parent_parser.add_argument_group('Formatter') 293 | parser.add_argument( 294 | '--with-command-id', action='store_true', default=False, 295 | help=""" 296 | Print command ID number. 297 | When this is set, --format option has no effect. 298 | If --with-session-id is also specified, session ID comes 299 | at the first column then command ID comes the next column. 300 | """) 301 | parser.add_argument( 302 | '--with-session-id', action='store_true', default=False, 303 | help=""" 304 | Print session ID number. 305 | When this is set, --format option has no effect. 306 | See also: --with-command-id 307 | """) 308 | parser.add_argument( 309 | '--format', default=r'{command}\n', 310 | help=""" 311 | Python string formatter. Available keys: 312 | command, exit_code, pipestatus (a list), start, stop, cwd, 313 | command_history_id, session_history_id. 314 | See also: 315 | http://docs.python.org/library/string.html#format-string-syntax 316 | """) 317 | parser.add_argument( 318 | '-f', dest='format_level', action='count', default=0, 319 | help=""" 320 | Set formatting detail. This can be given multiple times to 321 | make more detailed output. For example, giving it once 322 | equivalent to passing --with-command-id and one more -f means 323 | adding --with-session-id. 324 | """) 325 | 326 | # Misc 327 | parser = parent_parser.add_argument_group('Misc') 328 | parser.add_argument( 329 | '--output', default='-', type=argparse.FileType('w'), 330 | help=""" 331 | Output file to write the results in. Default is stdout. 332 | """) 333 | 334 | 335 | commands = [ 336 | ('search', search_add_arguments, search_run), 337 | ] 338 | -------------------------------------------------------------------------------- /rash/show.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | def show_run(command_history_id): 18 | """ 19 | Show detailed command history by its ID. 20 | """ 21 | from pprint import pprint 22 | from .config import ConfigStore 23 | from .database import DataBase 24 | db = DataBase(ConfigStore().db_path) 25 | with db.connection(): 26 | for ch_id in command_history_id: 27 | crec = db.get_full_command_record(ch_id) 28 | pprint(crec.__dict__) 29 | print("") 30 | 31 | 32 | def show_add_arguments(parser): 33 | parser.add_argument( 34 | 'command_history_id', nargs='+', type=int, 35 | help=""" 36 | Integer ID of command history. 37 | """) 38 | 39 | 40 | commands = [ 41 | ('show', show_add_arguments, show_run), 42 | ] 43 | -------------------------------------------------------------------------------- /rash/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tkf/rash/585da418ec37dd138f1a4277718b6f507e9536a2/rash/tests/__init__.py -------------------------------------------------------------------------------- /rash/tests/test_indexer.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | import tempfile 19 | import shutil 20 | import json 21 | 22 | from ..config import ConfigStore 23 | from ..indexer import Indexer 24 | from ..utils.pathutils import mkdirp 25 | from .utils import BaseTestCase 26 | 27 | 28 | class TestIndexer(BaseTestCase): 29 | 30 | def setUp(self): 31 | self.base_path = tempfile.mkdtemp(prefix='rash-test-') 32 | self.cfstore = ConfigStore(self.base_path) 33 | 34 | def tearDown(self): 35 | shutil.rmtree(self.base_path) 36 | 37 | def get_indexer(self, keep_json=True, check_duplicate=True): 38 | return Indexer(self.cfstore, check_duplicate, keep_json) 39 | 40 | def prepare_records(self, **records): 41 | if set(records) > set(['command', 'init', 'exit']): 42 | raise ValueError( 43 | 'Unknown record type in {0}'.format(list(records))) 44 | paths = [] 45 | for (rectype, data_list) in records.items(): 46 | for (i, data) in enumerate(data_list): 47 | json_path = os.path.join(self.cfstore.record_path, 48 | rectype, 49 | '{0:05d}.json'.format(i)) 50 | mkdirp(os.path.dirname(json_path)) 51 | with open(json_path, 'w') as f: 52 | json.dump(data, f) 53 | paths.append(json_path) 54 | return paths 55 | 56 | def get_dummy_records(self, num_command=1, num_init=1, num_exit=1): 57 | gen = lambda i, **kwds: dict(session_id='SID-{0}'.format(i), **kwds) 58 | return dict( 59 | command=[gen(i, start=i, stop=i + 1) for i in range(num_command)], 60 | init=[gen(i, start=i) for i in range(num_init)], 61 | exit=[gen(i, stop=i) for i in range(num_exit)], 62 | ) 63 | 64 | def test_find_record_files(self): 65 | indexer = self.get_indexer() 66 | self.assertEqual(list(indexer.find_record_files()), []) 67 | desired_paths = self.prepare_records(**self.get_dummy_records()) 68 | actual_paths = list(indexer.find_record_files()) 69 | self.assertSetEqual(set(actual_paths), set(desired_paths)) 70 | 71 | def test_index_all_and_keep_json(self): 72 | desired_paths = self.prepare_records(**self.get_dummy_records()) 73 | indexer = self.get_indexer() 74 | indexer.index_all() 75 | actual_paths = list(indexer.find_record_files()) 76 | self.assertSetEqual(set(actual_paths), set(desired_paths)) 77 | 78 | def test_index_all_and_discard_json(self): 79 | self.prepare_records(**self.get_dummy_records()) 80 | indexer = self.get_indexer(keep_json=False) 81 | indexer.index_all() 82 | actual_paths = list(indexer.find_record_files()) 83 | self.assertEqual(actual_paths, []) 84 | -------------------------------------------------------------------------------- /rash/tests/test_query.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | from ..config import Configuration 17 | from ..query import expand_query 18 | from .utils import BaseTestCase 19 | 20 | 21 | class TestExpandQuery(BaseTestCase): 22 | 23 | def setUp(self): 24 | self.config = Configuration() 25 | self.config.search.alias['test'] = ["--include-pattern", "*test*"] 26 | self.config.search.alias['build'] = ["--include-pattern", "*build*"] 27 | 28 | def test_alias_no_query_no_expansion(self): 29 | kwds = expand_query(self.config, {}) 30 | self.assertEqual(kwds.get('include_pattern', []), []) 31 | self.assertEqual(kwds['pattern'], []) 32 | 33 | def test_alias_expansion(self): 34 | kwds = expand_query(self.config, {'pattern': ['test']}) 35 | self.assertEqual(kwds['include_pattern'], ['*test*']) 36 | self.assertEqual(kwds['pattern'], []) 37 | 38 | def test_alias_should_add_option(self): 39 | kwds = expand_query(self.config, {'include_pattern': ['*make*'], 40 | 'pattern': ['test']}) 41 | self.assertEqual(kwds['include_pattern'], ['*make*', '*test*']) 42 | self.assertEqual(kwds['pattern'], []) 43 | 44 | def test_alias_two_expansions(self): 45 | kwds = expand_query(self.config, {'pattern': ['test', 'build']}) 46 | self.assertEqual(kwds['include_pattern'], ['*test*', '*build*']) 47 | self.assertEqual(kwds['pattern'], []) 48 | -------------------------------------------------------------------------------- /rash/tests/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import unittest 18 | import functools 19 | from contextlib import contextmanager 20 | 21 | from ..utils.py3compat import zip_longest 22 | 23 | 24 | class BaseTestCase(unittest.TestCase): 25 | 26 | if not hasattr(unittest.TestCase, 'assertIn'): 27 | def assertIn(self, member, container): 28 | self.assertTrue(member in container) 29 | 30 | if not hasattr(unittest.TestCase, 'assertNotIn'): 31 | def assertNotIn(self, member, container): 32 | self.assertTrue(member not in container) 33 | 34 | if not hasattr(unittest.TestCase, 'assertIsInstance'): 35 | def assertIsInstance(self, obj, cls, msg=None): 36 | self.assertTrue(isinstance(obj, cls), msg) 37 | 38 | if not hasattr(unittest.TestCase, 'assertSetEqual'): 39 | def assertSetEqual(self, set1, set2, msg=None): 40 | self.assertEqual(set1, set2, msg) 41 | 42 | try: 43 | skipIf = unittest.skipIf 44 | except AttributeError: 45 | 46 | def skipIf(condition, reason): 47 | def decorator(func): 48 | @functools.wraps(func) 49 | def wrapper(*args, **kwds): 50 | if condition: 51 | print("Skipping {0} because:".format(func.__name__)) 52 | print(reason) 53 | else: 54 | return func(*args, **kwds) 55 | return wrapper 56 | return decorator 57 | 58 | 59 | @contextmanager 60 | def monkeypatch(obj, name, attr): 61 | """ 62 | Context manager to replace attribute named `name` in `obj` with `attr`. 63 | """ 64 | orig = getattr(obj, name) 65 | setattr(obj, name, attr) 66 | yield 67 | setattr(obj, name, orig) 68 | 69 | 70 | def zip_dict(dictionary, fillvalue=None): 71 | """ 72 | Zip values in `dictionary` and yield dictionaries with same keys. 73 | 74 | >>> list(zip_dict({'a': [1, 2, 3], 'b': [4, 5]})) 75 | [{'a': 1, 'b': 4}, {'a': 2, 'b': 5}, {'a': 3, 'b': None}] 76 | 77 | """ 78 | (keys, lists) = zip(*dictionary.items()) 79 | for values in zip_longest(*lists, fillvalue=fillvalue): 80 | yield dict(zip(keys, values)) 81 | -------------------------------------------------------------------------------- /rash/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Submodules that are not specific to this project. 3 | """ 4 | 5 | # Copyright (C) 2013- Takafumi Arakaki 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /rash/utils/confutils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | import platform 19 | 20 | 21 | def get_config_directory(appname): 22 | """ 23 | Get OS-specific configuration directory. 24 | 25 | :type appname: str 26 | :arg appname: capitalized name of the application 27 | 28 | """ 29 | if platform.system().lower() == 'windows': 30 | path = os.path.join(os.getenv('APPDATA') or '~', appname, appname) 31 | elif platform.system().lower() == 'darwin': 32 | path = os.path.join('~', 'Library', 'Application Support', appname) 33 | else: 34 | path = os.path.join(os.getenv('XDG_CONFIG_HOME') or '~/.config', 35 | appname.lower()) 36 | return os.path.expanduser(path) 37 | -------------------------------------------------------------------------------- /rash/utils/iterutils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import itertools 18 | 19 | from .py3compat import zip 20 | 21 | 22 | def nonempty(iterative): 23 | """ 24 | True if `iterative` returns at least one element. 25 | 26 | >>> nonempty(iter([1])) 27 | True 28 | >>> nonempty(iter([])) 29 | False 30 | 31 | """ 32 | for _ in iterative: 33 | return True 34 | return False 35 | 36 | 37 | def repeat(item, num): 38 | return itertools.islice(itertools.repeat(item), num) 39 | 40 | 41 | def _backward_shifted_predicate(predicate, num, iterative, include_zero=True): 42 | queue = [] 43 | for elem in iterative: 44 | if predicate(elem): 45 | for q in queue: 46 | yield True 47 | yield include_zero 48 | queue = [] 49 | else: 50 | queue.append(elem) 51 | if len(queue) > num: 52 | queue.pop(0) 53 | yield False 54 | for _ in queue: 55 | yield False 56 | 57 | 58 | def _forward_shifted_predicate(predicate, num, iterative, include_zero=True): 59 | counter = 0 60 | for elem in iterative: 61 | if predicate(elem): 62 | counter = num 63 | yield include_zero 64 | elif counter > 0: 65 | yield True 66 | counter -= 1 67 | else: 68 | yield False 69 | 70 | 71 | def include_before(predicate, num, iterative): 72 | """ 73 | Return elements in `iterative` including `num`-before elements. 74 | 75 | >>> list(include_before(lambda x: x == 'd', 2, 'abcded')) 76 | ['b', 'c', 'd', 'e', 'd'] 77 | 78 | """ 79 | (it0, it1) = itertools.tee(iterative) 80 | ps = _backward_shifted_predicate(predicate, num, it1) 81 | return (e for (e, p) in zip(it0, ps) if p) 82 | 83 | 84 | def include_after(predicate, num, iterative): 85 | """ 86 | Return elements in `iterative` including `num`-after elements. 87 | 88 | >>> list(include_after(lambda x: x == 'b', 2, 'abcbcde')) 89 | ['b', 'c', 'b', 'c', 'd'] 90 | 91 | """ 92 | (it0, it1) = itertools.tee(iterative) 93 | ps = _forward_shifted_predicate(predicate, num, it1) 94 | return (e for (e, p) in zip(it0, ps) if p) 95 | 96 | 97 | def include_context(predicate, num, iterative): 98 | """ 99 | Return elements in `iterative` including `num` before and after elements. 100 | 101 | >>> ''.join(include_context(lambda x: x == '!', 2, 'bb!aa__bb!aa')) 102 | 'bb!aabb!aa' 103 | 104 | """ 105 | (it0, it1, it2) = itertools.tee(iterative, 3) 106 | psf = _forward_shifted_predicate(predicate, num, it1) 107 | psb = _backward_shifted_predicate(predicate, num, it2) 108 | return (e for (e, pf, pb) in zip(it0, psf, psb) if pf or pb) 109 | -------------------------------------------------------------------------------- /rash/utils/pathutils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | 19 | 20 | def mkdirp(path): 21 | """ 22 | Make directory at `path` if it does not exist. 23 | """ 24 | if not os.path.isdir(path): 25 | os.makedirs(path) 26 | -------------------------------------------------------------------------------- /rash/utils/py3compat.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | import sys 19 | PY3 = (sys.version_info[0] >= 3) 20 | 21 | try: 22 | getcwd = os.getcwdu 23 | except AttributeError: 24 | getcwd = os.getcwd 25 | 26 | 27 | try: 28 | from contextlib import nested 29 | except ImportError: 30 | from contextlib import contextmanager 31 | 32 | @contextmanager 33 | def nested(*managers): 34 | if managers: 35 | with managers[0] as ctx: 36 | with nested(*managers[1:]) as rest: 37 | yield (ctx,) + rest 38 | else: 39 | yield () 40 | 41 | try: 42 | from itertools import izip_longest as zip_longest 43 | except ImportError: 44 | from itertools import zip_longest 45 | 46 | try: 47 | from itertools import izip as zip 48 | except ImportError: 49 | zip = zip 50 | -------------------------------------------------------------------------------- /rash/utils/sqlconstructor.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | import itertools 17 | 18 | from .iterutils import repeat 19 | 20 | 21 | def concat_expr(operator, conditions): 22 | """ 23 | Concatenate `conditions` with `operator` and wrap it by (). 24 | 25 | It returns a string in a list or empty list, if `conditions` is empty. 26 | 27 | """ 28 | expr = " {0} ".format(operator).join(conditions) 29 | return ["({0})".format(expr)] if expr else [] 30 | 31 | 32 | def adapt_matcher(matcher): 33 | if isinstance(matcher, str): 34 | return matcher.format 35 | else: 36 | return matcher 37 | 38 | 39 | def negate(matcher): 40 | return lambda *args: 'NOT ' + matcher(*args) 41 | 42 | 43 | class SQLConstructor(object): 44 | 45 | """ 46 | SQL constructor. 47 | 48 | >>> sc = SQLConstructor('table', ['c1', 'c2']) 49 | >>> (sql, params, keys) = sc.compile() 50 | >>> sql 51 | 'SELECT c1, c2 FROM table' 52 | 53 | >>> sc.add_or_matches('{0} = {1}', 'c1', [111, 222]) 54 | >>> (sql, params, keys) = sc.compile() 55 | >>> sql 56 | 'SELECT c1, c2 FROM table WHERE (c1 = ? OR c1 = ?)' 57 | >>> params 58 | [111, 222] 59 | 60 | """ 61 | 62 | def __init__(self, join_source, columns, keys=None, 63 | group_by=None, having=None, 64 | limit=None, 65 | table_alias=None): 66 | self.__init_vars() 67 | 68 | self.join_source = join_source 69 | self.columns = columns[:] 70 | self.keys = columns[:] if keys is None else keys[:] 71 | self.having = having or [] 72 | self.group_by = group_by or [] 73 | self.limit = limit 74 | self.table_alias = table_alias 75 | 76 | self.column_params = [] 77 | self.join_params = [] 78 | self.params = [] 79 | self.conditions = [] 80 | 81 | def __init_vars(self): 82 | self._ordering = [] 83 | 84 | def join(self, source, op='LEFT JOIN', on=''): 85 | """ 86 | Join `source`. 87 | 88 | >>> sc = SQLConstructor('main', ['c1', 'c2']) 89 | >>> sc.join('sub', 'JOIN', 'main.id = sub.id') 90 | >>> (sql, params, keys) = sc.compile() 91 | >>> sql 92 | 'SELECT c1, c2 FROM main JOIN sub ON main.id = sub.id' 93 | 94 | It is possible to pass another `SQLConstructor` as a source. 95 | 96 | >>> sc = SQLConstructor('main', ['c1', 'c2']) 97 | >>> sc.add_or_matches('{0} = {1}', 'c1', [111]) 98 | >>> subsc = SQLConstructor('sub', ['d1', 'd2']) 99 | >>> subsc.add_or_matches('{0} = {1}', 'd1', ['abc']) 100 | >>> sc.join(subsc, 'JOIN', 'main.id = sub.id') 101 | >>> sc.add_column('d1') 102 | >>> (sql, params, keys) = sc.compile() 103 | >>> print(sql) # doctest: +NORMALIZE_WHITESPACE 104 | SELECT c1, c2, d1 FROM main 105 | JOIN ( SELECT d1, d2 FROM sub WHERE (d1 = ?) ) 106 | ON main.id = sub.id 107 | WHERE (c1 = ?) 108 | 109 | `params` is set appropriately to include parameters for joined 110 | source: 111 | 112 | >>> params 113 | ['abc', 111] 114 | 115 | Note that `subsc.compile` is called when `sc.join(subsc, ...)` 116 | is called. Therefore, calling `subsc.add_` does not 117 | effect `sc`. 118 | 119 | :type source: str or SQLConstructor 120 | :arg source: table 121 | :type op: str 122 | :arg op: operation (e.g., 'JOIN') 123 | :type on: str 124 | :arg on: on clause. `source` ("right" source) can be 125 | referred using `{r}` formatting field. 126 | 127 | """ 128 | if isinstance(source, SQLConstructor): 129 | (sql, params, _) = source.compile() 130 | self.join_params.extend(params) 131 | jsrc = '( {0} )'.format(sql) 132 | if source.table_alias: 133 | jsrc += ' AS ' + source.table_alias 134 | on = on.format(r=source.table_alias) 135 | else: 136 | jsrc = source 137 | on = on.format(r=source) 138 | constraint = 'ON {0}'.format(on) if on else '' 139 | self.join_source = ' '.join([self.join_source, op, jsrc, constraint]) 140 | 141 | @property 142 | def sql_where(self): 143 | if self.conditions: 144 | return 'WHERE {0}'.format(" AND ".join(self.conditions)) 145 | 146 | @property 147 | def sql_group_by(self): 148 | if self.group_by: 149 | return 'GROUP BY {0}'.format(', '.join(self.group_by)) 150 | 151 | @property 152 | def sql_having(self): 153 | if self.having: 154 | return 'HAVING {0}'.format(' AND '.join(self.having)) 155 | 156 | @property 157 | def sql_order_by(self): 158 | if self._ordering: 159 | terms = map(' '.join, self._ordering) 160 | return 'ORDER BY {0}'.format(', '.join(terms)) 161 | 162 | sql_limit = '' 163 | 164 | @property 165 | def sql(self): 166 | return ' '.join(filter(None, [ 167 | 'SELECT', ', '.join(self.columns), 'FROM', self.join_source, 168 | self.sql_where, 169 | self.sql_group_by, 170 | self.sql_having, 171 | self.sql_order_by, 172 | self.sql_limit, 173 | ])) 174 | 175 | def compile(self): 176 | """ 177 | Compile SQL and return 3-tuple ``(sql, params, keys)``. 178 | 179 | Example usage:: 180 | 181 | (sql, params, keys) = sc.compile() 182 | for row in cursor.execute(sql, params): 183 | record = dict(zip(keys, row)) 184 | 185 | """ 186 | params = self.column_params + self.join_params + self.params 187 | if self.limit and self.limit >= 0: 188 | self.sql_limit = 'LIMIT ?' 189 | params += [self.limit] 190 | return (self.sql, params, self.keys) 191 | 192 | @staticmethod 193 | def _adapt_params(params): 194 | if isinstance(params, (tuple, list)): 195 | return params 196 | elif params is None: 197 | return [] 198 | else: 199 | return [params] 200 | 201 | @staticmethod 202 | def _default_flatten(numq): 203 | if numq == 1: 204 | return lambda x: x 205 | else: 206 | return lambda x: itertools.chain(*x) 207 | 208 | def add_and_matches(self, matcher, lhs, params, numq=1, flatten=None): 209 | """ 210 | Add AND conditions to match to `params`. 211 | 212 | :type matcher: str or callable 213 | :arg matcher: if `str`, `matcher.format` is used. 214 | :type lhs: str 215 | :arg lhs: the first argument to `matcher`. 216 | :type params: list 217 | :arg params: each element should be able to feed into sqlite '?'. 218 | :type numq: int 219 | :arg numq: number of parameters for each condition. 220 | :type flatten: None or callable 221 | :arg flatten: when `numq > 1`, it should return a list of 222 | length `numq * len(params)`. 223 | 224 | """ 225 | params = self._adapt_params(params) 226 | qs = ['?'] * numq 227 | flatten = flatten or self._default_flatten(numq) 228 | expr = repeat(adapt_matcher(matcher)(lhs, *qs), len(params)) 229 | self.conditions.extend(expr) 230 | self.params.extend(flatten(params)) 231 | 232 | def add_or_matches(self, matcher, lhs, params, numq=1, flatten=None): 233 | """ 234 | Add OR conditions to match to `params`. See `add_and_matches`. 235 | """ 236 | params = self._adapt_params(params) 237 | qs = ['?'] * numq 238 | flatten = flatten or self._default_flatten(numq) 239 | expr = repeat(adapt_matcher(matcher)(lhs, *qs), len(params)) 240 | self.conditions.extend(concat_expr('OR', expr)) 241 | self.params.extend(flatten(params)) 242 | 243 | def add_matches(self, matcher, lhs, 244 | match_params=[], include_params=[], exclude_params=[], 245 | numq=1, flatten=None): 246 | """ 247 | Quick way to call `add_or_matches` and `add_and_matches`. 248 | """ 249 | matcher = adapt_matcher(matcher) 250 | notmatcher = negate(matcher) 251 | self.add_and_matches(matcher, lhs, match_params, numq, flatten) 252 | self.add_or_matches(matcher, lhs, include_params, numq, flatten) 253 | self.add_and_matches(notmatcher, lhs, exclude_params, numq, flatten) 254 | 255 | def uniquify_by(self, column, chooser=None, aggregate='MAX'): 256 | """ 257 | Group by `column` and run `aggregate` function on `chooser` column. 258 | """ 259 | self.group_by.append(column) 260 | if chooser: 261 | i = self.columns.index(chooser) 262 | self.columns[i] = '{0}({1})'.format(aggregate, self.columns[i]) 263 | 264 | def add_group_by(self, condition): 265 | self.group_by.append(condition) 266 | 267 | def add_having(self, condition): 268 | self.having.append(condition) 269 | 270 | def add_column(self, column, key=None, params=[]): 271 | self.columns.append(column) 272 | self.keys.append(key or column) 273 | self.column_params.extend(params) 274 | 275 | def move_where_clause_to_column(self, column='condition', key=None): 276 | """ 277 | Move whole WHERE clause to a column named `column`. 278 | """ 279 | if self.conditions: 280 | expr = " AND ".join(self.conditions) 281 | params = self.params 282 | self.params = [] 283 | self.conditions = [] 284 | else: 285 | expr = '1' 286 | params = [] 287 | self.add_column('({0}) AS {1}'.format(expr, column), 288 | key or column, 289 | params) 290 | 291 | def order_by(self, expr, order='ASC'): 292 | if expr is None: 293 | return 294 | self._ordering.append((expr, order)) 295 | -------------------------------------------------------------------------------- /rash/utils/termdetection.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import os 18 | 19 | 20 | def subdict_by_key_prefix(dct, prefix): 21 | items = [] 22 | for (k, v) in dct.items(): 23 | if k.startswith(prefix): 24 | items.append((k, v)) 25 | return dict(items) 26 | 27 | 28 | def detect_terminal(_environ=os.environ): 29 | """ 30 | Detect "terminal" you are using. 31 | 32 | First, this function checks if you are in tmux, byobu, or screen. 33 | If not it uses $COLORTERM [#]_ if defined and fallbacks to $TERM. 34 | 35 | .. [#] So, if you are in Gnome Terminal you have "gnome-terminal" 36 | instead of "xterm-color"". 37 | 38 | """ 39 | if _environ.get('TMUX'): 40 | return 'tmux' 41 | elif subdict_by_key_prefix(_environ, 'BYOBU'): 42 | return 'byobu' 43 | elif _environ.get('TERM').startswith('screen'): 44 | return _environ['TERM'] 45 | elif _environ.get('COLORTERM'): 46 | return _environ['COLORTERM'] 47 | else: 48 | return _environ.get('TERM') 49 | 50 | 51 | if __name__ == '__main__': 52 | print(detect_terminal()) 53 | -------------------------------------------------------------------------------- /rash/utils/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tkf/rash/585da418ec37dd138f1a4277718b6f507e9536a2/rash/utils/tests/__init__.py -------------------------------------------------------------------------------- /rash/utils/tests/test_termdetection.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import unittest 18 | 19 | from ..termdetection import detect_terminal 20 | 21 | 22 | class TestTerminalDetection(unittest.TestCase): 23 | 24 | def assert_terminal(self, terminal, environ): 25 | self.assertEqual(detect_terminal(_environ=environ), terminal) 26 | 27 | def test_tmux(self): 28 | self.assert_terminal( 29 | 'tmux', 30 | {'TMUX': 'some value', 'TERM': 'screen'}) 31 | 32 | def test_byobu(self): 33 | self.assert_terminal( 34 | 'byobu', 35 | {'BYOBU_WINDOWS': 'some/path', 'TERM': 'screen'}) 36 | 37 | def test_screen(self): 38 | self.assert_terminal( 39 | 'screen', 40 | {'TERM': 'screen', 'COLORTERM': 'gnome-terminal'}) 41 | 42 | def test_gnome_terminal(self): 43 | self.assert_terminal( 44 | 'gnome-terminal', 45 | {'COLORTERM': 'gnome-terminal', 'TERM': 'xterm-color'}) 46 | 47 | def test_fallback(self): 48 | self.assert_terminal('xterm', {'TERM': 'xterm'}) 49 | -------------------------------------------------------------------------------- /rash/utils/timeutils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import time 18 | import datetime 19 | 20 | try: 21 | from parsedatetime import parsedatetime 22 | HAS_PARSEDATETIME = True 23 | except: 24 | HAS_PARSEDATETIME = False 25 | 26 | 27 | def parse_datetime(string): 28 | if not HAS_PARSEDATETIME: 29 | return 30 | cal = parsedatetime.Calendar() 31 | dates = cal.parse(string) 32 | if dates: 33 | return datetime.datetime.utcfromtimestamp(time.mktime(dates[0])) 34 | 35 | 36 | def parse_duration(string): 37 | """ 38 | Parse human readable duration. 39 | 40 | >>> parse_duration('1m') 41 | 60 42 | >>> parse_duration('7 days') == 7 * 24 * 60 * 60 43 | True 44 | 45 | """ 46 | if string.isdigit(): 47 | return int(string) 48 | try: 49 | return float(string) 50 | except ValueError: 51 | pass 52 | string = string.rstrip() 53 | for (suf, mult) in DURATION_SUFFIX_MAP.items(): 54 | if string.lower().endswith(suf): 55 | try: 56 | return parse_duration(string[:-len(suf)].strip()) * mult 57 | except TypeError: 58 | return 59 | 60 | 61 | DURATION_SUFFIX_MAP = { 62 | 'minute': 60, 63 | 'hour': 60 * 60, 64 | 'day': 60 * 60 * 24, 65 | } 66 | 67 | 68 | def _add_duration(): 69 | dsm = DURATION_SUFFIX_MAP 70 | additional = {} 71 | for (suf, mult) in dsm.items(): 72 | additional[suf[0]] = mult 73 | additional[suf + 's'] = mult 74 | dsm['min'] = dsm['minute'] 75 | dsm.update(additional) 76 | 77 | _add_duration() 78 | -------------------------------------------------------------------------------- /rash/watchrecord.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013- Takafumi Arakaki 2 | 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | import time 18 | import signal 19 | 20 | try: 21 | from watchdog.events import ( 22 | FileSystemEventHandler, FileCreatedEvent) 23 | assert FileSystemEventHandler # fool pyflakes 24 | except ImportError: 25 | # Dummy class for making this module importable: 26 | FileSystemEventHandler = object 27 | 28 | 29 | class RecordHandler(FileSystemEventHandler): 30 | 31 | def __init__(self, indexer, **kwds): 32 | self.__indexer = indexer 33 | super(RecordHandler, self).__init__(**kwds) 34 | 35 | def on_created(self, event): 36 | if isinstance(event, FileCreatedEvent): 37 | self.__indexer.index_record(event.src_path) 38 | 39 | 40 | def raise_keyboardinterrupt(_signum, _frame): 41 | raise KeyboardInterrupt 42 | 43 | 44 | def install_sigterm_handler(): 45 | signal.signal(signal.SIGTERM, raise_keyboardinterrupt) 46 | 47 | 48 | def watch_record(indexer, use_polling=False): 49 | """ 50 | Start watching `cfstore.record_path`. 51 | 52 | :type indexer: rash.indexer.Indexer 53 | 54 | """ 55 | if use_polling: 56 | from watchdog.observers.polling import PollingObserver as Observer 57 | Observer # fool pyflakes 58 | else: 59 | from watchdog.observers import Observer 60 | 61 | event_handler = RecordHandler(indexer) 62 | observer = Observer() 63 | observer.schedule(event_handler, path=indexer.record_path, recursive=True) 64 | indexer.logger.debug('Start observer.') 65 | observer.start() 66 | try: 67 | while True: 68 | time.sleep(1) 69 | except KeyboardInterrupt: 70 | indexer.logger.debug('Got KeyboardInterrupt. Stopping observer.') 71 | observer.stop() 72 | indexer.logger.debug('Joining observer.') 73 | observer.join() 74 | indexer.logger.debug('Finish watching record.') 75 | -------------------------------------------------------------------------------- /rash_cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from rash.cli import main 3 | if __name__ == '__main__': 4 | main() 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from distutils.core import setup 5 | 6 | import rash 7 | 8 | PY3 = (sys.version_info[0] >= 3) 9 | 10 | install_requires = [ 11 | 'argparse', 12 | ] 13 | if not PY3: 14 | install_requires.extend([ 15 | 'parsedatetime', 16 | 'watchdog', 17 | ]) 18 | 19 | 20 | setup( 21 | name='rash', 22 | version=rash.__version__, 23 | packages=['rash', 'rash.utils', 24 | 'rash.tests', 'rash.utils.tests', 'rash.functional_tests'], 25 | package_data={ 26 | # See also ./MANIFEST.in 27 | 'rash': [ 28 | 'schema.sql', 29 | os.path.join('ext', '*sh'), 30 | ], 31 | }, 32 | author=rash.__author__, 33 | author_email='aka.tkf@gmail.com', 34 | url='https://github.com/tkf/rash', 35 | license=rash.__license__, 36 | description='Rash Advances Shell History', 37 | long_description=rash.__doc__, 38 | keywords='history, shell, search', 39 | classifiers=[ 40 | "Development Status :: 3 - Alpha", 41 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 42 | "Programming Language :: Python", 43 | "Programming Language :: Python :: 2", 44 | "Programming Language :: Python :: 2.6", 45 | "Programming Language :: Python :: 2.7", 46 | "Programming Language :: Python :: 3", 47 | "Programming Language :: Python :: 3.2", 48 | "Programming Language :: Unix Shell", 49 | "Topic :: System :: Shells", 50 | "Topic :: System :: Logging" 51 | # see: http://pypi.python.org/pypi?%3Aaction=list_classifiers 52 | ], 53 | install_requires=install_requires, 54 | entry_points={ 55 | 'console_scripts': ['rash = rash.cli:main'], 56 | }, 57 | ) 58 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26, py27, py32, py33, py34, doctest 3 | 4 | [testenv] 5 | deps = 6 | nose 7 | commands = nosetests --with-doctest rash [] 8 | changedir = {envtmpdir} 9 | 10 | [testenv:doctest] 11 | # Some doctests are not run probably because it is not accessible via 12 | # __doc__. To run these doctest, I use Sphinx's doctest extension. 13 | # Note that currently this test is run against one Python interpreter. 14 | # I can choose interpreter by using basepython (for exmaple, 15 | # `basepython = python3.2`), but let's keep tox.ini simple for now. 16 | deps = 17 | sphinx 18 | sphinxcontrib-programoutput 19 | commands = 20 | sphinx-build -b doctest -d doc/build/doctrees doc/source doc/build/doctest 21 | changedir = {toxinidir} 22 | --------------------------------------------------------------------------------