├── staging ├── setup │ ├── hooks │ │ ├── commit-msg │ │ ├── pre-commit │ │ ├── post-commit │ │ ├── prepare-commit-msg │ │ ├── pre-commit.d │ │ │ ├── git-lint │ │ │ ├── oc-dev-lint │ │ │ ├── fail │ │ │ ├── trailing-whitespace │ │ │ ├── python-no-tabs │ │ │ ├── python-docstrings │ │ │ ├── no-invisible-spaces │ │ │ ├── ascii-filename │ │ │ └── nose-tests │ │ ├── post-commit.d │ │ │ └── congrats │ │ ├── prepare-commit-msg.d │ │ │ └── jira-reminder │ │ ├── commit-msg.d │ │ │ └── no-insult │ │ └── run │ ├── python │ │ ├── system.txt │ │ └── run │ ├── pkg │ │ └── run │ ├── missing.py │ ├── yum │ │ └── run │ ├── packages.py │ ├── homebrew │ │ └── run │ ├── setup │ ├── apt │ │ └── run │ ├── README.md │ ├── base │ │ └── run │ ├── packages.yaml │ ├── lib_setup.sh │ └── bootstrap.sh ├── commands │ ├── dev │ │ ├── testdata │ │ │ └── lint │ │ │ │ ├── package │ │ │ │ ├── __init__.py │ │ │ │ ├── a │ │ │ │ │ └── __init__.py │ │ │ │ └── b │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── b1 │ │ │ │ │ └── __init__.py │ │ │ │ ├── two-modules │ │ │ │ ├── a │ │ │ │ │ └── __init__.py │ │ │ │ └── b │ │ │ │ │ └── __init__.py │ │ │ │ └── two-modules-and-one-nested │ │ │ │ ├── a │ │ │ │ └── __init__.py │ │ │ │ ├── b │ │ │ │ └── __init__.py │ │ │ │ └── d │ │ │ │ └── d2 │ │ │ │ └── __init__.py │ │ ├── __init__.py │ │ ├── repos │ │ │ ├── grep.py │ │ │ ├── pull.py │ │ │ ├── status.py │ │ │ ├── push.py │ │ │ ├── __init__.py │ │ │ ├── checkout.py │ │ │ └── gc.py │ │ ├── tests.py │ │ ├── test_lint.py │ │ ├── pycompile.py │ │ ├── newlines.py │ │ ├── jsondiff.py │ │ ├── hosts.py │ │ ├── hookup.py │ │ └── lint.py │ ├── __init__.py │ ├── local │ │ ├── __init__.py │ │ └── wtf │ │ │ ├── __init__.py │ │ │ └── fix_osx_routes.py │ ├── misc │ │ ├── __init__.py │ │ ├── dtruss.py │ │ ├── crc32.py │ │ ├── sets.py │ │ ├── syntax.py │ │ └── date.py │ ├── config │ │ ├── __init__.py │ │ ├── edit.py │ │ ├── show.py │ │ ├── rm.py │ │ └── add.py │ ├── session │ │ ├── __init__.py │ │ ├── replay.py │ │ └── record.py │ ├── noop.py │ ├── setup.py │ ├── interactive.py │ ├── bash.py │ ├── edit.py │ ├── diag.py │ └── refresh.py ├── conf │ ├── ssh_config │ ├── yourtool.yaml │ └── pylintrc ├── README.md └── shell │ ├── csh_profile.sh │ ├── bash_profile.sh │ └── lib.sh ├── OSSMETADATA ├── src └── python │ ├── MANIFEST.in │ ├── cligraphy │ ├── __init__.py │ └── core │ │ ├── tracking.py │ │ ├── capture │ │ ├── session.capnp │ │ ├── termsize.py │ │ ├── __init__.py │ │ ├── fmt_capnp.py │ │ └── ptysnoop.py │ │ ├── lib │ │ ├── cli_ui.py │ │ ├── times.py │ │ ├── __init__.py │ │ └── ssh.py │ │ ├── util.py │ │ ├── decorators.py │ │ ├── log.py │ │ ├── reporting.py │ │ ├── __init__.py │ │ ├── parsers.py │ │ └── cli.py │ ├── setup.py │ └── requirements.txt ├── .hooks ├── README.md ├── .editorconfig ├── AUTHORS ├── .gitignore ├── .travis.yml ├── TODO └── LICENSE.txt /staging/setup/hooks/commit-msg: -------------------------------------------------------------------------------- 1 | run -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit: -------------------------------------------------------------------------------- 1 | run -------------------------------------------------------------------------------- /staging/setup/hooks/post-commit: -------------------------------------------------------------------------------- 1 | run -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=privatecollab 2 | -------------------------------------------------------------------------------- /staging/setup/hooks/prepare-commit-msg: -------------------------------------------------------------------------------- 1 | run -------------------------------------------------------------------------------- /src/python/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | -------------------------------------------------------------------------------- /staging/setup/python/system.txt: -------------------------------------------------------------------------------- 1 | virtualenv>=13.1.0 2 | -------------------------------------------------------------------------------- /staging/commands/dev/testdata/lint/package/__init__.py: -------------------------------------------------------------------------------- 1 | """Dummy module for testing""" 2 | -------------------------------------------------------------------------------- /staging/commands/dev/testdata/lint/package/a/__init__.py: -------------------------------------------------------------------------------- 1 | """Dummy module for testing""" 2 | -------------------------------------------------------------------------------- /staging/commands/dev/testdata/lint/package/b/__init__.py: -------------------------------------------------------------------------------- 1 | """Dummy module for testing""" 2 | -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit.d/git-lint: -------------------------------------------------------------------------------- 1 | # 2 | # Runs git-lint 3 | # 4 | 5 | git lint 6 | -------------------------------------------------------------------------------- /staging/commands/dev/testdata/lint/package/b/b1/__init__.py: -------------------------------------------------------------------------------- 1 | """Dummy module for testing""" 2 | -------------------------------------------------------------------------------- /staging/commands/dev/testdata/lint/two-modules/a/__init__.py: -------------------------------------------------------------------------------- 1 | """Dummy module for testing""" 2 | -------------------------------------------------------------------------------- /staging/commands/dev/testdata/lint/two-modules/b/__init__.py: -------------------------------------------------------------------------------- 1 | """Dummy module for testing""" 2 | -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit.d/oc-dev-lint: -------------------------------------------------------------------------------- 1 | # 2 | # Runs oc dev lint 3 | # 4 | oc dev lint 5 | -------------------------------------------------------------------------------- /staging/conf/ssh_config: -------------------------------------------------------------------------------- 1 | Host * 2 | SendEnv LANG LC_* 3 | EscapeChar none 4 | Protocol 2 5 | -------------------------------------------------------------------------------- /staging/commands/dev/testdata/lint/two-modules-and-one-nested/a/__init__.py: -------------------------------------------------------------------------------- 1 | """Dummy module for testing""" 2 | -------------------------------------------------------------------------------- /staging/commands/dev/testdata/lint/two-modules-and-one-nested/b/__init__.py: -------------------------------------------------------------------------------- 1 | """Dummy module for testing""" 2 | -------------------------------------------------------------------------------- /staging/commands/dev/testdata/lint/two-modules-and-one-nested/d/d2/__init__.py: -------------------------------------------------------------------------------- 1 | """Dummy module for testing""" 2 | -------------------------------------------------------------------------------- /staging/commands/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """ 5 | Cligraphy tools 6 | """ 7 | -------------------------------------------------------------------------------- /src/python/cligraphy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """ 5 | Cligraphy tools 6 | """ 7 | -------------------------------------------------------------------------------- /staging/README.md: -------------------------------------------------------------------------------- 1 | This folder contains bits and pieces that are part of the initial open-source drop but aren't ready for general use. 2 | -------------------------------------------------------------------------------- /staging/commands/local/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """ 5 | Cligraphy tools 6 | """ 7 | -------------------------------------------------------------------------------- /.hooks: -------------------------------------------------------------------------------- 1 | pre-commit.d/ascii-filename 2 | pre-commit.d/trailing-whitespace 3 | pre-commit.d/python-no-tabs 4 | pre-commit.d/python-docstrings 5 | -------------------------------------------------------------------------------- /staging/commands/local/wtf/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """ 5 | Cligraphy tools 6 | """ 7 | -------------------------------------------------------------------------------- /staging/commands/misc/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """ 5 | Misc commands 6 | 7 | Misc commands 8 | """ 9 | -------------------------------------------------------------------------------- /staging/commands/dev/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """ 5 | Dev commands 6 | 7 | Development related comands 8 | """ 9 | -------------------------------------------------------------------------------- /staging/setup/hooks/post-commit.d/congrats: -------------------------------------------------------------------------------- 1 | # 2 | # Print a nice message after a commit is made (example hook fragment) 3 | # 4 | 5 | echo "YAY FOR COMMITS!" 6 | -------------------------------------------------------------------------------- /staging/commands/config/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """Configuration 5 | 6 | Configuration related commands 7 | """ 8 | -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit.d/fail: -------------------------------------------------------------------------------- 1 | # 2 | # Fails a commit (example fragment) 3 | # 4 | 5 | oc_err "The fail hook fragment is listed in .hooks, failing commit" 6 | exit 1 7 | -------------------------------------------------------------------------------- /staging/commands/session/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """ 5 | Session commands 6 | 7 | Record and replay terminal sessions 8 | """ 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | cligraphy - A command line toolkit 2 | ================================== 3 | 4 | # Overview 5 | 6 | cligraphy is a python framework to help develop, share and run command line tools. 7 | 8 | 9 | -------------------------------------------------------------------------------- /staging/setup/hooks/prepare-commit-msg.d/jira-reminder: -------------------------------------------------------------------------------- 1 | # 2 | # Adds a comment to commit messages prior to editing (example fragment) 3 | # 4 | 5 | echo "# Reminder: include JIRA ticket numbers in your commit, if any!" >> $1 6 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 4 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file with this pattern: 2 | # 3 | # For individuals: 4 | # Name 5 | # 6 | # For organizations: 7 | # Organization 8 | 9 | Netflix, Inc <*@netflix.com> 10 | -------------------------------------------------------------------------------- /staging/commands/dev/repos/grep.py: -------------------------------------------------------------------------------- 1 | from nflx_oc.commands.dev.repos import run_for_all_repos 2 | 3 | 4 | def configure(parser): 5 | parser.add_argument('pattern') 6 | 7 | 8 | def main(args): 9 | run_for_all_repos("git grep '%s'" % args.pattern) 10 | -------------------------------------------------------------------------------- /staging/commands/dev/repos/pull.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix 3 | 4 | 5 | """Pull all repos from stash 6 | """ 7 | 8 | from nflx_oc.commands.dev.repos import run_for_all_repos 9 | 10 | 11 | def main(): 12 | run_for_all_repos('git pull') 13 | -------------------------------------------------------------------------------- /staging/commands/dev/repos/status.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix 3 | 4 | 5 | """Show status for all repos 6 | """ 7 | 8 | from nflx_oc.commands.dev.repos import run_for_all_repos 9 | 10 | 11 | def main(): 12 | run_for_all_repos('git status') 13 | -------------------------------------------------------------------------------- /staging/commands/dev/repos/push.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix 3 | 4 | 5 | """Push all repos to stash 6 | """ 7 | 8 | from nflx_oc.commands.dev.repos import run_for_all_repos 9 | 10 | 11 | def main(): 12 | run_for_all_repos('git push origin master') 13 | -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit.d/trailing-whitespace: -------------------------------------------------------------------------------- 1 | # 2 | # Rejects commits that add whitespace errors 3 | # 4 | 5 | set +e 6 | git diff-index --check --cached ${AGAINST} -- 7 | status=$? 8 | set -e 9 | 10 | if test $status -ne 0; then 11 | oc_err "Commit would introduce whitespace errors" 12 | fi 13 | -------------------------------------------------------------------------------- /staging/setup/hooks/commit-msg.d/no-insult: -------------------------------------------------------------------------------- 1 | # 2 | # Checks for bad language in a commit message (example hook fragment) 3 | # 4 | 5 | set +e 6 | cat $1 | grep -i fuck 1>/dev/null 2>&1 7 | status=$? 8 | set -e 9 | 10 | if test $status -eq 0; then 11 | oc_err "Commit message contains coarse language" 12 | fi 13 | -------------------------------------------------------------------------------- /staging/commands/config/edit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """Edit local oc configuration 5 | """ 6 | 7 | import os 8 | from cligraphy.core import edit_configuration 9 | 10 | def main(args): 11 | def do(filename): 12 | os.system('$EDITOR %s' % filename) 13 | edit_configuration('oc', do) 14 | -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit.d/python-no-tabs: -------------------------------------------------------------------------------- 1 | # 2 | # Rejects commits that add tab characters to python files 3 | # 4 | 5 | set +e 6 | git diff-index -u --cached ${AGAINST} -- '*.py' | grep -C 20 -e "^+.*$(printf '\t')" 7 | status=$? 8 | set -e 9 | 10 | if test $status -eq 0; then 11 | oc_err "Tabs are now allowed in python code" 12 | fi 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.dmg 2 | *.egg 3 | *.egg-info 4 | *.mo 5 | *.py[cod] 6 | *.save 7 | *.so 8 | *~ 9 | .coverage 10 | .DS_Store 11 | .gradle 12 | .idea 13 | .installed.cfg 14 | .mr.developer.cfg 15 | .project 16 | .pydevproject 17 | .tox 18 | .vagrant 19 | \#*\# 20 | __pycache__ 21 | build 22 | develop-eggs 23 | dist 24 | eggs 25 | nosetests.xml 26 | parts 27 | pip-log.txt 28 | sdist 29 | var 30 | -------------------------------------------------------------------------------- /staging/setup/pkg/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ${CLIGRAPHY_REPO_PATH}/setup/lib_setup.sh 4 | oc_setup_init_app pkg 5 | 6 | if test $(uname) = 'FreeBSD'; then 7 | echo "Disabled" 8 | # oc_run sudo pkg install -y $(~/.cligraphy/python-envs/oc/bin/python ${CLIGRAPHY_REPO_PATH}/setup/packages.py ${CLIGRAPHY_SETUP_APP_PATH}/../packages.yaml) 9 | fi 10 | 11 | oc_success 12 | -------------------------------------------------------------------------------- /staging/commands/noop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """A noop command for testing""" 5 | 6 | import time 7 | import logging 8 | 9 | 10 | def configure(parser): 11 | parser.add_argument('sleep', nargs='?', type=int) 12 | 13 | 14 | def main(args): 15 | if args.sleep: 16 | logging.info('Sleeping %d seconds', args.sleep) 17 | time.sleep(args.sleep) 18 | -------------------------------------------------------------------------------- /staging/commands/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix, Inc. 3 | 4 | """Run setup 5 | 6 | Runs all setup steps (or just a subset) 7 | """ 8 | 9 | import os 10 | 11 | 12 | def configure(parser): 13 | parser.add_argument('fragment', nargs='*', help='fragment to setup') 14 | 15 | 16 | def main(args): 17 | os.system('${CLIGRAPHY_REPO_PATH}/setup/setup %s' % ' '.join((args.fragment))) 18 | -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit.d/python-docstrings: -------------------------------------------------------------------------------- 1 | # 2 | # Rejects commits that add triple-simple-quotes to python files 3 | # 4 | 5 | set +e 6 | git diff-index -u --cached ${AGAINST} -- '*.py' | grep -C 20 -e "^\+.*'''" 7 | status=$? 8 | set -e 9 | 10 | if test $status -eq 0; then 11 | oc_err "For triple-quoted strings, always use double quote characters to be consistent with the docstring convention in PEP 257." 12 | fi 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | matrix: 3 | include: 4 | - python: '2.7' 5 | cache: 6 | directories: 7 | - .pip_download_cache 8 | env: 9 | global: 10 | - PIP_DOWNLOAD_CACHE=".pip_download_cache" 11 | before_script: 12 | - cd src/python 13 | - pip install -r requirements.txt 14 | - python setup.py develop 15 | script: 16 | - nosetests 17 | notifications: 18 | email: 19 | - stefan@kentik.com 20 | - achu@netflix.com 21 | -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit.d/no-invisible-spaces: -------------------------------------------------------------------------------- 1 | # 2 | # Rejects commits that add invisible characters to source code 3 | # 4 | 5 | set +e 6 | git diff-index -u --cached ${AGAINST} -- '*.py' '*.java' '*.groovy' '*.sql' | grep --color -a -C 1 -E '\x{200B}' 7 | status=$? 8 | set -e 9 | 10 | if test $status -eq 0; then 11 | oc_err "Non-printable characters like non-breaking space are not allowed in source code files" 12 | fi 13 | -------------------------------------------------------------------------------- /staging/commands/interactive.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def ipython(): 5 | from IPython.terminal.ipapp import TerminalIPythonApp 6 | app = TerminalIPythonApp.instance() 7 | app.initialize(argv=[]) # argv=[] instructs IPython to ignore sys.argv 8 | app.start() 9 | 10 | 11 | def main(): 12 | try: 13 | ipython() 14 | except ImportError: 15 | logging.warning('ipython not available, using built-in console') 16 | import code 17 | code.interact(local=locals()) 18 | -------------------------------------------------------------------------------- /staging/setup/missing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | import sys 4 | 5 | 6 | def read(fname): 7 | with open(fname, 'r') as fp: 8 | return set([ x.strip() for x in fp.readlines()]) 9 | 10 | 11 | def main(): 12 | installed = read(sys.argv[1]) 13 | wanted = read(sys.argv[2]) 14 | todo = wanted - installed 15 | if not todo: 16 | return 17 | with open(sys.argv[3], 'w') as fp: 18 | fp.write("\n".join(todo)) 19 | 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /staging/commands/dev/repos/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """ 5 | Repos commands 6 | 7 | Source code repos related commands 8 | """ 9 | 10 | import os 11 | from cligraphy.core import ctx 12 | 13 | 14 | def run_for_all_repos(shell_command): 15 | """Run the given command in each repository""" 16 | for repo in ctx.conf.repos.list.keys(): 17 | print '='*12, repo 18 | os.system('cd %s/%s && %s' % (ctx.conf.repos.root, repo, shell_command)) 19 | print '' 20 | -------------------------------------------------------------------------------- /staging/commands/misc/dtruss.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """dtruss a process without running it as root""" 4 | 5 | import os 6 | import time 7 | 8 | 9 | def configure(args): 10 | args.add_argument('command', help='command line', nargs='+') 11 | 12 | 13 | def main(args): 14 | os.system('sudo -p "Sudo password: " echo') 15 | pid = os.fork() 16 | if pid == 0: 17 | time.sleep(0.5) 18 | os.execlp(args.command[0], *args.command) 19 | else: 20 | os.system('sudo dtruss -f -p %d' % (pid)) 21 | -------------------------------------------------------------------------------- /staging/setup/yum/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ${CLIGRAPHY_REPO_PATH}/setup/lib_setup.sh 4 | oc_setup_init_app yum 5 | 6 | if test $(uname) = 'Linux'; then 7 | 8 | if test $(lsb_release --id -s) = 'Fedora'; then 9 | 10 | # FIXME - wrong python interpreter if you workon another project and run oc setup 11 | oc_run sudo yum -y install $(~/.cligraphy/python-envs/oc/bin/python ${CLIGRAPHY_REPO_PATH}/setup/packages.py ${CLIGRAPHY_SETUP_APP_PATH}/../packages.yaml) 12 | 13 | fi 14 | 15 | fi 16 | 17 | oc_success 18 | -------------------------------------------------------------------------------- /src/python/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013-2018 Netflix, Inc. 3 | 4 | from setuptools import setup 5 | 6 | setup( 7 | name = 'cligraphy', 8 | version = '0.0.8', 9 | description = 'Cligraphy Command line tools', 10 | long_description = 'Cligraphy Command line tools', 11 | author = 'Netflix, Inc.', 12 | author_email = '', # OPEN SOURCE TODO 13 | include_package_data=True, 14 | license='Apache 2.0', 15 | zip_safe=False, 16 | setup_requires=[ 17 | 'setupmeta' 18 | ], 19 | 20 | ) 21 | -------------------------------------------------------------------------------- /staging/commands/dev/repos/checkout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix 3 | 4 | 5 | """Check out OC repositories 6 | """ 7 | 8 | 9 | from cligraphy.core import ctx 10 | import logging 11 | import os 12 | 13 | 14 | def main(): 15 | for repo, origin in ctx.conf.repos.list.items(): 16 | dest = '%s/%s' % (ctx.conf.repos.root, repo) 17 | if not os.path.exists(dest): 18 | os.system('git clone %s %s' % (origin, dest)) 19 | else: 20 | logging.debug("%s already exists, not cloning", dest) 21 | -------------------------------------------------------------------------------- /src/python/requirements.txt: -------------------------------------------------------------------------------- 1 | argcomplete>=1.0.0 2 | attrdict>=2.0.0 3 | certifi>=2015.11.20 4 | colorclass>=2.2.0 5 | colorlog>=2.6.0 6 | coverage>=4.0.3 7 | decorator==4.0.11 8 | enum34>=1.1.1 9 | faulthandler>=2.4 10 | filelock>=2.0.5 11 | future>=0.15.2 12 | futures>=3.0.3 13 | logilab-common>=1.1.0 14 | nose>=1.3.7 15 | paramiko>=1.16.0 16 | pathlib==1.0.1 17 | pytz>=2015.7 18 | PyYAML>=3.12 19 | redis>=2.10.5 20 | remember>=0.1 21 | requests>=2.11.1 22 | requests_cache>=0.4.10 23 | setproctitle>=1.1.9 24 | sh>=1.11 25 | six>=1.10.0 26 | subprocess32>=3.2.7 27 | termcolor>=1.1.0 28 | terminaltables>=3.0.0 29 | tzlocal>=1.2 30 | -------------------------------------------------------------------------------- /staging/commands/dev/repos/gc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix 3 | 4 | """Run git gc on all repos""" 5 | 6 | from nflx_oc.commands.dev.repos import run_for_all_repos 7 | 8 | 9 | def configure(parser): 10 | parser.add_argument('--aggressive', help='Pass --aggressive to git gc', action='store_true', default=False) 11 | parser.add_argument('--fsck', help='Run git fsck before gc', action='store_true', default=False) 12 | 13 | 14 | def main(args): 15 | if args.fsck: 16 | run_for_all_repos('git fsck') 17 | run_for_all_repos('git gc' + (' --aggressive' if args.aggressive else '')) 18 | -------------------------------------------------------------------------------- /staging/commands/dev/tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix 3 | 4 | 5 | """Test your code! 6 | nosetests wrapper 7 | """ 8 | 9 | import os 10 | 11 | 12 | def configure(args): 13 | args.add_argument('-c', '--coverage', help='Show test coverage', action='store_true') 14 | args.add_argument('-v', '--verbose', help='Verbose output', action='store_true') 15 | 16 | 17 | def main(args): 18 | command = ['nosetests'] 19 | 20 | if args.coverage: 21 | command.append('--with-coverage') 22 | 23 | if args.verbose: 24 | command.append('--verbose') 25 | 26 | os.system(' '.join(command)) 27 | -------------------------------------------------------------------------------- /staging/commands/misc/crc32.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """Compute crc32 5 | """ 6 | 7 | import zlib 8 | 9 | 10 | def configure(parser): 11 | parser.add_argument('filename_list', metavar='FILENAME', nargs='+', help='Filename, or - for stdin') 12 | 13 | 14 | def crc32(filename, filep): 15 | cksum = 0 16 | while True: 17 | data = filep.read(1024*1024) 18 | if not data: 19 | break 20 | cksum = zlib.crc32(data, cksum) 21 | print '%s %X' % (filename, cksum & 0xFFFFFFFF) 22 | 23 | 24 | def main(args): 25 | for filename in args.filename_list: 26 | with open('/dev/stdin' if filename == '-' else filename, 'rb') as filep: 27 | crc32(filename, filep) 28 | -------------------------------------------------------------------------------- /staging/setup/packages.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import yaml 4 | import sys 5 | import platform 6 | 7 | 8 | def select(packages, os): 9 | for pack, overrides in packages.iteritems(): 10 | selection = overrides.get(os, pack) if overrides else pack 11 | if not isinstance(selection, basestring): 12 | for item in selection: 13 | print item 14 | else: 15 | print selection 16 | 17 | 18 | def main(): 19 | os = platform.system().lower() 20 | 21 | if os == 'linux': 22 | os = platform.dist()[0].lower() 23 | 24 | packages = yaml.load(open(sys.argv[1])) 25 | select(packages['all'], os) 26 | select(packages.get(os, {}), os) 27 | 28 | 29 | if __name__ == '__main__': 30 | main() 31 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/tracking.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """Session and execution tracking""" 5 | 6 | 7 | from collections import namedtuple 8 | import uuid 9 | import os 10 | 11 | TrackingInformation = namedtuple('TrackingInformation', 'session_uuid,execution_uuid') 12 | 13 | 14 | def get_tracking(): 15 | """Initializes our tracking information""" 16 | session_uuid = os.getenv('CLIGRAPHY_SESSION_UUID', None) 17 | if session_uuid is None: 18 | # Create new session 19 | session_uuid = str(uuid.uuid4()) 20 | os.environ['CLIGRAPHY_SESSION_UUID'] = session_uuid 21 | 22 | return TrackingInformation(session_uuid=session_uuid, execution_uuid=str(uuid.uuid4())) 23 | 24 | TRACKING = get_tracking() 25 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | * license 2 | 3 | /* 4 | * Copyright 2016 Netflix, Inc. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | 21 | * move lint functions to lib 22 | -------------------------------------------------------------------------------- /staging/commands/session/replay.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | from cligraphy.core.capture import fmt_capnp 5 | 6 | import time 7 | import sys 8 | 9 | 10 | def configure(parser): 11 | parser.add_argument('-s', '--speedup', help='Speedup factor', default=1.0, type=float) 12 | parser.add_argument('filename') 13 | 14 | 15 | def main(args): 16 | player = fmt_capnp.CapnpSessionPlayer(args.filename) 17 | print player.session 18 | 19 | #for e in player.session_capnp.Event.read_multiple_packed(player.fpin): 20 | # print e 21 | 22 | for interval, data in player.play(): 23 | time.sleep(interval/args.speedup) 24 | if data is not None: 25 | sys.stdout.write(data) 26 | sys.stdout.flush() 27 | -------------------------------------------------------------------------------- /staging/commands/session/record.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | from cligraphy.core import capture, decorators 5 | from cligraphy.core.capture import fmt_capnp 6 | import logging 7 | import os 8 | 9 | 10 | def shell(): 11 | """ Run out command 12 | """ 13 | cmdline = ['/bin/bash', '/bin/bash', '-l'] 14 | logging.info("Running [%s] in pid %d", ' '.join(cmdline), os.getpid()) 15 | os.execl(*cmdline) 16 | 17 | 18 | @decorators.tag(decorators.Tag.interactive) 19 | def main(args): 20 | logging.basicConfig(level=logging.INFO) 21 | recorder = fmt_capnp.CapnpSessionRecorder() 22 | print 'Session start - recording in %s' % (recorder.filename) 23 | capture.spawn_and_record(recorder, shell, None) 24 | print 'Session done - recorded in %s' % (recorder.filename) 25 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/capture/session.capnp: -------------------------------------------------------------------------------- 1 | @0xcbbea7cb6eb29a56; 2 | 3 | struct Session { 4 | username @0 :Text; 5 | timestamp @1 :UInt64; 6 | windowSize @2 :WindowSize; 7 | environment @3 :List(EnvVar); 8 | } 9 | 10 | struct WindowSize { 11 | columns @0 :UInt16; 12 | lines @1 :UInt16; 13 | } 14 | 15 | struct EnvVar { 16 | name @0 :Text; 17 | value @1 :Text; 18 | } 19 | 20 | struct Event { 21 | timecode @0 :Float32; 22 | type @1 :Type; 23 | 24 | union { 25 | data @2 :Text; 26 | status @3 :UInt16; 27 | windowSize @4 :WindowSize; 28 | } 29 | 30 | enum Type { 31 | userInput @0; 32 | ptyInput @1; 33 | sessionEnd @2; 34 | windowResized @3; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /staging/shell/csh_profile.sh: -------------------------------------------------------------------------------- 1 | setenv CLIGRAPHY_PYTHON_ENV_ROOT ~/.cligraphy/python-envs 2 | setenv PATH ${PATH}:${CLIGRAPHY_REPO_PATH}/bin 3 | 4 | source ${CLIGRAPHY_PYTHON_ENV_ROOT}/oc/bin/activate.csh 5 | 6 | # 7 | # Handy oc shortcuts 8 | # 9 | 10 | # We explicitely alias oc to the full path in our virtualenv so that 'oc' commands work when we're working in another virtualenv, 11 | # and we alias to python -m as the default wrapper adds 0.1s 12 | alias oc '${CLIGRAPHY_PYTHON_ENV_ROOT}/oc/bin/python -m cligraphy.core.cli' 13 | alias repos 'oc dev repos' 14 | alias lint 'oc dev lint' 15 | alias tests 'oc dev tests' 16 | 17 | # 18 | # Handy aliases 19 | # 20 | 21 | alias json="python -c 'import json; import sys; print json.dumps(json.load(open(sys.argv[1]) if len(sys.argv) > 1 else sys.stdin), indent=4)'" 22 | alias rpurge='find . -name *~ -exec rm -i {} \;' 23 | -------------------------------------------------------------------------------- /staging/setup/homebrew/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ${CLIGRAPHY_REPO_PATH}/setup/lib_setup.sh 4 | oc_setup_init_app homebrew 5 | 6 | if test $(uname) = 'Darwin'; then 7 | 8 | oc_info "Installing xcode command line tools if necessary" 9 | xcode-select --install || oc_info "Xcode command lines tools already installed" 10 | 11 | if ! test -x /usr/local/bin/brew; then 12 | 13 | oc_info "Installing homebrew" 14 | ruby ${CLIGRAPHY_SETUP_APP_PATH}/install.rb 15 | 16 | fi 17 | 18 | cd /usr/local 19 | 20 | oc_run brew update 21 | 22 | # FIXME - wrong python interpreter if you workon another project and run oc setup 23 | oc_run_ignore_fail brew upgrade 24 | oc_run brew install $(~/.cligraphy/python-envs/oc/bin/python ${CLIGRAPHY_REPO_PATH}/setup/packages.py ${CLIGRAPHY_SETUP_APP_PATH}/../packages.yaml) 25 | 26 | fi 27 | 28 | oc_success 29 | -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit.d/ascii-filename: -------------------------------------------------------------------------------- 1 | # 2 | # Rejects filenames that are not fully ASCII 3 | # 4 | 5 | # Cross platform projects tend to avoid non-ASCII filenames; prevent 6 | # them from being added to the repository. We exploit the fact that the 7 | # printable range starts at the space character and ends with tilde. 8 | 9 | # Note that the use of brackets around a tr range is ok here, (it's 10 | # even required, for portability to Solaris 10's /usr/bin/tr), since 11 | # the square bracket bytes happen to fall in the designated range. 12 | 13 | if test $(git diff --cached --name-only --diff-filter=A -z ${AGAINST} | 14 | LC_ALL=C tr -d '[ -~]\0' | wc -c | tr -d '[[:space:]]' ) != 0 15 | then 16 | cat <<\EOF 17 | Error: Attempt to add a non-ASCII file name. 18 | This can cause problems if you want to work with people on other platforms. 19 | To be portable it is advisable to rename the file. 20 | EOF 21 | exit 1 22 | fi 23 | -------------------------------------------------------------------------------- /staging/setup/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | 4 | if test "$1" = "-h" -o "$1" = "--help" -o "$1" = "help"; then 5 | echo "setup [fragment...]: sets up environment for the oc tool suite" 6 | exit 0 7 | elif test -z "$1"; then 8 | PARTS=( 9 | base 10 | python 11 | homebrew 12 | apt 13 | yum 14 | pkg 15 | ) 16 | else 17 | declare -a PARTS=("$@") 18 | fi 19 | 20 | function failed { 21 | echo FAILED 22 | exit 1 23 | } 24 | 25 | if test -z "$CLIGRAPHY_REPO_PATH"; then 26 | echo "CLIGRAPHY_REPO_PATH environmental variable not set" 27 | failed 28 | fi 29 | 30 | source "${CLIGRAPHY_REPO_PATH}/setup/lib_setup.sh" 31 | oc_setup_init 32 | oc_no_root 33 | 34 | export CLIGRAPHY_LOG 35 | for ((i = 0; i < ${#PARTS[@]}; i++)); do 36 | PART=${PARTS[$i]} 37 | /bin/echo -n "Running ${PART} ... " 38 | oc_run ${CLIGRAPHY_REPO_PATH}/setup/${PARTS[$i]}/run && echo OK || failed ${PART} 39 | done 40 | -------------------------------------------------------------------------------- /staging/setup/apt/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ${CLIGRAPHY_REPO_PATH}/setup/lib_setup.sh 4 | oc_setup_init_app apt 5 | 6 | if test $(uname) = 'Linux'; then 7 | 8 | if test $(lsb_release --id -s) = 'Ubuntu'; then 9 | 10 | export DEBIAN_FRONTEND=noninteractive 11 | 12 | dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d: -f1 > installed.txt 13 | ~/.cligraphy/python-envs/oc/bin/python ${CLIGRAPHY_REPO_PATH}/setup/packages.py ${CLIGRAPHY_SETUP_APP_PATH}/../packages.yaml > wanted.txt 14 | 15 | oc_run /usr/bin/env python2.7 ${CLIGRAPHY_SETUP_APP_PATH}/../missing.py installed.txt wanted.txt todo.txt 16 | 17 | if test -f todo.txt; then 18 | 19 | oc_run sudo apt-get update 20 | # FIXME - wrong python interpreter if you workon another project and run oc setup 21 | oc_run sudo apt-get -q -y install $(cat todo.txt) 22 | fi 23 | 24 | fi 25 | 26 | fi 27 | 28 | oc_success 29 | -------------------------------------------------------------------------------- /staging/setup/README.md: -------------------------------------------------------------------------------- 1 | oc/setup - Set up and maintain your environment 2 | =============================================== 3 | 4 | This module takes care of bootstraping the oc toolset, and of maintaining up to date. 5 | 6 | Basic structure 7 | --------------- 8 | 9 | In base/, we create our base directory structure ($HOME/oc) and make sure your bash profile has the bits necessary for oc tools to work. 10 | 11 | Native packages: Homebrew / Apt / Yum 12 | -------------------------------------- 13 | 14 | On OSX, we use homebrew to install most native packages. 15 | We run homebrew from a private fork tn order to control and manage versioning. 16 | 17 | On other systems, non-python packages are installed by the package manager. 18 | 19 | The list of packages we install is in packages.yaml 20 | 21 | Python 22 | ------ 23 | 24 | Installs a recent pip in the system path if that's missing. 25 | 26 | Also creates a virtualenv for oc tools and install the tools (and their dependencies). 27 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/lib/cli_ui.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2015, 2016 Netflix, Inc. 3 | 4 | def prompt_int(prompt, value=None, default=None): 5 | """Prompt user for an int value""" 6 | while not isinstance(value, int): 7 | try: 8 | value = int(value) 9 | except TypeError: 10 | value = raw_input(prompt) 11 | if value == '': 12 | value = default 13 | return value 14 | 15 | 16 | def prompt_enter_choice(prompt, values, exceptions=None): 17 | prompt = '%s (%s)? ' % (prompt, '/'.join(values)) 18 | values = { value.upper(): value for value in values } 19 | while True: 20 | try: 21 | value = raw_input(prompt).upper() 22 | except BaseException as e: 23 | if exceptions and type(e) in exceptions: 24 | print 25 | return exceptions[type(e)] 26 | else: 27 | raise 28 | if value in values: 29 | return values[value] 30 | -------------------------------------------------------------------------------- /staging/commands/bash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 2015 Netflix, Inc. 4 | 5 | """Run bash inside the octools environment.""" 6 | 7 | import os 8 | import tempfile 9 | 10 | RC = r""" 11 | export PS1="\u@\h:\w> " 12 | 13 | echo "Activating octools with ${CLIGRAPHY_REPO_PATH}/shell/oc_bash_profile.sh ..." 14 | source ${CLIGRAPHY_REPO_PATH}/shell/oc_bash_profile.sh 15 | 16 | cd ${CLIGRAPHY_REPO_PATH} 17 | """ 18 | 19 | def bash(command=None): 20 | """Start bash a custom rc file""" 21 | with tempfile.NamedTemporaryFile() as tmpfp: 22 | tmpfp.write(RC) 23 | tmpfp.flush() 24 | final_command = '/usr/bin/env bash --noprofile --rcfile %s' % tmpfp.name 25 | if command: 26 | final_command += (' -c %s' % command) 27 | os.system(final_command) 28 | 29 | 30 | def configure(parser): 31 | parser.add_argument('-c', '--command', help='Command to be executed (with bash -c, instead of starting an interactive shell)') 32 | 33 | def main(args): 34 | bash(args.command) 35 | -------------------------------------------------------------------------------- /staging/commands/dev/test_lint.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # (C) Netflix 2014 3 | 4 | """Lint command tests 5 | """ 6 | 7 | from nflx_oc.commands.dev import lint 8 | 9 | import os 10 | 11 | import unittest 12 | 13 | 14 | class TestModuleDiscovery(unittest.TestCase): 15 | """""" 16 | 17 | def setUp(self): 18 | self.test_data_root = os.path.join(os.path.dirname(lint.__file__), 'testdata/lint') 19 | self.prevdir = os.getcwd() 20 | os.chdir(self.test_data_root) 21 | 22 | def tearDown(self): 23 | os.chdir(self.prevdir) 24 | 25 | def test_module_discovery(self): 26 | """Test our lint wrapper module discovery 27 | """ 28 | self.assertEqual(lint.find_python_modules('package'), ['package']) 29 | self.assertEqual(lint.find_python_modules('two-modules'), ['two-modules/a', 'two-modules/b']) 30 | self.assertEqual(lint.find_python_modules('two-modules-and-one-nested'), 31 | ['two-modules-and-one-nested/a', 'two-modules-and-one-nested/b', 'two-modules-and-one-nested/d/d2']) 32 | -------------------------------------------------------------------------------- /staging/conf/yourtool.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # cligraphy tool configuration file 3 | # ---------------------------------------------------------- 4 | # 5 | # This is the default, shared, global configuration for your end-user tool based on cligraphy. 6 | # Most of these are *required* (as in, the code does not provide defaults) 7 | # Users should put overrides in ~/.yourtool/yourtool.yaml 8 | # 9 | 10 | commands: 11 | yourtool_module.commands: # Our core commands live here 12 | 13 | repos: 14 | git_proto: ssh 15 | git_root: '%cfg.repos.git_proto%://git@repos.domain.net' 16 | list: 17 | 18 | report: 19 | enabled: false 20 | server: https://cligraphy-backend.domain.net 21 | max_output_size: 8192 22 | 23 | ssh: 24 | proxy: 25 | enabled: true 26 | multiplexing: true 27 | host: bastion.domain.net 28 | config: "%cfg.tool.repo_path%/conf/ssh_config" 29 | args: "-x" 30 | multi: 31 | command: csshX {hosts} 32 | 33 | ipmi: 34 | ipmitool: "/usr/local/bin/ipmitool" 35 | user: ipmiusername 36 | escapechar: '^' 37 | 38 | refresh: 39 | tips: true 40 | -------------------------------------------------------------------------------- /staging/commands/misc/sets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env/python 2 | 3 | """Print the intersection or union of two line-delimited data sets 4 | """ 5 | 6 | 7 | OPERATIONS = { 8 | 'union': lambda left, right: left.union(right), 9 | 'inter': lambda left, right: left.intersection(right), 10 | 'ldiff': lambda left, right: left - right, 11 | 'rdiff': lambda left, right: right - left, 12 | } 13 | 14 | def read_dataset(filename): 15 | result = set() 16 | with open(filename, 'r') as fpin: 17 | while True: 18 | line = fpin.readline() 19 | if not line: 20 | break 21 | result.add(line[:-1]) 22 | return result 23 | 24 | 25 | def configure(parser): 26 | parser.add_argument('-o', '--operation', help='Set operation', choices=OPERATIONS.keys(), default='inter') 27 | parser.add_argument('left', help='Left-side data set') 28 | parser.add_argument('right', help='Right-side data set') 29 | 30 | 31 | def main(args): 32 | left = read_dataset(args.left) 33 | right = read_dataset(args.right) 34 | print '\n'.join(OPERATIONS[args.operation](left, right)) 35 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/capture/termsize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """Unix terminal utils 5 | """ 6 | 7 | import fcntl 8 | import termios 9 | import struct 10 | import sys 11 | import os 12 | 13 | 14 | def _ioctl_get_window_size(fd): 15 | """Calls TIOCGWINSZ for the given fd 16 | """ 17 | try: 18 | return struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) 19 | except IOError: 20 | return 21 | 22 | 23 | def get_terminal_size(): 24 | """Get current terminal size (best effort) 25 | """ 26 | cr = _ioctl_get_window_size(0) or _ioctl_get_window_size(1) or _ioctl_get_window_size(2) 27 | if not cr: 28 | with os.open(os.ctermid(), os.O_RDONLY) as fd: 29 | cr = _ioctl_get_window_size(fd) 30 | return int(cr[0]), int(cr[1]) 31 | 32 | 33 | def set_terminal_size(lines, columns): 34 | """Set current terminal size 35 | """ 36 | winsize = struct.pack("HHHH", lines, columns, 0, 0) 37 | fcntl.ioctl(1, termios.TIOCSWINSZ, winsize) 38 | sys.stdout.write("\x1b[8;{lines};{columns}t".format(lines=lines, columns=columns)) 39 | -------------------------------------------------------------------------------- /staging/setup/base/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Installs base oc tool things like directories, bash profile 3 | 4 | source ${CLIGRAPHY_REPO_PATH}/setup/lib_setup.sh 5 | oc_setup_init_app base 6 | 7 | mkdir -p ~/.cligraphy 8 | mkdir -p ~/.cligraphy/backup 9 | mkdir -p ~/.cligraphy/python-envs 10 | mkdir -p ~/.cligraphy/run 11 | 12 | if ! test -a ~/.bash_profile; then 13 | echo "export CLIGRAPHY_REPO_PATH=${CLIGRAPHY_REPO_PATH}" > ~/.bash_profile 14 | echo 'source ${CLIGRAPHY_REPO_PATH}/shell/oc_bash_profile.sh' >> ~/.bash_profile 15 | else 16 | if test $(oc_capture_ignore_fail grep -c oc_bash_profile ~/.bash_profile) -eq 0; then 17 | 18 | # Build new bash profile 19 | set +e 20 | 21 | cat ~/.bash_profile | grep '#!' > bash_profile 22 | echo "export CLIGRAPHY_REPO_PATH=${CLIGRAPHY_REPO_PATH}" >> bash_profile 23 | echo 'source ${CLIGRAPHY_REPO_PATH}/shell/oc_bash_profile.sh' >> bash_profile 24 | cat ~/.bash_profile | grep -v '#!' >> bash_profile 25 | 26 | set -e 27 | 28 | cp -a ~/.bash_profile ~/.cligraphy/backup/bash_profile 29 | cp bash_profile ~/.bash_profile 30 | fi 31 | fi 32 | 33 | oc_success 34 | -------------------------------------------------------------------------------- /staging/commands/config/show.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """Show current configuration 5 | 6 | Show oc configuration. By default the effective (auto+shared+custom) configuration is shown. 7 | """ 8 | 9 | from cligraphy.core import ctx, dictify_recursive 10 | import yaml 11 | import logging 12 | 13 | 14 | def configure(args): 15 | args.add_argument('--layer', type=str, default='', help='Only show the specified configuration layer') 16 | args.add_argument('--json', action='store_true', help='Output in json format') 17 | 18 | 19 | def main(args): 20 | if args.layer: 21 | selected = ctx.cligraph.conf_layers[args.layer][1] 22 | else: 23 | selected = ctx.cligraph.conf 24 | 25 | if not selected: 26 | if args.layer: 27 | logging.error("No configuration defined at layer [%s]", args.layer) 28 | else: 29 | logging.error("No configuration defined") 30 | return 31 | 32 | selected = dictify_recursive(selected) 33 | 34 | if args.json: 35 | import json 36 | print json.dumps(selected, indent=4) 37 | else: 38 | print yaml.dump(selected, width=50, indent=4, default_flow_style=False) 39 | -------------------------------------------------------------------------------- /staging/conf/pylintrc: -------------------------------------------------------------------------------- 1 | # 2 | # example pylintrc 3 | # 4 | 5 | # add linter plugins to your tool, and load them here 6 | #[MASTER] 7 | #load-plugins=yourtool.lint 8 | 9 | [BASIC] 10 | variable-rgx=[a-z_][a-z0-9_]{1,30}$ 11 | function-rgx=[a-z_][a-z0-9_]{1,36}$ 12 | method-rgx=[a-z_][a-z0-9_]{1,36}$ 13 | argument-rgx=[a-z_][a-z0-9_]{1,30}$ 14 | no-docstring-rgx=__.*__|main|configure|Test.* 15 | 16 | [FORMAT] 17 | max-line-length=180 18 | indent-string=' ' 19 | 20 | [MESSAGES CONTROL] 21 | disable=locally-disabled,no-self-use,too-few-public-methods,too-many-public-methods,too-many-return-statements,too-many-arguments,abstract-class-not-used,abstract-class-little-used,star-args,similarities,bad-whitespace,superfluous-parens,import-error,fixme,docstring-period,docstring-args,docstring-oneliner 22 | 23 | [IMPORTS] 24 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 25 | 26 | [REPORTS] 27 | output-format=colorized 28 | reports=no 29 | msg-template={abspath}:{line}:{column} {obj}: {msg} ({symbol}) 30 | 31 | [DESIGN] 32 | max-attributes = 12 33 | max-locals = 18 34 | max-branchs = 15 35 | max-statements = 60 36 | max-parents = 10 37 | 38 | [MISCELLANEOUS] 39 | notes=TODO,FIXME,XXX 40 | 41 | [TYPECHECK] 42 | ignored-classes=EasyDict,AttrDict 43 | generated-members=next,json 44 | -------------------------------------------------------------------------------- /staging/commands/config/rm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """Remove a configuration key 5 | 6 | Remove a configuration key. By default, your custom configuration file will be modified. 7 | """ 8 | 9 | 10 | from cligraphy.core import ctx, find_node, write_configuration_file 11 | import collections 12 | 13 | 14 | def configure(args): 15 | args.add_argument('-f', '--force', action='store_true', help='force deletion (of eg. keys that have sub-keys)') 16 | args.add_argument('--layer', type=str, default='custom', help='Perform operation on the specified configuration layer') 17 | args.add_argument('name', type=str, help='Configuration key path') 18 | 19 | 20 | def main(args): 21 | root = ctx.cligraph.conf_layers[args.layer][1] 22 | parts = args.name.split('.') 23 | node = find_node(root, parts[:-1]) 24 | 25 | if node is None or parts[-1] not in node: 26 | print 'no such configuration key %s' % (args.name) 27 | return 1 28 | 29 | value = node.pop(parts[-1]) 30 | if isinstance(value, collections.Mapping) and not args.force: 31 | print 'key %s has sub keys, not removing. use -f to force.' % (args.name) 32 | return 2 33 | 34 | write_configuration_file(ctx.cligraph.conf_layers[args.layer][0], root) 35 | -------------------------------------------------------------------------------- /staging/setup/hooks/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Run appropriate git hooks based on a .hooks at the top of the repository 4 | # 5 | 6 | export CLIGRAPHY_LOG=/dev/stdout 7 | 8 | source ${CLIGRAPHY_REPO_PATH}/setup/lib_setup.sh 9 | oc_setup_init_app oc-git-hook-run 10 | 11 | HOOK_NAME=$(basename ${0}) 12 | REPO_PATH=${CLIGRAPHY_PREV_PWD} 13 | 14 | FRAG_CONF=${REPO_PATH}/.hooks 15 | if test -e ${REPO_PATH}/.hooks; then 16 | IFS=$'\n' GLOBIGNORE='*' : ; FRAGMENTS_ENABLED=($(<${FRAG_CONF})) 17 | else 18 | declare -a FRAGMENTS_ENABLED=(none) 19 | fi 20 | 21 | cd ${REPO_PATH} 22 | oc_debug "Running [${HOOK_NAME}] hook fragments in repo [${REPO_PATH}]: ${FRAGMENTS_ENABLED[*]}" 23 | 24 | if git rev-parse --verify HEAD >/dev/null 2>&1 25 | then 26 | AGAINST=HEAD 27 | else 28 | # Initial commit: diff against an empty tree object 29 | AGAINST=4b825dc642cb6eb9a060e54bf8d69288fbee4904 30 | fi 31 | 32 | for frag in ${FRAGMENTS_ENABLED[@]}; do 33 | case ${frag} in ${HOOK_NAME}*) 34 | frag_file=${CLIGRAPHY_REPO_PATH}/setup/hooks/${frag} 35 | if test -e ${frag_file}; then 36 | oc_info "Running ${frag}" 37 | source ${frag_file} 38 | else 39 | oc_err "Could not find hook fragment ${frag} defined in ${FRAG_CONF}" 40 | fi 41 | esac 42 | done 43 | 44 | oc_success 45 | -------------------------------------------------------------------------------- /staging/commands/local/wtf/fix_osx_routes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """ 5 | Fix bogus routes left by Juniper Network Connect on OSX 6 | """ 7 | 8 | import tempfile 9 | import os 10 | 11 | 12 | def exec_bash_code(code): 13 | with tempfile.NamedTemporaryFile() as tmpfp: 14 | tmpfp.write(code) 15 | tmpfp.flush() 16 | os.system('/bin/bash %s' % tmpfp.name) 17 | 18 | 19 | def main(): 20 | exec_bash_code("""#!/usr/bin/env bash 21 | IFS=' 22 | ' 23 | interfaces=$(networksetup -listnetworkserviceorder | grep -E '\(\d+|\*\)' | cut -d' ' -f2- | grep -iE 'ethernet|wi-fi|iPhone') 24 | 25 | enabled=() 26 | echo "Listing enabled interfaces" 27 | for i in ${interfaces}; do 28 | if test $(networksetup -getnetworkserviceenabled "${i}") = 'Enabled'; then 29 | echo " ${i}" 30 | enabled+=(${i}) 31 | else 32 | enabled+=(${i}) 33 | fi 34 | done 35 | 36 | echo 37 | echo "Will use sudo to disable and re-enable interfaces listed above." 38 | echo 39 | echo "Continue? (enter, or ctrl-c to abort)" 40 | read junk 41 | 42 | echo "Disabling..." 43 | for i in ${enabled[@]}; do 44 | sudo networksetup -setnetworkserviceenabled ${i} off 45 | done 46 | 47 | echo "Done disabling, sleeping 2s..." 48 | sleep 2 49 | 50 | echo "Re-enabling..." 51 | for i in ${enabled[@]}; do 52 | sudo networksetup -setnetworkserviceenabled ${i} on 53 | done 54 | """) 55 | -------------------------------------------------------------------------------- /staging/commands/config/add.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """Set a configuration key 5 | 6 | Set a configuration key. By default, your custom configuration file will be modified. 7 | """ 8 | 9 | 10 | from cligraphy.core import ctx, find_node, write_configuration_file 11 | 12 | TYPES = { 13 | 'int': int, 14 | 'float': float, 15 | 'str': str, 16 | 'bool': lambda value: value in ('true', 'True', '1', 'yes') 17 | } 18 | 19 | 20 | def configure(args): 21 | args.add_argument('--layer', type=str, default='custom', help='Perform operation on the specified configuration layer') 22 | args.add_argument('-t', '--type', type=str, default='str', help='value type', choices=TYPES.keys()) 23 | args.add_argument('name', type=str, help='Configuration key path') 24 | args.add_argument('value', type=str, help='Value') 25 | 26 | 27 | def main(args): 28 | root = ctx.cligraph.conf_layers[args.layer][1] or {} 29 | parts = args.name.split('.') 30 | node = find_node(root, parts[:-1], add=True) 31 | 32 | if node is None: 33 | print 'could not find key %s' % (args.name) 34 | return 1 35 | 36 | if node.get(parts[-1], None) == args.value: 37 | print '%s is already set to value %s' % (args.name, args.value) 38 | return 0 39 | 40 | node[parts[-1]] = TYPES[args.type](args.value) 41 | write_configuration_file(ctx.cligraph.conf_layers[args.layer][0], root) 42 | -------------------------------------------------------------------------------- /staging/commands/misc/syntax.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """Check syntax 5 | """ 6 | 7 | import json 8 | import yaml 9 | 10 | 11 | def autodetect(filename, filep): 12 | """Try to detect language based on filename extension""" 13 | ext = filename.rpartition('.')[-1] 14 | checker = LANGUAGES.get(ext, None) 15 | if checker is None: 16 | raise Exception('Unknown file extension [%s]' % ext) 17 | checker(filename, filep) 18 | 19 | 20 | def json_check(_, filep): 21 | """Parses json to find syntax errors""" 22 | try: 23 | json.load(filep) 24 | except ValueError as exc: 25 | print 'JSON syntax error: %s' % exc 26 | 27 | 28 | def yaml_check(_, filep): 29 | """Parses yaml to find syntax errors""" 30 | try: 31 | yaml.load(filep) 32 | except yaml.scanner.ScannerError as exc: 33 | print 'YAML syntax error: %s -%s' % (exc.problem, exc.problem_mark) 34 | 35 | 36 | LANGUAGES = { 37 | 'auto': autodetect, 38 | 'json': json_check, 39 | 'yaml': yaml_check 40 | } 41 | 42 | 43 | def configure(parser): 44 | parser.add_argument('-l', dest='checker', default='auto', choices=LANGUAGES.keys()) 45 | parser.add_argument('filenames', metavar='FILENAME', nargs='+') 46 | 47 | 48 | def main(args): 49 | for filename in args.filenames: 50 | with open(filename, 'rb') as filep: 51 | LANGUAGES[args.checker](filename, filep) 52 | -------------------------------------------------------------------------------- /staging/setup/packages.yaml: -------------------------------------------------------------------------------- 1 | # Package list for oc tools 2 | # If a given definition has no sub-nodes, that name is used as the package name everywhere (shortcut for packages that have the same name 3 | # on ubuntu and brew.) Subnodes indicate overrides for certain systems 4 | 5 | all: 6 | coreutils: 7 | git: 8 | glib: 9 | ubuntu: libglib2.0-dev 10 | ipcalc: 11 | iperf: 12 | ipmitool: 13 | jq: 14 | libffi: 15 | ubuntu: libffi-dev 16 | fedora: libffi-devel 17 | mercurial: 18 | most: 19 | mtr: 20 | mysql: 21 | ubuntu: libmysqlclient-dev # really we only need the client lib 22 | fedora: mysql-libs # best effort googling, not tested 23 | node: 24 | nmap: 25 | pcre: 26 | ubuntu: 27 | - libpcre3 28 | - pcregrep 29 | pkg-config: 30 | fedora: pkgconfig 31 | postgresql: 32 | ubuntu: 33 | - libpq-dev 34 | - postgresql-client 35 | fedora: postgresql-libs # best effort googling, not tested 36 | freebsd: postgresql-libpqxx 37 | redis: 38 | ubuntu: redis-server 39 | rpl: 40 | tmux: 41 | watch: 42 | ubuntu: procps 43 | fedora: procps-ng 44 | freebsd: cmdwatch 45 | wget: 46 | xz: 47 | ubuntu: xz-utils 48 | freebsd: # installed by default 49 | conserver: 50 | fedora: conserver-client 51 | ubuntu: conserver-client 52 | darwin: 53 | csshx: 54 | ubuntu: 55 | smem: 56 | expect: 57 | libjpeg-dev: 58 | -------------------------------------------------------------------------------- /staging/commands/edit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """Edit an oc command 5 | 6 | Open the source code for an oc command in your $EDITOR 7 | """ 8 | 9 | 10 | import os 11 | import os.path 12 | import logging 13 | 14 | from cligraphy.core import ctx 15 | from cligraphy.core import decorators 16 | 17 | 18 | def configure(parser): 19 | parser.add_argument('-p', '--path', help='just show the path of the file that would be edited', action='store_true') 20 | parser.add_argument('terms', nargs='+', metavar='term', help='oc command (as a fuzzy sequence of terms, eg: oca log search)') 21 | 22 | 23 | @decorators.tag(decorators.Tag.interactive) 24 | def main(args): 25 | import cligraphy.core.cli 26 | import cligraphy.core.parsers 27 | from cligraphy.core.parsers import SmartCommandMapParser 28 | from cligraphy.core.util import undecorate_func 29 | 30 | parser = SmartCommandMapParser() 31 | for namespace, command_map in ctx.cligraph.get_command_maps(): 32 | parser.add_command_map(namespace, command_map) 33 | 34 | _, func = parser.pre_parse_args(args.terms) 35 | orig_func, _ = undecorate_func(func) 36 | filename = orig_func.__code__.co_filename 37 | 38 | print 'Command is defined in %s inside folder %s' % (filename, os.path.dirname(filename)) 39 | if not args.path: 40 | logging.info('Editing %s', filename) 41 | editor = ctx.conf.get('editor', os.getenv('EDITOR', None)) 42 | assert editor is not None, 'You have no configured editor - define $EDITOR or run eg. "oc conf add editor vim"' 43 | os.system('%s %s' % (editor, filename)) 44 | -------------------------------------------------------------------------------- /staging/setup/hooks/pre-commit.d/nose-tests: -------------------------------------------------------------------------------- 1 | git stash -q --keep-index 2 | 3 | set +e 4 | 5 | # cache results of git diff 6 | git_diff=$(git diff --cached --name-only) 7 | 8 | # run any tests available in the same directories as our changes 9 | modified_dirs=$(for i in $(echo "$git_diff" | grep -o '.*/' | sort | uniq); do if test -d $i; then echo $i; fi; done) 10 | if ! test -z "${modified_dirs}"; then 11 | echo -n "Executing any unit tests in " 12 | echo ${modified_dirs} 13 | nosetests -a '!slow' ${modified_dirs} 14 | status=$? 15 | else 16 | # no directories with changes, maybe there's python code in python/ ? 17 | if test -d python; then 18 | echo "Executing any unit tests in python/" 19 | nosetests -a '!slow' python/ 20 | status=$? 21 | 22 | else 23 | # fallback - just run nosetests and hope something happens 24 | echo "Executing any unit tests in current directory" 25 | nosetests -a '!slow' 26 | status=$? 27 | fi 28 | fi 29 | 30 | # only run additional tests if we didn't fail previously 31 | if test $status -eq 0; then 32 | # additionally run tests (if any) in the mirroring tests/ directory tree 33 | modified_dirs=$(for i in $(echo "$git_diff" | grep -o '/.*/' | sort | uniq); do if test -d "tests$i"; then echo "tests$i"; fi; done) 34 | if ! test -z "${modified_dirs}"; then 35 | echo -n "Executing any unit tests in " 36 | echo ${modified_dirs} 37 | nosetests -a '!slow' ${modified_dirs} 38 | status=$? 39 | fi 40 | fi 41 | 42 | set -e 43 | 44 | git stash pop -q 45 | 46 | if test $status -ne 0; then 47 | oc_err "Unit tests failed" 48 | fi 49 | -------------------------------------------------------------------------------- /staging/shell/bash_profile.sh: -------------------------------------------------------------------------------- 1 | export CLIGRAPHY_PYTHON_ENV_ROOT=~/.cligraphy/python-envs 2 | 3 | export WORKON_HOME=${CLIGRAPHY_PYTHON_ENV_ROOT} 4 | 5 | source ${CLIGRAPHY_PYTHON_ENV_ROOT}/oc/bin/activate 6 | 7 | # enable virtualenvwrapper if it's installed 8 | if test -f ${VIRTUAL_ENV}/bin/virtualenvwrapper.sh; then 9 | source ${VIRTUAL_ENV}/bin/virtualenvwrapper.sh 10 | fi 11 | 12 | # 13 | # Handy functions 14 | # 15 | 16 | function asn { 17 | dig +short $(echo $1 | awk -F. '{ print $4"."$3"."$2"."$1".origin.asn.cymru.com" }') TXT 18 | } 19 | 20 | function hr { 21 | for i in $(seq 1 $(tput cols)); do 22 | echo -n '=' 23 | done 24 | echo 25 | } 26 | 27 | # 28 | # Handy oc shortcuts 29 | # 30 | 31 | # We explicitely alias oc to the full path in our virtualenv so that 'oc' commands work when we're working in another virtualenv, 32 | # and we alias to python -m as the default wrapper adds 0.1s 33 | alias oc='${CLIGRAPHY_PYTHON_ENV_ROOT}/oc/bin/python -m cligraphy.core.cli' 34 | alias repos='oc dev repos' 35 | alias lint='oc dev lint' 36 | alias tests='oc dev tests' 37 | alias rebash='source ~/.bash_profile; hash -r' 38 | 39 | # 40 | # Handy aliases 41 | # 42 | 43 | alias json="python -c 'import json; import sys; print json.dumps(json.load(open(sys.argv[1]) if len(sys.argv) > 1 else sys.stdin), indent=4)'" 44 | alias rpurge='find . -name *~ -exec rm -i {} \;' 45 | 46 | # 47 | # oc command completion 48 | # 49 | 50 | source ${CLIGRAPHY_REPO_PATH}/shell/bash_complete.sh 51 | 52 | # 53 | # Shellshock reporting - probably want to remove this some time in 2015 54 | # 55 | 56 | env x='() { :;}; echo WARNING - Your shell is vulnerable to shellshock - http://go/shellshock' bash -c "echo -n" 57 | -------------------------------------------------------------------------------- /staging/commands/dev/pycompile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """Precompile python modules""" 5 | 6 | from nflx_oc.commands.dev.lint import find_git_root, find_python_modules, normalize_path 7 | 8 | import compileall 9 | import logging 10 | import subprocess 11 | import os.path 12 | 13 | 14 | def clean_and_compile(path): 15 | path = os.path.abspath(normalize_path(path)) 16 | 17 | logging.debug('Cleaning %s', path) 18 | command = "find %s -name '*.pyc' -delete -o -name '*.pyo' -delete" % path 19 | try: 20 | subprocess.check_output(command, shell=True) 21 | except subprocess.CalledProcessError, cpe: 22 | logging.exception('Exception while cleaning %s' % path) 23 | 24 | logging.debug('Compiling %s', path) 25 | command = "python -O -m compileall -q %s" % path 26 | try: 27 | subprocess.check_output(command, shell=True) 28 | except subprocess.CalledProcessError, cpe: 29 | logging.error('Exception while compiling %s. Output follows:\n%s', path, cpe.output) 30 | 31 | 32 | def configure(args): 33 | args.add_argument('-g', '--git-root', action='store_true', help='precompile from git repository root') 34 | args.add_argument('-n', '--dry-run', action='store_true', help='show paths that would be precompile') 35 | args.add_argument('paths', metavar='path', nargs='*', help='paths to precompile') 36 | 37 | 38 | def main(args): 39 | if args.git_root: 40 | path = find_git_root('.') 41 | if not path: 42 | print 'no git root found' 43 | return 1 44 | else: 45 | path = '.' 46 | 47 | paths = args.paths if args.paths else find_python_modules(path) 48 | for path in paths: 49 | if args.dry_run: 50 | continue 51 | 52 | clean_and_compile(path) 53 | -------------------------------------------------------------------------------- /staging/setup/lib_setup.sh: -------------------------------------------------------------------------------- 1 | # OC Setup library 2 | 3 | source ${CLIGRAPHY_REPO_PATH}/shell/lib.sh 4 | 5 | function oc_setup_init { 6 | if test -z "${CLIGRAPHY_LOG-}"; then 7 | # NB: these XXXXX are here for compat with ubuntu's mktemp 8 | CLIGRAPHY_LOG=$(mktemp -t ocsetup-log-${app}-XXXXXX) 9 | fi 10 | 11 | trap oc_setup_cleanup EXIT 12 | } 13 | 14 | function oc_setup_cleanup { 15 | code=${?} 16 | if test ${code} -ne 0; then 17 | echo "Exiting with status ${code}" 18 | echo "--- execution log ---" 19 | cat ${CLIGRAPHY_LOG} 20 | echo "--- end execution log ---" 21 | fi 22 | } 23 | 24 | function oc_setup_init_app { 25 | 26 | oc_setup_init 27 | oc_strict 28 | 29 | oc_args $# 1 30 | declare -r app=$1 31 | 32 | # general 33 | # NB: these XXXXX are here for compat with ubuntu's mktemp 34 | CLIGRAPHY_TMP=$(mktemp -d -t ocsetup-${app}-XXXXXX) 35 | CLIGRAPHY_PREV_PWD=${PWD} 36 | cd ${CLIGRAPHY_TMP} 37 | 38 | # setup-app specific 39 | CLIGRAPHY_SETUP_APP=${app} 40 | CLIGRAPHY_SETUP_APP_PATH=${CLIGRAPHY_REPO_PATH}/setup/${CLIGRAPHY_SETUP_APP} 41 | 42 | # trap 43 | CLIGRAPHY_CLEANUP_HOOKS[0]="true" 44 | trap 'oc_setup_cleanup_app ${LINENO}' EXIT 45 | 46 | oc_debug init done 47 | } 48 | 49 | function oc_add_cleanup_hook { 50 | index=${#CLIGRAPHY_CLEANUP_HOOKS[@]} 51 | eval CLIGRAPHY_CLEANUP_HOOKS[${index}]='"$@"' 52 | } 53 | 54 | function oc_setup_cleanup_app { 55 | 56 | code=${?} 57 | line=${1} 58 | 59 | oc_debug "exit trap called from line ${1}, code ${code}" 60 | 61 | for ((i = 0; i < ${#CLIGRAPHY_CLEANUP_HOOKS[@]}; i++)); do 62 | ${CLIGRAPHY_CLEANUP_HOOKS[$i]} 63 | done 64 | 65 | cd ${CLIGRAPHY_PREV_PWD} 66 | rm -rf ${CLIGRAPHY_TMP} 67 | 68 | oc_debug cleanup done 69 | 70 | if test ${CLIGRAPHY_SUCCESS-0} -ne 1; then 71 | exit 1 72 | fi 73 | } 74 | -------------------------------------------------------------------------------- /staging/commands/dev/newlines.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """Append missing newlines to the end of source code files 5 | """ 6 | 7 | 8 | import os 9 | import stat 10 | 11 | 12 | SOURCE_CODE_EXTENSIONS = set(('py',)) # 'css','js','html',... 13 | 14 | 15 | def walk(path): 16 | """Wraps os.walk""" 17 | result = [] 18 | for root, _, filenames in os.walk(path): 19 | for name in filenames: 20 | result.append(os.path.join(root, name)) 21 | return result 22 | 23 | 24 | def get_last_byte(name): 25 | """Return the last byte in a file""" 26 | with open(name, 'r') as infp: 27 | infp.seek(-1, 2) 28 | return infp.read(1) 29 | 30 | 31 | def configure(args): 32 | args.add_argument('-n', '--dry-run', action='store_true', help='dry run') 33 | args.add_argument('name_list', metavar='NAME', nargs='+', help='file or directory name') 34 | 35 | 36 | def main(args): 37 | files = [] 38 | for name in args.name_list: 39 | name = os.path.abspath(name) 40 | fstat = os.stat(name) 41 | if stat.S_ISDIR(fstat.st_mode): 42 | files.extend(walk(name)) 43 | else: 44 | files.append(name) 45 | 46 | source_code_files = [ name for name in files if name.rpartition('.')[-1] in SOURCE_CODE_EXTENSIONS ] 47 | missing_last_newline = [ name for name in source_code_files if get_last_byte(name) != '\n' ] 48 | 49 | if args.dry_run: 50 | print 'Missing newlines at the end of %d files:' % len(missing_last_newline) 51 | for name in missing_last_newline: 52 | print ' ', name 53 | else: 54 | for name in missing_last_newline: 55 | if os.access(name, os.W_OK): 56 | print 'Fixing', name 57 | with open(name, 'a') as fpout: 58 | fpout.write('\n') 59 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/lib/times.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """Human-friendly time period parsing 5 | """ 6 | 7 | import time 8 | import math 9 | from datetime import datetime 10 | from dateutil import parser 11 | 12 | UNITS = { 13 | 'month': 3600*24*31, # FIXME 1 month is not really 31 days 14 | 'week': 3600*24*7, 15 | 'day': 3600*24, 16 | 'hour': 3600, 17 | 'minute': 60, 18 | 'second': 1, 19 | } 20 | 21 | UNITS['mon'] = UNITS['month'] 22 | UNITS['w'] = UNITS['week'] 23 | UNITS['d'] = UNITS['day'] 24 | UNITS['h'] = UNITS['hour'] 25 | UNITS['min'] = UNITS['minute'] 26 | UNITS['m'] = UNITS['minute'] 27 | UNITS['sec'] = UNITS['second'] 28 | UNITS['s'] = UNITS['second'] 29 | 30 | def parse_time(human_time): 31 | """Parse a human-friendly time expression, such as 1d or now, and return a unix timestamp 32 | """ 33 | human_time = human_time.lower() 34 | now = int(time.time()) 35 | if human_time == 'now': 36 | return now 37 | try: 38 | unit = human_time[-1] 39 | value = int(human_time[:-1]) 40 | return now - (value * UNITS[unit]) 41 | except: 42 | raise Exception('Dont know what to do with time [%s]' % human_time) 43 | 44 | def parse_datetime(human_time): 45 | """Parse a human-friendly time expression (such as 1d or now) 46 | or datetime formatted as string (such as 2016-01-20 20:00) 47 | and return a datetime object 48 | """ 49 | try: 50 | return datetime.fromtimestamp(parse_time(human_time)) 51 | except: 52 | try: 53 | dt = parser.parse(human_time) 54 | return dt 55 | except: 56 | raise Exception('Dont know what to do with time [%s]' % human_time) 57 | 58 | def align_times(start, end, ratio=0.02): 59 | """Align timestamps on a reasonable boundary 60 | """ 61 | assert start < end, 'Start time must be before end time' 62 | boundary = int(math.ceil((end - start) * ratio)) 63 | return int(start / boundary) * boundary, int(end / (boundary + 1)) * (boundary + 1) 64 | -------------------------------------------------------------------------------- /staging/commands/dev/jsondiff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | import requests 5 | 6 | import sys 7 | import json 8 | 9 | 10 | def json_read(filename): 11 | if filename.startswith('http'): 12 | return requests.get(filename).json() 13 | else: 14 | with open(filename, 'r') as fp: 15 | return json.load(fp) 16 | 17 | 18 | def print_diff(print_func, left, right): 19 | 20 | if left is not None and right is not None: 21 | 22 | if type(left) != type(right): 23 | print_func('Differing types: %r(%s) -> %r(%s)' % (left, type(left).__name__, right, type(right).__name__)) 24 | return 25 | 26 | if type(left) == list: 27 | # right is also a list 28 | if not left: 29 | # right is also empty, no diffs 30 | return 31 | # non empty list, diff elements 32 | for index, (ileft, iright) in enumerate(zip(left, right)): 33 | print_diff(lambda x: print_func('Item %d: %s' % (index, x)), ileft, iright) 34 | 35 | elif type(left) == dict: 36 | leftkeys = set(left.keys()) 37 | rightkeys = set(right.keys()) 38 | 39 | for added in rightkeys - leftkeys: 40 | print_func('Added %s=%r' % (added, right[added])) 41 | 42 | for removed in leftkeys - rightkeys: 43 | print_func('Removed %s=%r' % (removed, left[removed])) 44 | 45 | for common in rightkeys.intersection(leftkeys): 46 | if left[common] != right[common]: 47 | print_diff(lambda x: print_func('Changed %s: %s' % (common, x)), left[common], right[common]) 48 | 49 | else: 50 | print_func('Changed %r -> %r' % (left, right)) 51 | 52 | 53 | def configure(args): 54 | args.add_argument('left', metavar='FILENAME', help='left-side file') 55 | args.add_argument('right', metavar='FILENAME', help='right-side file') 56 | 57 | 58 | def main(args): 59 | left = json_read(args.left) 60 | right = json_read(args.right) 61 | 62 | print_diff(lambda x: sys.stdout.write('%s\n' % x), left, right) 63 | -------------------------------------------------------------------------------- /staging/setup/python/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ${CLIGRAPHY_REPO_PATH}/setup/lib_setup.sh 4 | oc_setup_init_app python 5 | 6 | # 7 | # System python needs pip, virtualenv 8 | # ----------------------------------- 9 | # 10 | 11 | # Escape any virtual env 12 | unset VIRTUAL_ENV 13 | unset PYTHON_HOME 14 | 15 | declare -r oldpath=$PATH 16 | export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/git/bin: 17 | 18 | pip_location=$(which pip || exit 0) 19 | 20 | # Install pip if missing 21 | if ! test -x "${pip_location}"; then 22 | oc_info "Installing pip in python system path" 23 | oc_sudo /usr/bin/env python2.7 ${CLIGRAPHY_SETUP_APP_PATH}/get-pip 24 | pip_location=$(which pip) 25 | else 26 | if test $(oc_capture_ignore_fail ${pip_location} --version | grep -c 'python 2.7') -eq 0; then 27 | oc_err "${pip_location} doesn't seem to be using python 2.7, which is required" 28 | fi 29 | fi 30 | 31 | ${pip_location} freeze > system.installed.txt 32 | oc_run /usr/bin/env python2.7 ${CLIGRAPHY_SETUP_APP_PATH}/../missing.py system.installed.txt ${CLIGRAPHY_SETUP_APP_PATH}/system.txt system.todo.txt 33 | 34 | if test -s system.todo.txt; then 35 | oc_sudo ${pip_location} install --upgrade -r system.todo.txt 36 | fi 37 | 38 | PATH=${oldpath} 39 | 40 | 41 | # 42 | # Create and initialize our virtualenv 43 | # ------------------------------------------- 44 | # 45 | 46 | if ! test -d ~/.cligraphy/python-envs/oc; then 47 | oc_info "Creating virtualenv ~/.cligraphy/python-envs/oc" 48 | oc_run virtualenv ~/.cligraphy/python-envs/oc 49 | fi 50 | 51 | set +u 52 | source ~/.cligraphy/python-envs/oc/bin/activate 53 | 54 | 55 | # 56 | # Install oc tools 57 | # ------------------------------------------- 58 | # 59 | 60 | # Make a 1st pass with pip install -r so all dependencies, including things we might install via git (-e git+ssh://...) which are 61 | # not fully understood by python setup.py develop (eg. setup will complain if they're missing, but won't be able to install them) 62 | oc_run_ignore_fail cd ${CLIGRAPHY_SETUP_APP_PATH}"/../.." && pip install -r requirements.txt 63 | 64 | # Now run develop 65 | oc_run cd ${CLIGRAPHY_SETUP_APP_PATH}"/../.." && python setup.py develop 66 | 67 | # pytz is slow to load - unzip it (http://stackoverflow.com/questions/20500910/first-call-to-pytz-timezone-is-slow-in-virtualenv) 68 | oc_run_ignore_fail pip unzip pytz 69 | 70 | set +u 71 | deactivate 72 | 73 | oc_success 74 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/capture/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """Terminal session capture""" 5 | 6 | from cligraphy.core.capture import ptysnoop 7 | 8 | from abc import abstractmethod 9 | 10 | 11 | class Recorder(object): 12 | 13 | def start(self): 14 | pass 15 | 16 | def end(self, exitcode=0): 17 | """Finish a session record 18 | """ 19 | 20 | def record_window_resize(self, lines, columns): 21 | """Record a window resizing event 22 | """ 23 | 24 | @abstractmethod 25 | def record_user_input(self, data): 26 | """record user input separately from terminal output 27 | """ 28 | 29 | @abstractmethod 30 | def record_server_output(self, data): 31 | """record terminal output (user + server originated) 32 | """ 33 | 34 | 35 | class Player(object): 36 | """Plays a recorded session 37 | """ 38 | 39 | @abstractmethod 40 | def close(self): 41 | """Close this player 42 | """ 43 | 44 | @abstractmethod 45 | def play(self): 46 | """Generator of session playback events - basically (timecode, data) tuples 47 | """ 48 | 49 | 50 | class NoopOutputRecorder(Recorder): 51 | 52 | def output_as_string(self): 53 | return '(output discarded)' 54 | 55 | 56 | class BufferingOutputRecorder(Recorder): 57 | """Recorder that buffers output up to a maximum size""" 58 | 59 | def __init__(self, max_output_size): 60 | super(BufferingOutputRecorder, self).__init__() 61 | self._remaining_size = max_output_size 62 | self._total_size = 0 63 | self._buffer = [] 64 | 65 | @abstractmethod 66 | def record_server_output(self, data): 67 | """record terminal output (user + server originated) 68 | """ 69 | self._total_size += len(data) 70 | if self._remaining_size > 0: 71 | part = data[:self._remaining_size] 72 | self._buffer.append(part) 73 | self._remaining_size = max(0, self._remaining_size - len(part)) 74 | 75 | @property 76 | def total_size(self): 77 | return self._total_size 78 | 79 | def output_as_string(self): 80 | return ''.join(self._buffer) 81 | 82 | 83 | def spawn_and_record(recorder, func, parent_func, *args, **kwargs): 84 | script = ptysnoop.Script(recorder) 85 | return script.run(func, parent_func, *args, **kwargs) 86 | -------------------------------------------------------------------------------- /staging/commands/dev/hosts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix 3 | 4 | """Manage /etc/hosts static entries for development 5 | """ 6 | 7 | import tempfile 8 | import subprocess 9 | import logging 10 | 11 | MARKER = '# added by oc dev dns' 12 | 13 | 14 | def configure(args): 15 | args.add_argument('-n', '--dry-run', action='store_true', help='dry run') 16 | args.add_argument('action', choices=['add', 'clear'], help='sub-command to run') 17 | args.add_argument('name_list', metavar='NAME', nargs='*', help='entry name') 18 | 19 | 20 | def sudo_overwrite_file(filename, contents): 21 | """Overwrite file as root using sudo 22 | """ 23 | with tempfile.NamedTemporaryFile() as tmpfp: 24 | tmpfp.write(contents) 25 | tmpfp.flush() 26 | subprocess.check_call('cat %s | sudo -p "Enter sudo password to overwite %s: " tee %s' % (tmpfp.name, filename, filename), shell=True) 27 | 28 | 29 | def add_hosts_entries(args): 30 | """Add one or more static entries in /etc/hosts. Ignore existing names. 31 | """ 32 | contents = open('/etc/hosts', 'r').read() 33 | 34 | filtered_name_list = [] 35 | for name in args.name_list: 36 | if name in contents: 37 | logging.info('An entry for %s is already in /etc/hosts - skipping', name) 38 | else: 39 | filtered_name_list.append(name) 40 | 41 | if not filtered_name_list: 42 | logging.info('No names to add') 43 | return 44 | 45 | for name in filtered_name_list: 46 | contents = contents + '\n' + '127.0.0.1 %s %s' % (name, MARKER) 47 | 48 | contents += '\n' 49 | 50 | if args.dry_run: 51 | print contents 52 | else: 53 | sudo_overwrite_file('/etc/hosts', contents) 54 | 55 | 56 | def reset_hosts_entries(args): 57 | """Add one or more static entries in /etc/hosts. Ignore existing names. 58 | """ 59 | contents = open('/etc/hosts', 'r').readlines() 60 | filtered_lines = [ line.strip() for line in contents if MARKER not in line ] 61 | 62 | contents = '\n'.join(filtered_lines) 63 | 64 | if args.dry_run: 65 | print contents 66 | else: 67 | sudo_overwrite_file('/etc/hosts', contents) 68 | 69 | 70 | def main(args): 71 | if args.action == 'add': 72 | add_hosts_entries(args) 73 | elif args.action == 'clear': 74 | reset_hosts_entries(args) 75 | else: 76 | raise Exception('Unknown action %s' % args.action) 77 | -------------------------------------------------------------------------------- /staging/commands/dev/hookup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """Configure oc git hooks on a repository""" 5 | 6 | from nflx_oc.commands.dev.lint import find_git_root 7 | 8 | import os 9 | import logging 10 | 11 | def configure(args): 12 | args.add_argument('-n', '--dry-run', action='store_true', help="don't actually do anything") 13 | args.add_argument('paths', metavar='path', nargs='*', help='paths of repositories to configure') 14 | 15 | def get_oc_hooks_path(): 16 | return os.path.join(os.getenv('OC_REPO_PATH'), 'setup', 'hooks') 17 | 18 | def main(args): 19 | target_hooks_path = get_oc_hooks_path() 20 | if not os.path.isdir(target_hooks_path): 21 | logging.error("Could not find oc hooks under $OC_REPO_PATH (%s)", target_hooks_path) 22 | return 1 23 | 24 | paths = args.paths or (find_git_root('.'),) 25 | 26 | preflight = [hookup_repo(path, target_hooks_path, preflight=True) for path in paths] 27 | if not all(preflight): 28 | logging.error("Preflight failed") 29 | return 1 30 | 31 | if not args.dry_run: 32 | for path in paths: 33 | hookup_repo(path, target_hooks_path, preflight=False) 34 | 35 | 36 | def hookup_repo(repo_path, target_hooks_path, preflight=True): 37 | """Replaces a git repo's .git/hooks folder by a symlink to the given target_hooks_path""" 38 | 39 | git_path = os.path.join(repo_path, '.git') 40 | repo_hooks_path = os.path.join(git_path, 'hooks') 41 | 42 | if not os.path.isdir(git_path): 43 | logging.error('[%s] is not a suitable git repository (no .git directory found)', repo_path) 44 | return False 45 | 46 | if os.path.islink(repo_hooks_path): 47 | link_dest = os.path.realpath(repo_hooks_path) 48 | if link_dest == target_hooks_path: 49 | logging.info('[%s] is already hooked to [%s]', repo_path, link_dest) 50 | return True 51 | else: 52 | logging.error('[%s] is hooked to [%s] (not [%s])', repo_path, link_dest, target_hooks_path) 53 | return False 54 | 55 | if not os.path.isdir(repo_hooks_path): 56 | logging.error('[%s] is not a suitable git repository (no .git/hooks directory found)', repo_path) 57 | return False 58 | 59 | if preflight: 60 | logging.info('Preflight OK for [%s] (%s->%s)', repo_path, repo_hooks_path, target_hooks_path) 61 | else: 62 | logging.info('Hooking up [%s] (%s->%s)', repo_path, repo_hooks_path, target_hooks_path) 63 | os.rename(repo_hooks_path, repo_hooks_path + '_pre_hookup') 64 | os.symlink(target_hooks_path, repo_hooks_path) 65 | 66 | return True 67 | -------------------------------------------------------------------------------- /staging/shell/lib.sh: -------------------------------------------------------------------------------- 1 | # OC Setup - Common bash shell library 2 | 3 | function oc_log { 4 | #logger -t "$@" 5 | echo "$@" >>${CLIGRAPHY_LOG-/dev/null} 6 | } 7 | 8 | function oc_debug { 9 | if test ${CLIGRAPHY_DEBUG-0} -eq 1; then 10 | oc_log "${CLIGRAPHY_SETUP_APP-setup} [DEBUG] $@" 11 | fi 12 | } 13 | 14 | function oc_info { 15 | oc_log "${CLIGRAPHY_SETUP_APP-setup} [INFO ] $@" 16 | } 17 | 18 | function oc_warn { 19 | oc_log "${CLIGRAPHY_SETUP_APP-setup} [WARN ] $@" 20 | } 21 | 22 | function oc_err { 23 | oc_log "${CLIGRAPHY_SETUP_APP-setup} [ERR ] $@" 24 | oc_failure 25 | } 26 | 27 | function oc_no_root { 28 | if test $UID -eq 0; then 29 | oc_err "Do not run this script as root!" 30 | fi 31 | } 32 | 33 | function oc_success { 34 | CLIGRAPHY_SUCCESS=1 35 | exit 0 36 | } 37 | 38 | function oc_failure { 39 | CLIGRAPHY_SUCCESS=0 40 | exit 1 41 | } 42 | 43 | # Strict (and default) shell error checking: 44 | # - exit on command failures 45 | # - treat unbound variables as errors (use ${missing-default} !) 46 | function oc_strict { 47 | set -e 48 | set -u 49 | } 50 | 51 | # Function args checking 52 | function oc_args { 53 | if test $1 -ne $2; then 54 | oc_err "Wrong argument count, expected $1, got $2" 55 | exit 1 56 | fi 57 | } 58 | 59 | function oc_capture { 60 | oc_debug "oc_capture: [$@] --- output follows ---" 61 | out=$("$@" 2>&1) 62 | oc_debug "${out}" 63 | oc_debug "oc_capture: --- end of output ---" 64 | echo "$out" 65 | } 66 | 67 | 68 | function oc_capture_ignore_fail { 69 | oc_debug "oc_capture: [$@] --- output follows ---" 70 | set +e 71 | out=$("$@" 2>&1) 72 | set -e 73 | oc_debug "${out}" 74 | oc_debug "oc_capture: --- end of output ---" 75 | echo "$out" 76 | } 77 | 78 | 79 | # Run a command, capturing output in our log 80 | function oc_run { 81 | oc_debug "oc_run: [$@] --- output follows ---" 82 | 83 | set +e 84 | "$@" 1>>${CLIGRAPHY_LOG-/dev/null} 2>&1 85 | declare -r ret=${?} 86 | set -e 87 | 88 | oc_debug "oc_run: --- end of output ---" 89 | 90 | if test ${ret} -ne 0; then 91 | oc_warn "Command failed with exit code ${ret}" 92 | exit ${ret} 93 | fi 94 | } 95 | 96 | function oc_run_ignore_fail { 97 | oc_debug "oc_run: [$@] --- output follows ---" 98 | 99 | set +e 100 | "$@" 1>>${CLIGRAPHY_LOG-/dev/null} 2>&1 101 | declare -r ret=${?} 102 | set -e 103 | 104 | oc_debug "oc_run: --- end of output ---" 105 | } 106 | 107 | # Run a command as root, capturing output in our log 108 | function oc_sudo { 109 | c="$@" 110 | oc_run sudo -p "sudo[$c'] password: " "$@" 111 | } 112 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix, Inc. 3 | 4 | """Utility classes 5 | """ 6 | 7 | from contextlib import contextmanager 8 | import logging 9 | import signal 10 | import sys 11 | 12 | 13 | class TimeoutError(Exception): 14 | """Timeout Error""" 15 | pass 16 | 17 | 18 | @contextmanager 19 | def timeout(seconds, error_message='Timeout'): 20 | """Timeout context manager using SIGALARM.""" 21 | def _handle_timeout(signum, frame): # pylint:disable=unused-argument,missing-docstring 22 | raise TimeoutError(error_message) 23 | if seconds > 0: 24 | signal.signal(signal.SIGALRM, _handle_timeout) 25 | signal.alarm(seconds) 26 | try: 27 | yield 28 | finally: 29 | if seconds > 0: 30 | signal.alarm(0) 31 | 32 | 33 | def undecorate_func(func, decorators=None): 34 | """Finc the actual func behind any number of decorators 35 | """ 36 | if decorators is None: 37 | decorators = [] 38 | if hasattr(func, 'original_func'): 39 | decorators.append(func) 40 | return undecorate_func(getattr(func, 'original_func'), decorators) 41 | else: 42 | return func, decorators 43 | 44 | 45 | def try_import(module_name): 46 | """Attempt to import the given module (by name), returning a tuple (True, module object) or (False,None) on ImportError""" 47 | try: 48 | module = __import__(module_name) 49 | return True, module 50 | except ImportError: 51 | return False, None 52 | 53 | 54 | def call_chain(chain, *args, **kwargs): 55 | if len(chain) == 1: 56 | return chain[0](*args, **kwargs) 57 | elif len(chain) == 2: 58 | return chain[1](lambda: chain[0](*args, **kwargs)) 59 | elif len(chain) == 3: 60 | return chain[2](lambda: chain[1](lambda: chain[0](*args, **kwargs))) 61 | else: 62 | raise Exception("call_chain is a hack and doesn't support chains longer than 3") 63 | 64 | 65 | def profiling_wrapper(func): 66 | import cProfile, StringIO, pstats 67 | pr = cProfile.Profile() 68 | pr.enable() 69 | try: 70 | func() 71 | finally: 72 | pr.disable() 73 | s = StringIO.StringIO() 74 | sortby = 'cumulative' 75 | ps = pstats.Stats(pr, stream=s).sort_stats(sortby) 76 | ps.print_stats() 77 | print s.getvalue() 78 | 79 | 80 | def pdb_wrapper(func): 81 | try: 82 | return func() 83 | except Exception: 84 | import pdb 85 | import traceback 86 | etype, value, tb = sys.exc_info() 87 | logging.info('Top level exception caught, entering debugger') 88 | traceback.print_exc() 89 | pdb.post_mortem(tb) 90 | raise 91 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/decorators.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix, Inc. 3 | 4 | """Decorators.""" 5 | 6 | import cligraphy.core.parsers 7 | from cligraphy.core.util import undecorate_func 8 | 9 | from functools import wraps 10 | import logging 11 | 12 | from enum import Enum 13 | 14 | 15 | class Tag(Enum): 16 | """Enumeration of the decorator tags we support.""" 17 | beta = 1 18 | disabled = 2 19 | strict = 3 20 | interactive = 4 21 | 22 | 23 | class DisabledCommandException(Exception): 24 | """Raised when a @disabled command is invoked.""" 25 | pass 26 | 27 | 28 | def tag(tag_enum, reason='No reason given'): 29 | """Decorator that disables a command.""" 30 | # pylint:disable=missing-docstring,unused-argument 31 | def actual_decorator(func): 32 | @wraps(func) 33 | def wrapper(*args, **kwargs): 34 | func(*args, **kwargs) 35 | wrapper.tag = tag_enum 36 | wrapper.original_func = func 37 | return wrapper 38 | return actual_decorator 39 | 40 | 41 | def disabled(reason='No reason given'): 42 | """Decorator that disables a command.""" 43 | # pylint:disable=missing-docstring,unused-argument 44 | def actual_decorator(func): 45 | @wraps(func) 46 | def wrapper(*args, **kwargs): 47 | raise DisabledCommandException('This command is disabled: %s' % reason) 48 | wrapper.tag = Tag.disabled 49 | wrapper.original_func = func 50 | return wrapper 51 | return actual_decorator 52 | 53 | 54 | def beta(reason='No reason given'): 55 | """Decorator for unstable commands - user will be prompted to confirm before execution.""" 56 | # pylint:disable=missing-docstring,unused-argument 57 | def actual_decorator(func): 58 | @wraps(func) 59 | def wrapper(*args, **kwargs): 60 | message = 'WARNING - this command is marked as beta: %s - are you sure you want to continue (y/N)? ' % reason 61 | logging.warning(message) 62 | confirm = raw_input(message) 63 | if confirm and confirm.lower() in ('y', 'yes'): 64 | func(*args, **kwargs) 65 | else: 66 | print 'Cancelled' 67 | wrapper.tag = Tag.beta 68 | wrapper.original_func = func 69 | return wrapper 70 | return actual_decorator 71 | 72 | 73 | def strict(reason='No reason given'): 74 | """Command decorator that disables fuzzy matching.""" 75 | # pylint:disable=missing-docstring,unused-argument 76 | def actual_decorator(func): 77 | @wraps(func) 78 | def wrapper(*args, **kwargs): 79 | if cligraphy.core.parsers.FUZZY_PARSED: 80 | logging.error('This command must be called by its exact name (%s). Please run as %s', 81 | reason, cligraphy.core.parsers.FUZZY_PARSED[0][-1]) 82 | return 83 | else: 84 | func(*args, **kwargs) 85 | wrapper.tag = Tag.strict 86 | wrapper.original_func = func 87 | return wrapper 88 | return actual_decorator 89 | 90 | 91 | def get_tags(decorated_func): 92 | """Return the set of decorator tags applied to decorated_func.""" 93 | _, decorators = undecorate_func(decorated_func) 94 | return set([ x.tag for x in decorators ]) 95 | -------------------------------------------------------------------------------- /staging/commands/misc/date.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix, Inc. 3 | 4 | """Date/Time tool with timezone support 5 | 6 | Date/Time tool with timezone support 7 | """ 8 | 9 | 10 | import pytz 11 | import tzlocal 12 | import datetime 13 | import calendar 14 | 15 | DEFAULT_TIME_ZONES = ['Pacific/Honolulu', 16 | 'Pacific/Auckland', 17 | 'Australia/Sydney', 18 | 'Asia/Tokyo', 19 | 'America/Los_Angeles', 20 | 'America/Phoenix', 21 | 'America/Denver', 22 | 'America/Chicago', 23 | 'America/New_York', 24 | 'Europe/Dublin', 25 | 'Europe/London', 26 | 'Europe/Paris', 27 | 'Europe/Helsinki', 28 | 'Europe/Stockholm', 29 | 'Europe/Berlin', 30 | ] 31 | 32 | 33 | def pad(values, width): 34 | """Format a list of values to a certain width""" 35 | values = [ str(data) for data in values ] 36 | return ''.join([ data + (' ' * (width - len(data))) for data in values ]) 37 | 38 | 39 | def show(dt, show_zones): 40 | """Display a datetime object adjusted to the given timezones""" 41 | 42 | seen_zones = set() 43 | stamps = [] 44 | 45 | def add(dt): # pylint:disable=missing-docstring 46 | if dt.tzname() not in seen_zones: 47 | stamps.append(dt) 48 | seen_zones.add(dt.tzname()) 49 | # 50 | add(dt) 51 | add(dt.astimezone(tzlocal.get_localzone())) 52 | add(dt.astimezone(pytz.utc)) 53 | for tz in show_zones: 54 | add(dt.astimezone(pytz.timezone(tz))) 55 | 56 | stamps.sort(key=lambda x: int(x.strftime("%s"))) 57 | 58 | for stamp in stamps: 59 | print pad((stamp.tzinfo.zone, 60 | stamp.strftime("%H:%M:%S") + (' *' if stamp.tzinfo.zone == 'local' else ''), 61 | stamp.strftime("%Z"), 62 | stamp.strftime("%z"), 63 | stamp.strftime("%A"), 64 | stamp.strftime("%Y-%m-%d")), 24) 65 | 66 | print '' 67 | print 'Posix UTC timestamp:', calendar.timegm(dt.utctimetuple()) 68 | 69 | 70 | def configure(parser): 71 | parser.add_argument('-d', '--delta', metavar='DAYS', type=int, default=None, 72 | help='delta in days, can be negative (eg. "-d -4" to show information for 4 days ago)') 73 | parser.add_argument('-l', '--localize', metavar='OLSONCODE', default=None, 74 | help='force timezone (defaults to system local timezone, or UTC if a stamp is specified)') 75 | parser.add_argument('-z', dest='zones', metavar='OLSONCODE', nargs='+', 76 | default=DEFAULT_TIME_ZONES, 77 | help='show time in additional timezones') 78 | parser.add_argument('stamp', nargs='?', type=int, default=None, 79 | help='unix timestamp to use (instead of now)') 80 | 81 | 82 | def main(args): 83 | 84 | tz = None 85 | 86 | if args.stamp is not None: 87 | dt = datetime.datetime.utcfromtimestamp(args.stamp) 88 | tz = pytz.utc 89 | else: 90 | dt = datetime.datetime.now() 91 | 92 | if args.delta is not None: 93 | dt = dt + datetime.timedelta(days=args.delta) 94 | 95 | if args.localize: 96 | tz = pytz.timezone(args.localize) 97 | elif tz is None: 98 | tz = tzlocal.get_localzone() 99 | 100 | show(tz.localize(dt), show_zones=args.zones) 101 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/lib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2015, 2016 Netflix, Inc. 3 | 4 | import logging 5 | import time 6 | import os.path 7 | 8 | from decorator import decorator 9 | from pathlib import Path 10 | 11 | 12 | def retry(retry_count, exceptions, log_message, retry_sleep=0, backoff=1, maxdelay=None): 13 | """ Decorator to implement retry logic 14 | :retry_count int: number of attempts before aborting on failure. 15 | :exceptions tuple: exception classes to trap and retry on. 16 | :retry_sleep float or int: sets the initial delay between attempts in seconds. 17 | :backoff float or int: sets the factor by which the delay should lengthen after each failure. 18 | backoff must be greater than 1, or else it isn't really a backoff. 19 | :maxdelay int: exponential backoffs can get pretty lengthy. This limits the maximum delay 20 | 21 | """ 22 | @decorator 23 | def _retry(f, *args, **kwargs): 24 | _tries, _delay = retry_count, retry_sleep 25 | while _tries > 0: 26 | try: 27 | return f(*args, **kwargs) 28 | except (exceptions) as e: 29 | logging.debug('Failed to %s. Attempt: %s/%s', log_message, retry_count + 1 - _tries, retry_count) 30 | _tries -= 1 31 | if _tries == 0: 32 | logging.error('Failed to %s with %s attempts', log_message, retry_count) 33 | raise(e) 34 | time.sleep(_delay) 35 | _delay *= backoff 36 | if maxdelay and _delay > maxdelay: 37 | _delay = maxdelay 38 | return _retry 39 | 40 | 41 | def chunks(l, n): 42 | """ Yield successive n-sized chunks from l. 43 | """ 44 | for i in xrange(0, len(l), n): 45 | yield l[i:i + n] 46 | 47 | 48 | def is_file(parser, item): 49 | """ 50 | Takes an argparse.ArgumentParser instance and value that you want 51 | validated. If it is a file, returns a resolved pathlib.PosixPath object. 52 | If it isn't, prints the appropriate error and aborts execution via the 53 | ArguementParser. 54 | 55 | :param parser: ArgumentParser that you are trying to validate the item from 56 | :type parser: argparse.ArgumentParser 57 | :param item: path to file to test if is a file 58 | :type item: str 59 | :return: resolved Path object 60 | :rtype: pathlib.PosixPath 61 | """ 62 | f = Path(os.path.expanduser(item)) 63 | try: 64 | f = f.resolve() 65 | except IOError: 66 | parser.error('The file {file!r} does not exist'.format(file=item)) 67 | if not f.is_file(): 68 | parser.error('{item!r} is not a file'.format(item=item)) 69 | try: 70 | with f.open('r') as fp: 71 | pass 72 | except IOError as e: 73 | parser.error('{item!r} cannot be opened for reading: {err!s}'.format( 74 | item=item, err=e 75 | )) 76 | return f 77 | 78 | def get_user_choice(prompt, choices, case_lower=True): 79 | """ prompt user to make a choice. converts choices to lowercase unicode strings 80 | prompt: str: 81 | choices: list of str or int: 82 | case_lower: defaults to True for lower case enforcement 83 | return: unicode choice 84 | """ 85 | if case_lower: 86 | _vals = [unicode(x).lower() for x in choices] 87 | else: 88 | _vals = [unicode(x) for x in choices] 89 | while True: 90 | if case_lower: 91 | _input = unicode(raw_input(prompt)).lower() 92 | else: 93 | _input = unicode(raw_input(prompt)) 94 | if _input in _vals: 95 | return _input 96 | -------------------------------------------------------------------------------- /staging/commands/dev/lint.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Netflix 3 | 4 | 5 | """Lint your code! 6 | Pylint wrapper 7 | """ 8 | 9 | import stat 10 | import os 11 | import re 12 | import logging 13 | import subprocess 14 | import sys 15 | 16 | IGNORE = r'^(\..*|.*egg-info|dist|build|testdata)$' 17 | 18 | 19 | def _find_all_python_modules(path, ignore_re, results): 20 | """Find all folders under (and including) path that contain an __init__.py file""" 21 | logging.debug('visiting %s', path) 22 | dirs = [] 23 | for name in os.listdir(path): 24 | if ignore_re.match(name): 25 | logging.debug('ignored %s', name) 26 | continue 27 | if stat.S_ISDIR(os.stat(os.path.join(path, name)).st_mode): 28 | dirs.append(name) 29 | elif name == '__init__.py': 30 | results.add(path) 31 | for dirname in dirs: 32 | _find_all_python_modules(os.path.join(path, dirname), ignore_re, results) 33 | 34 | 35 | def find_python_modules(path): 36 | """Find unique python modules, grouping hierarchies""" 37 | ignore_re = re.compile(IGNORE) 38 | results = set() 39 | _find_all_python_modules(path, ignore_re, results) 40 | kept = set() 41 | for path in results: 42 | prefix = path.rpartition('/')[0] 43 | if prefix not in results: 44 | kept.add(path) 45 | return sorted(list(kept)) 46 | 47 | 48 | def find_git_root(path): 49 | """Find the first git root in path and its parents""" 50 | if os.path.exists(os.path.join(path, '.git')): 51 | return path 52 | if not path or path == '/': 53 | return None 54 | return find_git_root(os.path.abspath(os.path.join(path, '..'))) 55 | 56 | 57 | def normalize_path(path): 58 | """Normalize paths - as of now only transforms . into ../$CWD""" 59 | if path == '.': 60 | return os.path.join('..', os.getcwd().split('/')[-1]) 61 | else: 62 | return path 63 | 64 | 65 | def interact(lines, editor='subl'): 66 | """Sublime mode: show lint violations one by one, opening sublime at the proper location, and waiting for enter between lines""" 67 | print '--- Press enter to show the next error. Note that line offsets will diverge as you add/remove lines from the code -- ' 68 | for line in lines: 69 | if line.startswith('/') and ':' in line: 70 | print line 71 | os.system('%s %s' % (editor, line.split(' ')[0])) 72 | sys.stdin.readline() 73 | 74 | 75 | def configure(args): 76 | args.add_argument('-g', '--git-root', action='store_true', help='lint from git repository root') 77 | args.add_argument('-n', '--dry-run', action='store_true', help='show paths that would be linted') 78 | args.add_argument('-i', '--interactive', action='store_true', help='iterate over issues, opening sublime at the right location') 79 | args.add_argument('-e', '--errors-only', action='store_true', help='show errors only') 80 | args.add_argument('paths', metavar='path', nargs='*', help='paths to lint') 81 | 82 | 83 | def main(args): 84 | lines = [] 85 | 86 | git_root = find_git_root('.') 87 | 88 | if args.git_root: 89 | path = git_root 90 | if not path: 91 | print 'no git root found' 92 | return 1 93 | else: 94 | path = '.' 95 | paths = args.paths if args.paths else find_python_modules(path) 96 | for path in paths: 97 | if args.dry_run: 98 | print path 99 | continue 100 | path = normalize_path(path) 101 | 102 | pylintrc = '.pylintrc' 103 | if not os.path.exists(pylintrc): 104 | if git_root: 105 | pylintrc = os.path.join(git_root, pylintrc) 106 | if not os.path.exists(pylintrc): 107 | pylintrc = '%s/conf/pylintrc' % os.getenv('OC_REPO_PATH') 108 | 109 | #FIXME(stf/oss) venv name 110 | command = '${OC_PYTHON_ENV_ROOT}/octools-cligraphy/bin/pylint --rcfile %s %s %s' % (pylintrc, '--errors-only' if args.errors_only else '', path) 111 | logging.debug('Linting %s with %s', path, command) 112 | try: 113 | subprocess.check_output(command, shell=True) 114 | except subprocess.CalledProcessError, cpe: 115 | lines.extend(cpe.output.split('\n')) 116 | 117 | if lines: 118 | if args.interactive: 119 | interact(lines) 120 | else: 121 | print '\n'.join(lines) 122 | print 'Issues were found' 123 | sys.exit(1) 124 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/capture/fmt_capnp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """Session recording and replay 5 | """ 6 | 7 | from cligraphy.core import capture 8 | from cligraphy.core.capture.termsize import get_terminal_size, set_terminal_size 9 | 10 | import os.path 11 | import os 12 | import time 13 | 14 | 15 | LOG_ROOT = '/tmp/session' 16 | 17 | 18 | def session_capnp(): 19 | """Load out capnproto schema 20 | """ 21 | import capnp 22 | capnp.remove_import_hook() 23 | return capnp.load(os.path.join(capture.__path__[0], 'session.capnp')) 24 | 25 | 26 | class CapnpSessionRecorder(capture.Recorder): 27 | """Records a pty session 28 | """ 29 | 30 | def __init__(self): 31 | self.last_ts = time.time() 32 | self.out_fp = None 33 | self.session_capnp = session_capnp() 34 | tm = time.gmtime() 35 | dirname = '%s/%04d/%02d/%02d/%02d' % (LOG_ROOT, tm.tm_year, tm.tm_mon, tm.tm_mday, tm.tm_hour) 36 | try: 37 | umask = os.umask(2) 38 | os.makedirs(dirname) 39 | os.umask(umask) 40 | except OSError: 41 | pass 42 | self.filename = dirname + '/%s-%d.log' % (os.getenv('USER'), time.time() * 1000) 43 | 44 | def start(self): 45 | """Start a session record 46 | """ 47 | self.out_fp = open(self.filename, 'wb') 48 | 49 | session = self.session_capnp.Session.new_message() 50 | 51 | session.username = os.getenv('USER') 52 | session.timestamp = int(time.time() * 1000) 53 | 54 | window_size = session.init('windowSize') 55 | window_size.lines, window_size.columns = get_terminal_size() 56 | 57 | environ = os.environ.items() 58 | session_env = session.init('environment', len(environ)) 59 | for index, item in enumerate(environ): 60 | session_env[index].name = item[0] 61 | session_env[index].value = item[1] 62 | 63 | session.write(self.out_fp) 64 | self.out_fp.flush() 65 | self.last_ts = time.time() 66 | 67 | def _timecode(self): 68 | """Returns the current time code 69 | """ 70 | now = time.time() 71 | ret = max(now - self.last_ts, 0.0) 72 | self.last_ts = now 73 | return ret 74 | 75 | def end(self, exitcode=0): 76 | """Finish a session record 77 | """ 78 | event = self.session_capnp.Event.new_message() 79 | event.timecode = self._timecode() 80 | event.type = 'sessionEnd' 81 | event.status = exitcode 82 | event.write_packed(self.out_fp) 83 | self.out_fp.close() 84 | 85 | def record_window_resize(self, lines, columns): 86 | """Record a window resizing event 87 | """ 88 | event = self.session_capnp.Event.new_message() 89 | event.timecode = self._timecode() 90 | event.type = 'windowResized' 91 | window_size = event.init('windowSize') 92 | window_size.lines, window_size.columns = lines, columns 93 | event.write_packed(self.out_fp) 94 | 95 | def record_user_input(self, data): 96 | """record user input separately from terminal output 97 | """ 98 | event = self.session_capnp.Event.new_message() 99 | event.timecode = self._timecode() 100 | event.type = 'userInput' 101 | event.data = data 102 | event.write_packed(self.out_fp) 103 | 104 | def record_server_output(self, data): 105 | """record terminal output (user + server originated) 106 | """ 107 | event = self.session_capnp.Event.new_message() 108 | event.timecode = self._timecode() 109 | event.type = 'ptyInput' 110 | event.data = data 111 | event.write_packed(self.out_fp) 112 | 113 | 114 | class CapnpSessionPlayer(capture.Player): 115 | """Plays a recorded session 116 | """ 117 | 118 | def __init__(self, filename): 119 | self.filename = filename 120 | self.session_capnp = session_capnp() 121 | self.fpin = open(self.filename, 'rb') 122 | self.session = self.session_capnp.Session.read(self.fpin) 123 | 124 | def close(self): 125 | """Close this player 126 | """ 127 | self.fpin.close() 128 | 129 | def play(self): 130 | """Generator of session playback events - basically (timecode, data) tuples 131 | """ 132 | set_terminal_size(self.session.windowSize.lines, self.session.windowSize.columns) 133 | 134 | skipped = 0 135 | for event in self.session_capnp.Event.read_multiple_packed(self.fpin): 136 | if event.type == 'ptyInput': 137 | yield event.timecode + skipped, event.data 138 | skipped = 0 139 | elif event.type == 'windowResized': 140 | set_terminal_size(event.windowSize.lines, event.windowSize.columns) 141 | yield event.timecode + skipped, None 142 | skipped = 0 143 | else: 144 | skipped += event.timecode 145 | self.close() 146 | -------------------------------------------------------------------------------- /staging/commands/diag.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2015 Netflix, Inc. 3 | 4 | """Grab and export interesting information to help diagnose octools issues""" 5 | 6 | from cligraphy.core import ctx, dictify_recursive 7 | 8 | import getpass 9 | import importlib 10 | import json 11 | import logging 12 | import os 13 | import os.path 14 | import platform 15 | import socket 16 | import subprocess 17 | import sys 18 | import time 19 | import traceback 20 | 21 | import base64 22 | import StringIO 23 | import gzip 24 | 25 | 26 | import requests 27 | 28 | 29 | def _add_file_part(parts, name, filename): 30 | """Add a file to the diagnostics parts list, or an error message if the file could not be read""" 31 | try: 32 | with open(filename, 'r') as fp: 33 | parts.append((name, True, fp.read())) 34 | except: # pylint:disable=bare-except 35 | parts.append((name, False, traceback.format_exc())) 36 | 37 | 38 | def _add_output_part(parts, name, command, **kwargs): 39 | """Add the output of a command to the parts list, or an error message if the command failed""" 40 | try: 41 | parts.append((name, True, subprocess.check_output(command, stderr=subprocess.STDOUT, **kwargs))) 42 | except subprocess.CalledProcessError as cpe: 43 | parts.append((name, False, 'Command returned with status code %d. Output: [%s]' % (cpe.returncode, cpe.output))) 44 | except: # pylint:disable=bare-except 45 | parts.append((name, False, traceback.format_exc())) 46 | 47 | 48 | def _jsonify(value): 49 | """Stringify a value if it cannot be serialized to json, otherwise leave it alone""" 50 | try: 51 | json.dumps(value) 52 | return value 53 | except TypeError: 54 | return str(value) 55 | 56 | 57 | def _add_modinfo_part(parts, module_name): 58 | """Add information about a module to the parts list""" 59 | try: 60 | mod = importlib.import_module(module_name) 61 | details = { 62 | str(k): _jsonify(v) for k, v in mod.__dict__.items() if k not in ('__builtins__',) 63 | } 64 | parts.append(('modinfo:%s' % module_name, True, details)) 65 | except: # pylint:disable=bare-except 66 | parts.append(('modinfo:%s' % module_name, False, traceback.format_exc())) 67 | 68 | 69 | def configure(parser): 70 | parser.add_argument("-n", "--dryrun", help="print generated diagnostics manifest, don't upload it", action="store_true") 71 | 72 | 73 | def main(args): 74 | logging.info('Creating diagnostic manifest...') 75 | 76 | os.chdir('/tmp') 77 | diag = {} 78 | 79 | diag['user'] = getpass.getuser() 80 | diag['host'] = socket.gethostname() 81 | diag['ts'] = time.time() 82 | diag['python.version'] = tuple(sys.version_info) 83 | diag['platform'] = platform.platform() 84 | diag['environ'] = dict(os.environ) 85 | diag['conf'] = dictify_recursive(ctx.conf) 86 | 87 | parts = [] 88 | diag['parts'] = parts 89 | 90 | 91 | _add_file_part(parts, 'requirements.txt', os.path.join(ctx.conf.tool.repo_path, 'requirements.txt')) 92 | _add_output_part(parts, 'pip freeze', ['pip', 'freeze']) 93 | 94 | _add_modinfo_part(parts, 'requests') 95 | 96 | os.chdir(ctx.conf.tool.repo_path) 97 | _add_output_part(parts, 'git.branch', 'git rev-parse --abbrev-ref HEAD'.split()) 98 | _add_output_part(parts, 'git.status', 'git status --porcelain'.split()) 99 | _add_output_part(parts, 'git.version', 'git --version'.split()) 100 | _add_output_part(parts, 'git.ls', 'ls -la .git'.split()) 101 | 102 | _add_output_part(parts, 'uptime', ['uptime']) 103 | 104 | _add_output_part(parts, 'ssh.keys', ['ssh-add', '-l']) 105 | 106 | _add_output_part(parts, 'net.ifconfig', ['ifconfig']) 107 | _add_output_part(parts, 'net.routes', ['netstat', '-nr']) 108 | _add_file_part(parts, 'net.resolv.conf', '/etc/resolv.conf') 109 | 110 | _add_output_part(parts, 'sys.ps', ['ps', 'aux']) 111 | _add_output_part(parts, 'sys.df', ['df', '-m']) 112 | _add_output_part(parts, 'sys.filevault', ['fdesetup', 'status']) 113 | 114 | _add_output_part(parts, 'net.ifconfig', ['ifconfig']) 115 | _add_output_part(parts, 'net.routes', ['netstat', '-nr']) 116 | _add_file_part(parts, 'net.resolv.conf', '/etc/resolv.conf') 117 | 118 | _add_output_part(parts, 'sys.ps', ['ps', 'aux']) 119 | _add_output_part(parts, 'sys.df', ['df', '-m']) 120 | _add_output_part(parts, 'sys.filevault', ['fdesetup', 'status']) 121 | 122 | _add_file_part(parts, 'sys.usr_local_bin_pip', '/usr/local/bin/pip') 123 | 124 | logging.info('Done creating diagnostic manifest') 125 | 126 | if args.dryrun or not ctx.conf.report.enabled: 127 | print json.dumps(diag, indent=4) 128 | else: 129 | buff = StringIO.StringIO() 130 | with gzip.GzipFile(fileobj=buff, mode="w", compresslevel=9) as fp: 131 | json.dump(diag, fp) 132 | body = base64.b64encode(buff.getvalue()) 133 | 134 | logging.info('Uploading diagnostic manifest (body is %d bytes)', len(body)) 135 | endpoint = '%s/api/v0.1/diagnostic' % ctx.conf.report.server 136 | response = requests.post(endpoint, data={ 137 | 'user_email': ctx.conf.user.email, 138 | 'body': body, 139 | 'stamp': int(time.time()), 140 | }, timeout=60) 141 | response.raise_for_status() 142 | logging.info('Diagnostic manifest posted successfully') 143 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/log.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013, 2014 Netflix, Inc. 3 | 4 | """Command line tools entry point""" 5 | 6 | from cligraphy.core.util import try_import 7 | 8 | from remember import memoize 9 | 10 | import json 11 | import logging 12 | import logging.config 13 | import os 14 | import sys 15 | 16 | 17 | # new log level for, you guessed it, dryrun logging. 18 | # See: http://stackoverflow.com/a/13638084 19 | DRYRUN_num = 15 20 | DRYRUN_name = 'DRYRUN' 21 | 22 | 23 | def _dryrun(self, message, *args, **kws): 24 | if self.isEnabledFor(DRYRUN_num): 25 | self._log(DRYRUN_num, message, args, **kws) 26 | 27 | 28 | class JsonHandler(logging.Handler): 29 | 30 | def __init__(self, destination, *args, **kwargs): 31 | self.destination = destination 32 | self.recurse = False 33 | super(JsonHandler, self).__init__(*args, **kwargs) 34 | 35 | def emit(self, record): 36 | if self.recurse: 37 | return 38 | try: 39 | self.recurse = True 40 | self._emit(record) 41 | finally: 42 | self.recurse = False 43 | 44 | def _emit(self, record, first_attempt=True): 45 | try: 46 | msg = record.msg % record.args 47 | except TypeError: 48 | msg = str(record.msg) + ' (formatting error)' 49 | 50 | output = {} 51 | try: 52 | output = { 53 | 'ts': int(record.created * 1000), 54 | 'level': record.levelno, 55 | 'msg': msg, 56 | 57 | # module, function, ... 58 | 'module': record.module, 59 | 'file': record.filename, 60 | 'func': record.funcName, 61 | 'line': record.lineno, 62 | 63 | # process and thread information 64 | 'tid': record.thread, 65 | 'tname': record.threadName, 66 | 'pid': record.process, 67 | 'pname': record.processName, 68 | } 69 | 70 | if hasattr(record, 'extra'): 71 | if first_attempt: 72 | output['extra'] = record.extra 73 | else: 74 | output['extra'] = {'_omitted': True} # We tried to serialize our record once and got an error, so omit ctx now 75 | 76 | if record.exc_info and record.exc_info[0] and record.exc_info[1]: 77 | output['exc_type'] = record.exc_info[0].__name__ 78 | output['exc_msg'] = repr(record.exc_info[1].message) 79 | tb = [] 80 | cur = record.exc_info[2] 81 | while cur: 82 | frame = cur.tb_frame 83 | tb.append((frame.f_code.co_filename, frame.f_code.co_name, frame.f_lineno)) 84 | cur = cur.tb_next 85 | output['exc_tb'] = tb 86 | 87 | try: 88 | msg = json.dumps(output) 89 | self.destination.log_event('log', record.created, msg) 90 | except (TypeError, OverflowError): 91 | if first_attempt: 92 | return self.emit(record, False) # Try again, without ctx this time 93 | else: 94 | raise 95 | 96 | except (KeyboardInterrupt, SystemExit): 97 | raise 98 | except: # pylint:disable=bare-except 99 | print 'log.structured: serialization error - details follow' 100 | print(output) 101 | self.handleError(record) 102 | 103 | 104 | def silence_verbose_loggers(): 105 | logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN) 106 | 107 | 108 | @memoize.memoize() 109 | def _get_dict_config(): 110 | dict_config = { 111 | 'version': 1, 112 | 'disable_existing_loggers': False, 113 | 'formatters': { 114 | 'default': { 115 | 'datefmt': '%Y-%m-%d %H:%M:%S', 116 | 'format': "%(asctime)s %(levelname)-8s %(module)s %(funcName)s %(message)s" 117 | }, 118 | }, 119 | 'handlers': { 120 | 'defaultHandler': { 121 | 'level': 'DEBUG', 122 | 'class': 'logging.StreamHandler', 123 | 'formatter': 'default' 124 | }, 125 | }, 126 | 'loggers': { 127 | '': { 128 | 'handlers': (['defaultHandler']), 129 | 'level': 'WARN', 130 | }, 131 | } 132 | } 133 | 134 | if os.isatty(sys.stderr.fileno()) and try_import('colorlog')[0]: 135 | import colorlog 136 | colorlog.default_log_colors.update({DRYRUN_name: 'blue'}) 137 | dict_config['formatters']['colors'] = { 138 | '()': 'colorlog.ColoredFormatter', 139 | 'datefmt': '%Y-%m-%d %H:%M:%S', 140 | 'format': "%(log_color)s%(asctime)s %(levelname)-8s%(reset)s %(purple)s%(module)s/%(funcName)s%(reset)s %(message)s" 141 | } 142 | dict_config['handlers']['defaultHandler']['formatter'] = 'colors' 143 | 144 | return dict_config 145 | 146 | 147 | def setup_logging(level=None): 148 | """Configure logging""" 149 | logging.addLevelName(DRYRUN_num, DRYRUN_name) 150 | logging.Logger.dryrun = _dryrun 151 | try: 152 | logging.config.dictConfig(_get_dict_config()) 153 | logging.captureWarnings(True) 154 | silence_verbose_loggers() 155 | if level is not None: 156 | logging.getLogger().setLevel(level) 157 | except Exception: # pylint:disable=broad-except 158 | logging.basicConfig(level=logging.WARN) 159 | logging.warn('Could not configure logging, using basicConfig', exc_info=True) 160 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/reporting.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | from cligraphy.core.tracking import TRACKING 5 | 6 | import threading 7 | from Queue import Queue, Empty, Full 8 | import collections 9 | 10 | import requests 11 | 12 | import logging 13 | import os 14 | import socket 15 | import time 16 | 17 | ReportingEvent = collections.namedtuple('ReportingEvent', 'name,args,kwargs') 18 | 19 | 20 | class Reporter(object): 21 | reporting_event_cls = ReportingEvent 22 | 23 | def report_command_start(self, command_line): 24 | """Report command execution""" 25 | pass 26 | 27 | def report_command_output(self, output): 28 | """Report command output""" 29 | pass 30 | 31 | def report_command_exit(self, exit_code): 32 | """Report command exit code""" 33 | pass 34 | 35 | def start(self): 36 | """Start this reporter""" 37 | pass 38 | 39 | def stop(self): 40 | """Stop this reporter""" 41 | pass 42 | 43 | 44 | class NoopReporter(Reporter): 45 | """Reporter that does nothing""" 46 | 47 | 48 | class ThreadedReporter(Reporter, threading.Thread): 49 | """Base class for reporters""" 50 | 51 | idle_period = 10 52 | 53 | def __init__(self): 54 | super(Reporter, self).__init__(group=None, name='oc-reporter-thread') 55 | self.daemon = True 56 | self.queue = Queue(maxsize=16) 57 | self.stop_event = threading.Event() 58 | 59 | def run(self): 60 | logging.debug('Starting reporter thread mainloop') 61 | while True: 62 | try: 63 | event = self.queue.get(block=True, timeout=self.idle_period) 64 | if event: 65 | if event.name == 'stop': 66 | return 67 | func = getattr(self, '_report_%s' % event.name, None) 68 | if func: 69 | func(*event.args, **event.kwargs) 70 | except Empty: 71 | if self.stop_event.is_set(): 72 | return 73 | else: 74 | try: 75 | self._report_idle() 76 | except Exception: 77 | pass 78 | 79 | def _report(self, event_name, *args, **kwargs): 80 | event = self.reporting_event_cls(name=event_name, args=args, kwargs=kwargs) 81 | try: 82 | self.queue.put(event, block=False) 83 | except Full: 84 | logging.debug('Could not put event (name=%s) in reporting queue', event_name, exc_info=True) 85 | 86 | def _report_idle(self): 87 | pass 88 | 89 | def report_command_start(self, command_line): 90 | """Report command execution""" 91 | self._report('command_start', command_line) 92 | 93 | def report_command_output(self, output): 94 | """Report command output""" 95 | self._report('command_output', output) 96 | 97 | def report_command_exit(self, exit_code): 98 | """Report command exit code""" 99 | self._report('command_exit', exit_code) 100 | 101 | def start(self): 102 | threading.Thread.start(self) 103 | 104 | def stop(self, timeout=1.5): 105 | self.stop_event.set() 106 | self._report('stop') 107 | logging.debug('Waiting on reporter thread for up to %s second', timeout) 108 | self.join(timeout=timeout) 109 | if self.isAlive(): 110 | logging.debug('Reported thread is still running') 111 | 112 | 113 | def disable_ssl_warnings(): 114 | import requests.packages.urllib3 115 | requests.packages.urllib3.disable_warnings() 116 | 117 | 118 | class ToolsPadReporter(ThreadedReporter): 119 | 120 | timeout = 0.75 121 | 122 | def __init__(self, cligraph): 123 | super(ToolsPadReporter, self).__init__() 124 | self.cligraph = cligraph 125 | self.requests_session = requests.Session() 126 | self.requests_session.headers.update({'User-Agent': 'octools:%s@%s' % (os.getenv('USER'), socket.gethostname())}) 127 | self.created = False 128 | self.session_endpoint = None 129 | self.create_event_endpoint = None 130 | self.idle_endpoint = '%s' % self.cligraph.conf.report.server 131 | self.create_session_endpoint = '%s/api/v0.1/execution' % self.cligraph.conf.report.server 132 | disable_ssl_warnings() 133 | 134 | def _report_idle(self): 135 | self.requests_session.head(self.idle_endpoint) 136 | 137 | def _report_command_start(self, command_line): 138 | try: 139 | response = self.requests_session.post(self.create_session_endpoint, data={ 140 | 'uuid': TRACKING.execution_uuid, 141 | 'session_uuid': TRACKING.session_uuid, 142 | 'user_email': self.cligraph.conf.user.email, 143 | 'start_stamp': int(time.time()), 144 | 'command_line': ' '.join(command_line) 145 | }, timeout=self.timeout) 146 | if response.status_code == 201: 147 | self.created = True 148 | self.session_endpoint = '%s%s' % (self.cligraph.conf.report.server, response.json()['uri']) 149 | self.create_event_endpoint = '%s/event' % (self.session_endpoint) 150 | else: 151 | response.raise_for_status() 152 | 153 | except requests.exceptions.RequestException: 154 | logging.debug('Could not report command start', exc_info=True) 155 | 156 | def _report_command_output(self, output): 157 | if not self.created: 158 | return 159 | try: 160 | response = self.requests_session.post(self.create_event_endpoint, data={ 161 | 'stamp': int(time.time()), 162 | 'type': 'output', 163 | 'body': output, 164 | }, timeout=self.timeout) 165 | response.raise_for_status() 166 | except requests.exceptions.RequestException: 167 | logging.debug('Could not report comand output', exc_info=True) 168 | 169 | def _report_command_exit(self, exit_code): 170 | if not self.created: 171 | return 172 | try: 173 | response = self.requests_session.patch(self.session_endpoint, data={ 174 | 'end_stamp': int(time.time()), 175 | 'exit_code': exit_code, 176 | }, timeout=self.timeout) 177 | response.raise_for_status() 178 | except requests.exceptions.RequestException: 179 | logging.debug('Could not report command exit', exc_info=True) 180 | -------------------------------------------------------------------------------- /staging/commands/refresh.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix 3 | 4 | """Refresh oc tools install 5 | 6 | refresh updates your oc tools install: 7 | - runs "git pull" on your octools repository 8 | - installs/reinstalls/upgrades python dependencies 9 | - precompiles python files 10 | - refreshes commands cache 11 | 12 | Quick mode disables most of this and only refreshes the commands cache. 13 | """ 14 | 15 | import subprocess 16 | import os 17 | import sys 18 | import random 19 | import urlparse 20 | 21 | from pip.req.req_install import parse_editable 22 | from pip.utils import get_installed_distributions 23 | 24 | from cligraphy.core.parsers import NO_HELP 25 | from cligraphy.core import ctx 26 | 27 | from nflx_oc.commands.dev.pycompile import clean_and_compile 28 | from nflx_oc.commands.dev.hookup import get_oc_hooks_path, hookup_repo 29 | 30 | 31 | TIPS = [ 32 | "You can disable these random tips: oc conf add -t bool refresh.tips false" 33 | ] 34 | 35 | 36 | def _build_tips(command_maps): 37 | def _explore(command_map, path): 38 | for name, node in command_map.iteritems(): 39 | if node.get('type') == 'cmd': 40 | full_name = ' '.join(path + (name,)) 41 | node_help = node.get('help') 42 | if node_help not in (None, NO_HELP) and len(node_help) > 1: 43 | TIPS.append('%s: %s' % (full_name, node_help)) 44 | else: 45 | TIPS.append('Looks like "%s" is missing a help string... You could help out by adding it!' % (full_name)) 46 | else: 47 | _explore(node, path + (name,)) 48 | for namespace, command_map in command_maps: 49 | _explore(command_map['commands'], filter(None, ('oc', namespace,))) 50 | 51 | 52 | def _install_deps(venv_prefix, filename): 53 | """""" 54 | reqs = [] 55 | installed_locations = { x.key: x.location for x in get_installed_distributions() } 56 | with open(filename, 'r') as fp: 57 | for line in fp: 58 | line = line.strip() 59 | if not line or line.startswith('#'): 60 | continue 61 | if line.startswith('-i '): 62 | continue 63 | if line.startswith('-e'): 64 | # skip req if we have a local copy 65 | editable = parse_editable(line[3:]) 66 | req_name = editable[0] 67 | url = editable[1] 68 | path = urlparse.urlsplit(url).path 69 | repo_name = path.rpartition('/')[-1] 70 | if repo_name.endswith('.git'): 71 | repo_name = repo_name[:-4] 72 | local_checkout_path = os.path.join(ctx.conf.repos.root, repo_name) 73 | if os.path.exists(local_checkout_path): 74 | # we have a local checkout, check that it's what we use in our venv 75 | installed_location = installed_locations.get(req_name) 76 | if local_checkout_path == installed_location: 77 | print '[refresh] info: using local checkout %s for %s' % (local_checkout_path, req_name) 78 | continue 79 | else: 80 | if installed_location is not None: 81 | print '[refresh] notice: local checkout of %s exists in %s but another install is active in octools (%s)' % (req_name, local_checkout_path, installed_location) 82 | reqs.append(line) 83 | 84 | final_req_fname = os.path.join(ctx.conf.user.dotdir, 'requirements.txt') # FIXME(stf/oss) 85 | with open(final_req_fname, 'w') as fp: 86 | fp.write('\n'.join(reqs)) 87 | 88 | command_line = [ 89 | os.path.join(venv_prefix, 'pip'), 90 | 'install', 91 | '-r', final_req_fname 92 | ] 93 | env = { 'PIP_EXISTS_ACTION': 's' } 94 | env.update(os.environ) 95 | try: 96 | subprocess.check_output(command_line, env=env, stderr=subprocess.STDOUT) 97 | except subprocess.CalledProcessError as cpe: 98 | print "[refresh] %s failed:" % (' '.join(command_line)) 99 | print cpe.output 100 | raise 101 | 102 | 103 | def _check_for_issues(): 104 | # git vulnerabilities before 2.7.4 105 | try: 106 | git_version = subprocess.check_output(['git', '--version']).strip() 107 | git_version_int = int(''.join(git_version.rpartition(' ')[-1].split('.')[0:3])) 108 | if git_version_int < 274: 109 | print '[refresh] git versions under 2.7.4 are vulnerable - upgrade now! (on osx, brew update && brew upgrade git)' 110 | except: 111 | pass 112 | 113 | 114 | def refresh(cligraph, quick): 115 | oldcwd = os.getcwd() 116 | try: 117 | os.chdir(os.getenv('OC_REPO_PATH')) 118 | 119 | _check_for_issues() 120 | 121 | if not os.access('.', os.W_OK): 122 | print '[refresh] repository is not writeable (shared install?), forcing quick mode' 123 | quick = True 124 | 125 | if not quick: 126 | print '[refresh] refreshing repository...' 127 | branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() 128 | subprocess.check_output(['git', 'pull', 'origin', branch]) 129 | 130 | venv = os.getenv('VIRTUAL_ENV') 131 | if venv: 132 | venv_prefix = os.path.dirname(sys.executable) 133 | print '[refresh] refreshing python dependencies...' 134 | _install_deps(venv_prefix, 'requirements.txt') 135 | subprocess.check_output((os.path.join(venv_prefix, 'python') + ' setup.py develop --no-deps').split(' ')) 136 | else: 137 | print '[refresh] $VIRTUAL_ENV is not defined, not refreshing python dependencies.' 138 | 139 | print '[refresh] precompiling python files...' 140 | clean_and_compile('.') 141 | 142 | print '[refresh] hooking up your repo...' 143 | hookup_repo(os.getenv('OC_REPO_PATH'), get_oc_hooks_path(), preflight=False) 144 | 145 | print '[refresh] refreshing commands cache...' 146 | command_maps = cligraph.get_command_maps(autodiscover=True) 147 | _build_tips(command_maps) 148 | finally: 149 | os.chdir(oldcwd) 150 | 151 | if ctx.conf.refresh.tips: 152 | print 'Random tip:\n %s' % (random.choice(TIPS)) 153 | 154 | 155 | def configure(parser): 156 | parser.add_argument('-q', '--quick', help="Quick mode: only refresh commands cache, don't refresh repository or dependencies", 157 | action='store_const', default=False, const=True) 158 | 159 | 160 | def main(args): 161 | refresh(ctx.cligraph, args.quick) 162 | -------------------------------------------------------------------------------- /staging/setup/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # octools bootstrap 3 | # curl -L go/octools.sh | bash 4 | 5 | 6 | function exit_trap { 7 | cat <<\EOF 8 | ------------------------------------------------------------------------------ 9 | 10 | ▄██████████████▄▐█▄▄▄▄█▌ 11 | ██████▌▄▌▄▐▐▌███▌▀▀██▀▀ octools bootstrap script exited unexpectedly 12 | ████▄█▌▄▌▄▐▐▌▀███▄▄█▌ 13 | ▄▄▄▄▄██████████████▀ 14 | 15 | ------------------------------------------------------------------------------ 16 | EOF 17 | } 18 | 19 | trap exit_trap EXIT 20 | 21 | # resize terminal to 42x100 22 | printf '\e[8;42;100t' 23 | clear 24 | 25 | cat <<\EOF 26 | ------------------------------------------------------------------------------ 27 | 28 | ██████╗ ██████╗████████╗ ██████╗ ██████╗ ██╗ ███████╗ 29 | ██╔═══██╗██╔════╝╚══██╔══╝██╔═══██╗██╔═══██╗██║ ██╔════╝ 30 | ██║ ██║██║ ██║ ██║ ██║██║ ██║██║ ███████╗ 31 | ██║ ██║██║ ██║ ██║ ██║██║ ██║██║ ╚════██║ 32 | ╚██████╔╝╚██████╗ ██║ ╚██████╔╝╚██████╔╝███████╗███████║ 33 | ╚═════╝ ╚═════╝ ╚═╝ ╚═════╝ ╚═════╝ ╚══════╝╚══════╝ 34 | 35 | ██████╗ ██████╗ ██████╗ ████████╗███████╗████████╗██████╗ █████╗ ██████╗ 36 | ██╔══██╗██╔═══██╗██╔═══██╗╚══██╔══╝██╔════╝╚══██╔══╝██╔══██╗██╔══██╗██╔══██╗ 37 | ██████╔╝██║ ██║██║ ██║ ██║ ███████╗ ██║ ██████╔╝███████║██████╔╝ 38 | ██╔══██╗██║ ██║██║ ██║ ██║ ╚════██║ ██║ ██╔══██╗██╔══██║██╔═══╝ 39 | ██████╔╝╚██████╔╝╚██████╔╝ ██║ ███████║ ██║ ██║ ██║██║ ██║██║ 40 | ╚═════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚══════╝ ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ 41 | 42 | ------------------------------------------------------------------------------ 43 | 44 | This script is intended to run on fresh new computers. It will install a basic 45 | set of dependencies and configure octools. 46 | 47 | ------------------------------------------------------------------------------ 48 | 49 | Hit enter to continue, or CTRL-C 50 | EOF 51 | 52 | read junk /tmp/oc-bootstrap/brew.installed 148 | /usr/local/bin/python ${REPO_PATH}/cligraphy-core/setup/packages.py ${REPO_PATH}/cligraphy-core/setup/packages.yaml >/tmp/oc-bootstrap/brew.wanted 149 | /usr/local/bin/python ${REPO_PATH}/cligraphy-core/setup/missing.py /tmp/oc-bootstrap/brew.installed /tmp/oc-bootstrap/brew.wanted /tmp/oc-bootstrap/brew.missing 150 | if test -f /tmp/oc-bootstrap/brew.missing; then 151 | cat /tmp/oc-bootstrap/brew.missing | xargs $BREW install 152 | fi 153 | 154 | echo "Creating our octools venv..." 155 | 156 | # switch to octools 157 | if test -d ~/.cligraphy/python-envs/oc; then 158 | rm -rf ~/.cligraphy/python-envs/oc 159 | fi 160 | 161 | /usr/local/bin/virtualenv ~/.cligraphy/python-envs/oc 162 | source ~/.cligraphy/python-envs/oc/bin/activate 163 | 164 | pip install -U pip setuptools wheel 165 | 166 | # make sure pip builds against brew-installed headers and libs, gmp and ssl in particular 167 | export CFLAGS='-I/usr/local/include -L/usr/local/lib -L/usr/local/opt/openssl/lib' 168 | 169 | echo "Installing python dependencies in the octools venv..." 170 | 171 | 172 | # OPENSOURCE TODO 173 | 174 | cd ${REPO_PATH}/octools 175 | #pip install --trusted-host pypi.domain.net -i https://pypi.domain.net/pypi/ -r requirements.txt 176 | python setup.py develop --no-deps 177 | 178 | echo "Setting up your bash profile..." 179 | export CLIGRAPHY_REPO_PATH=${REPO_PATH}/octools 180 | ./setup/setup base 181 | 182 | echo "Refreshing commands..." 183 | source ./shell/oc_bash_profile.sh 184 | oc --autodiscover refresh -q 185 | 186 | 187 | trap EXIT 188 | 189 | cat <<\EOF 190 | 191 | ------------------------------------------------------------------------------ 192 | 193 | All done! 194 | Please start a new terminal session and close this one 195 | 196 | ------------------------------------------------------------------------------ 197 | 198 | EOF 199 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """ 5 | Cligraphy tools 6 | """ 7 | 8 | from attrdict import AttrDict 9 | 10 | import yaml 11 | try: 12 | from yaml import CLoader as Loader, CDumper as Dumper 13 | except ImportError: 14 | from yaml import Loader, Dumper 15 | 16 | import collections 17 | import os 18 | import os.path 19 | import re 20 | import logging 21 | 22 | 23 | class Context(object): 24 | 25 | def __init__(self): 26 | self._cligraph = None 27 | 28 | @property 29 | def cligraph(self): 30 | return self._cligraph 31 | 32 | @cligraph.setter 33 | def cligraph(self, value): 34 | self._cligraph = value 35 | 36 | @property 37 | def conf(self): 38 | return self._cligraph.conf 39 | 40 | @property 41 | def parser(self): 42 | return self._cligraph.parser 43 | 44 | 45 | 46 | ctx = Context() 47 | 48 | 49 | __CONF_SUBST_RE = re.compile(r'(%cfg.([a-zA-Z0-9.-_]+)%)') 50 | 51 | 52 | def dictify_recursive(obj): 53 | """Transforms an attrdict tree into regular python dicts""" 54 | for key, val in obj.iteritems(): 55 | if isinstance(val, AttrDict): 56 | obj[key] = dictify_recursive(val) 57 | return dict(**obj) 58 | 59 | 60 | def update_recursive(base, overlay): 61 | """Resursively update base with values from overlay 62 | """ 63 | for key, val in overlay.iteritems(): 64 | if isinstance(val, collections.Mapping): 65 | base[key] = update_recursive(base.get(key, {}), val) 66 | else: 67 | base[key] = overlay[key] 68 | return base 69 | 70 | 71 | def find_node(node, path, add=False): 72 | """Find a node in a dict tree. Path is the broken down path (array). Add missing nodes if add=True.""" 73 | for part in path: 74 | if not node or not part in node: 75 | if add: 76 | node[part] = {} 77 | else: 78 | return None 79 | node = node[part] 80 | return node 81 | 82 | 83 | def get(root, confkey): 84 | """Config key getter""" 85 | return find_node(root, confkey.split('.')) 86 | 87 | 88 | def _resolve_config(root, subst_re, node=None): 89 | """recursive helper for resolve_config""" 90 | if node is None: 91 | node = root 92 | remaining = [] 93 | for key, val in node.iteritems(): 94 | if isinstance(val, collections.Mapping): 95 | remaining.extend(_resolve_config(root, subst_re, node=val)) 96 | elif isinstance(val, basestring): 97 | for match, confkey in subst_re.findall(val): 98 | confval = get(root, confkey) 99 | if confval is not None and (not isinstance(confval, basestring) or subst_re.search(confval) is None): 100 | node[key] = val = val.replace(match, str(confval)) 101 | else: 102 | remaining.append('%s: %s' % (key, val)) 103 | return remaining 104 | 105 | 106 | def resolve_config(cfg): 107 | """performs variable substitution in a configuration tree""" 108 | remaining = [] 109 | for _ in range(16): # 16 maximum substitution passes 110 | remaining_prev = remaining 111 | remaining = _resolve_config(cfg, __CONF_SUBST_RE) 112 | if not remaining or len(remaining) == len(remaining_prev): 113 | break 114 | if remaining: 115 | raise Exception('Incorrect configuration file: could not resolve some configuration variables: %s' % remaining) 116 | 117 | 118 | def automatic_configuration(cligraph, layer_name): 119 | auto_data = { 120 | 'tool': { 121 | 'name': cligraph.tool_name, 122 | 'shortname': cligraph.tool_shortname, 123 | 'version': '1.0', 124 | 'repo_path': cligraph.tool_path, 125 | }, 126 | 'repos': { 127 | 'root': os.path.abspath(os.path.join(cligraph.tool_path, '..')), 128 | }, 129 | } 130 | username = os.getenv('USER') 131 | if username: 132 | auto_data['user'] = { 133 | 'name': username, 134 | 'email': '%s@domain.net' % (username) # OPEN SOURCE TODO 135 | } 136 | else: 137 | auto_data['user'] = { 138 | 'name': 'unknown', 139 | 'email': 'octools-unknown-user@domain.net' # OPEN SOURCE TODO 140 | } 141 | auto_data['user']['dotdir'] = os.path.abspath(os.path.expanduser('~/.' + cligraph.tool_shortname)) 142 | return auto_data 143 | 144 | 145 | def read_configuration(cligraph, custom_suffix=''): 146 | """Read configuration dict for the given tool 147 | """ 148 | 149 | cfg = {} 150 | layers = collections.OrderedDict() 151 | layers['auto'] = [automatic_configuration, None] 152 | layers['shared'] = [os.path.join(cligraph.tool_path, 'conf/%s.yaml' % cligraph.tool_shortname), None] 153 | layers['custom'] = [os.path.join(os.path.abspath(os.path.expanduser('~/.' + cligraph.tool_shortname)), '%s.yaml%s' % (cligraph.tool_shortname, custom_suffix)), None] 154 | 155 | for layer_name, layer_data in layers.items(): 156 | if callable(layer_data[0]): 157 | layer = layer_data[0](cligraph, layer_name) 158 | else: 159 | if not os.path.exists(layer_data[0]): 160 | continue 161 | with open(layer_data[0], 'r') as filep: 162 | layer = yaml.load(filep, Loader=Loader) 163 | layers[layer_name][1] = layer 164 | if layer: 165 | update_recursive(cfg, layer) 166 | 167 | resolve_config(cfg) 168 | return AttrDict(**cfg), layers 169 | 170 | 171 | def write_configuration_file(filename, conf): 172 | """Write a config dict to the specified file, in yaml format""" 173 | import shutil 174 | 175 | try: 176 | shutil.copy2(filename, '%s.back' % filename) 177 | except IOError: 178 | pass 179 | 180 | try: 181 | os.makedirs(os.path.dirname(filename)) 182 | except OSError: 183 | pass 184 | 185 | with open(filename, 'w') as filep: 186 | yaml.dump(conf, filep, indent=4, default_flow_style=False, Dumper=Dumper) 187 | 188 | 189 | def edit_configuration(tool_name, callback): 190 | """Wrapper for configuration editing tools 191 | """ 192 | import shutil 193 | 194 | filename = os.path.join(USER_DOTDIR, '%s.yaml' % tool_name) 195 | edit_filename = '%s.edit' % filename 196 | 197 | if os.path.exists(edit_filename): 198 | logging.warn('Unfinished edit file %s exists, editing that one.', edit_filename) 199 | else: 200 | if os.path.exists(filename): 201 | shutil.copy2(filename, edit_filename) 202 | else: 203 | with open(edit_filename, 'w') as fp: 204 | fp.write('\n') 205 | 206 | callback(edit_filename) 207 | 208 | read_configuration(tool_name, custom_suffix='.edit') # validate that configuration is still readable 209 | 210 | if os.path.exists(filename): 211 | shutil.copy2(filename, '%s.back' % filename) 212 | os.rename(edit_filename, filename) 213 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/capture/ptysnoop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Netflix, Inc. 3 | 4 | """Capture interactive unix terminal activity 5 | 6 | Basically a python reimplementation of ye old 'script' utility 7 | """ 8 | 9 | import errno 10 | import fcntl 11 | import logging 12 | import os 13 | import select 14 | import signal 15 | import struct 16 | import termios 17 | import threading 18 | import multiprocessing 19 | import time 20 | 21 | STDIN, STDOUT, STDERR = 0, 1, 2 22 | DEFAULT_BUFF_SIZE = 128 23 | WAKEUP = '!' 24 | 25 | 26 | def xwrite(fileno, data): 27 | """Write data to fileno 28 | """ 29 | offset = 0 30 | remaining = len(data) 31 | while remaining > 0: 32 | count = os.write(fileno, data[offset:]) 33 | remaining -= count 34 | offset += count 35 | 36 | 37 | class Script(object): 38 | """A python reimplementation of the script utility 39 | """ 40 | 41 | def __init__(self, recorder, buff_size=DEFAULT_BUFF_SIZE, idle_timeout=0, select_timeout=1): 42 | self.recorder = recorder 43 | self.buff_size = buff_size 44 | self.idle_timeout = idle_timeout 45 | self.select_timeout = select_timeout 46 | self.activity_stamp = time.time() 47 | self.master = None 48 | self.slave = None 49 | self.child_pid = None 50 | self.start_event = multiprocessing.Event() 51 | self.stop_event = threading.Event() 52 | self.resize_event = threading.Event() 53 | self.tcattr = None 54 | self.wakeup_r, self.wakeup_w = os.pipe() 55 | 56 | def _on_stdin_input(self): 57 | """User typing something 58 | """ 59 | data = os.read(STDIN, self.buff_size) 60 | 61 | if len(data) == 0: # EOF 62 | self.stop_event.set() 63 | else: 64 | xwrite(self.master, data) 65 | self.recorder.record_user_input(data) 66 | 67 | def _on_pty_input(self): 68 | """Some data has been displayed on screen 69 | """ 70 | data = os.read(self.master, self.buff_size) 71 | 72 | if len(data) == 0: 73 | self.stop_event.set() 74 | else: 75 | xwrite(1, data) 76 | self.recorder.record_server_output(data) 77 | 78 | def _on_wakeup(self): 79 | """Our main thread woke us up 80 | """ 81 | os.read(self.wakeup_r, 1) 82 | 83 | if self.resize_event.is_set(): 84 | self.resize_event.clear() 85 | data = fcntl.ioctl(STDIN, termios.TIOCGWINSZ, '0123') 86 | lines, columns = struct.unpack('hh', data) 87 | self.recorder.record_window_resize(lines, columns) 88 | fcntl.ioctl(self.slave, termios.TIOCSWINSZ, data) 89 | 90 | def _io_loop(self): 91 | """I/O loop (executed as a parent process thread) 92 | """ 93 | 94 | self.start_event.set() 95 | 96 | io_actions = { 97 | STDIN: self._on_stdin_input, 98 | self.master: self._on_pty_input, 99 | self.wakeup_r: self._on_wakeup, 100 | } 101 | 102 | rlist = io_actions.keys() 103 | 104 | self.recorder.start() 105 | 106 | while not self.stop_event.is_set(): 107 | try: 108 | activity = select.select(rlist, [], [], self.select_timeout)[0] 109 | except select.error as err: 110 | assert err.args[0] != errno.EINTR, 'Should not be getting interrupted syscalls in thread' 111 | raise 112 | 113 | if activity: 114 | self.activity_stamp = time.time() 115 | elif self.idle_timeout > 0: 116 | if time.time() > self.activity_stamp + self.idle_timeout: 117 | self.stop_event.set() 118 | 119 | for active_fd in activity: 120 | try: 121 | io_actions[active_fd]() 122 | except OSError as ose: 123 | assert ose.errno != errno.EINTR, 'Should not be getting interrupted syscalls in thread' 124 | raise 125 | 126 | def _on_sigchild(self, *args, **kwargs): # pylint:disable=unused-argument 127 | """SIGCHILD handler 128 | """ 129 | signal.signal(signal.SIGCHLD, self._on_sigchild) 130 | # logging.debug('Got sigchild') 131 | self.stop_event.set() 132 | os.write(self.wakeup_w, WAKEUP) 133 | 134 | def _on_sigwinch(self, *args, **kwargs): # pylint:disable=unused-argument 135 | """SIGWINCH handler 136 | """ 137 | # logging.debug('Got sigwinch') 138 | self.resize_event.set() 139 | os.write(self.wakeup_w, WAKEUP) 140 | 141 | def _run_parent(self): 142 | """Parent process main loop 143 | """ 144 | io_thread = threading.Thread(group=None, target=self._io_loop, name='ptysnoop_io_thread') 145 | io_thread.start() 146 | 147 | signal.signal(signal.SIGCHLD, self._on_sigchild) 148 | signal.signal(signal.SIGWINCH, self._on_sigwinch) 149 | 150 | while not self.stop_event.is_set(): 151 | self.stop_event.wait(10) 152 | 153 | io_thread.join() 154 | 155 | def _open_pty(self): 156 | """Create a PTY 157 | """ 158 | # get our terminal params 159 | self.tcattr = termios.tcgetattr(STDIN) 160 | winsize = fcntl.ioctl(STDIN, termios.TIOCGWINSZ, '0123') 161 | # open a pty 162 | self.master, self.slave = os.openpty() 163 | # set the slave's terminal params 164 | termios.tcsetattr(self.slave, termios.TCSANOW, self.tcattr) 165 | fcntl.ioctl(self.slave, termios.TIOCSWINSZ, winsize) 166 | 167 | def _setup_slave_pty(self): 168 | """Set suitable tty options for our pty slave 169 | """ 170 | os.setsid() 171 | fcntl.ioctl(self.slave, termios.TIOCSCTTY, 0) 172 | os.close(self.master) 173 | os.dup2(self.slave, STDIN) 174 | os.dup2(self.slave, STDOUT) 175 | os.dup2(self.slave, STDERR) # FIXME can we handle stderr better? 176 | os.close(self.slave) 177 | 178 | def _fix_tty(self): 179 | """Set suitable tty options 180 | """ 181 | assert self.tcattr is not None 182 | iflag, oflag, cflag, lflag, ispeed, ospeed, chars = self.tcattr # pylint:disable=unpacking-non-sequence 183 | # equivalent to cfmakeraw 184 | iflag &= ~(termios.IGNBRK | termios.BRKINT | termios.PARMRK | termios.ISTRIP | termios.INLCR | 185 | termios.IGNCR | termios.ICRNL | termios.IXON) 186 | oflag &= ~termios.OPOST 187 | lflag &= ~(termios.ECHO | termios.ECHONL | termios.ICANON | termios.ISIG | termios.IEXTEN) 188 | cflag &= ~(termios.CSIZE | termios.PARENB) 189 | cflag |= termios.CS8 190 | termios.tcsetattr(STDIN, termios.TCSANOW, [iflag, oflag, cflag, lflag, ispeed, ospeed, chars]) 191 | 192 | def _done(self): 193 | """Nicely close our logs, reset the terminal and exit 194 | """ 195 | os.close(self.master) 196 | termios.tcsetattr(STDIN, termios.TCSADRAIN, self.tcattr) 197 | 198 | pid, status = os.waitpid(self.child_pid, os.WNOHANG) 199 | if pid != 0: 200 | assert pid == self.child_pid 201 | exit_code = status >> 8 202 | self.recorder.end(exit_code) 203 | return exit_code 204 | else: 205 | logging.warn('waitpid(%d) returned %d %d', self.child_pid, pid, status) 206 | 207 | def run(self, callback, parent_callback, *args, **kwargs): 208 | """Setup, fork and run 209 | """ 210 | 211 | if threading.active_count() > 1: 212 | threads = threading.enumerate() 213 | logging.warning('Programming error: there are %d active threads (list follows)', len(threads)) 214 | for thread in threads: 215 | logging.warning(' - %s', thread) 216 | logging.warning('Creating threads before forking can lead to issues and should be avoided') 217 | 218 | self._open_pty() 219 | self._fix_tty() 220 | 221 | pid = os.fork() 222 | if pid == 0: 223 | # child 224 | self._setup_slave_pty() 225 | self.start_event.wait() 226 | callback(*args, **kwargs) 227 | else: 228 | # parent 229 | try: 230 | self.child_pid = pid 231 | if parent_callback: 232 | parent_callback() 233 | self._run_parent() 234 | finally: 235 | return self._done() 236 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2016 Netflix 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/lib/ssh.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2015 Netflix, Inc. 3 | 4 | """ 5 | Wrappers and helpers around paramiko, with two general goals: 6 | - concentrate dependency on paramiko in this module 7 | - support our sometimes intricate uses cases (eg. efficient bastion hopping) 8 | """ 9 | 10 | from cligraphy.core import ctx 11 | 12 | import paramiko 13 | import OpenSSL.crypto as openssl 14 | 15 | import os 16 | import logging 17 | import time 18 | 19 | DEFAULT_TIMEOUT = 5 20 | DEFAULT_COMMAND_TIMEOUT = 60 21 | 22 | def _setup(): 23 | # basically silence paramiko transport 24 | logging.getLogger('paramiko.transport').setLevel(logging.CRITICAL) 25 | 26 | 27 | _setup() 28 | 29 | 30 | class OpenSSLRSAKey(paramiko.rsakey.RSAKey): 31 | """For speed, use openssl instead of paramiko's slow pycrypto to sign our auth message""" 32 | 33 | def _from_private_key_file(self, filename, password): 34 | # paramiko key loading 35 | data = self._read_private_key_file('RSA', filename, password) 36 | self._decode_key(data) 37 | # openssl key loading 38 | with open(filename, 'rt') as fp: 39 | self._ssl_key = openssl.load_privatekey(openssl.FILETYPE_PEM, fp.read(), password) 40 | 41 | def sign_ssh_data(self, data): 42 | sig = openssl.sign(self._ssl_key, data, 'sha1') 43 | m = paramiko.message.Message() 44 | m.add_string('ssh-rsa') 45 | m.add_string(sig) 46 | return m 47 | 48 | def get_proxy_client(password=None): 49 | """Creates a new client object connected to ${ssh.proxy.host}""" 50 | return ssh_client_builder().hostname(ctx.conf.ssh.proxy.host).password(password).allow_agent().ssh_config().connect() 51 | 52 | 53 | def get_awsprod_client(password=None): 54 | """Creates a new client object connected to an awsprod bastion""" 55 | return ssh_client_builder().hostname('awsprod').password(password).allow_agent().ssh_config().connect() 56 | 57 | 58 | def get_client(proxy_client, dest, username='root', port=22, agent=True, keys=None, password=None): 59 | """Creates a new client object connected to a remove server, by hopping through a bastion host using the given proxy_client""" 60 | 61 | if proxy_client: 62 | proxy_transport = proxy_client.get_transport() 63 | fwd = proxy_transport.open_channel("direct-tcpip", (dest, port), ('127.0.0.1', 0)) 64 | else: 65 | fwd = None 66 | 67 | client = paramiko.SSHClient() 68 | client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 69 | 70 | connected = False 71 | 72 | if not connected and agent: 73 | logging.debug('Trying agent auth') 74 | try: 75 | client.connect(dest, username=username, sock=fwd, timeout=DEFAULT_TIMEOUT, allow_agent=True) 76 | connected = True 77 | except paramiko.ssh_exception.SSHException: 78 | logging.debug('Could not auth using agent') 79 | 80 | 81 | if not connected and keys: 82 | logging.debug('Trying key auth') 83 | # FIXME: here we reconnect for each key attempt; don't do that and use the same connection instead. 84 | for key in keys: 85 | try: 86 | client.connect(dest, username=username, sock=fwd, timeout=DEFAULT_TIMEOUT, allow_agent=False, pkey=key) 87 | connected = True 88 | break 89 | except paramiko.ssh_exception.SSHException: 90 | continue 91 | 92 | if not connected and password: 93 | logging.debug('Trying password auth') 94 | try: 95 | client.connect(dest, username=username, sock=fwd, timeout=DEFAULT_TIMEOUT, allow_agent=False, password=password) 96 | connected = True 97 | except paramiko.ssh_exception.SSHException: 98 | logging.debug('Could not auth using password') 99 | 100 | if not connected: 101 | raise Exception('Could not authenticate') 102 | 103 | return client 104 | 105 | 106 | class _SSHClientBuilder(object): 107 | 108 | def __init__(self): 109 | super(_SSHClientBuilder, self).__init__() 110 | self._hostname = None 111 | self._port = 22 112 | self._username = None 113 | self._password = None 114 | self._config = paramiko.SSHConfig() 115 | self._client_class = paramiko.SSHClient 116 | self._missing_host_key_policy = paramiko.AutoAddPolicy 117 | self._timeout = DEFAULT_TIMEOUT 118 | self._banner_timeout = DEFAULT_TIMEOUT 119 | self._allow_agent = None 120 | self._proxy_command = None 121 | self._sock = None 122 | 123 | def ssh_config(self, filename='~/.ssh/config'): 124 | """Parse the given ssh config file. Optional. 125 | No config file is parsed by default, but if this is called without a filename, ~/.ssh/config will be parsed.""" 126 | absolute_filename = os.path.expanduser(filename) 127 | if not os.path.exists(absolute_filename): 128 | logging.warning('ssh config file %s does not exist, skipping it', absolute_filename) 129 | return self 130 | logging.debug('Reading ssh configuration from %s', absolute_filename) 131 | with open(absolute_filename, 'r') as fpin: 132 | self._config.parse(fpin) 133 | return self 134 | 135 | def client(self, client_class): 136 | """Sets the client class to build with. Optional, defaults to paramiko.SSHClient""" 137 | logging.debug('Setting client class to %s', client_class) 138 | self._client_class = client_class 139 | return self 140 | 141 | def missing_host_key_policy(self, policy_class): 142 | """Set the client class to build with. Optional, defaults to paramiko.AutoAddPolicy""" 143 | logging.debug('Setting missing host key policy to %s', policy_class) 144 | self._missing_host_key_policy = policy 145 | return self 146 | 147 | def hostname(self, hostname): 148 | """Set the destination hostname. Mandatory""" 149 | logging.debug('Setting hostname to %s', hostname) 150 | self._hostname = hostname 151 | return self 152 | 153 | def username(self, username): 154 | """Set the username. Optional, defaults to $USER""" 155 | logging.debug('Setting username to %s', username) 156 | self._username = username 157 | return self 158 | 159 | def password(self, password): 160 | """Set a password. Optional, no default""" 161 | logging.debug('Setting password (password not shown here for security reasons)') 162 | self._password = password 163 | return self 164 | 165 | def port(self, port): 166 | """Set the destination port number. Optional, defaults to 22""" 167 | logging.debug('Setting port to %d', port) 168 | self._port = port 169 | return self 170 | 171 | def timeout(self, timeout=5, banner_timeout=5): 172 | logging.debug('Setting timeouts: timeout=%d, banner_timeout=%d', timeout, banner_timeout) 173 | self._timeout = timeout 174 | self._banner_timeout = banner_timeout 175 | return self 176 | 177 | def allow_agent(self, allow_agent=True): 178 | """Allow or disallow ssh agent. Optional. Defaults to true""" 179 | self._allow_agent = allow_agent 180 | return self 181 | 182 | def proxy(self, proxy_command): 183 | self._proxy_command = proxy_command 184 | return self 185 | 186 | def sock(self, sock): 187 | self._sock = sock 188 | return self 189 | 190 | def connect(self): 191 | """Finish building the client and connects to the target server, returning a paramiko client object""" 192 | assert self._client_class 193 | assert self._hostname is not None, 'destination hostname was not specified' 194 | client = self._client_class() 195 | if self._missing_host_key_policy: 196 | client.set_missing_host_key_policy(self._missing_host_key_policy()) 197 | 198 | config_data = self._config.lookup(self._hostname) 199 | ssh_kwargs = { 200 | 'timeout': self._timeout, 201 | 'banner_timeout': self._banner_timeout, 202 | 'port': self._port 203 | } 204 | 205 | # unless one is explicitely specified with .user(), get our username from configuration, defaulting to $USER 206 | if self._username is None: 207 | ssh_kwargs['username'] = config_data.get('user', os.getenv('USER')) 208 | else: 209 | ssh_kwargs['username'] = self._username 210 | 211 | if self._password is not None: 212 | ssh_kwargs['password'] = self._password 213 | 214 | if 'proxycommand' in config_data: 215 | ssh_kwargs['sock'] = paramiko.ProxyCommand(config_data['proxycommand']) 216 | elif self._proxy_command is not None: 217 | ssh_kwargs['sock'] = paramiko.ProxyCommand(self._proxy_command) 218 | 219 | if config_data.get('identity_file') is not None: 220 | ssh_kwargs['key_filename'] = config_data.get('identity_file') 221 | 222 | # unless explicitely specified with .allow_agent, allow agent by default unless identitiesonly is yes in our config 223 | if self._allow_agent is None: 224 | ssh_kwargs['allow_agent'] = config_data.get('identitiesonly', 'no') != 'yes' 225 | else: 226 | ssh_kwargs['allow_agent'] = self._allow_agent 227 | 228 | if self._sock is not None: 229 | ssh_kwargs['sock'] = self._sock 230 | 231 | logging.debug('Connecting to %s with options %s', config_data['hostname'], ssh_kwargs) 232 | client.connect(config_data['hostname'], **ssh_kwargs) 233 | return client 234 | 235 | def ssh_client_builder(): 236 | return _SSHClientBuilder() 237 | 238 | 239 | class TaskException(Exception): 240 | pass 241 | 242 | class ConnectException(TaskException): 243 | pass 244 | 245 | class CommandException(TaskException): 246 | pass 247 | 248 | 249 | def exec_command(client, command, command_timeout=DEFAULT_COMMAND_TIMEOUT, 250 | feed_data=None, get_files=[], put_files=[], 251 | io_step_s=0.1, io_chunk_size=1024, 252 | out=None, err=None): 253 | """Execute a command on the given client, grabbing stdout/stderr and the exit status code""" 254 | transport = client.get_transport() 255 | channel = transport.open_session() 256 | sftp = None 257 | if get_files or put_files: 258 | sftp = paramiko.SFTPClient.from_transport(transport) 259 | sftp.get_channel().settimeout(command_timeout) 260 | 261 | try: 262 | if put_files: 263 | for path, content in put_files.items(): 264 | with sftp.open(path, 'wb') as remote_fp: 265 | remote_fp.write(content) 266 | 267 | channel.exec_command(command) 268 | if feed_data is not None: 269 | channel.settimeout(command_timeout) 270 | channel.sendall(feed_data) 271 | 272 | channel.shutdown_write() 273 | channel.settimeout(io_step_s) 274 | out = out if out is not None else [] 275 | err = err if err is not None else [] 276 | deadline = time.time() + command_timeout 277 | while time.time() <= deadline and not channel.exit_status_ready(): 278 | ready = channel.recv_ready(), channel.recv_stderr_ready() 279 | if not ready[0] and not ready[1]: 280 | time.sleep(io_step_s) 281 | continue 282 | if ready[0]: 283 | out.append(channel.recv(io_chunk_size)) 284 | if ready[1]: 285 | err.append(channel.recv_stderr(io_chunk_size)) 286 | 287 | if not channel.exit_status_ready(): 288 | raise CommandException("Timeout during command execution") 289 | 290 | while channel.recv_ready(): 291 | out.append(channel.recv(io_chunk_size)) 292 | while channel.recv_stderr_ready(): 293 | out.append(channel.recv_stderr(io_chunk_size)) 294 | 295 | status = channel.recv_exit_status() 296 | 297 | get_files_result = {} 298 | if get_files: 299 | for path in get_files: 300 | with sftp.open(path, 'rb') as remote_fp: 301 | get_files_result[path] = remote_fp.read() 302 | 303 | return out, err, status, get_files_result 304 | finally: 305 | if sftp: 306 | sftp.close() 307 | channel.close() 308 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/parsers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013, 2104 Netflix, Inc. 3 | 4 | 5 | import argparse 6 | import copy 7 | import functools 8 | import imp 9 | import importlib 10 | import json 11 | import logging 12 | import os 13 | import os.path 14 | import pkgutil 15 | import sys 16 | import time 17 | from contextlib import contextmanager 18 | 19 | 20 | try: 21 | import gevent.monkey as gevent_monkey 22 | except ImportError: 23 | gevent_monkey = None 24 | 25 | NO_HELP = 'No help :(' 26 | 27 | FUZZY_PARSED = [] 28 | RECENT_SUB_PARSERS = [] 29 | UNAVAILABLE_MODULES = [] 30 | 31 | 32 | class CustomDescriptionFormatter(argparse.RawTextHelpFormatter): 33 | 34 | def _get_help_string(self, action): 35 | help = action.help 36 | if '(default' not in help and type(action) not in (argparse._StoreConstAction, argparse._StoreTrueAction, argparse._StoreFalseAction): 37 | if action.default is not argparse.SUPPRESS: 38 | defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE] 39 | if action.option_strings or action.nargs in defaulting_nargs: 40 | help += ' (default: %(default)s)' 41 | return help 42 | 43 | 44 | class ParserError(Exception): 45 | 46 | def __init__(self, parser, message): 47 | self.parser = parser 48 | self.message = message 49 | 50 | def report(self, force_message=None): 51 | if RECENT_SUB_PARSERS: 52 | RECENT_SUB_PARSERS[-1].print_help(sys.stderr) 53 | else: 54 | self.parser.print_help(sys.stderr) 55 | sys.stderr.write('\n') 56 | if force_message: 57 | sys.stderr.write(force_message) 58 | else: 59 | sys.stderr.write('%s: error: %s\n' % (self.parser.prog, self.message)) 60 | sys.exit(2) 61 | 62 | 63 | def split_args(args): 64 | """Split command line args in 3 groups: 65 | - head, containing the initial options 66 | - body, containing everything after head, up to the first option 67 | - tail, containing everything after body 68 | """ 69 | 70 | head = [] 71 | body = [] 72 | tail = [] 73 | for arg in args: 74 | if arg.startswith('-'): 75 | if body: 76 | tail.append(arg) 77 | else: 78 | head.append(arg) 79 | else: 80 | if tail: 81 | tail.append(arg) 82 | else: 83 | body.append(arg) 84 | return head, body, tail 85 | 86 | 87 | def attempt_fuzzy_matching(args, candidates): 88 | head, body, tail = split_args(args) 89 | 90 | if not body: 91 | return None, None 92 | 93 | import re 94 | matches = [] 95 | 96 | while body: 97 | partial_body = body 98 | fuzzy_body = '.*'.join(partial_body) 99 | for candidate in candidates: 100 | if re.search(fuzzy_body, candidate): 101 | matches.append(candidate) 102 | if matches: 103 | break 104 | tail.insert(0, body.pop()) 105 | 106 | if not matches: 107 | logging.debug('No fuzzy matches for command line args %s', args) 108 | return None, matches 109 | 110 | if len(matches) > 1: 111 | logging.debug('Multiple fuzzy matches for command line args %s: %s', args, matches) 112 | return None, matches 113 | 114 | new_args = [] 115 | new_args.extend(head) 116 | new_args.extend(matches[0].split(' ')) 117 | new_args.extend(tail) 118 | return new_args, matches 119 | 120 | 121 | class BaseParser(argparse.ArgumentParser): 122 | 123 | def __init__(self, *args, **kwargs): 124 | argparse.ArgumentParser.__init__(self, *args, **kwargs) 125 | 126 | def error(self, message): 127 | raise ParserError(self, message) 128 | 129 | def parse_known_args(self, args=None, namespace=None): 130 | RECENT_SUB_PARSERS.append(self) 131 | return argparse.ArgumentParser.parse_known_args(self, args, namespace) 132 | 133 | def _check_value(self, action, value): 134 | if action.choices is not None and value not in action.choices: # special handling for a nicer "choice" error message 135 | available = ', '.join(map(str, action.choices)) 136 | if len(available) > 20: 137 | available = '\n%s' % available 138 | msg = 'Invalid choice [%s], choose from: %s' % (value, available) 139 | raise argparse.ArgumentError(action, msg) 140 | else: 141 | super(BaseParser, self)._check_value(action, value) 142 | 143 | 144 | class SmartCommandMapParser(BaseParser): 145 | 146 | def __init__(self, *args, **kwargs): 147 | super(SmartCommandMapParser, self).__init__(*args, **kwargs) 148 | self.root_sub = self.add_subparsers(help='Available sub-commands', parser_class=BaseParser) 149 | self.sub_map = {'': self.root_sub} 150 | self.flat_map = {} 151 | 152 | def add_namespace(self, namespace, parent): 153 | desc = '%s sub-command group' % namespace.capitalize() 154 | temp = parent.add_parser(namespace, 155 | help=desc, 156 | description=desc, 157 | formatter_class=CustomDescriptionFormatter, 158 | add_help=False) 159 | temp.add_argument('-h', '--help', dest='_help', action=argparse._HelpAction) 160 | sub = temp.add_subparsers(help='Available sub-commands', parser_class=BaseParser) 161 | self.sub_map[namespace] = sub 162 | return sub 163 | 164 | def add_item(self, subparser, module_name, item, command_path): 165 | name, node = item 166 | if node.get('type') == 'cmd': 167 | parser = subparser.add_parser(name, 168 | help=node.get('help'), 169 | description=node.get('desc', node.get('help')), 170 | formatter_class=CustomDescriptionFormatter, 171 | add_help=False) 172 | if node.get('error'): 173 | def _func(*args, **kwargs): 174 | logging.error('This command is unavailable: %s', node.get('desc')) 175 | logging.error('NB: after fixing the issue, remember to run oc refresh again') 176 | sys.exit(1) 177 | parser.set_defaults(_func=_func) 178 | else: 179 | parser.set_defaults(_func=functools.partial(finish_parser, copy.copy(parser), module_name + '.' + name)) 180 | self.flat_map[(command_path + ' ' + name).strip()] = module_name + '.' + name 181 | else: 182 | sub = self.add_namespace(name, subparser) 183 | for sub_node in node.iteritems(): 184 | self.add_item(sub, module_name + '.' + name, sub_node, command_path + ' ' + name) 185 | 186 | def add_command_map(self, namespace, command_map): 187 | sub = self.sub_map.get(namespace, None) 188 | if sub is None: 189 | sub = self.add_namespace(namespace, self.root_sub) 190 | 191 | module_name = command_map['module'] 192 | for item in command_map['commands'].iteritems(): 193 | self.add_item(sub, module_name, item, namespace) 194 | 195 | def pre_parse_args(self, args): 196 | """Perform our first pass parsing of the given command line arguments. 197 | Try fuzzy matching if we can't parse the command line as is. 198 | Return the final args (eg. possibly corrected after fuzzy matching) and the actual command function to be executed. 199 | """ 200 | try: 201 | parsed_args, _ = self.parse_known_args(args) 202 | except ParserError as pe: 203 | # if we get 'too few arguments here', user is referencing a command group, and we shouldn't fuzzy match 204 | if pe.message == 'too few arguments': 205 | pe.report() 206 | logging.debug('Could not parse command line args [%s], attempting fuzzy matching', args) 207 | fixed_args, matches = attempt_fuzzy_matching(args, self.flat_map.keys()) 208 | if fixed_args: 209 | logging.info('Your input "%s" matches "%s"', ' '.join(args), ' '.join(fixed_args)) 210 | FUZZY_PARSED.append((' '.join(args), ' '.join(fixed_args))) 211 | args = fixed_args 212 | parsed_args, _ = self.parse_known_args(args) 213 | elif matches: 214 | message = 'Your command line matched the following existing commands:\n %s\n' % ('\n '.join(matches)) 215 | pe.report(force_message=message) 216 | else: 217 | pe.report() 218 | 219 | return args, parsed_args._func() 220 | 221 | def parse_args(self, args=None): 222 | if args is None: 223 | args = sys.argv[1:] 224 | fixed_args, _ = self.pre_parse_args(args) 225 | return super(SmartCommandMapParser, self).parse_args(fixed_args) 226 | 227 | 228 | def detect_monkey_patch(): 229 | return gevent_monkey and len(gevent_monkey.saved) > 0 230 | 231 | 232 | def error_module(module_name, exc): 233 | message = '%s: %s' % (exc.__class__.__name__, exc) 234 | UNAVAILABLE_MODULES.append((module_name, message)) 235 | mod = imp.new_module(module_name) 236 | mod.__file__ = '__%s_error_stub__' % module_name 237 | mod.__doc__ = message 238 | mod.__error__ = True 239 | return mod 240 | 241 | @contextmanager 242 | def import_time_reporting(module_name): 243 | start = time.time() 244 | yield 245 | elapsed = time.time() - start 246 | if elapsed > 0.5: 247 | logging.info('slow import: module %s took %.2f seconds' % (module_name, elapsed)) 248 | 249 | 250 | def find_command_modules(prefix, path): 251 | """Returns a list of all command modules and packages""" 252 | prefix = '%s.' % (prefix) 253 | for loader, module_name, is_pkg in pkgutil.iter_modules(path, prefix=prefix): 254 | try: 255 | with import_time_reporting(module_name): 256 | mod = importlib.import_module(module_name) 257 | 258 | if detect_monkey_patch(): 259 | logging.error('gevent monkey patching detected after module %s was loaded', module_name) 260 | raise Exception('monkey patching is not allowed in oc command modules') 261 | 262 | if getattr(mod, 'main', None): 263 | yield mod 264 | 265 | if is_pkg: 266 | with import_time_reporting(module_name): 267 | pkg = importlib.import_module(module_name) 268 | for mod in find_command_modules(pkg.__name__, pkg.__path__): 269 | yield mod 270 | except KeyboardInterrupt: 271 | raise 272 | except ImportError as ie: 273 | yield error_module(module_name, ie) 274 | except SyntaxError as se: 275 | yield error_module(module_name, se) 276 | except BaseException as be: 277 | yield error_module(module_name, be) 278 | 279 | 280 | def finish_parser(parser, module_name): 281 | logging.debug('Build actual parser for module %s', module_name) 282 | module = importlib.import_module(module_name) 283 | 284 | if hasattr(module, 'configure'): 285 | module.configure(parser) 286 | 287 | parser.set_defaults(_func=module.main) 288 | parser.add_argument('-h', '--help', dest='_help', action=argparse._HelpAction) 289 | return module.main 290 | 291 | 292 | class AutoDiscoveryCommandMap(object): 293 | """Automatically builds a commands map for our oc sub commands""" 294 | 295 | def __init__(self, cligraph, root_module_name): 296 | self.cligraph = cligraph 297 | self.root_node = {} 298 | self.package_nodes = {'': self.root_node} 299 | self.root_module_name = root_module_name 300 | 301 | def parse_help(self, module): 302 | """Parse docstring to generate usage""" 303 | doc = getattr(module, '__doc__', None) 304 | if doc is None: 305 | return NO_HELP, 'No description provided, you could add one! Code is probably located here: %s' % (module.__file__.replace('pyc', 'py')) 306 | else: 307 | halp, _, desc = doc.strip().replace('%', '%%').partition('\n') 308 | halp = halp.strip() 309 | if not desc: 310 | desc = halp 311 | return halp, desc 312 | 313 | def get_node(self, package_name): 314 | """Get or create a sub node""" 315 | logging.debug('Looking for package node [%s]', package_name) 316 | complete_package_name = package_name 317 | sub = self.package_nodes.get(package_name, None) 318 | if sub is None: 319 | logging.debug('-- not found') 320 | if '.' in package_name: 321 | parent_name, _, package_name = package_name.partition('.') 322 | parent = self.get_node(parent_name) 323 | else: 324 | parent = self.root_node 325 | 326 | sub = parent[package_name] = self.package_nodes[complete_package_name] = {} 327 | return sub 328 | 329 | def build(self, force_autodiscover=False): 330 | root_module = importlib.import_module(self.root_module_name) 331 | 332 | cached_command_map_filename = os.getenv('%s_COMMANDS_CACHE' % (self.cligraph.conf.tool.shortname.upper()), 333 | os.path.join(self.cligraph.conf.user.dotdir, 'commands.json')) 334 | if not force_autodiscover and os.path.exists(cached_command_map_filename): 335 | try: 336 | with open(cached_command_map_filename) as fpin: 337 | command_map = json.load(fpin) 338 | return command_map 339 | except ValueError: 340 | logging.warning("Could not parse existing commands cache %s, ignoring it", cached_command_map_filename) 341 | 342 | for module in find_command_modules(self.root_module_name, root_module.__path__): 343 | package_name, _, module_name = module.__name__.rpartition('.') 344 | assert package_name.startswith(self.root_module_name) 345 | package_name = package_name[len(self.root_module_name)+1:] 346 | logging.debug('Configuring parser for %s.%s', package_name, module_name) 347 | halp, desc = self.parse_help(module) 348 | data = {'type': 'cmd', 'help': halp, 'desc': desc} 349 | if getattr(module, '__error__', False): 350 | data['error'] = True 351 | self.get_node(package_name)[module_name] = data 352 | 353 | if UNAVAILABLE_MODULES: 354 | logging.warning('The following modules are not available:') 355 | for name, msg in UNAVAILABLE_MODULES: 356 | logging.warning(' %s: %s', name, msg) 357 | 358 | command_map = {'module': self.root_module_name, 'commands': self.root_node} 359 | 360 | if os.access(os.path.dirname(cached_command_map_filename), os.W_OK) and (not os.path.exists(cached_command_map_filename) 361 | or os.access(cached_command_map_filename, os.W_OK)): 362 | logging.debug('Writing command map json to %s', cached_command_map_filename) 363 | 364 | cached_command_map_filename_new = cached_command_map_filename + '.new' 365 | with open(cached_command_map_filename_new, 'w') as fpout: 366 | json.dump(command_map, fpout, indent=4) 367 | os.rename(cached_command_map_filename_new, cached_command_map_filename) 368 | else: 369 | logging.warning('Not updating commands cache (%s is not writeable)', cached_command_map_filename) 370 | logging.warning('Tip: are you using a shared install of octools? If so, no need to run oc refresh.') 371 | 372 | return command_map 373 | -------------------------------------------------------------------------------- /src/python/cligraphy/core/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 2013 Netflix, Inc. 4 | 5 | 6 | """Command line tools entry point.""" 7 | 8 | from cligraphy.core import capture, decorators, read_configuration, ctx 9 | from cligraphy.core.log import setup_logging 10 | from cligraphy.core.parsers import AutoDiscoveryCommandMap, SmartCommandMapParser, ParserError, CustomDescriptionFormatter 11 | from cligraphy.core.reporting import ToolsPadReporter, NoopReporter 12 | from cligraphy.core.util import undecorate_func, pdb_wrapper, profiling_wrapper, call_chain 13 | 14 | from remember.memoize import memoize 15 | 16 | import faulthandler 17 | from setproctitle import setproctitle # pylint:disable=no-name-in-module 18 | 19 | import argcomplete 20 | import argparse 21 | import logging 22 | import logging.config 23 | import os 24 | import signal 25 | import subprocess 26 | import sys 27 | 28 | 29 | def _warn_about_bad_non_ascii_chars(args): 30 | """Detect non-ascii variants of some interesting characters, such as — instead of -""" 31 | bad_chars = ( 32 | u'—', 33 | u'…', 34 | u'“', 35 | u'”', 36 | u'\u200b', # zero-width space 37 | ) 38 | try: 39 | line = ' '.join(arg.decode(sys.stdout.encoding or 'UTF-8') for arg in args) 40 | bad = [ char in bad_chars for char in line ] 41 | if any(bad): 42 | logging.warning('Your command line contains %d bad unicode character(s), did you copy/paste from a tool that garbles text?', len(bad)) 43 | logging.warning('> ' + line) 44 | logging.warning('> ' + ''.join('^' if isbad else ' ' for isbad in bad)) 45 | return 46 | except Exception: # pylint:disable=broad-except 47 | logging.warning('Exception while trying to detect bad unicode characters in command line args; continuing...', exc_info=True) 48 | 49 | 50 | def _warn_about_bad_path(env_root, path): 51 | """Warn about mistakes in $PATH""" 52 | if not env_root: 53 | logging.warning('Running oc outside of its virtualenv is supported, but untested. Please report any bugs!') 54 | return 55 | env_bin = os.path.join(env_root, 'bin') 56 | elements = path.split(':') 57 | try: 58 | position = elements.index(env_bin) 59 | if position != 0: 60 | logging.warning('Your oc virtualenv, %s, is not the first element of your $PATH. Things might be broken.', env_root) 61 | except ValueError: 62 | logging.warning('Your oc virtualenv, %s, is not listed in your $PATH. Things are likely broken.', env_root) 63 | return 64 | 65 | 66 | class _VersionAction(argparse.Action): 67 | """Shows last commit information""" 68 | def __call__(self, *args, **kwargs): 69 | try: 70 | os.chdir(ctx.conf.tool.repo_path) 71 | last_commit = subprocess.check_output(['git', 'log', '-1'], stderr=subprocess.PIPE) 72 | except (subprocess.CalledProcessError, OSError): 73 | last_commit = '(could not get more recent commit information)' 74 | print '%s v%s\n%s' % (ctx.conf.tool.name, ctx.conf.tool.version, last_commit) 75 | sys.exit(1) 76 | 77 | 78 | class Cligraph(object): 79 | reporter_cls = ToolsPadReporter 80 | 81 | def __init__(self, name, shortname, path): 82 | assert name 83 | assert shortname 84 | assert path 85 | self.tool_name = name 86 | self.tool_shortname = shortname 87 | self.tool_path = path 88 | self.conf, self.conf_layers = read_configuration(self) 89 | ctx.cligraph = self 90 | self.reporter = None 91 | 92 | def setup_reporter(self, args): 93 | """ 94 | Setups up the reporter using ``self.reporter_cls`` class variable. 95 | NOTE: A reporter must be configured for Cligraphy to operate properly. 96 | If reporting is disabled via the config, a :class:`NoopReporter` is used instead. 97 | 98 | :param args: The parsed arguments for the command 99 | :type args: :class:`argparse.Namespace` 100 | :return: None 101 | """ 102 | if self.conf.report.enabled: 103 | self.reporter = self.reporter_cls(self) 104 | else: 105 | self.reporter = NoopReporter() 106 | 107 | # FIXME(stf/oss): header names for octools 108 | def _setup_requests_audit_headers(self, command): 109 | """Setup requests user-agent and x-user/x-app headers. 110 | Just best effort - we don't care that much if this fails""" 111 | try: 112 | import requests 113 | def _default_user_agent(*args): 114 | return 'requests (cligraphy/%s)' % command 115 | requests.utils.default_user_agent = _default_user_agent 116 | 117 | base_default_headers = requests.utils.default_headers 118 | def _default_headers(*args): 119 | headers = base_default_headers() 120 | headers['X-User'] = os.getenv('USER') 121 | headers['X-App'] = 'cligraphy/%s' % command 122 | return headers 123 | requests.utils.default_headers = _default_headers 124 | requests.sessions.default_headers = _default_headers 125 | except: 126 | logging.warn("Could set up requests audit headers, continuing anyway") 127 | pass 128 | 129 | # pylint:disable=protected-access 130 | def _run(self, args): 131 | """Run command by calling the main() function correctly (how we pass args depends on its actual signature).""" 132 | # if the main method has been decorated, we need to look at the original function's argspec (but call the wrapper) 133 | import inspect 134 | orig_func, _ = undecorate_func(args._func) 135 | argspec = inspect.getargspec(orig_func) 136 | if len(argspec.args) == 0: 137 | return args._func() 138 | elif argspec.varargs or argspec.keywords or len(argspec.args) > 1: 139 | kwargs = dict(**vars(args)) 140 | for kw in kwargs.keys(): 141 | if kw.startswith('_'): 142 | logging.debug('removing internal arg %s', kw) 143 | del kwargs[kw] 144 | return args._func(**kwargs) 145 | else: 146 | if argspec.args[0] != 'args': 147 | raise Exception('Programming error in command: if main() only has one argument it must be called "args"') 148 | func = args._func 149 | for kw in vars(args).keys(): 150 | if kw.startswith('_') and kw not in ('_parser', '_cligraph'): #FIXME(stf/oss) maybe just expose cligraph 151 | logging.debug('removing internal arg %s', kw) 152 | delattr(args, kw) 153 | return func(args) 154 | 155 | def _run_command_process(self, args): 156 | """Command (child) process entry point. args contains the function to execute and all arguments.""" 157 | 158 | setup_logging(args._level) 159 | 160 | command = ' '.join(sys.argv[1:]) 161 | setproctitle('oc/command/%s' % command) 162 | faulthandler.register(signal.SIGUSR2, all_threads=True, chain=False) # pylint:disable=no-member 163 | self._setup_requests_audit_headers(command) 164 | 165 | ret = 1 166 | try: 167 | chain = [self._run] 168 | if args._profile: 169 | chain.append(profiling_wrapper) 170 | if args._pdb: 171 | chain.append(pdb_wrapper) 172 | ret = call_chain(chain, args) 173 | except SystemExit as exc: 174 | ret = exc.code 175 | except ParserError as pe: 176 | pe.report() 177 | except Exception: # pylint:disable=broad-except 178 | logging.exception('Top level exception in command process') 179 | finally: 180 | sys.exit(ret) 181 | 182 | 183 | def _parse_args(self): 184 | """Setup parser and parse cli arguments. 185 | NB! counter-intuitively, this function also messes around with logging levels. 186 | """ 187 | 188 | # We want some of our options to take effect as early as possible, as they affect command line parsing. 189 | # For these options we resort to some ugly, basic argv spotting 190 | 191 | if '--debug' in sys.argv: 192 | logging.getLogger().setLevel(logging.DEBUG) 193 | logging.debug('Early debug enabled') 194 | 195 | if '--verbose' in sys.argv or '-v' in sys.argv: 196 | logging.getLogger().setLevel(logging.INFO) 197 | 198 | autodiscover = False 199 | if '--autodiscover' in sys.argv: 200 | logging.debug('Autodiscover enabled') 201 | autodiscover = True 202 | 203 | parser = SmartCommandMapParser(prog=self.tool_shortname, 204 | description="Cligraphy command line tools", 205 | formatter_class=CustomDescriptionFormatter) 206 | 207 | self.parser = parser # expose to eg. ctx 208 | 209 | parser.add_argument('--version', action=_VersionAction, nargs=0, dest="_version") 210 | parser.add_argument("--debug", help="enable debuging output", dest="_level", action="store_const", const=logging.DEBUG) 211 | parser.add_argument("--pdb", help="run pdb on exceptions", dest="_pdb", action="store_true") 212 | parser.add_argument("--no-capture", help="(DEPRECATED) disable input/output capture", dest="_capture_deprecated", action="store_false", default=True) # DEPRECATED; left behind to avoid breaking existing references 213 | parser.add_argument("--enable-capture", help="enable input/output capture", dest="_capture", action="store_true", default=False) 214 | parser.add_argument("--no-reporting", help="disable reporting", dest="_reporting", action="store_false", default=True) 215 | parser.add_argument("--profile", help="enable profiling", dest="_profile", action="store_true", default=False) 216 | parser.add_argument("--autodiscover", help="re-discover commands and refresh cache (default: read cached commands list)", dest="_autodiscover", action="store_true") 217 | parser.add_argument("-v", "--verbose", help="enable informational output", dest="_level", action="store_const", const=logging.INFO) 218 | 219 | for namespace, command_map in self.get_command_maps(autodiscover): 220 | parser.add_command_map(namespace, command_map) 221 | 222 | argcomplete.autocomplete(parser) 223 | 224 | _warn_about_bad_non_ascii_chars(sys.argv) 225 | _warn_about_bad_path(os.getenv('VIRTUAL_ENV'), os.getenv('PATH')) 226 | 227 | args = parser.parse_args() 228 | args._parser = parser # deprecated 229 | 230 | # pylint:disable=protected-access 231 | if args._level is not None: 232 | logging.getLogger().setLevel(args._level) 233 | 234 | return args 235 | 236 | @memoize() 237 | def get_command_maps(self, autodiscover=False): 238 | """Get all the command maps defined in our configuration. 239 | 240 | If autodiscover is True (defaults to False), python commands will be autodiscovered (instead of simply being obtained 241 | from a cached command map). 242 | 243 | :param autodiscover: (default - `False`) Whether or not to autodiscover commands 244 | :type autodiscover: bool 245 | :return: A dictionary containing the command map 246 | :rtype: dict 247 | """ 248 | 249 | result = [] 250 | for module, options in self.conf.commands.items(): 251 | try: 252 | if options is None: 253 | options = {} 254 | logging.debug('Configuring commands module %s with options %s', module, options) 255 | 256 | opt_type = options.get('type', 'python') 257 | opt_namespace = options.get('namespace', '') 258 | 259 | if opt_type == 'python': 260 | result.append((opt_namespace, AutoDiscoveryCommandMap(self, module).build(force_autodiscover=autodiscover))) 261 | else: 262 | raise Exception('Dont know how to handle commands module with type [%s]', opt_type) 263 | except Exception as exc: # pylint:disable=broad-except 264 | logging.warning('Could not configure commands module [%s] defined in configuration: %s. Skipping it.', module, exc, 265 | exc_info=True) 266 | 267 | return result 268 | 269 | def before_command_start(self, args, recorder): 270 | """Called after the args have been parsed but before the command starts 271 | Reports command start using the configured reporter, sets the process 272 | title to ``oc/parent/join script args``, and registers a fault handler 273 | 274 | :param args: The parsed args the command was called with 275 | :type args: :class:`argparse.Namespace` 276 | :param recorder: The output recorder being used to capture command output 277 | :type recorder: :class:`capture.Recorder` 278 | :rtype: None 279 | """ 280 | # reporting: we send command executions report to a web service; 281 | # unless the report.enabled conf key is false 282 | self.reporter.report_command_start(sys.argv) 283 | 284 | setproctitle('oc/parent/%s' % ' '.join(sys.argv[1:])) 285 | faulthandler.register(signal.SIGUSR2, all_threads=True, chain=False) # pylint:disable=no-member 286 | 287 | def after_command_finish(self, args, recorder, status): 288 | """Called after a command has finished running. 289 | Reports command exit and output using the configured reporter. 290 | 291 | :param args: The parsed args the command was called with 292 | :type args: :class:`argparse.Namespace` 293 | :param recorder: The output recorder being used to capture command output 294 | :type recorder: :class:`capture.Recorder` 295 | :param status: The exit status code of the command run 296 | :type status: int 297 | :rtype: None 298 | """ 299 | # report execution details 300 | self.reporter.report_command_exit(status) 301 | self.reporter.report_command_output(recorder.output_as_string()) # TODO(stefan): also report total output size 302 | self.reporter.stop() 303 | 304 | def main(self): 305 | """Main oc wrapper entry point.""" 306 | 307 | setup_logging() 308 | 309 | try: 310 | args = self._parse_args() 311 | logging.debug("Parsed args: %r", vars(args)) 312 | except ParserError as pe: 313 | pe.report() 314 | 315 | if not os.isatty(sys.stdin.fileno()) or not os.isatty(sys.stdout.fileno()): 316 | logging.info('stdin or stdout is not a tty, disabling capture') 317 | args._capture = False 318 | 319 | self.setup_reporter(args) 320 | 321 | decs = decorators.get_tags(args._func) 322 | logging.debug("Decorator tags: %s", decs) 323 | 324 | if decorators.Tag.interactive in decs or not args._capture: 325 | recorder = capture.NoopOutputRecorder() 326 | else: 327 | recorder = capture.BufferingOutputRecorder(max_output_size=self.conf.report.max_output_size) 328 | 329 | self.before_command_start(args, recorder) 330 | 331 | if args._capture: 332 | # go ahead and run out command in a child process, recording all I/O 333 | logging.debug('Parent process %d ready to execute command process', os.getpid()) 334 | status = capture.spawn_and_record(recorder, self._run_command_process, 335 | self.reporter.start, args) 336 | logging.debug('Command process exited with status %r', status) 337 | else: 338 | self.reporter.start() 339 | try: 340 | self._run_command_process(args) 341 | except SystemExit as exc: 342 | status = exc.code 343 | 344 | self.after_command_finish(args, recorder, status) 345 | 346 | return status 347 | --------------------------------------------------------------------------------