├── .circleci └── config.yml ├── .gitattributes ├── .github └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── AUTHORS ├── CHANGELOG.md ├── DESIGN.md ├── LICENSE.txt ├── Makefile ├── README-v2.md ├── README.md ├── RELEASE_ENGINEERING.md ├── SPECS └── empty.spec ├── Version2-Ideas.md ├── bin ├── Makefile ├── _blackbox_common.sh ├── _blackbox_common_test.sh ├── _stack_lib.sh ├── blackbox_addadmin ├── blackbox_cat ├── blackbox_decrypt_all_files ├── blackbox_decrypt_file ├── blackbox_deregister_file ├── blackbox_diff ├── blackbox_edit ├── blackbox_edit_end ├── blackbox_edit_start ├── blackbox_initialize ├── blackbox_list_admins ├── blackbox_list_files ├── blackbox_listadmins ├── blackbox_postdeploy ├── blackbox_recurse ├── blackbox_register_new_file ├── blackbox_removeadmin ├── blackbox_shred_all_files ├── blackbox_update_all_files ├── blackbox_view └── blackbox_whatsnew ├── binv2 ├── blackbox_addadmin ├── blackbox_cat ├── blackbox_decrypt_all_files ├── blackbox_decrypt_file ├── blackbox_deregister_file ├── blackbox_diff ├── blackbox_edit ├── blackbox_edit_end ├── blackbox_edit_start ├── blackbox_initialize ├── blackbox_list_admins ├── blackbox_list_files ├── blackbox_listadmins ├── blackbox_postdeploy ├── blackbox_register_new_file ├── blackbox_removeadmin ├── blackbox_shred_all_files ├── blackbox_update_all_files ├── blackbox_view └── blackbox_whatsnew ├── blackbox.plugin.zsh ├── build └── build.go ├── cmd └── blackbox │ ├── blackbox.go │ ├── cli.go │ └── drive.go ├── docs ├── README.md ├── admin-ops.md ├── advanced.md ├── alternatives.md ├── backwards-compatibility.md ├── compatibility.md ├── dev-code-overview.md ├── dev.md ├── enable-repo.md ├── encryption.md ├── expired-keys.md ├── file-ops.md ├── full-command-list.md ├── git-tips.md ├── gnupg-tips.md ├── installation.md ├── role-accounts.md ├── subversion-tips.md ├── support.md ├── user-overview.md ├── why-is-this-important.md ├── with-ansible.md └── with-puppet.md ├── go.mod ├── go.sum ├── integrationTest ├── NOTES.txt ├── README.txt ├── asserts.go ├── integration_test.go ├── ithelpers.go └── test_data │ ├── 000-admin-list.txt │ ├── 000-file-list.txt │ ├── 000-status.txt │ ├── alice-cat-plain.txt │ ├── basic-status.txt │ ├── reencrypt-plain.txt │ └── status-noreg.txt ├── models ├── crypters.go └── vcs.go ├── pkg ├── bblog │ └── bblog.go ├── bbutil │ ├── filestats.go │ ├── rbio_test.go │ ├── runbash.go │ ├── shred.go │ ├── sortedfile_test.go │ ├── umask_posix.go │ └── umask_windows.go ├── box │ ├── box.go │ ├── boxutils.go │ ├── pretty_test.go │ └── verbs.go ├── commitlater │ └── commitlater.go ├── crypters │ ├── _all │ │ └── all.go │ ├── crypters.go │ └── gnupg │ │ ├── gnupg.go │ │ └── keychain.go ├── makesafe │ ├── makesafe.go │ └── makesafe_test.go └── vcs │ ├── _all │ └── all.go │ ├── git │ └── git.go │ ├── none │ └── none.go │ └── vcs.go └── tools ├── Portfile.template ├── auto_system_test ├── confidence_test.sh ├── macports_report_upgrade.sh ├── mk_deb_fpmdir ├── mk_deb_fpmdir.stack_blackbox.txt ├── mk_macports ├── mk_macports.vcs_blackbox.txt ├── mk_rpm_fpmdir ├── mk_rpm_fpmdir.stack_blackbox.txt ├── profile.d-usrblackbox-test.sh ├── profile.d-usrblackbox.sh └── test_functions.sh /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | workflows: 4 | version: 2 5 | build_and_test: 6 | jobs: 7 | - debian 8 | - ubuntu 9 | 10 | jobs: 11 | 12 | debian: 13 | docker: 14 | - image: debian:9.1 15 | steps: 16 | - checkout 17 | - run: 18 | name: 'Installing' 19 | command: | 20 | apt-get update -y 21 | apt-get install -y build-essential expect git gnupg2 pinentry-tty procps rpm ruby-dev libffi-dev 22 | gem install fpm 23 | - run: 24 | name: 'Cleaning' 25 | command: | 26 | rm -rf ~/.gpnupg 27 | make clean 28 | - run: 29 | name: 'Testing' 30 | command: | 31 | GPG=gpg2 make test 32 | make packages-deb 33 | make packages-rpm 34 | 35 | ubuntu: 36 | docker: 37 | - image: ubuntu:16.04 38 | steps: 39 | - checkout 40 | - run: 41 | name: 'Installing' 42 | command: | 43 | apt-get update -y 44 | apt-get install -y build-essential expect git gnupg2 pinentry-tty procps rpm ruby-dev libffi-dev 45 | gem install fpm 46 | - run: 47 | name: 'Cleaning' 48 | command: | 49 | rm -rf ~/.gpnupg 50 | make clean 51 | - run: 52 | name: 'Testing' 53 | command: | 54 | GPG=gpg2 make test 55 | make packages-deb 56 | make packages-rpm 57 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | bin/** text eol=lf 2 | tools/** text eol=lf 3 | Makefile text eol=lf 4 | Portfile text eol=lf 5 | blackbox.plugin.zsh text eol=lf 6 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | pull_request: 5 | branches: [ master ] 6 | push: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout repo 15 | uses: actions/checkout@v2 16 | with: 17 | fetch-depth: 0 18 | - name: Set up Go 19 | uses: actions/setup-go@v2 20 | with: 21 | go-version: ^1.15 22 | - name: Build binaries 23 | run: go run build/build.go 24 | - name: Run unit tests 25 | run: go test ./... 26 | - name: Run integration tests 27 | working-directory: integrationTest 28 | run: umask 0027 ; rm -rf /tmp/bbhome-* && go test -long -nocleanup 29 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: [published] 4 | 5 | name: release 6 | jobs: 7 | release: 8 | name: release 9 | runs-on: ubuntu-latest 10 | steps: 11 | 12 | - name: Get release 13 | id: get_release 14 | uses: bruceadams/get-release@v1.2.2 15 | env: 16 | GITHUB_TOKEN: ${{ github.token }} 17 | 18 | - name: Checkout repo 19 | uses: actions/checkout@v2 20 | with: 21 | fetch-depth: 0 22 | 23 | - name: Set up Go 24 | uses: actions/setup-go@v2 25 | with: 26 | go-version: ^1.15 27 | 28 | - name: Build binaries 29 | run: go run build/build.go 30 | 31 | - name: Upload blackbox-Darwin 32 | uses: actions/upload-release-asset@v1 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 35 | with: 36 | upload_url: ${{ steps.get_release.outputs.upload_url }} 37 | asset_path: ./blackbox-Darwin 38 | asset_name: blackbox-Darwin 39 | asset_content_type: application/octet-stream 40 | 41 | - name: Upload blackbox-Linux 42 | uses: actions/upload-release-asset@v1 43 | env: 44 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 45 | with: 46 | upload_url: ${{ steps.get_release.outputs.upload_url }} 47 | asset_path: ./blackbox-Linux 48 | asset_name: blackbox-Linux 49 | asset_content_type: application/octet-stream 50 | 51 | - name: Upload blackbox.exe 52 | uses: actions/upload-release-asset@v1 53 | env: 54 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 55 | with: 56 | upload_url: ${{ steps.get_release.outputs.upload_url }} 57 | asset_path: ./blackbox.exe 58 | asset_name: blackbox.exe 59 | asset_content_type: application/octet-stream 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | develop-eggs/ 12 | dist/ 13 | eggs/ 14 | lib/ 15 | lib64/ 16 | parts/ 17 | sdist/ 18 | var/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | 23 | # Installer logs 24 | pip-log.txt 25 | pip-delete-this-directory.txt 26 | 27 | # Unit test / coverage reports 28 | htmlcov/ 29 | .tox/ 30 | .coverage 31 | .cache 32 | nosetests.xml 33 | coverage.xml 34 | 35 | # Translations 36 | *.mo 37 | 38 | # Mr Developer 39 | .mr.developer.cfg 40 | .project 41 | .pydevproject 42 | 43 | # Rope 44 | .ropeproject 45 | 46 | # Django stuff: 47 | *.log 48 | *.pot 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # macOS 54 | .DS_Store 55 | 56 | # Blackbox 57 | bbintegration 58 | .*.swp 59 | /integrationTest/.blackbox 60 | 61 | # golang 62 | /vendor/ 63 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Tom Limoncelli 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Release v1.20220610 2 | 3 | NOTE: I don't have a lot of time to commit to this project. I'd gladly accept help, especially 4 | with improving the testing on various operating systems. 5 | 6 | Major feature: macOS users rejoice! Incompatibility with macOS Monterey 12.3 is fixed! (#347) 7 | 8 | * Add .gitattributes during repo initialization (#352) 9 | * Update zgen reference to zgenom (#350) 10 | * Improve test data generation (#348) 11 | * Fix 'chmod' for macOS Monterey 12.3 (#347) 12 | 13 | 14 | Release v1.20200429 15 | 16 | NOTE: While there is now support for NetBSD and SunOS/SmartOS, the 17 | release process only tests on macOS and CentOS7 because that's all I 18 | have access to. 19 | 20 | * Fix tools that break when ".." or "." are used in a path (#304) 21 | * Respect PREFIX variable for copy-install (#294) 22 | * Documentation: Add pkgsrc install instructions (#292) 23 | * Improve support for Windows (#291) 24 | * Clarify gpg version usage (#290) 25 | * Many documentation fixes 26 | * DOCUMENTATION: Promote 'getting started' to a section, enumerate steps (#283) 27 | * Commit changes to gitignore when deregistering (#282) 28 | * Add support for NetBSD and SunOS (SmartOS) 29 | * Defend against ShellShock 30 | 31 | 32 | Release v1.20181219 33 | 34 | * New OS support: Add support for NetBSD and SunOS (SmartOS) 35 | * Testing: Improve confidence test. 36 | * .blackbox is now the default config directory for new repos. (#272) 37 | * Add blackbox_decrypt_file (#270) 38 | * Improved compatibility: change"/bin/[x]" to "/usr/bin/env [x]" (#265) 39 | * Add blackbox_less. (#263) 40 | * add nix method of install (#261) 41 | * Linked setting up of GPG key (#260) 42 | 43 | 44 | Release v1.20180618 45 | 46 | * Restore `make manual-install` with warning. (#258) 47 | 48 | Release v1.20180615 49 | 50 | * Standardize on .blackbox for config. Use keyrings/live for backwards compatibility. 51 | * Store keys in .blackbox directory (#218) 52 | * Suggest committing changes to pubring.gpg when running blackbox_removeadmin (#248) 53 | * Fix typo (#246) 54 | * Improve installation instructions (#244) 55 | * Fix replacing-expired-keys link in README (#241) 56 | * Fix problems when gpg2 is installed next to gpg (#237) 57 | * Many documentation corrections, updates, etc. 58 | * Exclude default keyring from import (#223) 59 | * .gitattributes not always updated (PR#146) 60 | * Fix bugs related to updating .gitattributes (PR#146) 61 | * Update readme with CircleCI link (#216) 62 | * Run the tests on a CI (#215) 63 | * Fixed Alpine compatibility (chmod) (#212) 64 | * direct repobase message to stderr (#204) 65 | * Improve Windows compatibility 66 | * NEW: .gitattributes Set Unix-only files to eol=lf 67 | * Silence 'not changed' output during keychain import (#200) 68 | * Improve FreeBSD compatibility 69 | * shred_file() outputs warning message to stderr. (#192) 70 | * Don't complain about GPG_AGENT_INFO if using newer gpg-agent (#189) 71 | * [FreeBSD] Fix use of chmod (#180) 72 | * Requiring a file to be entered to finish editing (#175) 73 | * Remove the key from the keyring when removing an admin (#173) 74 | * Add FreeBSD support (#172) 75 | * Add list admins commandline tool. (#170) 76 | ignore backup files and secring.gpg in $BLACKBOXDATA (#169) 77 | Allow parallel shredding of files (#167) 78 | * Add/improve Mingw support 79 | * Make "make confidence" less fragile 80 | * And a lot, lot more. 81 | 82 | Release v1.20170309 83 | 84 | * "make test" is an alias for "make confidence" 85 | * macOS: make_tempdir must create shorter paths 86 | * Fix "make confidence" for newer version of Git 87 | * README.md: Add info about our new mailing list 88 | 89 | Release v1.20170611 90 | 91 | * confidence_test.sh verifies external tools exist 92 | * confidence_test.sh more reliable for non-UTF8 users 93 | * "make test" no longer prompts for passwords 94 | * blackbox works better when target directory lives in root (#194) 95 | * Add confidence_test.sh tests for admin operations 96 | * blackbox_list_admins fails (#193) 97 | * confidence_test.sh works better on FreeBSD 98 | * tools/confidence_test.sh: now works with gnupg-2.0 and gnupg-2.1 99 | * Blackbox now officially supports both gnupg-2.0 and gnupg-2.1 100 | * blackbox_shred_all_files: BUGFIX: Does not shred files with spaces 101 | * blackbox_removeadmin: disable gpg's confirmation 102 | * Sync mk_rpm_fpmdir from master 103 | 104 | Release v1.20170127 105 | 106 | * Starting CHANGELOG. 107 | -------------------------------------------------------------------------------- /DESIGN.md: -------------------------------------------------------------------------------- 1 | BlackBox Internals 2 | ================== 3 | 4 | The goal of the Go rewrite is to improve the usability and 5 | maintainability of Blackbox, meanwhile make it easier to implement new 6 | 7 | The system is built in distinct layers: view, controller, model. 8 | 9 | Suppose there is a subcommand "`foo`". `blackbox.go` parses the 10 | user's command line args and calls `cmdFoo()`, which is given 11 | everything it needs to do the operation. For example, it is given the 12 | filenames the user specified exactly; even if an empty list means "all 13 | files", at this layer the empty list is passed to the function. 14 | 15 | `cmdFoo()` contains the business logic of how the operation should be 16 | done: usually iterating over filenames and calling verb(s) for each 17 | one. For example if an empty file list means "all files", this is the 18 | layer that enumerates the files. 19 | 20 | `cmdFoo()` is implemented in the file `cmd_foo.go`. The caller of 21 | `cmdFoo()` should provide all data it needs to get the job done. 22 | `cmdFoo()` doesn't refer to global flags, they are passed to the 23 | function as parameters. Therefore the function has zero side-effects 24 | (except possibly logging) and can be called as library functions by 25 | other systems. This is the external (binary) API which should be 26 | relatively stable. 27 | 28 | `cmdFoo()` calls verbs that are in `bbutil/`. Some of those verbs are 29 | actually interfaces. For example, any VCS-related verbs are actually a 30 | Go interface which might be implemented one of many ways (Git, 31 | Subversion, Mercurial), GPG-functions may be implemented by shelling 32 | out to `gpg.exe` or by using Go's gpg library. 33 | 34 | They layers look like this: 35 | 36 | | View | `blackbox.go` | Parses User Commands, calls controller | 37 | | Controller | `cmd_*.go` | The business logic. Iterates and calls verbs | 38 | | Model | `pkg/bbutil` | Verbs | 39 | | Interfaces | `pkg/*` | Interfaces and their implementations | 40 | 41 | At least that's the goal. We'll see how well we can achieve this. 42 | 43 | 44 | Version 2.0 45 | =========== 46 | 47 | Software architecture. 48 | 49 | We try to keep the command-line parsing separate from the business 50 | logic and all plug-ins. This keeps things clean and easy to refactor. 51 | In fact layer 2 could be used as a stand-alone module for projects 52 | that want to embed blackbox actions. 53 | 54 | Layer 1: The command itself 55 | 56 | * cmd/blackbox/blackbox.go -- main() not much more 57 | * cmd/blackbox/cli.go -- Set up and call the ufave/cli flag parser 58 | * cmd/blackbox/drive.go -- Check # of arguments, conflicting flags, and then call the businss logic layer 59 | 60 | Layer 2: The business logic 61 | 62 | * pkg/box/box.go -- The interface to accessing .blackbox (admins, files, etc.) 63 | * pkg/box/verbs.go -- Verbs called by Layer 1. Just the verbs 64 | * pkg/box/boxutils.go -- Functions needed by the verbs 65 | 66 | Layer 3: The plug-ins 67 | 68 | * pkg/vcs/... -- Plug-ins for Git, (Mercurial, Subversion, Perforce,) and None 69 | * pkg/crypters/... -- Plug-ins for PGP access: GnuPG, (go-openpgp, others in the future) 70 | 71 | Layer 4: Support functions for use by Layer 3 72 | 73 | * pkg/bbutil/filestats.go -- File manipulations 74 | * pkg/bbutil/runbash.go -- Safely run external Linux commands 75 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014-2021 Stack Exchange, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL=/bin/sh 2 | PREFIX?=/usr/local 3 | PKGNAME=stack_blackbox 4 | BASEDIR?=$(HOME) 5 | OUTPUTDIR?="$(BASEDIR)/debbuild-${PKGNAME}" 6 | 7 | all: 8 | @echo 'Menu:' 9 | @echo ' make update Update any generated files' 10 | @echo ' make packages-rpm Make RPM packages' 11 | @echo ' make packages-deb Make DEB packages' 12 | @echo ' make symlinks-install Make symlinks in ${PREFIX}/bin/' 13 | @echo ' make copy-install Copy "bin" files to ${PREFIX}/bin/' 14 | @echo ' make copy-uninstall Remove blackbox files from ${PREFIX}/bin/' 15 | @echo ' make test Run tests' 16 | 17 | install: 18 | @echo 'To install, copy the files from bin to somewhere in your PATH.' 19 | @echo 'The README.md document gives more details.' 20 | @echo 'Or run "make" (with no options) for more info.' 21 | 22 | # The default package type is RPM. 23 | packages: packages-rpm 24 | 25 | # 26 | # RPM builds 27 | # 28 | 29 | # NOTE: mk_rpm_fpmdir.stack_blackbox.txt is the master list of files. All 30 | # other packages should generate their list from it. 31 | 32 | packages-rpm: 33 | cd tools && PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_rpm_fpmdir stack_blackbox mk_rpm_fpmdir.stack_blackbox.txt 34 | 35 | packages-rpm-debug: 36 | @echo BUILD: 37 | @PKGRELEASE=99 make packages 38 | @echo ITEMS TO BE PACKAGED: 39 | find $(BASEDIR)/rpmbuild-$(PKGNAME)/installroot -type f 40 | @echo ITEMS ACTUALLY IN PACKAGE: 41 | @rpm -qpl $$(cat $(BASEDIR)/rpmbuild-$(PKGNAME)/bin-packages.txt) 42 | 43 | local-rpm: 44 | @PKGRELEASE=1 make packages 45 | -@sudo rpm -e $(PKGNAME) 46 | sudo rpm -i $$(cat $(BASEDIR)/rpmbuild-$(PKGNAME)/bin-packages.txt) 47 | 48 | lock-rpm: 49 | sudo yum versionlock add $(PKGNAME) 50 | 51 | unlock-rpm: 52 | sudo yum versionlock clear 53 | 54 | # 55 | # Manual install 56 | # 57 | symlinks-install: 58 | @echo "Symlinking files from ./bin to ${PREFIX}/bin" 59 | @cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do ln -fs `pwd`/$$f $(PREFIX)/bin/$$f; done 60 | @echo 'Done.' 61 | 62 | manual-install: 63 | @echo '***************************************************************' 64 | @echo '* DEPRECATED *' 65 | @echo '* `make manual-install` is now called `make symlinks-install` *' 66 | @echo '***************************************************************' 67 | $(MAKE) symlinks-install 68 | 69 | copy-install: 70 | @echo "Copying files from ./bin to ${PREFIX}/bin" 71 | @cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do cp `pwd`/$$f $(PREFIX)/bin/$$f; done 72 | @echo 'Done.' 73 | 74 | copy-uninstall: 75 | @echo "Removing blackbox files from ${PREFIX}/bin" 76 | @cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do rm $(PREFIX)/bin/$$f; done 77 | @echo 'Done.' 78 | 79 | # 80 | # DEB builds 81 | # 82 | 83 | packages-deb: tools/mk_deb_fpmdir.stack_blackbox.txt 84 | cd tools && OUTPUTDIR=$(OUTPUTDIR) PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_deb_fpmdir stack_blackbox mk_deb_fpmdir.stack_blackbox.txt 85 | 86 | # Make mk_deb_fpmdir.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt: 87 | tools/mk_deb_fpmdir.stack_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt 88 | sed -e '/^#/d' -e 's@/usr/blackbox/bin/@/usr/bin/@g' -e '/profile.d-usrblackbox.sh/d' $@ 89 | 90 | packages-deb-debug: tools/mk_deb_fpmdir.stack_blackbox.txt 91 | @echo BUILD: 92 | @PKGRELEASE=99 make packages-deb 93 | @echo ITEMS TO BE PACKAGED: 94 | find ~/debbuild-$(PKGNAME)/installroot -type f 95 | @echo ITEMS ACTUALLY IN PACKAGE: 96 | @dpkg --contents $$(cat $(BASEDIR)/debbuild-$(PKGNAME)/bin-packages.txt) 97 | 98 | local-deb: 99 | @PKGRELEASE=1 make packages 100 | -@sudo dpkg -e $(PKGNAME) 101 | sudo dpkg -i $$(cat $(BASEDIR)/rpmbuild-$(PKGNAME)/bin-packages.txt) 102 | 103 | # 104 | # MacPorts builds 105 | # 106 | # To test: 107 | # rm -rf /tmp/foo ; mkdir -p /tmp/foo;make packages-macports DESTDIR=/tmp/foo;find /tmp/foo -ls 108 | 109 | # Make mk_macports.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt: 110 | tools/mk_macports.vcs_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt 111 | sed -e '/^#/d' -e 's@/usr/blackbox/bin/@bin/@g' -e '/profile.d-usrblackbox.sh/d' $@ 112 | 113 | # MacPorts expects to run: make packages-macports DESTDIR=${destroot} 114 | packages-macports: tools/mk_macports.vcs_blackbox.txt 115 | mkdir -p $(DESTDIR)/bin 116 | cd tools && ./mk_macports mk_macports.vcs_blackbox.txt 117 | 118 | # stow is a pretty easy way to manage simple local installs on GNU systems 119 | install-stow: 120 | mkdir -p /usr/local/stow/blackbox/bin 121 | cp bin/* /usr/local/stow/blackbox/bin 122 | rm /usr/local/stow/blackbox/bin/Makefile 123 | cd /usr/local/stow; stow -R blackbox 124 | uninstall-stow: 125 | cd /usr/local/stow; stow -D blackbox 126 | rm -rf /usr/local/stow/blackbox 127 | 128 | # Add other package types here. 129 | 130 | # 131 | # Updates 132 | # 133 | update: tools/mk_deb_fpmdir.stack_blackbox.txt tools/mk_macports.vcs_blackbox.txt 134 | 135 | clean: 136 | rm -f tools/mk_deb_fpmdir.stack_blackbox.txt tools/mk_macports.vcs_blackbox.txt 137 | 138 | # 139 | # System Test: 140 | # 141 | test: confidence 142 | confidence: 143 | @if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg should not exist. If it does, bugs may polute your .gnupg configuration. If the code has no bugs everything will be fine. Do you feel lucky?'; false ; fi 144 | @if which >/dev/null gpg-agent ; then pkill gpg-agent ; rm -rf /tmp/tmp.* ; fi 145 | @export PATH="$(PWD)/bin:$(PREFIX)/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:/usr/local/MacGPG2/bin:/opt/homebrew/bin:$(PATH)" ; tools/auto_system_test 146 | @if which >/dev/null gpg-agent ; then pkill gpg-agent ; fi 147 | @if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg was created which means the scripts might be poluting GnuPG configuration. Fix this bug.'; false ; fi 148 | -------------------------------------------------------------------------------- /README-v2.md: -------------------------------------------------------------------------------- 1 | BlackBox v2 2 | =========== 3 | 4 | WARNING: v2 is still experimental. It is in the same git repo as v1 5 | because the filenames do not overlap. Please do not mix the two. v1 6 | is in `bin`. v2 is in `cmd/blackbox` and `binv2`. 7 | 8 | Blackbox is an open source tool that enables you to safe store sensitive information in 9 | Git (or other) repos by encrypting them with GPG. Only the encrypted 10 | version of the file is available. You can be free to provide access 11 | to the repo, as but only people with the right GPG keys can access the 12 | encrypted data. 13 | 14 | Things you should **never** store in a repo without encryption: 15 | 16 | * TLS (SSL) certificates 17 | * Passwords 18 | * API keys 19 | * And more! 20 | 21 | Project Info: 22 | 23 | * [Overview](user-overview.md) 24 | * [Why is this important?](why-is-this-important.md) 25 | * [Support/Community](support.md) 26 | * [How BB encrypts](encryption.md) 27 | * [OS Compatibility](compatibility.md) 28 | * [Installation Instructions](installation.md) 29 | * [Alternatives](alternatives.md) 30 | 31 | User Info: 32 | 33 | * [Enabling Blackbox on a Repo](enable-repo.md) 34 | * [Enroll a file](enable-repo.md) 35 | * [Full Command List](full-command-list.md) 36 | * [Add/Remove users](admin-ops.md) 37 | * [Add/Remove files](file-ops.md) 38 | * [Advanced techiques](advanced.md) 39 | * [Use with Role Accounts](role-accounts.md) 40 | * [Backwards Compatibility](backwards-compatibility.md) 41 | * [Replacing expired keys](expired-keys.md) 42 | * [Git Tips](git-tips.md) 43 | * [SubVersion Tips](subversion-tips.md) 44 | * [GnuPG tips](gnupg-tips.md) 45 | * [Use with Ansible](with-ansible.md) 46 | * [Use with Puppet](with-puppet.md) 47 | 48 | For contributors: 49 | 50 | * [Developer Info](dev.md) 51 | * [Code overview](dev-code-overview.md) 52 | * [HOWTO: Add new OS support](dev-add-os-support.md) 53 | * [HOWTO: Add new VCS support](dev-add-vcs-support.md) 54 | 55 | 56 | A slide presentation about an older release [is on SlideShare](http://www.slideshare.net/TomLimoncelli/the-blackbox-project-sfae). 57 | 58 | Join our mailing list: [https://groups.google.com/d/forum/blackbox-project](https://groups.google.com/d/forum/blackbox-project) 59 | 60 | 61 | License 62 | ======= 63 | 64 | This content is released under the MIT License. 65 | See the [LICENSE.txt](LICENSE.txt) file. 66 | -------------------------------------------------------------------------------- /RELEASE_ENGINEERING.md: -------------------------------------------------------------------------------- 1 | Table of Contents: 2 | ================== 3 | 4 | - [Branches and Tags:](#branches-and-tags) 5 | - [Testing:](#testing) 6 | - [Build Tasks](#build-tasks) 7 | - [Stable Releases](#stable-releases) 8 | - [Production Releases](#production-releases) 9 | - [Updating MacPorts (automatic)](#updating-macports-automatic) 10 | - [Updating MacPorts (manual)](#updating-macports-manual) 11 | 12 | Branches and Tags: 13 | ================== 14 | 15 | There are 3 branches/tags: 16 | 17 | - **HEAD:** The cutting edge of development. 18 | - **tag stable:** Stable enough for use by most people. 19 | - **tag production:** Burned in long enough that we are confident it can be widely adopted. 20 | 21 | If you are packaging BlackBox for distribution, you should track the *tag production*. You might also want to provide a separate package that tracks *tag stable:* for early adopters. 22 | 23 | Testing 24 | ======= 25 | 26 | Tips: 27 | 28 | * macOS: `brew install gpg pinentry` 29 | * FreeBSD: `pkg install gpg gmake` 30 | * CentOS7: `yum install gpg` 31 | 32 | To run a suite of tests: 33 | 34 | ``` 35 | cd ~/src/github.com/StackExchange/blackbox 36 | make test 37 | ``` 38 | 39 | FYI: For FreeBSD, use `gmake test` 40 | 41 | 42 | Build Tasks 43 | =========== 44 | 45 | Stable Releases 46 | =============== 47 | 48 | Step 0. Test the software 49 | 50 | Run this command to run the unit and system tests: 51 | 52 | ``` 53 | make test 54 | ``` 55 | 56 | NOTE: The tests require pinentry-tty. On macOS with NIX this 57 | can be installed via: `nix-env -i pinentry` 58 | 59 | Marking the software to be "stable": 60 | 61 | Step 1. Update CHANGELOG.md 62 | 63 | Use "git log" to see what has changed and update CHANGELOG.md. 64 | 65 | For a new release, add: 66 | 67 | ``` 68 | echo Release v1.$(date +%Y%m%d) 69 | ``` 70 | 71 | Commit with: 72 | 73 | ``` 74 | git commit -m'Update CHANGELOG.md' CHANGELOG.md 75 | ``` 76 | 77 | Step 2. Tag it. 78 | 79 | ``` 80 | git pull 81 | git tag -d stable 82 | git push origin :stable 83 | git tag stable 84 | git push origin tag stable 85 | ``` 86 | 87 | Step 3. Mark your calendar 1 week from today to check to see if this should be promoted to production. 88 | 89 | Production Releases 90 | =================== 91 | 92 | If no bugs have been reported a full week after a stable tag has been pushed, mark the release to be "production". 93 | 94 | ``` 95 | git fetch 96 | git checkout stable 97 | git tag -d production 98 | git push origin :production 99 | git tag production 100 | git push origin tag production 101 | R="v1.$(date +%Y%m%d)" 102 | git tag "$R" 103 | git push origin tag "$R" 104 | ``` 105 | 106 | Step 4. Get credit! 107 | 108 | Record the fact that you did this release in your weekly accomplishments file. 109 | 110 | 111 | 112 | Updating MacPorts (automatic) 113 | ============================= 114 | 115 | Step 1: Generate the Portfile 116 | 117 | ``` 118 | tools/macports_report_upgrade.sh 1.20150222 119 | ``` 120 | 121 | This script will generate a file called `Portfile-vcs_blackbox.diff` and instructions on how to submit it as a update request. 122 | 123 | Step 2: Submit the update request. 124 | 125 | Submit the diff file as a bug as instructed. The instructions should look like this: 126 | 127 | - PLEASE OPEN A TICKET WITH THIS INFORMATION: https://trac.macports.org/newticket 128 | - Summary: `vcs_blackbox @1.20150222 Update to latest upstream` 129 | - Description: `New upstream of vcs_blackbox. 130 | github.setup and checksums updated.` 131 | - Type: `update` 132 | - Component: `ports` 133 | - Port: `vcs_blackbox` 134 | - Keywords: `maintainer haspatch` 135 | - Attach this file: `Portfile-vcs_blackbox.diff` 136 | 137 | Step 3: Watch for the update to happen. 138 | 139 | 140 | Updating MacPorts (manual) 141 | ========================== 142 | 143 | This is the old, manual, procedure. If the automated procedure fails to work, these notes may or may not be helpful. 144 | 145 | The ultimate result of the script should be the output of `diff -u Portfile.orig Portfile` which is sent as an attachment to MacPorts. The new `Portfile` should have these changes: 146 | 147 | 1. The `github.setup` line should have a new version number. 148 | 2. The `checksums` line(s) should have updated checksums. 149 | 150 | How to generate the checksums? 151 | 152 | The easiest way is to to make a Portfile with incorrect checksums, then run `sudo port -v checksum vcs_blackbox` to see what they should have been. Fix the file, and try again until the checksum command works. 153 | 154 | Next run `port lint vcs_blackbox` and make sure it has no errors. 155 | 156 | Some useful commands: 157 | 158 | Change repos in sources.conf: 159 | 160 | ``` 161 | sudo vi /opt/local/etc/macports/sources.conf 162 | Add this line early in the file: 163 | file:///var/tmp/ports 164 | ``` 165 | 166 | Add a local repo: 167 | 168 | ``` 169 | fgrep >/dev/null -x 'file:///var/tmp/ports' /opt/local/etc/macports/sources.conf || sudo sed -i -e '1s@^@file:///var/tmp/ports\'$'\n@' /opt/local/etc/macports/sources.conf 170 | ``` 171 | 172 | Remove the local repo: 173 | 174 | ``` 175 | sudo sed -i -e '\@^file:///var/tmp/ports@d' /opt/local/etc/macports/sources.conf 176 | ``` 177 | 178 | Test a Portfile: 179 | 180 | ``` 181 | sudo port uninstall vcs_blackbox 182 | sudo port clean --all vcs_blackbox 183 | rm -rf ~/.macports/opt/local/var/macports/sources/rsync.macports.org/release/tarballs/ports/security/vcs_blackbox/ 184 | rm -rf /var/tmp/ports 185 | mkdir -p /var/tmp/ports/security/vcs_blackbox 186 | cp Portfile /var/tmp/ports/security/vcs_blackbox 187 | cd /var/tmp/ports && portindex 188 | sudo port -v checksum vcs_blackbox 189 | sudo port install vcs_blackbox 190 | ``` 191 | -------------------------------------------------------------------------------- /SPECS/empty.spec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StackExchange/blackbox/867fe52b1d5bd64934a789b431c31329805bf144/SPECS/empty.spec -------------------------------------------------------------------------------- /Version2-Ideas.md: -------------------------------------------------------------------------------- 1 | # Ideas for BlackBox Version 2 2 | 3 | I'm writing this to solicit feedback and encourage discussion. 4 | 5 | Here are my thoughts on a "version 2" of BlackBox. This is where 6 | I list ideas that would require major changes to the system. They 7 | might break backwards compatibility, though usually not. 8 | 9 | BlackBox grew from a few simple shell scripts used at StackOverflow.com 10 | to a larger system used by dozens (hundreds?) of organizations. Not 11 | all the design decisions were "forward looking". 12 | 13 | These are the things I'd like to change someday. 14 | 15 | [TOC] 16 | 17 | ## Change the commmand names 18 | 19 | There should be one program, with subcommands that have names that make more sense: 20 | 21 | * `blackbox admin add ` 22 | * `blackbox admin list` 23 | * `blackbox admin remove ` 24 | * `blackbox cat ...` 25 | * `blackbox decrypt ...` 26 | * `blackbox diff ...` 27 | * `blackbox edit ...` 28 | * `blackbox encrypt ...` 29 | * `blackbox file add ...` 30 | * `blackbox file list` 31 | * `blackbox file remove ...` 32 | * `blackbox info` 33 | * `blackbox init` 34 | * `blackbox reencrypt` 35 | * `blackbox shred --all| ...` 36 | * `blackbox status --all| ...` 37 | 38 | Backwards compatibility: The old scripts will be rewritten to use the new commands. 39 | 40 | ## Change the "keyrings" directory 41 | 42 | The name `keyrings` was unfortunate. First, it should probably begin with a `.`. Second, it stores more than just keyrings. Lastly, I'm finding that in most cases we want many repos to refer to the same keyring, which is not supported very well. 43 | 44 | A better system would be: 45 | 46 | 1. If `$BLACKBOX_CONFIG` is set, use that directory. 47 | 2. If the repo base directory has a file called `.blackbox_external`, read that file as if you are reading `$BLACKBOX_CONFIG` 48 | 3. If the repo base directory has a `keyrings` directory, use that. 49 | 4. If the repo base directory has a `.blackbox` directory, use that. 50 | 51 | Some thoughts on `.blackbox_external`: 52 | I'm not sure what the format should be, but I want it to be simple and expandable. It should support support `../../dir/name` and `/long/path`. However some day we may want to include a Git URL and have the system automatically get the keychain from it. That means the format has to be something like directory:../dir/name so that later we can add git:the-url. 53 | 54 | NOTE: Maybe `.blackbox_external` should be `.blackbox/BLACKBOX_CONFIG`? 55 | 56 | Backwards compatibility: `keyrings` would be checked before `.blackbox`. 57 | 58 | ## System Test 59 | 60 | There needs to be a very complete system test. The `make test` we 61 | have now is great for something written in bash. 62 | 63 | It should be easy to make tests. Perhaps a directory of files, each 64 | specifying a test. We could make a little language for writing tests. 65 | 66 | # This test becomes the user "alice" and verifies that she 67 | # can encrypt a file, and decrypt it, with full fidelity. 68 | BECOME alice a 69 | BASH echo "foo contents" >foo.txt 70 | SHOULD_NOT_EXIST foo.txt.gpg 71 | BASH blackbox encrypt foo.txt 72 | SHOULD_NOT_EXIST foo.txt 73 | SHOULD_EXIST foo.txt.gpg 74 | BASH_WITH_PASSWORD a blackbox decrypt foo.txt 75 | SHOULD_EXIST foo.txt.gpg 76 | SHOULD_EXIST foo.txt 77 | SHOULD_CONTAIN foo.txt "foo contents\n" 78 | 79 | ## Plug-in support 80 | 81 | There should plug-ins support for: 82 | 83 | Repo type: 84 | 85 | * Git -- Using /usr/bin/git or git.exe 86 | * Subversion 87 | * Mercurial 88 | * None (repoless) 89 | * Autodetect 90 | 91 | Encryption software: 92 | 93 | * GnuPG -- using /usr/bin/gpg{,2} or gpg.exe 94 | * golang.org/x/crypto/openpgp 95 | 96 | ## JSON or .txt 97 | 98 | The files in .blackbox are mostly .txt files. Instead we should 99 | define a .json format, and only read the .txt file is the .json file 100 | doesn't exist. 101 | 102 | 103 | ## Repo-less mode 104 | 105 | I can't imagine storing files that aren't in a repo. I just put everything in repos lately. I use it more than I use NFS. That said, I have received feedback that people would like the ability to disable automatic committing of files. 106 | 107 | I prefer the file commits to be automatic because when they were manual, people often accidentally committed the plaintext file instead of the GPG file. Fixing such mistakes is a PITA and, of yeah, a big security nightmare. 108 | 109 | That said, I'm willing to have a "repo-less" mode. 110 | 111 | When this mode is triggered, no add/commit/ignore tasks are done. The search for the keyrings directory still uses `$BLACKBOX_CONFIG` but if that is unset it looks for `.blackbox_config` in the current directory, then recursively `..` until we hit `/`. 112 | 113 | I think (but I'm not sure) this would benefit the entire system because it would force us to re-think what VCS actions are done when. 114 | 115 | I think (but I'm not sure) that a simple way to implement this would be to add an environment variable that overrides the automatic VCS detection. When set to "none", all VCS operations would basically become no-ops. (This could be done by writing a plug-in that does nothing for all the `vcs_*` calls) 116 | 117 | 118 | Backwards compatibility: This would add a `none` VCS, not remove any existing functionality. 119 | 120 | 121 | ## Is "bash" the right language? 122 | 123 | `bash` is fairly universal. It even exists on Windows. However it is not the right language for large systems. Writing the acceptance tests is quite a bear. Managing `.gitignore` files in bash is impossible and the current implementation fails in many cases. 124 | 125 | `python` is my second favorite language. It would make the code cleaner and more testable. However it is not installed everywhere. I would also want to write it in Python3 (why start a new project in Python2?) but sadly Python3 is less common. It is a chicken vs. egg situation. 126 | 127 | `go` is my favorite language. I could probably rewrite this in go in a weekend. However, now the code is compiled, not interpreted. Therefore we lose the ability to just `git clone` and have the tools you want. Not everyone has a Go compiler installed on every machine. 128 | 129 | The system is basically unusable on Windows without Cygwin or MINGW. A rewrite in python or go would make it work better on Windows, which currently requires Cygwin or MinGW (which is a bigger investment than installing Python). On the other hand, maybe Ubuntu-on-Windows makes that a non-issue. 130 | 131 | As long as the code is in `bash` the configuration files like `blackbox-files.txt` and `blackbox-admins.txt` have problems. Filenames with carriage returns aren't supported. If this was in Python/Go/etc. those files could be json or some format with decent quoting and we could handle funny file names better. On the other hand, maybe it is best that we don't support funny filenames... we shouldn't enable bad behavior. 132 | 133 | How important is itto blackbox users that the system is written in `bash`? 134 | 135 | 136 | ## Ditch the project and use git-crypt 137 | 138 | People tell me that git-crypt is better because, as a plug-in, automagically supports `git diff`, `git log` and `git blame`. 139 | 140 | However, I've never used it so I don't have any idea whether git-crypt is any better than blackbox. 141 | 142 | Of course, git-crypt doesn't work with SVN, HG, or any other VCS. Is blackbox's strong point the fact that it support so many VCS systems? To be honest, it originally only supported HG and GIT because I was at a company that used HG but then changed to GIT. Supporting anything else was thanks to contributors. Heck, HG support hasn't even been tested recently (by me) since we've gone all git where I work. 143 | 144 | How important is this to BlackBox users? 145 | -------------------------------------------------------------------------------- /bin/Makefile: -------------------------------------------------------------------------------- 1 | all: _stack_lib.sh 2 | 3 | # Snatch _stack_lib.sh from another StackExchange project. 4 | _stack_lib.sh: ../../scripts/misc/_stack_lib.sh 5 | cp ../../scripts/misc/_stack_lib.sh $@ 6 | -------------------------------------------------------------------------------- /bin/_blackbox_common_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # _blackbox_common_test.sh -- Unit tests of functions from _blackbox_common.sh 5 | # 6 | 7 | set -e 8 | . "${0%/*}/_blackbox_common.sh" 9 | . tools/test_functions.sh 10 | 11 | PHASE 'Test cp-permissions: TestA' 12 | touch TestA TestB TestC TestD 13 | chmod 0347 TestA 14 | chmod 0700 TestB 15 | chmod 0070 TestC 16 | chmod 0070 TestD 17 | cp_permissions TestA TestB TestC 18 | # NOTE: cp_permissions is not touching TestD. 19 | assert_file_perm '--wxr--rwx' TestA 20 | assert_file_perm '--wxr--rwx' TestB 21 | assert_file_perm '--wxr--rwx' TestC 22 | assert_file_perm '----rwx---' TestD # TestD doesn't change. 23 | rm -f TestA TestB TestC TestD 24 | 25 | echo '========== DONE.' 26 | -------------------------------------------------------------------------------- /bin/_stack_lib.sh: -------------------------------------------------------------------------------- 1 | # Library functions for bash scripts at Stack Exchange. 2 | 3 | # NOTE: This file is open sourced. Do not put Stack-proprietary code here. 4 | 5 | # Usage: 6 | # 7 | # set -e 8 | # . _stack_lib.sh 9 | 10 | # ----- Utility Functions ----- 11 | 12 | function debugmsg() { 13 | # Log to stderr. 14 | echo 1>&2 LOG: "$@" 15 | : 16 | } 17 | 18 | function logit() { 19 | # Log to stderr. 20 | echo 1>&2 LOG: "$@" 21 | } 22 | 23 | function fail_out() { 24 | echo "FAILED:" "$*" 25 | echo 'Exiting...' 26 | exit 1 27 | } 28 | 29 | # on_exit and add_on_exit from http://www.linuxjournal.com/content/use-bash-trap-statement-cleanup-temporary-files 30 | # Usage: 31 | # add_on_exit rm -f /tmp/foo 32 | # add_on_exit echo "I am exiting" 33 | # tempfile=$(mktemp) 34 | # add_on_exit rm -f "$tempfile" 35 | function on_exit() 36 | { 37 | for i in "${on_exit_items[@]}" 38 | do 39 | eval $i 40 | done 41 | } 42 | 43 | function add_on_exit() 44 | { 45 | local n=${#on_exit_items[*]} 46 | on_exit_items[$n]="$*" 47 | if [[ $n -eq 0 ]]; then 48 | trap on_exit EXIT 49 | fi 50 | } 51 | 52 | function create_self_deleting_tempfile() { 53 | local filename 54 | 55 | case $(uname -s) in 56 | Darwin | FreeBSD ) 57 | : "${TMPDIR:=/tmp}" ; 58 | filename=$(mktemp -t _stacklib_.XXXXXXXX ) 59 | ;; 60 | Linux | CYGWIN* | MINGW* | NetBSD | SunOS ) 61 | filename=$(mktemp) 62 | ;; 63 | * ) 64 | echo 'ERROR: Unknown OS. Exiting. (create_self_deleting_tempfile)' 65 | exit 1 66 | ;; 67 | esac 68 | 69 | add_on_exit rm -f "$filename" 70 | echo "$filename" 71 | } 72 | 73 | function create_self_deleting_tempdir() { 74 | local filename 75 | 76 | case $(uname -s) in 77 | Darwin | FreeBSD ) 78 | : "${TMPDIR:=/tmp}" ; 79 | filename=$(mktemp -d -t _stacklib_.XXXXXXXX ) 80 | ;; 81 | Linux | CYGWIN* | MINGW* | NetBSD | SunOS ) 82 | filename=$(mktemp -d) 83 | ;; 84 | * ) 85 | echo 'ERROR: Unknown OS. Exiting. (create_self_deleting_tempdir)' 86 | exit 1 87 | ;; 88 | esac 89 | 90 | add_on_exit rm -rf "$filename" 91 | echo "$filename" 92 | } 93 | 94 | # Securely and portably create a temporary file that will be deleted 95 | # on EXIT. $1 is the variable name to store the result. 96 | function make_self_deleting_tempfile() { 97 | local __resultvar="$1" 98 | local name 99 | 100 | case $(uname -s) in 101 | Darwin | FreeBSD ) 102 | : "${TMPDIR:=/tmp}" ; 103 | name=$(mktemp -t _stacklib_.XXXXXXXX ) 104 | ;; 105 | Linux | CYGWIN* | MINGW* | NetBSD | SunOS ) 106 | name=$(mktemp) 107 | ;; 108 | * ) 109 | echo 'ERROR: Unknown OS. Exiting. (make_self_deleting_tempfile)' 110 | exit 1 111 | ;; 112 | esac 113 | 114 | add_on_exit rm -f "$name" 115 | eval $__resultvar="$name" 116 | } 117 | 118 | function make_tempdir() { 119 | local __resultvar="$1" 120 | local name 121 | 122 | case $(uname -s) in 123 | Darwin | FreeBSD ) 124 | : "${TMPDIR:=/tmp}" ; 125 | # The full path to the temp directory must be short. 126 | # This is used by blackbox's testing suite to make a fake GNUPGHOME, 127 | # which needs to fit within sockaddr_un.sun_path (see unix(7)). 128 | name=$(mktemp -d -t SO ) 129 | ;; 130 | Linux | CYGWIN* | MINGW* | NetBSD | SunOS ) 131 | name=$(mktemp -d) 132 | ;; 133 | * ) 134 | echo 'ERROR: Unknown OS. Exiting. (make_tempdir)' 135 | exit 1 136 | ;; 137 | esac 138 | 139 | eval $__resultvar="$name" 140 | } 141 | 142 | function make_self_deleting_tempdir() { 143 | local __resultvar="$1" 144 | local dname 145 | 146 | make_tempdir dname 147 | 148 | add_on_exit rm -rf "$dname" 149 | eval $__resultvar="$dname" 150 | } 151 | 152 | function fail_if_not_running_as_root() { 153 | if [[ $EUID -ne 0 ]]; then 154 | echo 'ERROR: This command should only be run as root.' 155 | echo 'Exiting...' 156 | exit 1 157 | fi 158 | } 159 | 160 | function fail_if_in_root_directory() { 161 | # Verify nobody has tricked us into being in "/". 162 | case $(uname -s) in 163 | Darwin | FreeBSD | NetBSD ) 164 | if [[ $(stat -f'%i' / ) == $(stat -f'%i' . ) ]] ; then 165 | echo 'SECURITY ALERT: The current directory is the root directory.' 166 | echo 'Exiting...' 167 | exit 1 168 | fi 169 | ;; 170 | Linux | CYGWIN* | MINGW* | SunOS ) 171 | if [[ $(stat -c'%i' / ) == $(stat -c'%i' . ) ]] ; then 172 | echo 'SECURITY ALERT: The current directory is the root directory.' 173 | echo 'Exiting...' 174 | exit 1 175 | fi 176 | ;; 177 | * ) 178 | echo 'ERROR: Unknown OS. Exiting. (fail_if_in_root_directory)' 179 | exit 1 180 | ;; 181 | esac 182 | } 183 | 184 | function semverParseInto() { 185 | local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' 186 | #MAJOR 187 | eval $2=`echo $1 | sed -e "s#$RE#\1#"` 188 | #MINOR 189 | eval $3=`echo $1 | sed -e "s#$RE#\2#"` 190 | #MINOR 191 | eval $4=`echo $1 | sed -e "s#$RE#\3#"` 192 | #SPECIAL 193 | eval $5=`echo $1 | sed -e "s#$RE#\4#"` 194 | } 195 | 196 | function semverEQ() { 197 | local MAJOR_A=0 198 | local MINOR_A=0 199 | local PATCH_A=0 200 | local SPECIAL_A=0 201 | 202 | local MAJOR_B=0 203 | local MINOR_B=0 204 | local PATCH_B=0 205 | local SPECIAL_B=0 206 | 207 | semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A 208 | semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B 209 | 210 | if [ $MAJOR_A -ne $MAJOR_B ]; then 211 | return 1 212 | fi 213 | 214 | if [ $MINOR_A -ne $MINOR_B ]; then 215 | return 1 216 | fi 217 | 218 | if [ $PATCH_A -ne $PATCH_B ]; then 219 | return 1 220 | fi 221 | 222 | if [[ "_$SPECIAL_A" != "_$SPECIAL_B" ]]; then 223 | return 1 224 | fi 225 | 226 | 227 | return 0 228 | 229 | } 230 | 231 | function semverLT() { 232 | local MAJOR_A=0 233 | local MINOR_A=0 234 | local PATCH_A=0 235 | local SPECIAL_A=0 236 | 237 | local MAJOR_B=0 238 | local MINOR_B=0 239 | local PATCH_B=0 240 | local SPECIAL_B=0 241 | 242 | semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A 243 | semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B 244 | 245 | if [ $MAJOR_A -lt $MAJOR_B ]; then 246 | return 0 247 | fi 248 | 249 | if [[ $MAJOR_A -le $MAJOR_B && $MINOR_A -lt $MINOR_B ]]; then 250 | return 0 251 | fi 252 | 253 | if [[ $MAJOR_A -le $MAJOR_B && $MINOR_A -le $MINOR_B && $PATCH_A -lt $PATCH_B ]]; then 254 | return 0 255 | fi 256 | 257 | if [[ "_$SPECIAL_A" == "_" ]] && [[ "_$SPECIAL_B" == "_" ]] ; then 258 | return 1 259 | fi 260 | if [[ "_$SPECIAL_A" == "_" ]] && [[ "_$SPECIAL_B" != "_" ]] ; then 261 | return 1 262 | fi 263 | if [[ "_$SPECIAL_A" != "_" ]] && [[ "_$SPECIAL_B" == "_" ]] ; then 264 | return 0 265 | fi 266 | 267 | if [[ "_$SPECIAL_A" < "_$SPECIAL_B" ]]; then 268 | return 0 269 | fi 270 | 271 | return 1 272 | 273 | } 274 | 275 | function semverGT() { 276 | semverEQ $1 $2 277 | local EQ=$? 278 | 279 | semverLT $1 $2 280 | local LT=$? 281 | 282 | if [ $EQ -ne 0 ] && [ $LT -ne 0 ]; then 283 | return 0 284 | else 285 | return 1 286 | fi 287 | } 288 | 289 | -------------------------------------------------------------------------------- /bin/blackbox_addadmin: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_addadmin -- Add an admin to the system 5 | # 6 | 7 | # Example: 8 | # blackbox_addadmin tal@example.com 9 | # 10 | 11 | set -e 12 | source "${0%/*}/_blackbox_common.sh" 13 | 14 | fail_if_not_in_repo 15 | 16 | KEYNAME="$1" 17 | : "${KEYNAME:?ERROR: First argument must be a keyname (email address)}" ; 18 | 19 | # Add the email address to the BB_ADMINS file. Remove any duplicates. 20 | # The file must exist for sort to act as we expect. 21 | touch "$BB_ADMINS" 22 | echo "$1" >> "$BB_ADMINS" 23 | sort -fdu -o "$BB_ADMINS" "$BB_ADMINS" 24 | 25 | 26 | # Add the user's key to the keychain. 27 | 28 | # Extract it: 29 | make_self_deleting_tempfile pubkeyfile 30 | 31 | # The second argument, if present, is the directory to find the GPG keys to be imported. 32 | if [[ -z $2 ]]; then 33 | $GPG --export -a "$KEYNAME" >"$pubkeyfile" 34 | else 35 | # TODO(tlim): This could probably be done with GNUPGHOME 36 | # but that affects all commands; we just want it to affect the key export. 37 | $GPG --homedir="$2" --export -a "$KEYNAME" >"$pubkeyfile" 38 | fi 39 | 40 | if [[ $(wc -l < "$pubkeyfile") = 0 ]]; then 41 | fail_out "GPG key '$KEYNAME' not found. Please create it with: $GPG --gen-key" 42 | exit 1 43 | fi 44 | 45 | # Import it: 46 | $GPG --no-permission-warning --homedir="$KEYRINGDIR" --import "$pubkeyfile" 47 | pubring_path=$(get_pubring_path) 48 | vcs_add "$pubring_path" "$KEYRINGDIR/trustdb.gpg" "$BB_ADMINS" 49 | 50 | # Make a suggestion: 51 | echo 52 | echo 53 | echo 'NEXT STEP: You need to manually check these in:' 54 | echo ' ' $VCS_TYPE commit -m\'NEW ADMIN: $KEYNAME\' "$BLACKBOXDATA/$(basename ${pubring_path})" "$BLACKBOXDATA/trustdb.gpg" "$BLACKBOXDATA/$BB_ADMINS_FILE" 55 | -------------------------------------------------------------------------------- /bin/blackbox_cat: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_cat -- Decrypt a file, cat it, shred it 5 | # 6 | set -e 7 | source "${0%/*}/_blackbox_common.sh" 8 | 9 | for param in "$@" ; do 10 | shreddable=0 11 | unencrypted_file=$(get_unencrypted_filename "$param") 12 | if [[ ! -e "$unencrypted_file" ]]; then 13 | "${BLACKBOX_HOME}/blackbox_edit_start" "$param" 14 | shreddable=1 15 | fi 16 | cat "$unencrypted_file" 17 | if [[ $shreddable = 1 ]]; then 18 | shred_file "$unencrypted_file" 19 | fi 20 | done 21 | -------------------------------------------------------------------------------- /bin/blackbox_decrypt_all_files: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_decrypt_all_files -- Decrypt all blackbox files (INTERACTIVE). 5 | # 6 | 7 | # Usage: 8 | # blackbox_decrypt_all_files [GROUP] 9 | # GROUP is optional. If supplied, the resulting files 10 | # are chgrp'ed to that group. 11 | 12 | # Since this is often run in a security-critical situation, we 13 | # force /usr/bin and /bin to the front of the PATH. 14 | export PATH=/usr/bin:/bin:"$PATH" 15 | 16 | set -e 17 | source "${0%/*}/_blackbox_common.sh" 18 | 19 | gpg_agent_notice 20 | exec blackbox_postdeploy "$@" 21 | -------------------------------------------------------------------------------- /bin/blackbox_decrypt_file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_decrypt_file -- Decrypt one or more blackbox files. 5 | # 6 | 7 | set -e 8 | source "${0%/*}/_blackbox_common.sh" 9 | 10 | if [ $# -eq 0 ]; then 11 | echo >&2 "Please provide at least one file to decrypt" 12 | exit 1 13 | fi 14 | 15 | "${BLACKBOX_HOME}/blackbox_edit_start" "$@" 16 | -------------------------------------------------------------------------------- /bin/blackbox_deregister_file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_deregister_file -- Remove a file from the blackbox system. 5 | # 6 | # Takes an encrypted file and removes it from the blackbox system. 7 | # The encrypted file will also be removed from the filesystem. 8 | # The unencrypted file, if it exists, will be left alone. 9 | 10 | set -e 11 | source "${0%/*}/_blackbox_common.sh" 12 | 13 | unencrypted_file=$(get_unencrypted_filename "$1") 14 | encrypted_file=$(get_encrypted_filename "$1") 15 | 16 | if [[ "$1" == "$unencrypted_file" ]]; then 17 | echo ERROR: Please only deregister encrypted files. 18 | exit 1 19 | fi 20 | 21 | echo ========== PLAINFILE "$unencrypted_file" 22 | echo ========== ENCRYPTED "$encrypted_file" 23 | 24 | fail_if_not_exists "$encrypted_file" "Please specify an existing file." 25 | 26 | prepare_keychain 27 | remove_filename_from_cryptlist "$unencrypted_file" 28 | vcs_remove "$encrypted_file" 29 | vcs_notice "$unencrypted_file" 30 | vcs_add "$BB_FILES" 31 | 32 | vcs_commit "Removing from blackbox: ${unencrypted_file}" "$BB_FILES" "$encrypted_file" "$(vcs_ignore_file_path)" 33 | echo "========== UPDATING VCS: DONE" 34 | echo "Local repo updated. Please push when ready." 35 | echo " $VCS_TYPE push" 36 | -------------------------------------------------------------------------------- /bin/blackbox_diff: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_diff -- Show all differences. 5 | # 6 | 7 | set -e 8 | source "${0%/*}/_blackbox_common.sh" 9 | 10 | gpg_agent_notice 11 | prepare_keychain 12 | 13 | modified_files=() 14 | modifications=() 15 | echo '========== DIFFING FILES: START' 16 | while IFS= read <&99 -r unencrypted_file; do 17 | unencrypted_file=$(get_unencrypted_filename "$unencrypted_file") 18 | encrypted_file=$(get_encrypted_filename "$unencrypted_file") 19 | fail_if_not_on_cryptlist "$unencrypted_file" 20 | if [[ -f "$unencrypted_file" ]]; then 21 | out=$(diff -u <($GPG --yes -q --decrypt "$encrypted_file") "$unencrypted_file" || true) 22 | if [ "$out" != "" ]; then 23 | modified_files+=("$unencrypted_file") 24 | modifications+=("$out") 25 | fi 26 | fi 27 | done 99<"$BB_FILES" 28 | modified_files_number=${#modified_files[@]} 29 | for (( i=0; i<${modified_files_number}; i++ )); do 30 | echo ========== PROCESSING '"'${modified_files[$i]}'"' 31 | echo -e "${modifications[$i]}\n" 32 | done 33 | echo '========== DIFFING FILES: DONE' 34 | 35 | fail_if_keychain_has_secrets 36 | 37 | echo '========== DONE.' 38 | 39 | if [ ${#modified_files[@]} -eq 0 ] ; then 40 | exit 0 41 | fi 42 | 43 | echo 'Likely next steps:' 44 | for f in "${modified_files[@]}" ; do 45 | echo " blackbox_edit_end" '"'$f'"' 46 | done 47 | -------------------------------------------------------------------------------- /bin/blackbox_edit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_edit -- Decrypt a file temporarily for edition, then re-encrypts it again 5 | # 6 | set -e 7 | source "${0%/*}/_blackbox_common.sh" 8 | 9 | for param in "$@" ; do 10 | 11 | unencrypted_file=$(get_unencrypted_filename "$param") 12 | encrypted_file=$(get_encrypted_filename "$param") 13 | 14 | echo >&2 ========== PLAINFILE '"'$unencrypted_file'"' 15 | echo >&2 ========== ENCRYPTED '"'$encrypted_file'"' 16 | 17 | if ! is_on_cryptlist "$encrypted_file" && ! is_on_cryptlist "$unencrypted_file" ; then 18 | read -r -p "Encrypt file $param? (y/n) " ans 19 | case "$ans" in 20 | y* | Y*) 21 | "${BLACKBOX_HOME}/blackbox_register_new_file" "$unencrypted_file" 22 | ;; 23 | *) 24 | echo >&2 'Skipping...' 25 | continue 26 | ;; 27 | esac 28 | fi 29 | "${BLACKBOX_HOME}/blackbox_edit_start" "$unencrypted_file" 30 | $EDITOR "$(get_unencrypted_filename "$unencrypted_file")" 31 | "${BLACKBOX_HOME}/blackbox_edit_end" "$unencrypted_file" 32 | 33 | done 34 | -------------------------------------------------------------------------------- /bin/blackbox_edit_end: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_edit_end -- Re-encrypt file after edits. 5 | # 6 | 7 | set -e 8 | source "${0%/*}/_blackbox_common.sh" 9 | 10 | next_steps=() 11 | 12 | if [ $# -eq 0 ]; then 13 | echo >&2 "Please provide at least one file for which editing has finished" 14 | exit 1 15 | fi 16 | 17 | for param in "$@" ; do 18 | 19 | unencrypted_file=$(get_unencrypted_filename "$param") 20 | encrypted_file=$(get_encrypted_filename "$param") 21 | 22 | echo >&2 ========== PLAINFILE '"'$unencrypted_file'"' 23 | echo >&2 ========== ENCRYPTED '"'$encrypted_file'"' 24 | 25 | fail_if_not_on_cryptlist "$unencrypted_file" 26 | fail_if_not_exists "$unencrypted_file" "No unencrypted version to encrypt!" 27 | fail_if_keychain_has_secrets 28 | 29 | encrypt_file "$unencrypted_file" "$encrypted_file" 30 | shred_file "$unencrypted_file" 31 | echo >&2 ========== UPDATED '"'$encrypted_file'"' 32 | next_steps+=( " $VCS_TYPE commit -m\"${encrypted_file} updated\" \"$encrypted_file\"" ) 33 | 34 | done 35 | 36 | echo >&2 "Likely next step:" 37 | for x in "${next_steps[@]}" 38 | do 39 | echo >&2 "$x" 40 | done 41 | -------------------------------------------------------------------------------- /bin/blackbox_edit_start: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_edit_start -- Decrypt a file for editing. 5 | # 6 | 7 | set -e 8 | source "${0%/*}/_blackbox_common.sh" 9 | 10 | if [ $# -eq 0 ]; then 11 | echo >&2 "Please provide at least one file to start editing" 12 | exit 1 13 | fi 14 | 15 | for param in "$@" ; do 16 | 17 | unencrypted_file=$(get_unencrypted_filename "$param") 18 | encrypted_file=$(get_encrypted_filename "$param") 19 | 20 | echo >&2 ========== PLAINFILE '"'$unencrypted_file'"' 21 | 22 | fail_if_not_on_cryptlist "$unencrypted_file" 23 | fail_if_not_exists "$encrypted_file" "This should not happen." 24 | if [[ ! -s "$unencrypted_file" ]]; then 25 | rm -f "$unencrypted_file" 26 | fi 27 | if [[ -f "$unencrypted_file" ]]; then 28 | echo >&2 SKIPPING: "$1" "Will not overwrite non-empty files." 29 | continue 30 | fi 31 | 32 | prepare_keychain 33 | # FIXME(tlim): prepare_keychain only needs to run once, outside of the loop. 34 | decrypt_file "$encrypted_file" "$unencrypted_file" 35 | 36 | done 37 | -------------------------------------------------------------------------------- /bin/blackbox_initialize: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_initialize -- Enable blackbox for a GIT or HG repo. 5 | # 6 | # 7 | # Example: 8 | # blackbox_initialize 9 | # 10 | 11 | set -e 12 | source "${0%/*}/_blackbox_common.sh" 13 | 14 | if [[ $1 != 'yes' ]]; then 15 | read -r -p "Enable blackbox for this $VCS_TYPE repo? (yes/no) " ans 16 | if [[ $ans = 'no' || $ans = 'n' || $ans = '' ]]; then 17 | echo 'Exiting...' 18 | exit 1 19 | fi 20 | fi 21 | 22 | if [[ $VCS_TYPE = "unknown" ]]; then 23 | echo 'Not in a known VCS directory' 24 | exit 1 25 | fi 26 | 27 | change_to_vcs_root 28 | 29 | echo VCS_TYPE: $VCS_TYPE 30 | vcs_ignore "${BLACKBOXDATA}/pubring.gpg~" "${BLACKBOXDATA}/pubring.kbx~" "${BLACKBOXDATA}/secring.gpg" 31 | 32 | # Make directories 33 | mkdir -p "${KEYRINGDIR}" 34 | vcs_add "${KEYRINGDIR}" 35 | touch "$BLACKBOXDATA/$BB_ADMINS_FILE" "$BLACKBOXDATA/$BB_FILES_FILE" 36 | vcs_add "$BLACKBOXDATA/$BB_ADMINS_FILE" "$BLACKBOXDATA/$BB_FILES_FILE" 37 | 38 | if [[ $VCS_TYPE = "git" ]]; then 39 | 40 | # Set .gitattributes so that Windows users don't break the admin files. 41 | FILE="$BLACKBOXDATA/.gitattributes" 42 | touch "$FILE" 43 | LINE='blackbox-admins.txt text eol=lf' 44 | grep -qF "$LINE" "$FILE" || echo "$LINE" >> "$FILE" 45 | LINE='blackbox-files.txt text eol=lf' 46 | grep -qF "$LINE" "$FILE" || echo "$LINE" >> "$FILE" 47 | vcs_add "$FILE" 48 | fi 49 | 50 | if [[ $VCS_TYPE = "svn" ]]; then 51 | echo 52 | echo 53 | echo '`subversion` automatically tracks the ignored files; you just need to commit.' 54 | else 55 | IGNOREFILE="$(vcs_ignore_file_path)" 56 | test -f "$IGNOREFILE" && vcs_add "$IGNOREFILE" 57 | 58 | # Make a suggestion: 59 | echo 60 | echo 61 | echo 'NEXT STEP: You need to manually check these in:' 62 | echo ' ' $VCS_TYPE commit -m\'INITIALIZE BLACKBOX\' "$BLACKBOXDATA" "$IGNOREFILE" 63 | fi 64 | -------------------------------------------------------------------------------- /bin/blackbox_list_admins: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_list_admins -- List authorized admins 5 | # 6 | set -e 7 | source "${0%/*}/_blackbox_common.sh" 8 | cat "$BB_ADMINS" 9 | -------------------------------------------------------------------------------- /bin/blackbox_list_files: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_list_files -- List files that black box is tracking 5 | # 6 | set -e 7 | source "${0%/*}/_blackbox_common.sh" 8 | cat "$BB_FILES" 9 | -------------------------------------------------------------------------------- /bin/blackbox_listadmins: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_listadmins -- List active admins for keyring 5 | # 6 | 7 | # Example: 8 | # blackbox_listadmins 9 | # 10 | 11 | set -e 12 | source "${0%/*}/_blackbox_common.sh" 13 | 14 | fail_if_not_in_repo 15 | 16 | 17 | # simply display the contents of the admins file 18 | cat "$BB_ADMINS" 19 | -------------------------------------------------------------------------------- /bin/blackbox_postdeploy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_postdeploy -- Decrypt all blackbox files. 5 | # 6 | 7 | # Usage: 8 | # blackbox_postdeploy.sh [GROUP] 9 | # GROUP is optional. If supplied, the resulting files 10 | # are chgrp'ed to that group. 11 | 12 | # Since this is often run in a security-critical situation, we 13 | # force /usr/bin and /bin to the front of the PATH. 14 | export PATH=/usr/bin:/bin:"$PATH" 15 | 16 | set -e 17 | source "${0%/*}/_blackbox_common.sh" 18 | 19 | if [[ "$1" == "" ]]; then 20 | FILE_GROUP="" 21 | else 22 | FILE_GROUP="$1" 23 | fi 24 | 25 | change_to_vcs_root 26 | prepare_keychain 27 | 28 | # Decrypt: 29 | echo '========== Decrypting new/changed files: START' 30 | while IFS= read <&99 -r unencrypted_file; do 31 | encrypted_file=$(get_encrypted_filename "$unencrypted_file") 32 | decrypt_file_overwrite "$encrypted_file" "$unencrypted_file" 33 | cp_permissions "$encrypted_file" "$unencrypted_file" 34 | if [[ ! -z "$FILE_GROUP" ]]; then 35 | chmod g+r "$unencrypted_file" 36 | chgrp "$FILE_GROUP" "$unencrypted_file" 37 | fi 38 | done 99<"$BB_FILES" 39 | 40 | echo '========== Decrypting new/changed files: DONE' 41 | -------------------------------------------------------------------------------- /bin/blackbox_recurse: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # proposed space for blackbox recurion...coming soon 3 | set -e 4 | source "${0%/*}/_blackbox_common.sh" 5 | 6 | echo "$REBOBASE" 7 | -------------------------------------------------------------------------------- /bin/blackbox_register_new_file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_register_new_file -- Enroll new file(s) in the blackbox system. 5 | # 6 | # Takes previously unencrypted file(s) and enrolls them into the blackbox 7 | # system. Each file will be kept in the repo as an encrypted file. On deployment 8 | # to systems that need the plaintext (unencrypted) versions, run 9 | # blackbox_postdeploy.sh to decrypt all the files. 10 | 11 | set -e 12 | source "${0%/*}/_blackbox_common.sh" 13 | 14 | function register_new_file() { 15 | unencrypted_file=$(get_unencrypted_filename "$1") 16 | encrypted_file=$(get_encrypted_filename "$1") 17 | 18 | if [[ "$1" == "$encrypted_file" ]]; then 19 | echo ERROR: Please only register unencrypted files. 20 | exit 1 21 | fi 22 | 23 | echo "========== PLAINFILE $unencrypted_file" 24 | echo "========== ENCRYPTED $encrypted_file" 25 | 26 | fail_if_not_exists "$unencrypted_file" "Please specify an existing file." 27 | fail_if_exists "$encrypted_file" "Will not overwrite." 28 | 29 | prepare_keychain 30 | encrypt_file "$unencrypted_file" "$encrypted_file" 31 | add_filename_to_cryptlist "$unencrypted_file" 32 | vcs_ignore "$unencrypted_file" 33 | 34 | # Is the unencrypted file already in HG? (ie. are we correcting a bad situation) 35 | SECRETSEXPOSED=$(is_in_vcs "${unencrypted_file}") 36 | echo "========== CREATED: ${encrypted_file}" 37 | echo "========== UPDATING REPO:" 38 | shred_file "$unencrypted_file" 39 | 40 | if [[ "$SECRETSEXPOSED" == "true" ]] ; then 41 | vcs_remove "$unencrypted_file" 42 | vcs_add "$encrypted_file" 43 | fi 44 | 45 | echo 'NOTE: "already tracked!" messages are safe to ignore.' 46 | vcs_add "$BB_FILES" "$encrypted_file" 47 | vcs_commit "registered in blackbox: ${unencrypted_file}" "$BB_FILES" "$encrypted_file" "$(vcs_ignore_file_path)" 48 | } 49 | 50 | for target in "$@"; do 51 | register_new_file "$target" 52 | done 53 | 54 | echo "========== UPDATING VCS: DONE" 55 | if [[ $VCS_TYPE = "svn" ]]; then 56 | echo "Local repo updated and file pushed to source control (unless an error was displayed)." 57 | else 58 | echo "Local repo updated. Please push when ready." 59 | echo " $VCS_TYPE push" 60 | fi 61 | -------------------------------------------------------------------------------- /bin/blackbox_removeadmin: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_removeadmin -- Remove an admin to the system 5 | # NOTE: Does not remove admin from the keyring. 6 | # 7 | 8 | # Example: 9 | # blackbox_removeadmin tal@example.com 10 | # 11 | 12 | set -e 13 | source "${0%/*}/_blackbox_common.sh" 14 | 15 | fail_if_not_in_repo 16 | 17 | KEYNAME="$1" 18 | : "${KEYNAME:?ERROR: First argument must be a keyname (email address)}" ; 19 | 20 | # Remove the email address from the BB_ADMINS file. 21 | remove_line "$BB_ADMINS" "$KEYNAME" 22 | 23 | 24 | # remove the admin key from the pubring 25 | $GPG --no-permission-warning --homedir="$KEYRINGDIR" --batch --yes --delete-key "$KEYNAME" || true 26 | pubring_path=$(get_pubring_path) 27 | vcs_add "$pubring_path" "$KEYRINGDIR/trustdb.gpg" "$BB_ADMINS" 28 | 29 | 30 | # Make a suggestion: 31 | echo 32 | echo 33 | echo 'NEXT STEP: Check these into the repo. Probably with a command like...' 34 | echo $VCS_TYPE commit -m\'REMOVED ADMIN: $KEYNAME\' "$BLACKBOXDATA/$(basename ${pubring_path})" "$BLACKBOXDATA/trustdb.gpg" "$BLACKBOXDATA/$BB_ADMINS_FILE" 35 | -------------------------------------------------------------------------------- /bin/blackbox_shred_all_files: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_shred_all_files -- shred all decrypted versions of encrypted files 5 | # 6 | # Shred: To securely delete a file. 7 | # 8 | # Typical uses: 9 | # After running blackbox_edit_start, deciding not to edit the file. 10 | # A developer that wants to securely clean up a workspace before deleting it. 11 | # An automated process that doesn't want to leave 12 | # plaintext (unencrypted) files laying around. 13 | # 14 | # NOTE: The output lists files that were decrypted and are being 15 | # shredded. For example, if you have many encrypted files but none 16 | # have been decrypted for editing, you will see an empty list. 17 | 18 | set -e 19 | source "${0%/*}/_blackbox_common.sh" 20 | 21 | change_to_vcs_root 22 | 23 | echo '========== FILES BEING SHREDDED:' 24 | 25 | exported_internal_shred_file() { 26 | source "$1/_blackbox_common.sh" 27 | #unencrypted_file=$(get_unencrypted_filename "$2") 28 | unencrypted_file="$2" 29 | if [[ -f "$unencrypted_file" ]]; then 30 | echo " SHRED: $unencrypted_file" 31 | shred_file "$unencrypted_file" 32 | else 33 | echo "NOT FOUND: $unencrypted_file" 34 | fi 35 | } 36 | 37 | export -f exported_internal_shred_file 38 | 39 | DEREFERENCED_BIN_DIR="${0%/*}" 40 | MAX_PARALLEL_SHRED=10 41 | 42 | bash_args= 43 | if bash --help | grep import-functions >/dev/null 2>/dev/null; then 44 | bash_args=--import-functions 45 | fi 46 | 47 | export IFS= 48 | tr '\n' '\0' <"$BB_FILES" | xargs -0 -I{} -P $MAX_PARALLEL_SHRED bash $bash_args -c "exported_internal_shred_file $DEREFERENCED_BIN_DIR \"{}\"" $DEREFERENCED_BIN_DIR/fake 49 | 50 | echo '========== DONE.' 51 | -------------------------------------------------------------------------------- /bin/blackbox_update_all_files: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_update_all_files -- Decrypt then re-encrypt all files. Useful after keys are changed. 5 | # 6 | 7 | set -e 8 | source "${0%/*}/_blackbox_common.sh" 9 | 10 | gpg_agent_notice 11 | disclose_admins 12 | prepare_keychain 13 | 14 | echo '========== ENCRYPTED FILES TO BE RE-ENCRYPTED:' 15 | while IFS= read <&99 -r unencrypted_file; do 16 | echo " $unencrypted_file.gpg" 17 | done 99<"$BB_FILES" 18 | 19 | echo '========== FILES IN THE WAY:' 20 | need_warning=false 21 | while IFS= read <&99 -r unencrypted_file; do 22 | unencrypted_file=$(get_unencrypted_filename "$unencrypted_file") 23 | encrypted_file=$(get_encrypted_filename "$unencrypted_file") 24 | if [[ -f "$unencrypted_file" ]]; then 25 | need_warning=true 26 | echo " $unencrypted_file" 27 | fi 28 | done 99<"$BB_FILES" 29 | if "$need_warning" ; then 30 | echo 31 | echo 'WARNING: This will overwrite any unencrypted files laying about.' 32 | read -r -p 'Press CTRL-C now to stop. ENTER to continue: ' 33 | else 34 | echo 'All OK.' 35 | fi 36 | 37 | echo '========== RE-ENCRYPTING FILES:' 38 | while IFS= read <&99 -r unencrypted_file; do 39 | unencrypted_file=$(get_unencrypted_filename "$unencrypted_file") 40 | encrypted_file=$(get_encrypted_filename "$unencrypted_file") 41 | echo ========== PROCESSING '"'$unencrypted_file'"' 42 | fail_if_not_on_cryptlist "$unencrypted_file" 43 | decrypt_file_overwrite "$encrypted_file" "$unencrypted_file" 44 | encrypt_file "$unencrypted_file" "$encrypted_file" 45 | shred_file "$unencrypted_file" 46 | done 99<"$BB_FILES" 47 | 48 | fail_if_keychain_has_secrets 49 | 50 | echo '========== COMMITING TO VCS:' 51 | while IFS= read <&99 -r unencrypted_file; do 52 | vcs_add "$unencrypted_file.gpg" 53 | done 99<"$BB_FILES" 54 | vcs_commit 'Re-encrypted keys' 55 | 56 | echo '========== DONE.' 57 | echo 'Likely next step:' 58 | echo " $VCS_TYPE push" 59 | -------------------------------------------------------------------------------- /bin/blackbox_view: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_view -- Decrypt a file, view it, shred it 5 | # 6 | set -e 7 | source "${0%/*}/_blackbox_common.sh" 8 | 9 | for param in "$@" ; do 10 | shreddable=0 11 | unencrypted_file=$(get_unencrypted_filename "$param") 12 | if [[ ! -e "$unencrypted_file" ]]; then 13 | "${BLACKBOX_HOME}/blackbox_edit_start" "$param" 14 | shreddable=1 15 | fi 16 | ${PAGER:-less} "$unencrypted_file" 17 | if [[ $shreddable = 1 ]]; then 18 | shred_file "$unencrypted_file" 19 | fi 20 | done 21 | -------------------------------------------------------------------------------- /bin/blackbox_whatsnew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_whatsnew - show what has changed in the last commit for a given file 5 | # 6 | 7 | set -e 8 | source "${0%/*}/_blackbox_common.sh" 9 | 10 | if [[ $# -ne 1 ]] 11 | then 12 | echo "Pass only 1 file at a time" 13 | exit 1 14 | fi 15 | 16 | fail_if_not_in_repo 17 | gpg_agent_notice 18 | 19 | COLUMNS=`tput cols` 20 | FILE=$1 21 | GIT="git log --abbrev-commit --pretty=oneline" 22 | CURR_COMMIT=`$GIT $FILE | head -1 | awk '{print $1}'` 23 | PREV_COMMIT=`$GIT ${CURR_COMMIT}~1 $FILE | head -1 | awk '{print $1}'` 24 | # Use colordiff if available 25 | if which colordiff > /dev/null 2>&1 26 | then DIFF="colordiff" 27 | else DIFF="diff" 28 | fi 29 | 30 | cat_commit() 31 | { 32 | COMMIT=$1 33 | git checkout $COMMIT $FILE 34 | echo "[$COMMIT] $FILE" 35 | echo "---------------------" 36 | "${BLACKBOX_HOME}/blackbox_cat" $FILE | sed '/========== PLAINFILE/,/========== EXTRACTING/d' 37 | } 38 | 39 | CURR_CONTENT=`cat_commit $CURR_COMMIT` 40 | PREV_CONTENT=`cat_commit $PREV_COMMIT` 41 | clear 42 | 43 | # For some unknown reason this command executes fine but return exit code 1 44 | $DIFF -y --width $COLUMNS \ 45 | <(echo "CURRENT" "$CURR_CONTENT" | fold -w $(( $COLUMNS / 2 - 4 )) ) \ 46 | <(echo "PREVIOUS" "$PREV_CONTENT" | fold -w $(( $COLUMNS / 2 - 4 )) ) 47 | 48 | git checkout $CURR_COMMIT $FILE 49 | echo 50 | -------------------------------------------------------------------------------- /binv2/blackbox_addadmin: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox admin add "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_cat: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox cat "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_decrypt_all_files: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox decrypt --all --agentcheck=true --overwrite "@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_decrypt_file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox decrypt --overwrite "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_deregister_file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox file remove --safe "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_diff: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox diff --diff "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_edit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox edit "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_edit_end: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox encrypt --shred "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_edit_start: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox decrypt "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_initialize: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox init "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_list_admins: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox admin list 3 | -------------------------------------------------------------------------------- /binv2/blackbox_list_files: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox file list 3 | -------------------------------------------------------------------------------- /binv2/blackbox_listadmins: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox admin list 3 | -------------------------------------------------------------------------------- /binv2/blackbox_postdeploy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | blackbox decrypt --all --overwrite --group "$1" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_register_new_file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox file add --shred "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_removeadmin: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox admin remove "$@" 3 | -------------------------------------------------------------------------------- /binv2/blackbox_shred_all_files: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox shred --all 3 | -------------------------------------------------------------------------------- /binv2/blackbox_update_all_files: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec blackbox reencrypt --all --agentcheck 3 | -------------------------------------------------------------------------------- /binv2/blackbox_view: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | blackbox cat "$@" | ${PAGER:-less} 3 | -------------------------------------------------------------------------------- /binv2/blackbox_whatsnew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # blackbox_whatsnew - show what has changed in the last commit for a given file 5 | # 6 | exec blackbox whatsnew "$@" 7 | exit 0 8 | 9 | set -e 10 | source "${0%/*}/_blackbox_common.sh" 11 | 12 | if [[ $# -ne 1 ]] 13 | then 14 | echo "Pass only 1 file at a time" 15 | exit 1 16 | fi 17 | 18 | fail_if_not_in_repo 19 | gpg_agent_notice 20 | 21 | COLUMNS=`tput cols` 22 | FILE=$1 23 | GIT="git log --abbrev-commit --pretty=oneline" 24 | CURR_COMMIT=`$GIT $FILE | head -1 | awk '{print $1}'` 25 | PREV_COMMIT=`$GIT ${CURR_COMMIT}~1 $FILE | head -1 | awk '{print $1}'` 26 | # Use colordiff if available 27 | if which colordiff > /dev/null 2>&1 28 | then DIFF="colordiff" 29 | else DIFF="diff" 30 | fi 31 | 32 | cat_commit() 33 | { 34 | COMMIT=$1 35 | git checkout $COMMIT $FILE 36 | echo "[$COMMIT] $FILE" 37 | echo "---------------------" 38 | "${BLACKBOX_HOME}/blackbox_cat" $FILE | sed '/========== PLAINFILE/,/========== EXTRACTING/d' 39 | } 40 | 41 | CURR_CONTENT=`cat_commit $CURR_COMMIT` 42 | PREV_CONTENT=`cat_commit $PREV_COMMIT` 43 | clear 44 | 45 | # For some unknown reason this command executes fine but return exit code 1 46 | $DIFF -y --width $COLUMNS \ 47 | <(echo "CURRENT" "$CURR_CONTENT" | fold -w $(( $COLUMNS / 2 - 4 )) ) \ 48 | <(echo "PREVIOUS" "$PREV_CONTENT" | fold -w $(( $COLUMNS / 2 - 4 )) ) 49 | 50 | git checkout $CURR_COMMIT $FILE 51 | echo 52 | -------------------------------------------------------------------------------- /blackbox.plugin.zsh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | # The MIT License (MIT) 3 | 4 | # Copyright (c) 2014 Stack Exchange, Inc. 5 | 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | 13 | # The above copyright notice and this permission notice shall be included in 14 | # all copies or substantial portions of the Software. 15 | 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | # THE SOFTWARE. 23 | 24 | # Make it easy to install with antigen and other frameworks that can cope 25 | # with antigen/oh-my-zsh bundles 26 | # 27 | # antigen bundle StackExchange/blackbox 28 | 29 | # Add our plugin's bin diretory to user's path 30 | PLUGIN_BIN="$(dirname $0)/bin" 31 | export PATH="${PATH}:${PLUGIN_BIN}" 32 | -------------------------------------------------------------------------------- /build/build.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | var sha = flag.String("sha", "", "SHA of current commit") 14 | 15 | var goos = flag.String("os", "", "OS to build (linux, windows, or darwin) Defaults to all.") 16 | 17 | func main() { 18 | flag.Parse() 19 | flags := fmt.Sprintf(`-s -w -X main.SHA="%s" -X main.BuildTime=%d`, getVersion(), time.Now().Unix()) 20 | pkg := "github.com/StackExchange/blackbox/v2/cmd/blackbox" 21 | 22 | build := func(out, goos string) { 23 | log.Printf("Building %s", out) 24 | cmd := exec.Command("go", "build", "-o", out, "-ldflags", flags, pkg) 25 | os.Setenv("GOOS", goos) 26 | os.Setenv("GO111MODULE", "on") 27 | cmd.Stderr = os.Stderr 28 | cmd.Stdout = os.Stdout 29 | err := cmd.Run() 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | } 34 | 35 | for _, env := range []struct { 36 | binary, goos string 37 | }{ 38 | {"blackbox-Linux", "linux"}, 39 | {"blackbox.exe", "windows"}, 40 | {"blackbox-Darwin", "darwin"}, 41 | } { 42 | if *goos == "" || *goos == env.goos { 43 | build(env.binary, env.goos) 44 | } 45 | } 46 | } 47 | 48 | func getVersion() string { 49 | if *sha != "" { 50 | return *sha 51 | } 52 | // check teamcity build version 53 | if v := os.Getenv("BUILD_VCS_NUMBER"); v != "" { 54 | return v 55 | } 56 | // check git 57 | cmd := exec.Command("git", "rev-parse", "HEAD") 58 | v, err := cmd.CombinedOutput() 59 | if err != nil { 60 | return "" 61 | } 62 | ver := strings.TrimSpace(string(v)) 63 | // see if dirty 64 | cmd = exec.Command("git", "diff-index", "--quiet", "HEAD", "--") 65 | err = cmd.Run() 66 | // exit status 1 indicates dirty tree 67 | if err != nil { 68 | if err.Error() == "exit status 1" { 69 | ver += "[dirty]" 70 | } else { 71 | log.Printf("!%s!", err.Error()) 72 | } 73 | } 74 | return ver 75 | } 76 | -------------------------------------------------------------------------------- /cmd/blackbox/blackbox.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | _ "github.com/StackExchange/blackbox/v2/pkg/crypters" 8 | _ "github.com/StackExchange/blackbox/v2/pkg/crypters/_all" 9 | _ "github.com/StackExchange/blackbox/v2/pkg/vcs" 10 | _ "github.com/StackExchange/blackbox/v2/pkg/vcs/_all" 11 | ) 12 | 13 | var dryRun bool 14 | 15 | func main() { 16 | app := flags() 17 | err := app.Run(os.Args) 18 | if err != nil { 19 | fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) 20 | os.Exit(1) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /cmd/blackbox/cli.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // cli.go -- Create urfave/cli datastructures and apply them. 4 | 5 | import ( 6 | "fmt" 7 | 8 | "github.com/urfave/cli/v2" 9 | 10 | "github.com/StackExchange/blackbox/v2/pkg/bbutil" 11 | ) 12 | 13 | func flags() *cli.App { 14 | app := cli.NewApp() 15 | app.Version = "2.0.0" 16 | app.Usage = "Maintain encrypted files in a VCS (Git, Hg, Svn)" 17 | 18 | defUmask := bbutil.Umask(0) 19 | bbutil.Umask(defUmask) 20 | defUmaskS := fmt.Sprintf("%04o", defUmask) 21 | 22 | app.Flags = []cli.Flag{ 23 | // &cli.BoolFlag{ 24 | // Name: "dry-run", 25 | // Aliases: []string{"n"}, 26 | // Usage: "show what would have been done", 27 | // Destination: &dryRun, 28 | // }, 29 | &cli.StringFlag{ 30 | Name: "vcs", 31 | Usage: "Use this VCS (GIT, NONE) rather than autodetect", 32 | EnvVars: []string{"BLACKBOX_VCS"}, 33 | }, 34 | &cli.StringFlag{ 35 | Name: "crypto", 36 | Usage: "Crypto back-end plugin", 37 | Value: "GnuPG", 38 | EnvVars: []string{"BLACKBOX_CRYPTO"}, 39 | }, 40 | &cli.StringFlag{ 41 | Name: "config", 42 | Usage: "Path to config", 43 | //Value: ".blackbox", 44 | EnvVars: []string{"BLACKBOX_CONFIGDIR", "BLACKBOXDATA"}, 45 | }, 46 | &cli.StringFlag{ 47 | Name: "team", 48 | Usage: "Use .blackbox-$TEAM as the configdir", 49 | EnvVars: []string{"BLACKBOX_TEAM"}, 50 | }, 51 | &cli.StringFlag{ 52 | Name: "editor", 53 | Usage: "editor to use", 54 | Value: "vi", 55 | EnvVars: []string{"EDITOR", "BLACKBOX_EDITOR"}, 56 | }, 57 | &cli.StringFlag{ 58 | Name: "umask", 59 | Usage: "umask to set when decrypting", 60 | Value: defUmaskS, 61 | EnvVars: []string{"BLACKBOX_UMASK", "DECRYPT_UMASK"}, 62 | }, 63 | &cli.BoolFlag{ 64 | Name: "debug", 65 | Usage: "Show debug output", 66 | EnvVars: []string{"BLACKBOX_DEBUG"}, 67 | }, 68 | } 69 | 70 | app.Commands = []*cli.Command{ 71 | 72 | // List items in the order they appear in the help menu. 73 | 74 | { 75 | Name: "decrypt", 76 | Aliases: []string{"de", "start"}, 77 | Usage: "Decrypt file(s)", 78 | Flags: []cli.Flag{ 79 | &cli.BoolFlag{Name: "all", Usage: "All registered files"}, 80 | &cli.BoolFlag{Name: "agentcheck", Usage: "Do not check for gpg-agent when using --all"}, 81 | &cli.StringFlag{Name: "group", Usage: "Set group ownership"}, 82 | &cli.BoolFlag{Name: "overwrite", Usage: "Overwrite plaintext if it exists"}, 83 | }, 84 | Action: func(c *cli.Context) error { return cmdDecrypt(c) }, 85 | }, 86 | 87 | { 88 | Name: "encrypt", 89 | Aliases: []string{"en", "end"}, 90 | Usage: "Encrypts file(s)", 91 | Flags: []cli.Flag{ 92 | &cli.BoolFlag{Name: "shred", Usage: "Remove plaintext afterwards"}, 93 | }, 94 | Action: func(c *cli.Context) error { return cmdEncrypt(c) }, 95 | }, 96 | 97 | { 98 | Name: "edit", 99 | Aliases: []string{"vi"}, 100 | Usage: "Runs $EDITOR on file(s) (decrypt if needed)", 101 | Action: func(c *cli.Context) error { return cmdEdit(c) }, 102 | }, 103 | 104 | { 105 | Name: "cat", 106 | Usage: "Output plaintext to stderr (decrypt if needed)", 107 | Action: func(c *cli.Context) error { return cmdCat(c) }, 108 | }, 109 | 110 | { 111 | Name: "diff", 112 | Usage: "Diffs against encrypted version", 113 | Flags: []cli.Flag{ 114 | &cli.BoolFlag{Name: "all", Usage: "all files"}, 115 | }, 116 | Action: func(c *cli.Context) error { return cmdDiff(c) }, 117 | }, 118 | 119 | { 120 | Name: "init", 121 | Category: "ADMINISTRATIVE", 122 | Usage: "Initialized blackbox for this repo", 123 | Action: func(c *cli.Context) error { return cmdInit(c) }, 124 | }, 125 | 126 | { 127 | Name: "admin", 128 | Category: "ADMINISTRATIVE", 129 | Usage: "Add/list/remove administrators", 130 | Subcommands: []*cli.Command{ 131 | { 132 | Name: "add", 133 | Usage: "Adds admin(s)", 134 | Action: func(c *cli.Context) error { return cmdAdminAdd(c) }, 135 | }, 136 | { 137 | Name: "list", 138 | Usage: "Lists admins", 139 | Action: func(c *cli.Context) error { return cmdAdminList(c) }, 140 | }, 141 | { 142 | Name: "remove", 143 | Usage: "Remove admin(s)", 144 | Action: func(c *cli.Context) error { return cmdAdminRemove(c) }, 145 | }, 146 | }, 147 | }, 148 | 149 | { 150 | Name: "file", 151 | Category: "ADMINISTRATIVE", 152 | Usage: "Add/list/remove files from the registry", 153 | Subcommands: []*cli.Command{ 154 | { 155 | Name: "add", 156 | Usage: "Registers file with the system", 157 | Flags: []cli.Flag{ 158 | &cli.BoolFlag{Name: "shred", Usage: "Remove plaintext afterwords"}, 159 | }, 160 | Action: func(c *cli.Context) error { return cmdFileAdd(c) }, 161 | }, 162 | { 163 | Name: "list", 164 | Usage: "Lists the registered files", 165 | Action: func(c *cli.Context) error { return cmdFileList(c) }, 166 | }, 167 | { 168 | Name: "remove", 169 | Usage: "Deregister file from the system", 170 | Action: func(c *cli.Context) error { return cmdFileRemove(c) }, 171 | }, 172 | }, 173 | }, 174 | 175 | { 176 | Name: "info", 177 | Category: "DEBUG", 178 | Usage: "Report what we know about this repo", 179 | Action: func(c *cli.Context) error { return cmdInfo(c) }, 180 | }, 181 | 182 | { 183 | Name: "shred", 184 | Usage: "Shred files, or --all for all registered files", 185 | Flags: []cli.Flag{ 186 | &cli.BoolFlag{Name: "all", Usage: "All registered files"}, 187 | }, 188 | Action: func(c *cli.Context) error { return cmdShred(c) }, 189 | }, 190 | 191 | { 192 | Name: "status", 193 | Category: "ADMINISTRATIVE", 194 | Usage: "Print status of files", 195 | Flags: []cli.Flag{ 196 | &cli.BoolFlag{Name: "name-only", Usage: "Show only names of the files"}, 197 | &cli.BoolFlag{Name: "all", Usage: "All registered files"}, 198 | &cli.StringFlag{Name: "type", Usage: "only list if status matching this string"}, 199 | }, 200 | Action: func(c *cli.Context) error { return cmdStatus(c) }, 201 | }, 202 | 203 | { 204 | Name: "reencrypt", 205 | Usage: "Decrypt then re-encrypt files (erases any plaintext)", 206 | Category: "ADMINISTRATIVE", 207 | Flags: []cli.Flag{ 208 | &cli.BoolFlag{Name: "all", Usage: "All registered files"}, 209 | &cli.BoolFlag{Name: "overwrite", Usage: "Overwrite plaintext if it exists"}, 210 | &cli.BoolFlag{Name: "agentcheck", Usage: "Do not check for gpg-agent when using --all"}, 211 | }, 212 | Action: func(c *cli.Context) error { return cmdReencrypt(c) }, 213 | }, 214 | 215 | { 216 | Name: "testing_init", 217 | Usage: "For use with integration test", 218 | Category: "INTEGRATION TEST", 219 | Action: func(c *cli.Context) error { return testingInit(c) }, 220 | }, 221 | 222 | // 223 | 224 | } 225 | 226 | return app 227 | } 228 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | BlackBox 2 | ======== 3 | 4 | Blackbox is an open source tool that enables you to safe store sensitive information in 5 | Git (or other) repos by encrypting them with GPG. Only the encrypted 6 | version of the file is available. You can be free to provide access 7 | to the repo, as but only people with the right GPG keys can access the 8 | encrypted data. 9 | 10 | Things you should **never** store in a repo without encryption: 11 | 12 | * TLS (SSL) certificates 13 | * Passwords 14 | * API keys 15 | * And more! 16 | 17 | Project Info: 18 | 19 | * [Overview](user-overview.md) 20 | * [Why is this important?](why-is-this-important.md) 21 | * [Support/Community](support.md) 22 | * [How BB encrypts](encryption.md) 23 | * [OS Compatibility](compatibility.md) 24 | * [Installation Instructions](installation.md) 25 | * [Alternatives](alternatives.md) 26 | 27 | User Info: 28 | 29 | * [Enabling Blackbox on a Repo](enable-repo.md) 30 | * [Enroll a file](enable-repo.md) 31 | * [Full Command List](full-command-list.md) 32 | * [Add/Remove users](admin-ops.md) 33 | * [Add/Remove files](file-ops.md) 34 | * [Advanced techiques](advanced.md) 35 | * [Use with Role Accounts](role-accounts.md) 36 | * [Backwards Compatibility](backwards-compatibility.md) 37 | * [Replacing expired keys](expired-keys.md) 38 | * [Git Tips](git-tips.md) 39 | * [SubVersion Tips](subversion-tips.md) 40 | * [GnuPG tips](gnupg-tips.md) 41 | * [Use with Ansible](with-ansible.md) 42 | * [Use with Puppet](with-puppet.md) 43 | 44 | For contributors: 45 | 46 | * [Developer Info](dev.md) 47 | * [Code overview](dev-code-overview.md) 48 | * [HOWTO: Add new OS support](dev-add-os-support.md) 49 | * [HOWTO: Add new VCS support](dev-add-vcs-support.md) 50 | 51 | 52 | A slide presentation about an older release [is on SlideShare](http://www.slideshare.net/TomLimoncelli/the-blackbox-project-sfae). 53 | 54 | Join our mailing list: [https://groups.google.com/d/forum/blackbox-project](https://groups.google.com/d/forum/blackbox-project) 55 | 56 | 57 | License 58 | ======= 59 | 60 | This content is released under the MIT License. 61 | See the [LICENSE.txt](LICENSE.txt) file. 62 | -------------------------------------------------------------------------------- /docs/admin-ops.md: -------------------------------------------------------------------------------- 1 | User Management 2 | =============== 3 | 4 | 5 | # Who are the current admins? 6 | 7 | ``` 8 | blackbox admin list 9 | ``` 10 | 11 | 12 | # Add a new user (admin) 13 | 14 | FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?" 15 | 16 | `.blackbox/blackbox-admins.txt` is a file that lists which users are able to decrypt files. (More pedantically, it is a list of the GnuPG key names that the file is encrypted for.) 17 | 18 | To join the list of people that can edit the file requires three steps; You create a GPG key and add it to the key ring. Then, someone that already has access adds you to the system. Lastly, you should test your access. 19 | 20 | ## Step 1: NEWPERSON creates a GPG key pair on a secure machine and add to public keychain. 21 | 22 | If you don't already have a GPG key, here's how to generate one: 23 | 24 | ``` 25 | gpg --gen-key 26 | ``` 27 | 28 | WARNING: New versions of GPG generate keys which are not understood by 29 | old versions of GPG. If you generate a key with a new version of GPG, 30 | this will cause problems for users of older versions of GPG. 31 | Therefore it is recommended that you either assure that everyone using 32 | Blackbox have the exact same version of GPG, or generate GPG keys 33 | using a version of GPG as old as the oldest version of GPG used by 34 | everyone using Blackbox. 35 | 36 | Pick defaults for encryption settings, 0 expiration. Pick a VERY GOOD 37 | passphrase. Store a backup of the private key someplace secure. For 38 | example, keep the backup copy on a USB drive that is locked in safe. 39 | Or, at least put it on a secure machine with little or no internet 40 | access, full-disk-encryption, etc. Your employer probably has rules 41 | about how to store such things. 42 | 43 | FYI: If generating the key is slow, this is usually because the system 44 | isn't generating enough entropy. Tip: Open another window on that 45 | machine and run this command: `ls -R /` 46 | 47 | Now that you have a GPG key, add yourself as an admin: 48 | 49 | ``` 50 | blackbox admin add KEYNAME 51 | ``` 52 | 53 | ...where "KEYNAME" is the email address listed in the gpg key you created previously. For example: 54 | 55 | ``` 56 | blackbox admin add tal@example.com 57 | ``` 58 | 59 | When the command completes successfully, instructions on how to commit these changes will be output. Run the command as given to commit the changes. It will look like this: 60 | 61 | ``` 62 | git commit -m'NEW ADMIN: tal@example.com' .blackbox/pubring.gpg .blackbox/trustdb.gpg .blackbox/blackbox-admins.txt 63 | ``` 64 | 65 | 66 | Then push it to the repo: 67 | 68 | ``` 69 | git push 70 | 71 | or 72 | 73 | ht push 74 | 75 | (or whatever is appropriate) 76 | ``` 77 | 78 | NOTE: Creating a Role Account? If you are adding the pubring.gpg of a role account, you can specify the directory where the pubring.gpg file can be found as a 2nd parameter: `blackbox admin add puppetmaster@puppet-master-1.example.com /path/to/the/dir` 79 | 80 | ## Step 2: AN EXISTING ADMIN accepts you into the system. 81 | 82 | Ask someone that already has access to re-encrypt the data files. This 83 | gives you access. They simply decrypt and re-encrypt the data without 84 | making any changes. 85 | 86 | Pre-check: Verify the new keys look good. 87 | 88 | ``` 89 | git pull # Or whatever is required for your system 90 | gpg --homedir=.blackbox --list-keys 91 | ``` 92 | 93 | For example, examine the key name (email address) to make sure it conforms to corporate standards. 94 | 95 | Import the keychain into your personal keychain and reencrypt: 96 | 97 | ``` 98 | gpg --import .blackbox/pubring.gpg 99 | blackbox reencrypt --all shred 100 | ``` 101 | 102 | Push the re-encrypted files: 103 | 104 | ``` 105 | git commit -a 106 | git push 107 | 108 | or 109 | 110 | hg commit 111 | hg push 112 | ``` 113 | 114 | ### Step 3: NEWPERSON tests. 115 | 116 | Make sure you can decrypt a file. (Suggestion: Keep a dummy file in 117 | VCS just for new people to practice on.) 118 | 119 | 120 | # Remove a user 121 | 122 | Simply run `blackbox admin remove` with their keyname then re-encrypt: 123 | 124 | Example: 125 | 126 | ``` 127 | blackbox admin remove olduser@example.com 128 | blackbox reencrypt --all shred 129 | ``` 130 | 131 | When the command completes, you will be given a reminder to check in the change and push it. 132 | 133 | Note that their keys will still be in the key ring, but they will go unused. If you'd like to clean up the keyring, use the normal GPG commands and check in the file. 134 | 135 | FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?" 136 | 137 | ``` 138 | gpg --homedir=.blackbox --list-keys 139 | gpg --homedir=.blackbox --delete-key olduser@example.com 140 | git commit -m'Cleaned olduser@example.com from keyring' .blackbox/* 141 | ``` 142 | 143 | FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?" 144 | 145 | The key ring only has public keys. There are no secret keys to delete. 146 | 147 | Remember that this person did have access to all the secrets at one time. They could have made a copy. Therefore, to be completely secure, you should change all passwords, generate new SSL keys, and so on just like when anyone that had privileged access leaves an organization. 148 | 149 | -------------------------------------------------------------------------------- /docs/advanced.md: -------------------------------------------------------------------------------- 1 | Advanced Techniques 2 | =================== 3 | 4 | 5 | # Using Blackbox without a repo 6 | 7 | If the files are copied out of a repo they can still be decrypted and 8 | edited. Obviously edits, changes to keys, and such will be lost if 9 | they are made outside the repo. Also note that commands are most 10 | likely to only work if run from the base directory (i.e. the parent to 11 | the .blackbox directory). 12 | 13 | Without a repo, all commands must be run from the same directory 14 | as the ".blackbox" directory. It might work otherwise but no 15 | promises. 16 | 17 | 18 | # Mixing gpg 1.x/2.0 and 2.2 19 | 20 | WARNING: Each version of GnuPG uses a different, and incompatible, 21 | binary format to store the keychain. When Blackbox was originally 22 | created, I didn't know this. Things are mostly upwards compatible. 23 | That said, if you have some admins with GnuPG 1.x and others with GnuPG 2.2, 24 | you may corrupt the keychain. 25 | 26 | A future version will store the keychain in an GnuPG-approved 27 | version-neutral format. 28 | 29 | 30 | # Having gpg and gpg2 on the same machine 31 | 32 | NOTE: This is not implemented at this time. TODO(tlim) Use GPG to find 33 | the binary. 34 | 35 | In some situations, team members or automated roles need to install gpg 36 | 2.x alongside the system gpg version 1.x to catch up with the team's gpg 37 | version. On Ubuntu 16, you can ```apt-get install gnupg2``` which 38 | installs the binary gpg2. If you want to use this gpg2 binary, run every 39 | blackbox command with GPG=gpg2. 40 | 41 | For example: 42 | 43 | ``` 44 | GPG=gpg2 blackbox_postdeploy 45 | ``` 46 | 47 | -------------------------------------------------------------------------------- /docs/alternatives.md: -------------------------------------------------------------------------------- 1 | Alternatives 2 | ============ 3 | 4 | Here are other open source packages that do something similar to 5 | BlackBox. If you like them better than BlackBox, please use them. 6 | 7 | - [git-crypt](https://www.agwa.name/projects/git-crypt/) 8 | - [Pass](http://www.zx2c4.com/projects/password-store/) 9 | - [Transcrypt](https://github.com/elasticdog/transcrypt) 10 | - [Keyringer](https://keyringer.pw/) 11 | - [git-secret](https://github.com/sobolevn/git-secret) 12 | 13 | git-crypt has the best git integration. Once set up it is nearly 14 | transparent to the users. However it only works with git. 15 | -------------------------------------------------------------------------------- /docs/backwards-compatibility.md: -------------------------------------------------------------------------------- 1 | Backwards Compatibility 2 | ======================= 3 | 4 | # Where is the configuration stored? .blackbox vs. keyrings/live 5 | 6 | Blackbox stores its configuration data in the `.blackbox` subdirectory. Older 7 | repos use `keyrings/live`. For backwards compatibility either will work. 8 | 9 | All documentation refers to `.blackbox`. 10 | 11 | You can convert an old repo by simply renaming the directory: 12 | 13 | ``` 14 | mv keyrings/live .blackbox 15 | rmdir keyrings 16 | ``` 17 | 18 | There is no technical reason to convert old repos except that it is less 19 | confusing to users. 20 | 21 | This change was made in commit 60e782a0, release v1.20180615. 22 | 23 | 24 | # How blackbox fines the config directory: 25 | 26 | ## Creating the repo: 27 | 28 | `blackbox init` creates the config directory in the root 29 | of the repo. Here's how it picks the name: 30 | 31 | - If `$BLACKBOX_TEAM` is set, `.blackbox-$BLACKBOX_TEAM` is used. 32 | - If the flag `--team ` is set, it uses `.blackbox-` 33 | - Otherwise, it uses `.blackbox` 34 | 35 | When searching for the configuration directory, the following 36 | locations are checked. First match wins. 37 | 38 | - `.blackbox-$BLACKBOX_TEAM` (only if `$BLACKBOX_TEAM` is set) 39 | - The value of `--config value` (if the flag is set) 40 | - `$BLACKBOX_CONFIGDIR` (the preferred env. variable to use) 41 | - `$BLACKBOXDATA` (for backwards compatibility with v1) 42 | - `.blackbox` 43 | - `keyrings/live` (for backwards compatibility) 44 | 45 | NOTE: The env variables and `--config` should be set to the full path 46 | to the config directory (i.e.: `/Users/tom/gitstuff/myrepo/.blackbox`). 47 | If it is set to a relative directory (i.e. `.blackbox` or 48 | `../myrepo/.blackbox`) most commands will break. 49 | 50 | NOTE: Why the change from `$BLACKBOXDATA` to `$BLACKBOX_CONFIGDIR`? We want 51 | all the env. variables to begin with the prefix `BLACKBOX_`. If v1 52 | supported another name, that is still supported. If you are starting 53 | with v2 and have no other users using v1, please use the `BLACKBOX_` 54 | prefix. 55 | 56 | -------------------------------------------------------------------------------- /docs/compatibility.md: -------------------------------------------------------------------------------- 1 | Compatibility 2 | ============= 3 | 4 | # Compatibility with Blackbox v1 5 | 6 | The command names all changed from v1 to v2. The `binv2` directory 7 | includes shell scripts that provide full backwards compatibility. 8 | 9 | # Supported Architectures 10 | 11 | Blackbox supports a plug-in archtecture to easily support multiple VCS 12 | system. Current support is for: 13 | 14 | ## Supported VCS/DVCS systems 15 | 16 | * git 17 | * "none" (repo-less use is supported) 18 | * WOULD LOVE VOLUNTEERS TO HELP ADD SUPPORT FOR: hg, svn, p4 19 | 20 | ## Supported GPG versions 21 | 22 | * Git 1.x and 2.0 23 | * Git 2.2 and higher 24 | * WOULD LOVE VOLUNTEERS TO HELP ADD SUPPORT FOR: 25 | golang.org/x/crypto/openpgp (this would make the code have no 26 | external dependencies) 27 | 28 | ## Supported Operating systems 29 | 30 | Blackbox should work on any Linux system with GnuPG installed. 31 | Blackbox simply looks for `gpg` in `$PATH`. 32 | 33 | Windows: It should work (but has not been extensively tested) on 34 | Windows WSL2. 35 | 36 | # Automated testing 37 | 38 | While many combinations work, we do automated tests 39 | on these combinations. If any of these fail it blocks the release: 40 | 41 | * macOS: GnuPG 2.2 executables from https://gpgtools.org/ 42 | * CentOS: GnuPG 2.0.x executables from the "base" or "updates" repo. 43 | 44 | Windows native: VOLUNTEER NEEDED to make a native Windows version 45 | (should be rather simple as Go does most of the work) 46 | 47 | NOTE: Version 1 worked on CentOS/RedHat, macOS, Gygwin, WinGW, NetBSD, 48 | and SmartOS. Hopefully we can achieve that broad level of support in 49 | the future. Any system that is supported by the Go language and 50 | has GuPG 2.0.x or higher binaries available should be easy to achieve. 51 | We'd also like to have automated testing for the same. 52 | 53 | # Windows Support 54 | 55 | BlackBox assumes that `blackbox-admins.txt` and `blackbox-files.txt` will have 56 | LF line endings. Windows users should be careful to configure Git or other systems 57 | to not convert or "fix" those files. 58 | 59 | If you use Git, add the following lines to your `.gitattributes` file: 60 | 61 | **/blackbox-admins.txt text eol=lf 62 | **/blackbox-files.txt text eol=lf 63 | 64 | The `blackbox init` (and newer versions of `blackbox_initialize`) 65 | will create an appropriate `.gitattributes` file for you. 66 | 67 | # Cygwin 68 | 69 | TODO: List what packages are required for building the software. 70 | 71 | TODO: List what packages are required for running the software. 72 | 73 | 74 | # MinGW 75 | 76 | MinGW (comes with Git for Windows) support requires the following: 77 | 78 | TODO: FILL IN any requirements 79 | -------------------------------------------------------------------------------- /docs/dev-code-overview.md: -------------------------------------------------------------------------------- 1 | Code Overview 2 | ============= 3 | 4 | Here is how the code is laid out. 5 | 6 | TODO(tlim): Add a diagram of the layers 7 | 8 | ``` 9 | cmd/blackbox/ The command line tool. 10 | blackbox.go main() 11 | cli.go Definition of all subcommands and flags 12 | drive.go Processes flags and calls functions in verbs.go 13 | NOTE: These are the only files that are aware of the 14 | flags. Everything else gets the flag data passed to it 15 | as a parameter. This way the remaining system can be 16 | used as a module. 17 | 18 | pkg/box/ High-level functions related to "the black box". 19 | verbs.go One function per subcommand. 20 | box.go Functions for manipulating the files in .blackbox 21 | boxutils.go Helper functions for the above. 22 | 23 | pkg/bblog/ Module that provides logging facilities. 24 | pkg/bbutil/ Functions that are useful to box, plug-ins, etc. 25 | pkg/tainedname/ Module for printing filenames escaped for Bash. 26 | 27 | models/vcs.go The interface that defines a VCS plug-in. 28 | models/crypters.go The interface that defines a GPG plug-in. 29 | 30 | pkg/crypters/ Plug-ins for GPG functionality. 31 | pkg/crypters/gnupg Plug-in that runs an external gpg binary (found via $PATH) 32 | 33 | pkg/vcs/ Plug-ins for VCS functionality. 34 | pkg/vcs/none Repo-less mode. 35 | pkg/vcs/git Git mode. 36 | ``` 37 | -------------------------------------------------------------------------------- /docs/dev.md: -------------------------------------------------------------------------------- 1 | Developer Info 2 | ============== 3 | 4 | Code submissions are gladly welcomed! The code is fairly easy to read. 5 | 6 | Get the code: 7 | 8 | ``` 9 | git clone git@github.com:StackExchange/blackbox.git 10 | ``` 11 | 12 | Test your changes: 13 | 14 | ``` 15 | go test ./... 16 | ``` 17 | 18 | This runs through a number of system tests. It creates a repo, 19 | encrypts files, decrypts files, and so on. You can run these tests to 20 | verify that the changes you made didn't break anything. You can also 21 | use these tests to verify that the system works with a new operating 22 | system. 23 | 24 | Please submit tests with code changes: 25 | 26 | The best way to change BlackBox is via Test Driven Development. First 27 | add a test to `tools/confidence.sh`. This test should fail, and 28 | demonstrate the need for the change you are about to make. Then fix 29 | the bug or add the feature you want. When you are done, `make 30 | confidence` should pass all tests. The PR you submit should include 31 | your code as well as the new test. This way the confidence tests 32 | accumulate as the system grows as we know future changes don't break 33 | old features. 34 | 35 | Note: More info about compatibility are on the [Compatibility Page](compatibility.md) 36 | 37 | -------------------------------------------------------------------------------- /docs/enable-repo.md: -------------------------------------------------------------------------------- 1 | Enabling Blackbox on a Repo 2 | =========================== 3 | 4 | Overview: 5 | 1. Run the initialization command 6 | 2. Add at least one admin. 7 | 3. Add files. (don't add files before the admins) 8 | 9 | The long version: 10 | 11 | 1. If you don't have a GPG key, set it up using instructions such as: 12 | [Set up GPG key](https://help.github.com/articles/generating-a-new-gpg-key/). \ 13 | Now you are ready to go. 14 | 15 | 1. `cd` into a Git, Mercurial, Subversion or Perforce repository and run `blackbox init`. 16 | 17 | 1. Add yourself with `blackbox admin add YOUR@EMAIL` 18 | 19 | 1. Commit the files as directed. 20 | 21 | That's it! 22 | 23 | At this point you should encrypt a file and make sure you can decrypt 24 | it. This verifies that everything is working as expected. 25 | 26 | 27 | 1. Pick a file to be encrypted. Since this is a test, you might want 28 | to create a test file. Call it `secret.txt` and edit the file 29 | so that it includes your mother's maiden name. Just kidding! 30 | Store this sentence: `This is my test file.` 31 | 32 | 2. Run `blackbox file add secret.txt` 33 | 34 | 3. Decode the encrypted version: `blackbox cat secret.txt` 35 | 36 | The "cat" subcommand only accesses the encrypted (`.gpg`) file and is 37 | a good way to see that the file was encrypted properly. You should 38 | see `This is my test file.` 39 | 40 | 4 Verify that editing the file works. 41 | 42 | To view and/or edit a file, run `blackbox edit --shred secret.txt` 43 | 44 | Now encrypt it and shred the original: 45 | 46 | ``` 47 | blackbox encrypt --shred secret.txt 48 | ``` 49 | 50 | Now make sure you can decrypt the new file: 51 | 52 | ``` 53 | blackbox cat secret.txt 54 | ``` 55 | 56 | You should see the changed text. 57 | 58 | Now commit and push `secret.txt.gpg` and you are done! 59 | -------------------------------------------------------------------------------- /docs/encryption.md: -------------------------------------------------------------------------------- 1 | How is the encryption done? 2 | =========================== 3 | 4 | GPG has many different ways to encrypt a file. BlackBox uses the mode 5 | that lets you specify a list of keys that can decrypt the message. 6 | 7 | If you have 5 people ("admins") that should be able to access the 8 | secrets, each creates a GPG key and adds their public key to the 9 | keychain. The GPG command used to encrypt the file lists all 5 key 10 | names, and therefore any 1 key can decrypt the file. 11 | 12 | Blackbox stores a copy of the public keys of all admins. It never 13 | stores the private keys. 14 | 15 | To remove someone's access, remove that admin's key name (i.e. email 16 | address) from the list of admins and re-encrypt all the files. They 17 | can still read the .gpg file (assuming they have access to the 18 | repository) but they can't decrypt it any more. 19 | 20 | *What if they kept a copy of the old repo before you removed access?* 21 | Yes, they can decrypt old versions of the file. This is why when an 22 | admin leaves the team, you should change all your passwords, SSL 23 | certs, and so on. You should have been doing that before BlackBox, 24 | right? 25 | 26 | *Why don't you use symmetric keys?* In other words, why mess with all 27 | this GPG key stuff and instead why don't we just encrypt all the files 28 | with a single passphrase. Yes, GPG supports that, but then we are 29 | managing a shared password, which is fraught with problems. If someone 30 | "leaves the team" we would have to communicate to everyone a new 31 | password. Now we just have to remove their key. This scales better. 32 | 33 | *How do automated processes decrypt without asking for a password?* 34 | GPG requires a passphrase on a private key. However, it permits the 35 | creation of subkeys that have no passphrase. For automated processes, 36 | create a subkey that is only stored on the machine that needs to 37 | decrypt the files. For example, at Stack Exchange, when our Continuous 38 | Integration (CI) system pushes a code change to our Puppet masters, 39 | they run `blackbox decrypt --all --overwrite` to decrypt all the files. 40 | The user that 41 | runs this code has a subkey that doesn't require a passphrase. Since 42 | we have many masters, each has its own key. And, yes, this means our 43 | Puppet Masters have to be very secure. However, they were already 44 | secure because, like, dude... if you can break into someone's puppet 45 | master you own their network. 46 | 47 | *If you use Puppet, why didn't you just use hiera-eyaml?* There are 4 48 | reasons: 49 | 50 | 1. This works with any Git or Mercurial repo, even if you aren't using Puppet. 51 | 2. hiera-eyaml decrypts "on demand" which means your Puppet Master now uses a lot of CPU to decrypt keys every time it is contacted. It slows down your master, which, in my case, is already slow enough. 52 | 3. This works with binary files, without having to ASCIIify them and paste them into a YAML file. Have you tried to do this with a cert that is 10K long and changes every few weeks? Ick. 53 | 4. hiera-eyaml didn't exist when I wrote this. (That's the real reason.) 54 | 55 | -------------------------------------------------------------------------------- /docs/expired-keys.md: -------------------------------------------------------------------------------- 1 | Replacing expired keys 2 | ====================== 3 | 4 | If someone's key has already expired, blackbox will stop 5 | encrypting. You see this error: 6 | 7 | ``` 8 | $ blackbox_edit_end modified_file.txt 9 | --> Error: can't re-encrypt because a key has expired. 10 | ``` 11 | 12 | FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?" 13 | 14 | You can also detect keys that are about to expire by issuing this command and manually reviewing the "expired:" dates: 15 | 16 | gpg --homedir=.blackbox --list-keys 17 | 18 | or... list UIDs that will expire within 1 month from today: (Warning: this also lists keys without an expiration date) 19 | 20 | gpg --homedir=.blackbox --list-keys --with-colons --fixed-list-mode | grep ^uid | awk -F: '$6 < '$(( $(date +%s) + 2592000)) 21 | 22 | Here's how to replace the key: 23 | 24 | - Step 1. Administrator removes expired user: 25 | 26 | Warning: This process will erase any unencrypted files that you were in the process of editing. Copy them elsewhere and restore the changes when done. 27 | 28 | ``` 29 | blackbox_removeadmin expired_user@example.com 30 | # This next command overwrites any changed unencrypted files. See warning above. 31 | blackbox_update_all_files 32 | git commit -m "Re-encrypt all files" 33 | gpg --homedir=.blackbox --delete-key expired_user@example.com 34 | git commit -m 'Cleaned expired_user@example.com from keyring' .blackbox/* 35 | git push 36 | ``` 37 | 38 | - Step 2. Expired user adds an updated key: 39 | 40 | ``` 41 | git pull 42 | blackbox_addadmin updated_user@example.com 43 | git commit -m'NEW ADMIN: updated_user@example.com .blackbox/pubring.gpg .blackbox/trustdb.gpg .blackbox/blackbox-admins.txt 44 | git push 45 | ``` 46 | 47 | - Step 3. Administrator re-encrypts all files with the updated key of the expired user: 48 | 49 | ``` 50 | git pull 51 | gpg --import .blackbox/pubring.gpg 52 | blackbox_update_all_files 53 | git commit -m "Re-encrypt all files" 54 | git push 55 | ``` 56 | 57 | - Step 4: Clean up: 58 | 59 | Any files that were temporarily copied in the first step so as to not be overwritten can now be copied back and re-encrypted with the `blackbox_edit_end` command. 60 | 61 | (Thanks to @chishaku for finding a solution to this problem!) 62 | 63 | -------------------------------------------------------------------------------- /docs/file-ops.md: -------------------------------------------------------------------------------- 1 | How to add/remove a file into the system? 2 | ========================================= 3 | 4 | # Adding files: 5 | 6 | - If you need to, start the GPG Agent: `eval $(gpg-agent --daemon)` 7 | - Add the file to the system: 8 | 9 | ``` 10 | blackbox file add path/to/file.name.key 11 | 12 | # If you want to delete the old plaintext: 13 | blackbox file add --shred path/to/file.name.key 14 | ``` 15 | 16 | Multiple file names can be specified on the command line: 17 | 18 | Example 1: Register 2 files: 19 | 20 | ``` 21 | blackbox file add --shred file1.txt file2.txt 22 | ``` 23 | 24 | Example 2: Register all the files in `$DIR`: 25 | 26 | ``` 27 | find $DIR -type f -not -name '*.gpg' -print0 | xargs -0 blackbox file add 28 | ``` 29 | 30 | 31 | # Removing files 32 | 33 | This command 34 | 35 | ``` 36 | blackbox file remove path/to/file.name.key 37 | ``` 38 | 39 | TODO(tlim): Add examples. 40 | 41 | # List files 42 | 43 | To see what files are currently enrolled in the system: 44 | 45 | ``` 46 | blackbox file list 47 | ``` 48 | 49 | You can also see their status: 50 | 51 | ``` 52 | blackbox status 53 | blackbox status just_one_file.txt 54 | blackbox status --type ENCRYPTED 55 | ``` 56 | -------------------------------------------------------------------------------- /docs/full-command-list.md: -------------------------------------------------------------------------------- 1 | Blackbox Command List 2 | ===================== 3 | 4 | ## Global Flags 5 | ### `--vcs` 6 | ### `--crypto` 7 | ### `--config` 8 | ### `--team` 9 | ### `--editor` 10 | ### `--umask` 11 | ### `--debug` 12 | ### `--help` 13 | ### `--help` 14 | ### `--version` 15 | ## User Commands 16 | ### `blackbox decrypt` 17 | ### `blackbox encrypt` 18 | ### `blackbox edit` 19 | ### `blackbox cat` 20 | ### `blackbox diff` 21 | ### `blackbox shred` 22 | ### `blackbox help` 23 | ## User Commands 24 | ### `blackbox init` 25 | ### `blackbox admin` 26 | ### `blackbox file` 27 | ### `blackbox status` 28 | ### `blackbox reencrypt` 29 | ## Debug 30 | ### `blackbox info` 31 | ## Integration Test (secret menu) 32 | ### `blackbox testing_init` 33 | 34 | TODO(tlim): Can we automatically generate this? The data is all in cli.go 35 | -------------------------------------------------------------------------------- /docs/git-tips.md: -------------------------------------------------------------------------------- 1 | GIT tips 2 | ======== 3 | 4 | 5 | # Configure git to show diffs in encrypted files 6 | 7 | It's possible to tell Git to decrypt versions of the file before running them through `git diff` or `git log`. To achieve this do: 8 | 9 | - Add the following to `.gitattributes` at the top of the git repository: 10 | 11 | ``` 12 | *.gpg diff=blackbox 13 | ``` 14 | 15 | - Add the following to `.git/config`: 16 | 17 | ``` 18 | [diff "blackbox"] 19 | textconv = gpg --use-agent -q --batch --decrypt 20 | ```` 21 | 22 | Commands like `git log -p file.gpg` and `git diff master --` will display as expected. 23 | -------------------------------------------------------------------------------- /docs/gnupg-tips.md: -------------------------------------------------------------------------------- 1 | GnuPG tips 2 | ========== 3 | 4 | # Common error messages 5 | 6 | * Message: `gpg: filename: skipped: No public key` 7 | * Solution: Usually this means there is an item in 8 | `.blackbox/blackbox-admins.txt` that is not the name of the key. 9 | Either something invalid was inserted (like a filename instead of a 10 | username) or a user has left the organization and their key was 11 | removed from the keychain, but their name wasn't removed from the 12 | blackbox-admins.txt file. 13 | 14 | * Message: `gpg: decryption failed: No secret key` 15 | * Solution: Usually means you forgot to re-encrypt the file with the new key. 16 | 17 | * Message: `Error: can't re-encrypt because a key has expired.` 18 | * Solution: A user's key has expired and can't be used to encrypt any more. Follow the [Replace expired keys](expired-keys.md) page. 19 | 20 | FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?" 21 | 22 | # GnuPG problems 23 | 24 | Blackbox is just a front-end to GPG. If you get into a problem with a 25 | key or file, you'll usually have better luck asking for advice on 26 | the gnupg users mailing list TODO: Get link to this list 27 | 28 | 29 | The author of Blackbox is not a GnuPG expert. He wrote Blackbox 30 | because it was better than trying to remember GPG's horrible flag 31 | names. 32 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | Installation Instructions 2 | ========================= 3 | 4 | Currently blackbox v2 is installed by compiling the code and 5 | copying the binary someplace: 6 | 7 | TODO: 8 | 9 | ``` 10 | git clone FILL IN 11 | ``` 12 | 13 | Future: We will have RPM, DEB, Chocolately packages. 14 | 15 | 16 | Next step: [Enable on a repo](enable-repo.md) 17 | 18 | -------------------------------------------------------------------------------- /docs/role-accounts.md: -------------------------------------------------------------------------------- 1 | Set up automated users or "role accounts" 2 | ========================================= 3 | 4 | TODO(tlim): I think this is overly complex. With GnuPG 2.2 and later, 5 | you can use `--password '' --quick-generate-key userid` and you are 6 | done. No need for subkeys. Maybe rework this? 7 | 8 | With role accounts, you have an automated system that needs to be able 9 | to decrypt secrets without a password. This means the security of your 10 | repo is based on how locked down the automation system is. This 11 | is risky, so be careful. 12 | 13 | 14 | i.e. This is how a Puppet Master can have access to the unencrypted data. 15 | 16 | FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?" 17 | 18 | An automated user (a "role account") is one that that must be able to decrypt without a passphrase. In general you'll want to do this for the user that pulls the files from the repo to the master. This may be automated with Jenkins CI or other CI system. 19 | 20 | GPG keys have to have a passphrase. However, passphrases are optional on subkeys. Therefore, we will create a key with a passphrase then create a subkey without a passphrase. Since the subkey is very powerful, it should be created on a very secure machine. 21 | 22 | There's another catch. The role account probably can't check files into Git/Mercurial. It probably only has read-only access to the repo. That's a good security policy. This means that the role account can't be used to upload the subkey public bits into the repo. 23 | 24 | Therefore, we will create the key/subkey on a secure machine as yourself. From there we can commit the public portions into the repo. Also from this account we will export the parts that the role account needs, copy them to where the role account can access them, and import them as the role account. 25 | 26 | ProTip: If asked to generate entropy, consider running this on the same machine in another window: `sudo dd if=/dev/sda of=/dev/null` 27 | 28 | For the rest of this doc, you'll need to make the following substitutions: 29 | 30 | - ROLEUSER: svc_deployacct or whatever your role account's name is. 31 | - NEWMASTER: the machine this role account exists on. 32 | - SECUREHOST: The machine you use to create the keys. 33 | 34 | NOTE: This should be more automated/scripted. Patches welcome. 35 | 36 | On SECUREHOST, create the puppet master's keys: 37 | 38 | ``` 39 | $ mkdir /tmp/NEWMASTER 40 | $ cd /tmp/NEWMASTER 41 | $ gpg --homedir . --gen-key 42 | Your selection? 43 | (1) RSA and RSA (default) 44 | What keysize do you want? (2048) DEFAULT 45 | Key is valid for? (0) DEFAULT 46 | 47 | # Real name: Puppet CI Deploy Account 48 | # Email address: svc_deployacct@hostname.domain.name 49 | ``` 50 | 51 | NOTE: Rather than a real email address, use the username@FQDN of the host the key will be used on. If you use this role account on many machines, each should have its own key. By using the FQDN of the host, you will be able to know which key is which. In this doc, we'll refer to username@FQDN as $KEYNAME 52 | 53 | Save the passphrase somewhere safe! 54 | 55 | Create a sub-key that has no password: 56 | 57 | ``` 58 | $ gpg --homedir . --edit-key svc_deployacct 59 | gpg> addkey 60 | (enter passphrase) 61 | Please select what kind of key you want: 62 | (3) DSA (sign only) 63 | (4) RSA (sign only) 64 | (5) Elgamal (encrypt only) 65 | (6) RSA (encrypt only) 66 | Your selection? 6 67 | What keysize do you want? (2048) 68 | Key is valid for? (0) 69 | Command> key 2 70 | (the new subkey has a "*" next to it) 71 | Command> passwd 72 | (enter the main key's passphrase) 73 | (enter an empty passphrase for the subkey... confirm you want to do this) 74 | Command> save 75 | ``` 76 | 77 | Now securely export this directory to NEWMASTER: 78 | 79 | ``` 80 | gpg --homedir . --export -a svc_sadeploy >/tmp/NEWMASTER/pubkey.txt 81 | tar cvf /tmp/keys.tar . 82 | rsync -avP /tmp/keys.tar NEWMASTER:/tmp/. 83 | ``` 84 | 85 | On NEWMASTER, receive the new GnuPG config: 86 | 87 | ``` 88 | sudo -u svc_deployacct bash 89 | mkdir -m 0700 -p ~/.gnupg 90 | cd ~/.gnupg && tar xpvf /tmp/keys.tar 91 | ``` 92 | 93 | 101 | 102 | Back on SECUREHOST, add the new email address to .blackbox/blackbox-admins.txt: 103 | 104 | ``` 105 | cd /path/to/the/repo 106 | blackbox_addadmin $KEYNAME /tmp/NEWMASTER 107 | ``` 108 | 109 | Verify that secring.gpg is a zero-length file. If it isn't, you have somehow added a private key to the keyring. Start over. 110 | 111 | ``` 112 | cd .blackbox 113 | ls -l secring.gpg 114 | ``` 115 | 116 | Commit the recent changes: 117 | 118 | ``` 119 | cd .blackbox 120 | git commit -m"Adding key for KEYNAME" pubring.gpg trustdb.gpg blackbox-admins.txt 121 | ``` 122 | 123 | Regenerate all encrypted files with the new key: 124 | 125 | ``` 126 | blackbox_update_all_files 127 | git status 128 | git commit -m"updated encryption" -a 129 | git push 130 | ``` 131 | 132 | On NEWMASTER, import the keys and decrypt the files: 133 | 134 | ``` 135 | sudo -u svc_sadeploy bash # Become the role account. 136 | gpg --import /etc/puppet/.blackbox/pubring.gpg 137 | export PATH=$PATH:/path/to/blackbox/bin 138 | blackbox_postdeploy 139 | sudo -u puppet cat /etc/puppet/hieradata/blackbox.yaml # or any encrypted file. 140 | ``` 141 | 142 | ProTip: If you get "gpg: decryption failed: No secret key" then you forgot to re-encrypt blackbox.yaml with the new key. 143 | 144 | On SECUREHOST, securely delete your files: 145 | 146 | ``` 147 | cd /tmp/NEWMASTER 148 | # On machines with the "shred" command: 149 | shred -u /tmp/keys.tar 150 | find . -type f -print0 | xargs -0 shred -u 151 | # All else: 152 | rm -rf /tmp/NEWMASTER 153 | ``` 154 | 155 | Also shred any other temporary files you may have made. 156 | 157 | 158 | 159 | -------------------------------------------------------------------------------- /docs/subversion-tips.md: -------------------------------------------------------------------------------- 1 | Subversion Tips 2 | =============== 3 | 4 | NOTE: This is from v1. Can someone that uses Subversion check 5 | this and update it? 6 | 7 | 8 | The current implementation will store the blackbox in `/keyrings` at 9 | the root of the entire repo. This will create an issue between 10 | environments that have different roots (i.e. checking out `/` on 11 | development vs `/releases/foo` in production). To get around this, you 12 | can `export BLACKBOX_REPOBASE=/path/to/repo` and set a specific base 13 | for your repo. 14 | 15 | This was originally written for git and supports a two-phase commit, 16 | in which `commit` is a local commit and "push" sends the change 17 | upstream to the version control server when something is registered or 18 | deregistered with the system. The current implementation will 19 | immediately `commit` a file (to the upstream subversion server) when 20 | you execute a `blackbox_*` command. 21 | 22 | -------------------------------------------------------------------------------- /docs/support.md: -------------------------------------------------------------------------------- 1 | Support 2 | ======= 3 | 4 | # Join our community! 5 | 6 | Join the [blackbox-project mailing list](https://groups.google.com/d/forum/blackbox-project)! 7 | 8 | 9 | 10 | # How to submit bugs or ask questions? 11 | 12 | We welcome questions, bug reports and feedback! 13 | 14 | The best place to start is to join the [blackbox-project mailing list](https://groups.google.com/d/forum/blackbox-project) and ask there. 15 | 16 | Bugs are tracked here in Github. Please feel free to [report bugs](https://github.com/StackExchange/blackbox/issues) yourself. 17 | -------------------------------------------------------------------------------- /docs/user-overview.md: -------------------------------------------------------------------------------- 1 | User Guide 2 | ========== 3 | 4 | # Overview 5 | 6 | Suppose you have a VCS repository (i.e. a Git or Mercurial repo) and 7 | certain files contain secrets such as passwords or SSL private keys. 8 | Often people just store such files "and hope that nobody finds them in 9 | the repo". That's not safe. Hope is not a strategy. 10 | 11 | With BlackBox, those files are stored encrypted using GPG. Access to 12 | the repo without also having the right GPG keys makes those files as worthless 13 | as random bits. As long as you keep your GPG keys safe, you don't 14 | have to worry about storing your VCS repo on an untrusted server or 15 | letting anyone clone the repo. 16 | 17 | Heck, even if you trust your server, now you don't have to trust the 18 | people that do backups of that server! 19 | 20 | Each person ("admin") of the system can decrypt all the files using 21 | their GPG key, which has its own passphrase. The authorized GPG keys 22 | can decrypt any file. This is better than systems that use one 23 | GPG key (and passphrase) that must be shared among a group of people. 24 | It is much better than having one passphrase for each file (I don't 25 | think anyone actually does that). 26 | 27 | Since any admin's GPG key can decrypt the files, if one person leaves 28 | the company, you don't have to communicate a new passphrase to everyone. 29 | Simply disable the one key that should no longer have access. 30 | The process for doing this is as easy as running 2 commands (1 to 31 | disable their key, 1 to re-encrypt all files.) Obviously if they kept 32 | a copy of the repo (and their own passphrase) before leaving the 33 | company, they have access to the secrets. However, you should rotate 34 | those secrets anyway. ("rotate secrets" means changing the passwords, 35 | regenerating TLS certs, and so on). 36 | 37 | # Sample session: 38 | 39 | First we are going to list the files currently in the blackbox. In 40 | this case, it is an SSH private key. 41 | 42 | ``` 43 | $ blackbox file list 44 | modules/log_management/files/id_rsa 45 | ``` 46 | 47 | Excellent! Our coworkers have already registered a file with the 48 | system. Let's decrypt it, edit it, and re-encrypt it. 49 | 50 | ``` 51 | $ blackbox decrypt modules/log_management/files/id_rsa 52 | ========== DECRYPTING "modules/log_management/files/id_rsa" 53 | $ vi modules/log_management/files/id_rsa 54 | ``` 55 | 56 | That was easy so far! 57 | 58 | When we encrypt it, Blackbox will not commit the changes, but it 59 | will give a hint that you should. It spells out the exact command you 60 | need to type and even proposes a commit message. 61 | 62 | ``` 63 | $ blackbox encrypt modules/log_management/files/id_rsa 64 | ========== ENCRYPTING "modules/log_management/files/id_rsa" 65 | 66 | NEXT STEP: You need to manually check these in: 67 | git commit -m"ENCRYPTED modules/log_management/files/id_rsa" modules/log_management/files/id_rsa.gpg 68 | ``` 69 | 70 | You can also use `blackbox edit ` to decrypt a file, edit it 71 | (it will call `$EDITOR`) and re-encrypt it. 72 | 73 | 74 | Now let's register a new file with the blackbox system. 75 | `data/pass.yaml` is a small file that stores a very important 76 | password. In this example, we had just stored the unecrypted 77 | password in our repo. That's bad. Let's encrypt it. 78 | 79 | ``` 80 | $ blackbox file add data/pass.yaml 81 | ========== SHREDDING ("/bin/rm", "-f"): "data/pass.yaml" 82 | 83 | NEXT STEP: You need to manually check these in: 84 | git commit -m"NEW FILES: data/pass.yaml" .gitignore keyrings/live/blackbox-files.txt modules/stacklb/pass.yaml modules/stacklb/pass.yaml.gpg 85 | ``` 86 | 87 | Before we commit the change, let's do a `git status` to see what else 88 | has changed. 89 | 90 | ``` 91 | $ git status 92 | On branch master 93 | Changes to be committed: 94 | (use "git restore --staged ..." to unstage) 95 | modified: .gitignore 96 | modified: keyrings/live/blackbox-files.txt 97 | deleted: modules/stacklb/pass.yaml 98 | new file: modules/stacklb/pass.yaml.gpg 99 | 100 | ``` 101 | 102 | Notice that a number of files were modified: 103 | 104 | * `.gitignore`: This file is updated to include the plaintext 105 | filename, so that you don't accidentally add it to the repo in the 106 | future. 107 | * `.blackbox/blackbox-files.txt`: The list of files that are registered with the system. 108 | * `data/pass.yaml`: The file we encrypted is deleted from the repo. 109 | * `data/pass.yaml.gpg`: The encrypted file is added to the repo. 110 | 111 | Even though pass.yaml was deleted from the repo, it is still in the 112 | repo's history. Anyone with an old copy of the repo, or a new copy 113 | that knows how to view the repo's history, can see the secret 114 | password. For that reason, you should change the password and 115 | re-encrypt the file. This is an important point. Blackbox is not 116 | magic and it doesn't have a "Men In Black"-style neuralizer that 117 | can make people forget the past. If someone leaves a project, you 118 | have to change the old passwords, etc. 119 | 120 | Those are the basics. Your next step might be: 121 | 122 | * TODO: How to enable Blackbox for a repo. 123 | * TODO: How to add yourself as an admin to a repo. 124 | * TODO: Complete list of [all blackbox commands](all-commands) 125 | -------------------------------------------------------------------------------- /docs/why-is-this-important.md: -------------------------------------------------------------------------------- 1 | Why encrypt your secrets? 2 | ========================= 3 | 4 | OBVIOUSLY we don't want secret things like SSL private keys and 5 | passwords to be leaked. 6 | 7 | NOT SO OBVIOUSLY when we store "secrets" in a VCS repo like Git or 8 | Mercurial, suddenly we are less able to share our code with other 9 | people. Communication between subteams of an organization is hurt. You 10 | can't collaborate as well. Either you find yourself emailing 11 | individual files around (yuck!), making a special repo with just the 12 | files needed by your collaborators (yuck!!), or just deciding that 13 | collaboration isn't worth all that effort (yuck!!!). 14 | 15 | The ability to be open and transparent about our code, with the 16 | exception of a few specific files, is key to the kind of collaboration 17 | that DevOps and modern IT practitioners need to do. 18 | -------------------------------------------------------------------------------- /docs/with-ansible.md: -------------------------------------------------------------------------------- 1 | How to use the secrets with Ansible? 2 | =================================== 3 | 4 | Ansible Vault provides functionality for encrypting both entire files 5 | and strings stored within files; however, keeping track of the 6 | password(s) required for decryption is not handled by this module. 7 | 8 | Instead one must specify a password file when running the playbook. 9 | 10 | Ansible example for password file: `my_secret_password.txt.gpg` 11 | 12 | ``` 13 | ansible-playbook --vault-password-file my_secret_password.txt site.yml 14 | ``` 15 | 16 | Alternatively, one can specify this in the 17 | `ANSIBLE_VAULT_PASSWORD_FILE` environment variable. 18 | 19 | -------------------------------------------------------------------------------- /docs/with-puppet.md: -------------------------------------------------------------------------------- 1 | How to use the secrets with Puppet? 2 | =================================== 3 | 4 | # Entire files: 5 | 6 | Entire files, such as SSL certs and private keys, are treated just 7 | like regular files. You decrypt them any time you push a new release 8 | to the puppet master. 9 | 10 | Example of an encrypted file named `secret_file.key.gpg` 11 | 12 | * Plaintext file is: `modules/${module_name}/files/secret_file.key` 13 | * Encrypted file is: `modules/${module_name}/files/secret_file.key.gpg` 14 | * Puppet sees it as: `puppet:///modules/${module_name}/secret_file.key` 15 | 16 | Puppet code that stores `secret_file.key` in `/etc/my_little_secret.key`: 17 | 18 | ``` 19 | file { '/etc/my_little_secret.key': 20 | ensure => 'file', 21 | owner => 'root', 22 | group => 'puppet', 23 | mode => '0760', 24 | source => "puppet:///modules/${module_name}/secret_file.key", # No ".gpg" 25 | } 26 | ``` 27 | 28 | # Small strings: 29 | 30 | For small strings such as passwords and API keys, it makes sense 31 | to store them in an (encrypted) YAML file which is then made 32 | available via hiera. 33 | 34 | For example, we use a file called `blackbox.yaml`. You can access the 35 | data in it using the hiera() function. 36 | 37 | *Setup:* 38 | 39 | Edit `hiera.yaml` to include "blackbox" to the search hierarchy: 40 | 41 | ``` 42 | :hierarchy: 43 | - ... 44 | - blackbox 45 | - ... 46 | ``` 47 | 48 | In blackbox.yaml specify: 49 | 50 | ``` 51 | --- 52 | module::test_password: "my secret password" 53 | ``` 54 | 55 | In your Puppet Code, access the password as you would any hiera data: 56 | 57 | ``` 58 | $the_password = hiera('module::test_password', 'fail') 59 | 60 | file {'/tmp/debug-blackbox.txt': 61 | content => $the_password, 62 | owner => 'root', 63 | group => 'root', 64 | mode => '0600', 65 | } 66 | ``` 67 | 68 | The variable `$the_password` will contain "my secret password" and can be used anywhere strings are used. 69 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/StackExchange/blackbox/v2 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 7 | github.com/mattn/go-runewidth v0.0.9 // indirect 8 | github.com/olekukonko/tablewriter v0.0.4 9 | github.com/sergi/go-diff v1.2.0 // indirect 10 | github.com/urfave/cli/v2 v2.2.0 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= 3 | github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= 4 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 5 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 10 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 11 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 12 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 13 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 14 | github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 15 | github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= 16 | github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 17 | github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= 18 | github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= 19 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 20 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 21 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 22 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 23 | github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= 24 | github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 25 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 26 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 27 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 28 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 29 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 30 | github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= 31 | github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= 32 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 33 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 34 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 35 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 36 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 37 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 38 | -------------------------------------------------------------------------------- /integrationTest/NOTES.txt: -------------------------------------------------------------------------------- 1 | 2 | This should accept VCS-type and --crypto flags. 3 | Then a shell script should run various combinations of VCS and crypters. 4 | 5 | # Startup 6 | * Create a repo (git, none) 7 | 8 | # Test basic operations: 9 | * As Alice: 10 | * initialize blackbox, add her keys to it, see that the usual files 11 | exist. See her name in bb-admins.txt 12 | * encrypt a file, see that the plaintext is deleted, see the file in bb-files.txt 13 | * decrypt the file, see the original plaintext is recovered. 14 | * Encrypt a file --noshred. 15 | * Decrypt the file, it should fail as the plaintext exists. 16 | * Remove the plaintext. 17 | * Decrypt the file, it should fail as the plaintext exists. 18 | 19 | # Test hand-off from Alice to Bob. 20 | * As Bob 21 | * add himself to the admins. 22 | * As Alice 23 | * Update-all-files 24 | * Create a new file. Encrypt it. 25 | * As Bob 26 | * Decrypt both files 27 | * Verify contents of the new file, and the file from previous. 28 | * Create a new file. Encrypt it. 29 | * As Alice: 30 | * Decrypt all files. 31 | * Verify contents of the 3 plaintext files. 32 | 33 | # Test a git-less directory 34 | * Copy the old repo somewhere. Remove the .git directory. 35 | * As Alice: 36 | * Decrypt all 37 | * Verify plaintext contents 38 | 39 | # Test post-deploy with/without GID 40 | * Back at the original repo: 41 | * Shred all 42 | * Run post-deploy. Verify. 43 | * Shred all 44 | * Run post-deploy with a custom GID. Verify. 45 | 46 | # Test removing an admin 47 | * As Bob: 48 | * removes Alice. (Verify) 49 | * Re-encrypt 50 | * Decrypt all & verify. 51 | * As alice 52 | * Decrypting should fail. 53 | 54 | # Test funny names and paths 55 | * my/path/to/relsecrets.txt 56 | * cwd=other/place ../../my/path/to/relsecrets.txt 57 | * !important!.txt 58 | * #andpounds.txt 59 | * stars*bars?.txt 60 | * space space.txt 61 | * Do add/encrypt/decrypt 62 | * Do blackbox_update_all_files 63 | * Do remove them all 64 | 65 | # When people start asking for commands to work with relative paths 66 | # Test from outside the repo 67 | * mkdir ../other/place 68 | * cd ../other/place 69 | * decrypt ../../secret1.txt 70 | * encrypt ../../secret1.txt 71 | 72 | # Test specific commands: 73 | # blackbox admins list 74 | # blackbox file list 75 | # blackbox status --name-only (create 1 of each "type") 76 | # blackbox status --type=FOO 77 | 78 | # These should all fail: 79 | # blackbox file list --all 80 | # blackbox file list blah 81 | # blackbox shred list --all 82 | # blackbox shred list blah 83 | 84 | 85 | 86 | rm -rf /tmp/bbhome-* && BLACKBOX_DEBUG=true go test -verbose -long -nocleanup 87 | rm -rf /tmp/bbhome-* && go test -long -nocleanup 88 | 89 | ( gbb && cd cmd/blackbox && go install ) && blackbox 90 | 91 | cd /tmp && rm -rf /tmp/bbhome-* ; mkdir /tmp/bbhome-1 ; cd /tmp/bbhome-1 && git init ; gitmeWork ; ( gbb && cd cmd/blackbox && go install ) && blackbox init yes && gitmeWork ; git commit -mm -a ; blackbox admin add tlimoncelli ; git commit -mnewadmin -a ; echo secrt > secret.txt ; blackbox file add secret.txt 92 | -------------------------------------------------------------------------------- /integrationTest/README.txt: -------------------------------------------------------------------------------- 1 | 2 | Each test does the following: 3 | 1. Copy the files from testdata/NNNN 4 | 2. Run the command in test_NNNN.sh 5 | 3. 6 | 7 | 8 | TEST ENROLLMENT: 9 | 10 | PHASE 'Alice creates a repo. She creates secret.txt.' 11 | PHASE 'Alice wants to be part of the secret system.' 12 | PHASE 'She creates a GPG key...' 13 | PHASE 'Initializes BB...' 14 | PHASE 'and adds herself as an admin.' 15 | PHASE 'Bob arrives.' 16 | PHASE 'Bob creates a gpg key.' 17 | PHASE 'Alice does the second part to enroll bob.' 18 | PHASE 'She enrolls bob.' 19 | PHASE 'She enrolls secrets.txt.' 20 | PHASE 'She decrypts secrets.txt.' 21 | PHASE 'She edits secrets.txt.' 22 | PHASE 'Alice copies files to a non-repo directory. (NO REPO)' 23 | PHASE 'Alice shreds these non-repo files. (NO REPO)' 24 | PHASE 'Alice decrypts secrets.txt (NO REPO).' 25 | PHASE 'Alice edits secrets.txt. (NO REPO EDIT)' 26 | PHASE 'Alice decrypts secrets.txt (NO REPO EDIT).' 27 | PHASE 'appears.' 28 | #PHASE 'Bob makes sure he has all new keys.' 29 | 30 | TEST INDIVIDUAL COMMANDS: 31 | 32 | PHASE 'Bob postdeploys... default.' 33 | PHASE 'Bob postdeploys... with a GID.' 34 | PHASE 'Bob cleans up the secret.' 35 | PHASE 'Bob removes Alice.' 36 | PHASE 'Bob reencrypts files so alice can not access them.' 37 | PHASE 'Bob decrypts secrets.txt.' 38 | PHASE 'Bob edits secrets.txt.' 39 | PHASE 'Bob decrypts secrets.txt VERSION 3.' 40 | PHASE 'Bob exposes a secret in the repo.' 41 | PHASE 'Bob corrects it by registering it.' 42 | PHASE 'Bob enrolls my/path/to/relsecrets.txt.' 43 | PHASE 'Bob decrypts relsecrets.txt.' 44 | PHASE 'Bob enrolls !important!.txt' 45 | PHASE 'Bob enrolls #andpounds.txt' 46 | PHASE 'Bob enrolls stars*bars?.txt' 47 | PHASE 'Bob enrolls space space.txt' 48 | PHASE 'Bob checks out stars*bars?.txt.' 49 | PHASE 'Bob checks out space space.txt.' 50 | PHASE 'Bob shreds all exposed files.' 51 | PHASE 'Bob updates all files.' 52 | PHASE 'Bob DEregisters mistake.txt' 53 | PHASE 'Bob enrolls multiple files: multi1.txt and multi2.txt' 54 | PHASE 'Alice returns. She should be locked out' 55 | PHASE 'Alice tries to decrypt secret.txt. Is blocked.' 56 | -------------------------------------------------------------------------------- /integrationTest/asserts.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/andreyvit/diff" 9 | ) 10 | 11 | func assertFileMissing(t *testing.T, name string) { 12 | t.Helper() 13 | _, err := os.Stat(name) 14 | if err != nil && os.IsNotExist(err) { 15 | return 16 | } 17 | if err == nil { 18 | t.Fatalf("assertFileMissing failed: %v exists", name) 19 | } 20 | t.Fatalf("assertFileMissing: %q: %v", name, err) 21 | } 22 | 23 | func assertFileExists(t *testing.T, name string) { 24 | t.Helper() 25 | _, err := os.Stat(name) 26 | if err == nil { 27 | return 28 | } 29 | if os.IsNotExist(err) { 30 | t.Fatalf("assertFileExists failed: %v not exist", name) 31 | } 32 | t.Fatalf("assertFileExists: file can't be accessed: %v: %v", name, err) 33 | } 34 | 35 | func assertFileEmpty(t *testing.T, name string) { 36 | t.Helper() 37 | c, err := ioutil.ReadFile(name) 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | if len(c) != 0 { 42 | t.Fatalf("got=%v want=%v: %v", len(c), 0, name) 43 | } 44 | } 45 | 46 | func assertFileContents(t *testing.T, name string, contents string) { 47 | t.Helper() 48 | c, err := ioutil.ReadFile(name) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | 53 | if w, g := contents, string(c); w != g { 54 | t.Errorf("assertFileContents(%q) mismatch (-got +want):\n%s", 55 | name, diff.LineDiff(g, w)) 56 | } 57 | } 58 | 59 | func assertFilePerms(t *testing.T, name string, perms os.FileMode) { 60 | t.Helper() 61 | s, err := os.Stat(name) 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | if s.Mode() != perms { 66 | t.Fatalf("got=%#o want=%#o: %v", s.Mode(), perms, name) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /integrationTest/test_data/000-admin-list.txt: -------------------------------------------------------------------------------- 1 | user1@example.com 2 | user2@example.com 3 | -------------------------------------------------------------------------------- /integrationTest/test_data/000-file-list.txt: -------------------------------------------------------------------------------- 1 | bar.txt 2 | foo.txt 3 | -------------------------------------------------------------------------------- /integrationTest/test_data/000-status.txt: -------------------------------------------------------------------------------- 1 | +-------------+------------------------+ 2 | | STATUS | NAME | 3 | +-------------+------------------------+ 4 | | BOTHMISSING | status-BOTHMISSING.txt | 5 | | DECRYPTED | status-DECRYPTED.txt | 6 | | ENCRYPTED | status-ENCRYPTED.txt | 7 | | GPGMISSING | status-GPGMISSING.txt | 8 | | SHREDDED | status-SHREDDED.txt | 9 | +-------------+------------------------+ 10 | -------------------------------------------------------------------------------- /integrationTest/test_data/alice-cat-plain.txt: -------------------------------------------------------------------------------- 1 | I am the foo.txt file! 2 | -------------------------------------------------------------------------------- /integrationTest/test_data/basic-status.txt: -------------------------------------------------------------------------------- 1 | +-----------+---------+ 2 | | STATUS | NAME | 3 | +-----------+---------+ 4 | | ENCRYPTED | foo.txt | 5 | +-----------+---------+ 6 | -------------------------------------------------------------------------------- /integrationTest/test_data/reencrypt-plain.txt: -------------------------------------------------------------------------------- 1 | I am the foo.txt file! 2 | -------------------------------------------------------------------------------- /integrationTest/test_data/status-noreg.txt: -------------------------------------------------------------------------------- 1 | +-----------+----------------------+ 2 | | STATUS | NAME | 3 | +-----------+----------------------+ 4 | | ENCRYPTED | status-ENCRYPTED.txt | 5 | | NOTREG | blah.txt | 6 | +-----------+----------------------+ 7 | -------------------------------------------------------------------------------- /models/crypters.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | // Crypter is gpg binaries, go-opengpg, etc. 4 | type Crypter interface { 5 | // Name returns the plug-in's canonical name. 6 | Name() string 7 | // Decrypt name+".gpg", possibly overwriting name. 8 | Decrypt(filename string, umask int, overwrite bool) error 9 | // Encrypt name, overwriting name+".gpg" 10 | Encrypt(filename string, umask int, receivers []string) (string, error) 11 | // Cat outputs a file, unencrypting if needed. 12 | Cat(filename string) ([]byte, error) 13 | // AddNewKey extracts keyname from sourcedir's GnuPG chain to destdir keychain. 14 | AddNewKey(keyname, repobasename, sourcedir, destdir string) ([]string, error) 15 | } 16 | -------------------------------------------------------------------------------- /models/vcs.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import "github.com/StackExchange/blackbox/v2/pkg/commitlater" 4 | 5 | // Vcs is git/hg/etc. 6 | type Vcs interface { 7 | // Name returns the plug-in's canonical name. 8 | Name() string 9 | // Discover returns true if we are a repo of this type; along with the Abs path to the repo root (or "" if we don't know). 10 | Discover() (bool, string) 11 | 12 | // SetFileTypeUnix informs the VCS that files should maintain unix-style line endings. 13 | SetFileTypeUnix(repobasedir string, files ...string) error 14 | // IgnoreAnywhere tells the VCS to ignore these files anywhere in the repo. 15 | IgnoreAnywhere(repobasedir string, files []string) error 16 | // IgnoreAnywhere tells the VCS to ignore these files, rooted in the base of the repo. 17 | IgnoreFiles(repobasedir string, files []string) error 18 | 19 | // CommitTitle sets the title of the next commit. 20 | CommitTitle(title string) 21 | // NeedsCommit queues up commits for later execution. 22 | NeedsCommit(message string, repobasedir string, names []string) 23 | // DebugCommits dumps a list of future commits. 24 | DebugCommits() commitlater.List 25 | // FlushCommits informs the VCS to do queued up commits. 26 | FlushCommits() error 27 | 28 | // TestingInitRepo initializes a repo of this type (for use by integration tests) 29 | TestingInitRepo() error 30 | } 31 | -------------------------------------------------------------------------------- /pkg/bblog/bblog.go: -------------------------------------------------------------------------------- 1 | package bblog 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "os" 7 | ) 8 | 9 | /* 10 | 11 | To use this, include the following lines in your .go file. 12 | 13 | var logErr *log.Logger 14 | var logDebug *log.Logger 15 | func init() { 16 | logErr = bblog.GetErr() 17 | logDebug = bblog.GetDebug(debug) 18 | } 19 | 20 | Or in a function: 21 | 22 | logErr := bblog.GetErr() 23 | logDebug := bblog.GetDebug(debug) 24 | logDebug.Printf("whatever: %v", err) 25 | 26 | */ 27 | 28 | var logErr *log.Logger 29 | var logDebug *log.Logger 30 | 31 | // GetErr returns a logger handle used for errors 32 | func GetErr() *log.Logger { 33 | if logErr == nil { 34 | logErr = log.New(os.Stderr, "", 0) 35 | } 36 | return logErr 37 | } 38 | 39 | // GetDebug returns a Logger handle used for debug info (output is discarded if viable=false) 40 | func GetDebug(visible bool) *log.Logger { 41 | if visible { 42 | logDebug = log.New(os.Stderr, "", 0) 43 | } else { 44 | // Invisible mode (i.e. display nothing) 45 | logDebug = log.New(ioutil.Discard, "", 0) 46 | } 47 | return logDebug 48 | } 49 | -------------------------------------------------------------------------------- /pkg/bbutil/filestats.go: -------------------------------------------------------------------------------- 1 | package bbutil 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "sort" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | // DirExists returns true if directory exists. 14 | func DirExists(path string) (bool, error) { 15 | stat, err := os.Stat(path) 16 | if err == nil { 17 | return stat.IsDir(), nil 18 | } 19 | if os.IsNotExist(err) { 20 | return false, nil 21 | } 22 | return true, err 23 | } 24 | 25 | // FileExistsOrProblem returns true if the file exists or if we can't determine its existence. 26 | func FileExistsOrProblem(path string) bool { 27 | _, err := os.Stat(path) 28 | if err == nil { 29 | return true 30 | } 31 | if os.IsNotExist(err) { 32 | return false 33 | } 34 | return true 35 | } 36 | 37 | // Touch updates the timestamp of a file. 38 | func Touch(name string) error { 39 | var err error 40 | _, err = os.Stat(name) 41 | if os.IsNotExist(err) { 42 | file, err := os.Create(name) 43 | if err != nil { 44 | return fmt.Errorf("TouchFile failed: %w", err) 45 | } 46 | file.Close() 47 | } 48 | 49 | currentTime := time.Now().Local() 50 | return os.Chtimes(name, currentTime, currentTime) 51 | } 52 | 53 | // ReadFileLines is like ioutil.ReadFile() but returns an []string. 54 | func ReadFileLines(filename string) ([]string, error) { 55 | b, err := ioutil.ReadFile(filename) 56 | if err != nil { 57 | return nil, err 58 | } 59 | s := string(b) 60 | s = strings.TrimSuffix(s, "\n") 61 | if s == "" { 62 | return []string{}, nil 63 | } 64 | l := strings.Split(s, "\n") 65 | return l, nil 66 | } 67 | 68 | // AddLinesToSortedFile adds a line to a sorted file. 69 | func AddLinesToSortedFile(filename string, newlines ...string) error { 70 | lines, err := ReadFileLines(filename) 71 | //fmt.Printf("DEBUG: read=%q\n", lines) 72 | if err != nil { 73 | return fmt.Errorf("AddLinesToSortedFile can't read %q: %w", filename, err) 74 | } 75 | if !sort.StringsAreSorted(lines) { 76 | return fmt.Errorf("AddLinesToSortedFile: file wasn't sorted: %v", filename) 77 | } 78 | lines = append(lines, newlines...) 79 | sort.Strings(lines) 80 | contents := strings.Join(lines, "\n") + "\n" 81 | //fmt.Printf("DEBUG: write=%q\n", contents) 82 | err = ioutil.WriteFile(filename, []byte(contents), 0o660) 83 | if err != nil { 84 | return fmt.Errorf("AddLinesToSortedFile can't write %q: %w", filename, err) 85 | } 86 | return nil 87 | } 88 | 89 | // AddLinesToFile adds lines to the end of a file. 90 | func AddLinesToFile(filename string, newlines ...string) error { 91 | lines, err := ReadFileLines(filename) 92 | if err != nil { 93 | return fmt.Errorf("AddLinesToFile can't read %q: %w", filename, err) 94 | } 95 | lines = append(lines, newlines...) 96 | contents := strings.Join(lines, "\n") + "\n" 97 | err = ioutil.WriteFile(filename, []byte(contents), 0o660) 98 | if err != nil { 99 | return fmt.Errorf("AddLinesToFile can't write %q: %w", filename, err) 100 | } 101 | return nil 102 | } 103 | 104 | // FindDirInParent looks for target in CWD, or .., or ../.., etc. 105 | func FindDirInParent(target string) (string, error) { 106 | // Prevent an infinite loop by only doing "cd .." this many times 107 | maxDirLevels := 30 108 | relpath := "." 109 | for i := 0; i < maxDirLevels; i++ { 110 | // Does relpath contain our target? 111 | t := filepath.Join(relpath, target) 112 | //logDebug.Printf("Trying %q\n", t) 113 | _, err := os.Stat(t) 114 | if err == nil { 115 | return t, nil 116 | } 117 | if !os.IsNotExist(err) { 118 | return "", fmt.Errorf("stat failed FindDirInParent (%q): %w", t, err) 119 | } 120 | // Ok, it really wasn't found. 121 | 122 | // If we are at the root, stop. 123 | if abs, err := filepath.Abs(relpath); err == nil && abs == "/" { 124 | break 125 | } 126 | // Try one directory up 127 | relpath = filepath.Join("..", relpath) 128 | } 129 | return "", fmt.Errorf("Not found") 130 | } 131 | -------------------------------------------------------------------------------- /pkg/bbutil/rbio_test.go: -------------------------------------------------------------------------------- 1 | package bbutil 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestRunBashInputOutput(t *testing.T) { 8 | 9 | in := "This is a test of the RBIO system.\n" 10 | bin := []byte(in) 11 | 12 | out, err := RunBashInputOutput(bin, "cat") 13 | sout := string(out) 14 | if err != nil { 15 | t.Error(err) 16 | } 17 | 18 | if in != sout { 19 | t.Errorf("not equal %q %q", in, out) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pkg/bbutil/runbash.go: -------------------------------------------------------------------------------- 1 | package bbutil 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/exec" 9 | ) 10 | 11 | // RunBash runs a Bash command. 12 | func RunBash(command string, args ...string) error { 13 | cmd := exec.Command(command, args...) 14 | cmd.Stdin = os.Stdin 15 | cmd.Stdout = os.Stdout 16 | cmd.Stderr = os.Stderr 17 | err := cmd.Start() 18 | if err != nil { 19 | log.Fatal(err) 20 | } 21 | err = cmd.Wait() 22 | if err != nil { 23 | return fmt.Errorf("RunBash cmd=%q err=%w", command, err) 24 | } 25 | return nil 26 | } 27 | 28 | // RunBashOutput runs a Bash command, captures output. 29 | func RunBashOutput(command string, args ...string) (string, error) { 30 | cmd := exec.Command(command, args...) 31 | cmd.Stdin = os.Stdin 32 | cmd.Stderr = os.Stderr 33 | out, err := cmd.Output() 34 | if err != nil { 35 | return "", fmt.Errorf("RunBashOutput err=%w", err) 36 | } 37 | return string(out), err 38 | } 39 | 40 | // RunBashOutputSilent runs a Bash command, captures output, discards stderr. 41 | func RunBashOutputSilent(command string, args ...string) (string, error) { 42 | cmd := exec.Command(command, args...) 43 | cmd.Stdin = os.Stdin 44 | // Leave cmd.Stderr unmodified and stderr is discarded. 45 | out, err := cmd.Output() 46 | if err != nil { 47 | return "", fmt.Errorf("RunBashOutputSilent err=%w", err) 48 | } 49 | return string(out), err 50 | } 51 | 52 | // RunBashInput runs a Bash command, sends input on stdin. 53 | func RunBashInput(input string, command string, args ...string) error { 54 | 55 | cmd := exec.Command(command, args...) 56 | cmd.Stdin = bytes.NewBuffer([]byte(input)) 57 | cmd.Stdout = os.Stdout 58 | cmd.Stderr = os.Stderr 59 | err := cmd.Run() 60 | if err != nil { 61 | return fmt.Errorf("RunBashInput err=%w", err) 62 | } 63 | return nil 64 | } 65 | 66 | // RunBashInputOutput runs a Bash command, sends input on stdin. 67 | func RunBashInputOutput(input []byte, command string, args ...string) ([]byte, error) { 68 | 69 | cmd := exec.Command(command, args...) 70 | cmd.Stdin = bytes.NewBuffer(input) 71 | cmd.Stderr = os.Stderr 72 | out, err := cmd.Output() 73 | if err != nil { 74 | return nil, fmt.Errorf("RunBashInputOutput err=%w", err) 75 | } 76 | return out, nil 77 | } 78 | -------------------------------------------------------------------------------- /pkg/bbutil/shred.go: -------------------------------------------------------------------------------- 1 | package bbutil 2 | 3 | // Pick an appropriate secure erase command for this operating system 4 | // or just delete the file with os.Remove(). 5 | 6 | // Code rewritten based https://codereview.stackexchange.com/questions/245072 7 | 8 | import ( 9 | "fmt" 10 | "io/ioutil" 11 | "os" 12 | "os/exec" 13 | ) 14 | 15 | var shredCmds = []struct { 16 | name, opts string 17 | }{ 18 | {"sdelete", "-a"}, 19 | {"shred", "-u"}, 20 | {"srm", "-f"}, 21 | {"rm", "-Pf"}, 22 | } 23 | 24 | func shredTemp(path, opts string) error { 25 | file, err := ioutil.TempFile("", "shredTemp.") 26 | if err != nil { 27 | return err 28 | } 29 | filename := file.Name() 30 | defer os.Remove(filename) 31 | defer file.Close() 32 | 33 | err = file.Close() 34 | if err != nil { 35 | return err 36 | } 37 | err = RunBash(path, opts, filename) 38 | if err != nil { 39 | return err 40 | } 41 | return nil 42 | } 43 | 44 | var shredPath, shredOpts = func() (string, string) { 45 | for _, cmd := range shredCmds { 46 | path, err := exec.LookPath(cmd.name) 47 | if err != nil { 48 | continue 49 | } 50 | err = shredTemp(path, cmd.opts) 51 | if err == nil { 52 | return path, cmd.opts 53 | } 54 | } 55 | return "", "" 56 | }() 57 | 58 | // ShredInfo reveals the shred command and flags (for "blackbox info") 59 | func ShredInfo() string { 60 | return shredPath + " " + shredOpts 61 | } 62 | 63 | // shredFile shreds one file. 64 | func shredFile(filename string) error { 65 | fi, err := os.Stat(filename) 66 | if err != nil { 67 | return err 68 | } 69 | if !fi.Mode().IsRegular() { 70 | err := fmt.Errorf("filename is not mode regular") 71 | return err 72 | } 73 | 74 | if shredPath == "" { 75 | // No secure erase command found. Default to a normal file delete. 76 | // TODO(tlim): Print a warning? Have a flag that causes this to be an error? 77 | return os.Remove(filename) 78 | } 79 | 80 | err = RunBash(shredPath, shredOpts, filename) 81 | if err != nil { 82 | return err 83 | } 84 | return nil 85 | } 86 | 87 | // ShredFiles securely erases a list of files. 88 | func ShredFiles(names []string) error { 89 | 90 | // TODO(tlim) DO the shredding in parallel like in v1. 91 | 92 | var eerr error 93 | for _, n := range names { 94 | _, err := os.Stat(n) 95 | if err != nil { 96 | if os.IsNotExist(err) { 97 | fmt.Printf("======= already gone: %q\n", n) 98 | continue 99 | } 100 | } 101 | fmt.Printf("========== SHREDDING: %q\n", n) 102 | e := shredFile(n) 103 | if e != nil { 104 | eerr = e 105 | fmt.Printf("ERROR: %v\n", e) 106 | } 107 | } 108 | return eerr 109 | } 110 | -------------------------------------------------------------------------------- /pkg/bbutil/sortedfile_test.go: -------------------------------------------------------------------------------- 1 | package bbutil 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestAddLinesToSortedFile(t *testing.T) { 10 | 11 | var tests = []struct { 12 | start string 13 | add []string 14 | expected string 15 | }{ 16 | { 17 | "", 18 | []string{"one"}, 19 | "one\n", 20 | }, 21 | { 22 | "begin\ntwo\n", 23 | []string{"at top"}, 24 | "at top\nbegin\ntwo\n", 25 | }, 26 | { 27 | "begin\ntwo\n", 28 | []string{"zbottom"}, 29 | "begin\ntwo\nzbottom\n", 30 | }, 31 | { 32 | "begin\ntwo\n", 33 | []string{"middle"}, 34 | "begin\nmiddle\ntwo\n", 35 | }, 36 | } 37 | 38 | for i, test := range tests { 39 | content := []byte(test.start) 40 | tmpfile, err := ioutil.TempFile("", "example") 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | tmpfilename := tmpfile.Name() 45 | defer os.Remove(tmpfilename) 46 | 47 | if _, err := tmpfile.Write(content); err != nil { 48 | t.Fatal(err) 49 | } 50 | if err := tmpfile.Close(); err != nil { 51 | t.Fatal(err) 52 | } 53 | AddLinesToSortedFile(tmpfilename, test.add...) 54 | expected := test.expected 55 | 56 | got, err := ioutil.ReadFile(tmpfilename) 57 | if err != nil { 58 | t.Fatal(err) 59 | } 60 | if expected != string(got) { 61 | t.Errorf("test %v: contents wrong:\nexpected: %q\n got: %q", i, expected, got) 62 | } 63 | os.Remove(tmpfilename) 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /pkg/bbutil/umask_posix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | package bbutil 5 | 6 | import "syscall" 7 | 8 | // Umask is a no-op on Windows, and calls syscall.Umask on all other 9 | // systems. On Windows it returns 0, which is a decoy. 10 | func Umask(mask int) int { 11 | return syscall.Umask(mask) 12 | } 13 | -------------------------------------------------------------------------------- /pkg/bbutil/umask_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package bbutil 5 | 6 | // Umask is a no-op on Windows, and calls syscall.Umask on all other 7 | // systems. On Windows it returns 0, which is a decoy. 8 | func Umask(mask int) int { 9 | return 0o000 10 | } 11 | -------------------------------------------------------------------------------- /pkg/box/boxutils.go: -------------------------------------------------------------------------------- 1 | package box 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "os/user" 8 | "path/filepath" 9 | "runtime" 10 | "strconv" 11 | "strings" 12 | 13 | "github.com/StackExchange/blackbox/v2/pkg/makesafe" 14 | ) 15 | 16 | // FileStatus returns the status of a file. 17 | func FileStatus(name string) (string, error) { 18 | /* 19 | DECRYPTED: File is decrypted and ready to edit (unknown if it has been edited). 20 | ENCRYPTED: GPG file is newer than plaintext. Indicates recented edited then encrypted. 21 | SHREDDED: Plaintext is missing. 22 | GPGMISSING: The .gpg file is missing. Oops? 23 | PLAINERROR: Can't access the plaintext file to determine status. 24 | GPGERROR: Can't access .gpg file to determine status. 25 | */ 26 | 27 | p := name 28 | e := p + ".gpg" 29 | ps, perr := os.Stat(p) 30 | es, eerr := os.Stat(e) 31 | if perr == nil && eerr == nil { 32 | if ps.ModTime().Before(es.ModTime()) { 33 | return "ENCRYPTED", nil 34 | } 35 | return "DECRYPTED", nil 36 | } 37 | 38 | if os.IsNotExist(perr) && os.IsNotExist(eerr) { 39 | return "BOTHMISSING", nil 40 | } 41 | 42 | if eerr != nil { 43 | if os.IsNotExist(eerr) { 44 | return "GPGMISSING", nil 45 | } 46 | return "GPGERROR", eerr 47 | } 48 | 49 | if perr != nil { 50 | if os.IsNotExist(perr) { 51 | return "SHREDDED", nil 52 | } 53 | } 54 | return "PLAINERROR", perr 55 | } 56 | 57 | func anyGpg(names []string) error { 58 | for _, name := range names { 59 | if strings.HasSuffix(name, ".gpg") { 60 | return fmt.Errorf( 61 | "no not specify .gpg files. Specify %q not %q", 62 | strings.TrimSuffix(name, ".gpg"), name) 63 | } 64 | } 65 | return nil 66 | } 67 | 68 | // func isChanged(pname string) (bool, error) { 69 | // // if .gpg exists but not plainfile: unchanged 70 | // // if plaintext exists but not .gpg: changed 71 | // // if plainfile < .gpg: unchanged 72 | // // if plainfile > .gpg: don't know, need to try diff 73 | 74 | // // Gather info about the files: 75 | 76 | // pstat, perr := os.Stat(pname) 77 | // if perr != nil && (!os.IsNotExist(perr)) { 78 | // return false, fmt.Errorf("isChanged(%q) returned error: %w", pname, perr) 79 | // } 80 | // gname := pname + ".gpg" 81 | // gstat, gerr := os.Stat(gname) 82 | // if gerr != nil && (!os.IsNotExist(perr)) { 83 | // return false, fmt.Errorf("isChanged(%q) returned error: %w", gname, gerr) 84 | // } 85 | 86 | // pexists := perr == nil 87 | // gexists := gerr == nil 88 | 89 | // // Use the above rules: 90 | 91 | // // if .gpg exists but not plainfile: unchanged 92 | // if gexists && !pexists { 93 | // return false, nil 94 | // } 95 | 96 | // // if plaintext exists but not .gpg: changed 97 | // if pexists && !gexists { 98 | // return true, nil 99 | // } 100 | 101 | // // At this point we can conclude that both p and g exist. 102 | // // Can't hurt to test that assertion. 103 | // if (!pexists) && (!gexists) { 104 | // return false, fmt.Errorf("Assertion failed. p and g should exist: pn=%q", pname) 105 | // } 106 | 107 | // pmodtime := pstat.ModTime() 108 | // gmodtime := gstat.ModTime() 109 | // // if plainfile < .gpg: unchanged 110 | // if pmodtime.Before(gmodtime) { 111 | // return false, nil 112 | // } 113 | // // if plainfile > .gpg: don't know, need to try diff 114 | // return false, fmt.Errorf("Can not know for sure. Try git diff?") 115 | // } 116 | 117 | func parseGroup(userinput string) (int, error) { 118 | if userinput == "" { 119 | return -1, fmt.Errorf("group spec is empty string") 120 | } 121 | 122 | // If it is a valid number, use it. 123 | i, err := strconv.Atoi(userinput) 124 | if err == nil { 125 | return i, nil 126 | } 127 | 128 | // If not a number, look it up by name. 129 | g, err := user.LookupGroup(userinput) 130 | if err == nil { 131 | i, err = strconv.Atoi(g.Gid) 132 | return i, nil 133 | } 134 | 135 | // Give up. 136 | return -1, err 137 | } 138 | 139 | // FindConfigDir tests various places until it finds the config dir. 140 | // If we can't determine the relative path, "" is returned. 141 | func FindConfigDir(reporoot, team string) (string, error) { 142 | 143 | candidates := []string{} 144 | if team != "" { 145 | candidates = append(candidates, ".blackbox-"+team) 146 | } 147 | candidates = append(candidates, ".blackbox") 148 | candidates = append(candidates, "keyrings/live") 149 | logDebug.Printf("DEBUG: candidates = %q\n", candidates) 150 | 151 | maxDirLevels := 30 // Prevent an infinite loop 152 | relpath := "." 153 | for i := 0; i < maxDirLevels; i++ { 154 | // Does relpath contain any of our directory names? 155 | for _, c := range candidates { 156 | t := filepath.Join(relpath, c) 157 | logDebug.Printf("Trying %q\n", t) 158 | fi, err := os.Stat(t) 159 | if err == nil && fi.IsDir() { 160 | return t, nil 161 | } 162 | if err == nil { 163 | return "", fmt.Errorf("path %q is not a directory: %w", t, err) 164 | } 165 | if !os.IsNotExist(err) { 166 | return "", fmt.Errorf("dirExists access error: %w", err) 167 | } 168 | } 169 | 170 | // If we are at the root, stop. 171 | if abs, _ := filepath.Abs(relpath); abs == "/" { 172 | break 173 | } 174 | // Try one directory up 175 | relpath = filepath.Join("..", relpath) 176 | } 177 | 178 | return "", fmt.Errorf("No .blackbox (or equiv) directory found") 179 | } 180 | 181 | func gpgAgentNotice() { 182 | // Is gpg-agent configured? 183 | if os.Getenv("GPG_AGENT_INFO") != "" { 184 | return 185 | } 186 | // Are we on macOS? 187 | if runtime.GOOS == "darwin" { 188 | // We assume the use of https://gpgtools.org, which 189 | // uses the keychain. 190 | return 191 | } 192 | 193 | // TODO(tlim): v1 verifies that "gpg-agent --version" outputs a version 194 | // string that is 2.1.0 or higher. It seems that 1.x is incompatible. 195 | 196 | fmt.Println("WARNING: You probably want to run gpg-agent as") 197 | fmt.Println("you will be asked for your passphrase many times.") 198 | fmt.Println("Example: $ eval $(gpg-agent --daemon)") 199 | fmt.Print("Press CTRL-C now to stop. ENTER to continue: ") 200 | input := bufio.NewScanner(os.Stdin) 201 | input.Scan() 202 | } 203 | 204 | func shouldWeOverwrite() { 205 | fmt.Println() 206 | fmt.Println("WARNING: This will overwrite any unencrypted files laying about.") 207 | fmt.Print("Press CTRL-C now to stop. ENTER to continue: ") 208 | input := bufio.NewScanner(os.Stdin) 209 | input.Scan() 210 | } 211 | 212 | // PrettyCommitMessage generates a pretty commit message. 213 | func PrettyCommitMessage(verb string, files []string) string { 214 | if len(files) == 0 { 215 | // This use-case should probably be an error. 216 | return verb + " (no files)" 217 | } 218 | rfiles := makesafe.RedactMany(files) 219 | m, truncated := makesafe.FirstFewFlag(rfiles) 220 | if truncated { 221 | return verb + ": " + m 222 | } 223 | return verb + ": " + m 224 | } 225 | -------------------------------------------------------------------------------- /pkg/box/pretty_test.go: -------------------------------------------------------------------------------- 1 | package box 2 | 3 | import "testing" 4 | 5 | func TestPrettyCommitMessage(t *testing.T) { 6 | long := "aVeryVeryLongLongLongStringStringString" 7 | for i, test := range []struct { 8 | data []string 9 | expected string 10 | }{ 11 | {[]string{}, `HEADING (no files)`}, 12 | {[]string{"one"}, `HEADING: one`}, 13 | {[]string{"one", "two"}, `HEADING: one two`}, 14 | {[]string{"one", "two", "three"}, `HEADING: one two three`}, 15 | {[]string{"one", "two", "three", "four"}, 16 | `HEADING: one two three four`}, 17 | {[]string{"one", "two", "three", "four", "five"}, 18 | `HEADING: one two three four five`}, 19 | {[]string{"has spaces.txt"}, `HEADING: "has spaces.txt"`}, 20 | {[]string{"two\n"}, `HEADING: "twoX"(redacted)`}, 21 | {[]string{"smile😁eyes"}, `HEADING: smile😁eyes`}, 22 | {[]string{"tab\ttab", "two very long strings.txt"}, 23 | `HEADING: "tabXtab"(redacted) "two very long strings.txt"`}, 24 | {[]string{long, long, long, long}, 25 | "HEADING: " + long + " " + long + " (and others)"}, 26 | } { 27 | g := PrettyCommitMessage("HEADING", test.data) 28 | if g == test.expected { 29 | //t.Logf("%03d: PASSED files=%q\n", i, test.data) 30 | t.Logf("%03d: PASSED", i) 31 | } else { 32 | t.Errorf("%03d: FAILED files==%q got=(%q) wanted=(%q)\n", i, test.data, g, test.expected) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pkg/commitlater/commitlater.go: -------------------------------------------------------------------------------- 1 | package commitlater 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type future struct { 8 | message string // Message that describes this transaction. 9 | dir string // Basedir of the files 10 | files []string // Names of the files 11 | display []string // Names as to be displayed to the user 12 | } 13 | 14 | // List of futures to be done in the future. 15 | type List struct { 16 | items []*future 17 | } 18 | 19 | // Add queues up a future commit. 20 | func (list *List) Add(message string, repobasedir string, files []string) { 21 | item := &future{ 22 | message: message, 23 | dir: repobasedir, 24 | files: files, 25 | } 26 | list.items = append(list.items, item) 27 | } 28 | 29 | func sameDirs(l *List) bool { 30 | if len(l.items) <= 1 { 31 | return true 32 | } 33 | for _, k := range l.items[1:] { 34 | if k.dir != l.items[0].dir { 35 | return false 36 | } 37 | } 38 | return true 39 | } 40 | 41 | // Flush executes queued commits. 42 | func (list *List) Flush( 43 | title string, 44 | fadd func([]string) error, 45 | fcommit func([]string, string, []string) error, 46 | ) error { 47 | 48 | // Just list the individual commit commands. 49 | if title == "" || len(list.items) < 2 || !sameDirs(list) { 50 | for _, fut := range list.items { 51 | err := fadd(fut.files) 52 | if err != nil { 53 | return fmt.Errorf("add files1 (%q) failed: %w", fut.files, err) 54 | } 55 | err = fcommit([]string{fut.message}, fut.dir, fut.files) 56 | if err != nil { 57 | return fmt.Errorf("commit files (%q) failed: %w", fut.files, err) 58 | } 59 | } 60 | return nil 61 | } 62 | 63 | // Create a long commit message. 64 | var m []string 65 | var f []string 66 | for _, fut := range list.items { 67 | err := fadd(fut.files) 68 | if err != nil { 69 | return fmt.Errorf("add files2 (%q) failed: %w", fut.files, err) 70 | } 71 | m = append(m, fut.message) 72 | f = append(f, fut.files...) 73 | } 74 | msg := []string{title} 75 | for _, mm := range m { 76 | msg = append(msg, " * "+mm) 77 | } 78 | err := fcommit(msg, list.items[0].dir, f) 79 | if err != nil { 80 | return fmt.Errorf("commit files (%q) failed: %w", f, err) 81 | } 82 | 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /pkg/crypters/_all/all.go: -------------------------------------------------------------------------------- 1 | package all 2 | 3 | import ( 4 | _ "github.com/StackExchange/blackbox/v2/pkg/crypters/gnupg" 5 | ) 6 | -------------------------------------------------------------------------------- /pkg/crypters/crypters.go: -------------------------------------------------------------------------------- 1 | package crypters 2 | 3 | import ( 4 | "sort" 5 | "strings" 6 | 7 | "github.com/StackExchange/blackbox/v2/models" 8 | ) 9 | 10 | // Crypter is the handle 11 | type Crypter interface { 12 | models.Crypter 13 | } 14 | 15 | // NewFnSig function signature needed by reg. 16 | type NewFnSig func(debug bool) (Crypter, error) 17 | 18 | // Item stores one item 19 | type Item struct { 20 | Name string 21 | New NewFnSig 22 | Priority int 23 | } 24 | 25 | // Catalog is the list of registered vcs's. 26 | var Catalog []*Item 27 | 28 | // SearchByName returns a Crypter handle for name. 29 | // The search is case insensitive. 30 | func SearchByName(name string, debug bool) Crypter { 31 | name = strings.ToLower(name) 32 | for _, v := range Catalog { 33 | //fmt.Printf("Trying %v %v\n", v.Name) 34 | if strings.ToLower(v.Name) == name { 35 | chandle, err := v.New(debug) 36 | if err != nil { 37 | return nil // No idea how that would happen. 38 | } 39 | //fmt.Printf("USING! %v\n", v.Name) 40 | return chandle 41 | } 42 | } 43 | return nil 44 | } 45 | 46 | // Register a new VCS. 47 | func Register(name string, priority int, newfn NewFnSig) { 48 | //fmt.Printf("CRYPTER registered: %v\n", name) 49 | item := &Item{ 50 | Name: name, 51 | New: newfn, 52 | Priority: priority, 53 | } 54 | Catalog = append(Catalog, item) 55 | 56 | // Keep the list sorted. 57 | sort.Slice(Catalog, func(i, j int) bool { return Catalog[j].Priority < Catalog[i].Priority }) 58 | } 59 | -------------------------------------------------------------------------------- /pkg/crypters/gnupg/gnupg.go: -------------------------------------------------------------------------------- 1 | package gnupg 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | 11 | "github.com/StackExchange/blackbox/v2/pkg/bblog" 12 | "github.com/StackExchange/blackbox/v2/pkg/bbutil" 13 | "github.com/StackExchange/blackbox/v2/pkg/crypters" 14 | ) 15 | 16 | var pluginName = "GnuPG" 17 | 18 | func init() { 19 | crypters.Register(pluginName, 100, registerNew) 20 | } 21 | 22 | // CrypterHandle is the handle 23 | type CrypterHandle struct { 24 | GPGCmd string // "gpg2" or "gpg" 25 | logErr *log.Logger 26 | logDebug *log.Logger 27 | } 28 | 29 | func registerNew(debug bool) (crypters.Crypter, error) { 30 | 31 | crypt := &CrypterHandle{ 32 | logErr: bblog.GetErr(), 33 | logDebug: bblog.GetDebug(debug), 34 | } 35 | 36 | // Which binary to use? 37 | path, err := exec.LookPath("gpg2") 38 | if err != nil { 39 | path, err = exec.LookPath("gpg") 40 | if err != nil { 41 | path = "gpg2" 42 | } 43 | } 44 | crypt.GPGCmd = path 45 | 46 | return crypt, nil 47 | } 48 | 49 | // Name returns my name. 50 | func (crypt CrypterHandle) Name() string { 51 | return pluginName 52 | } 53 | 54 | // Decrypt name+".gpg", possibly overwriting name. 55 | func (crypt CrypterHandle) Decrypt(filename string, umask int, overwrite bool) error { 56 | 57 | a := []string{ 58 | "--use-agent", 59 | "-q", 60 | "--decrypt", 61 | "-o", filename, 62 | } 63 | if overwrite { 64 | a = append(a, "--yes") 65 | } 66 | a = append(a, filename+".gpg") 67 | 68 | oldumask := bbutil.Umask(umask) 69 | err := bbutil.RunBash(crypt.GPGCmd, a...) 70 | bbutil.Umask(oldumask) 71 | return err 72 | } 73 | 74 | // Cat returns the plaintext or, if it is missing, the decrypted cyphertext. 75 | func (crypt CrypterHandle) Cat(filename string) ([]byte, error) { 76 | 77 | a := []string{ 78 | "--use-agent", 79 | "-q", 80 | "--decrypt", 81 | } 82 | 83 | // TODO(tlim): This assumes the entire gpg file fits in memory. If 84 | // this becomes a problem, re-implement this using exec Cmd.StdinPipe() 85 | // and feed the input in chunks. 86 | in, err := ioutil.ReadFile(filename + ".gpg") 87 | if err != nil { 88 | 89 | if os.IsNotExist(err) { 90 | // Encrypted file doesn't exit? Return the plaintext. 91 | return ioutil.ReadFile(filename) 92 | } 93 | 94 | return nil, err 95 | } 96 | 97 | return bbutil.RunBashInputOutput(in, crypt.GPGCmd, a...) 98 | } 99 | 100 | // Encrypt name, overwriting name+".gpg" 101 | func (crypt CrypterHandle) Encrypt(filename string, umask int, receivers []string) (string, error) { 102 | var err error 103 | 104 | crypt.logDebug.Printf("Encrypt(%q, %d, %q)", filename, umask, receivers) 105 | encrypted := filename + ".gpg" 106 | a := []string{ 107 | "--use-agent", 108 | "--yes", 109 | "--encrypt", 110 | "-o", encrypted, 111 | } 112 | for _, f := range receivers { 113 | a = append(a, "-r", f) 114 | } 115 | a = append(a, "--encrypt") 116 | a = append(a, filename) 117 | //err = bbutil.RunBash("ls", "-la") 118 | 119 | oldumask := bbutil.Umask(umask) 120 | crypt.logDebug.Printf("Args = %q", a) 121 | err = bbutil.RunBash(crypt.GPGCmd, a...) 122 | bbutil.Umask(oldumask) 123 | 124 | return encrypted, err 125 | } 126 | 127 | // AddNewKey extracts keyname from sourcedir's GnuPG chain to destdir keychain. 128 | // It returns a list of files that may have changed. 129 | func (crypt CrypterHandle) AddNewKey(keyname, repobasedir, sourcedir, destdir string) ([]string, error) { 130 | 131 | // $GPG --homedir="$2" --export -a "$KEYNAME" >"$pubkeyfile" 132 | args := []string{ 133 | "--export", 134 | "-a", 135 | } 136 | if sourcedir != "" { 137 | args = append(args, "--homedir", sourcedir) 138 | } 139 | args = append(args, keyname) 140 | crypt.logDebug.Printf("ADDNEWKEY: Extracting key=%v: gpg, %v\n", keyname, args) 141 | pubkey, err := bbutil.RunBashOutput("gpg", args...) 142 | if err != nil { 143 | return nil, err 144 | } 145 | if len(pubkey) == 0 { 146 | return nil, fmt.Errorf("Nothing found when %q exported from %q", keyname, sourcedir) 147 | } 148 | 149 | // $GPG --no-permission-warning --homedir="$KEYRINGDIR" --import "$pubkeyfile" 150 | args = []string{ 151 | "--no-permission-warning", 152 | "--homedir", destdir, 153 | "--import", 154 | } 155 | crypt.logDebug.Printf("ADDNEWKEY: Importing: gpg %v\n", args) 156 | // fmt.Printf("DEBUG: crypter ADD %q", args) 157 | err = bbutil.RunBashInput(pubkey, "gpg", args...) 158 | if err != nil { 159 | return nil, fmt.Errorf("AddNewKey failed: %w", err) 160 | } 161 | 162 | // Suggest: ${pubring_path} trustdb.gpg blackbox-admins.txt 163 | var changed []string 164 | 165 | // Prefix each file with the relative path to it. 166 | prefix, err := filepath.Rel(repobasedir, destdir) 167 | if err != nil { 168 | //fmt.Printf("FAIL (%v) (%v) (%v)\n", repobasedir, destdir, err) 169 | prefix = destdir 170 | } 171 | for _, file := range []string{"pubring.gpg", "pubring.kbx", "trustdb.gpg"} { 172 | path := filepath.Join(destdir, file) 173 | if bbutil.FileExistsOrProblem(path) { 174 | changed = append(changed, filepath.Join(prefix, file)) 175 | } 176 | } 177 | return changed, nil 178 | } 179 | -------------------------------------------------------------------------------- /pkg/crypters/gnupg/keychain.go: -------------------------------------------------------------------------------- 1 | package gnupg 2 | 3 | /* 4 | 5 | # How does Blackbox manage key rings? 6 | 7 | Blackbox uses the user's .gnupg directory for most actions, such as decrypting data. 8 | Decrypting requires the user's private key, which is stored by the user in their 9 | home directory (and up to them to store safely). 10 | Black box does not store the user's private key in the repo. 11 | 12 | When encrypting data, black needs the public key of all the admins, not just the users. 13 | To assure that the user's `.gnupg` has all these public keys, prior to 14 | encrypting data the public keys are imported from .blackbox, which stores 15 | a keychain that stores the public (not private!) keys of all the admins. 16 | 17 | FYI: v1 does this import before decrypting, because I didn't know any better. 18 | 19 | # Binary compatibility: 20 | 21 | When writing v1, we didn't realize that the pubkey.gpg file is a binary format 22 | that is not intended to be portable. In fact, it is intentionally not portable. 23 | This means that all admins must use the exact same version of GnuPG 24 | or the files (pubring.gpg or pubring.kbx) may get corrupted. 25 | 26 | In v2, we store the public keys in the portable ascii format 27 | in a file called `.blackbox/public-keys-db.asc`. 28 | It will also update the binary files if they exist. 29 | If `.blackbox/public-keys-db.asc` doesn't exist, it will be created. 30 | 31 | Eventually we will stop updating the binary files. 32 | 33 | # Importing public keys to the user 34 | 35 | How to import the public keys to the user's GPG system: 36 | 37 | If pubkeyring-ascii.txt exists: 38 | gpg --import pubkeyring-ascii.asc 39 | Else if pubring.kbx 40 | gpg --import pubring.kbx 41 | Else if pubring.gpg 42 | gpg --import pubring.gpg 43 | 44 | This is what v1 does: 45 | #if gpg2 is installed next to gpg like on ubuntu 16 46 | if [[ "$GPG" != "gpg2" ]]; then 47 | $GPG --export --no-default-keyring --keyring "$(get_pubring_path)" >"$keyringasc" 48 | $GPG --import "$keyringasc" 2>&1 | egrep -v 'not changed$' >&2 49 | Else 50 | $GPG --keyring "$(get_pubring_path)" --export | $GPG --import 51 | fi 52 | 53 | # How to add a key to the keyring? 54 | 55 | Old, binary format: 56 | # Get the key they want to add: 57 | FOO is a user-specified directory, otherwise $HOME/.gnupg: 58 | $GPG --homedir="FOO" --export -a "$KEYNAME" >TEMPFILE 59 | # Import into the binary files: 60 | KEYRINGDIR is .blackbox 61 | $GPG --no-permission-warning --homedir="$KEYRINGDIR" --import TEMPFILE 62 | # Git add any of these files if they exist: 63 | pubring.gpg pubring.kbx trustdb.gpg blackbox-admins.txt 64 | # Tell the user to git commit them. 65 | 66 | New, ascii format: 67 | # Get the key to be added. Write to a TEMPFILE 68 | FOO is a user-specified directory, otherwise $HOME/.gnupg: 69 | $GPG --homedir="FOO" --export -a "$KEYNAME" >TEMPFILE 70 | # Make a tempdir called TEMPDIR 71 | # Import the pubkeyring-ascii.txt to TEMPDIR's keyring. (Skip if file not found) 72 | # Import the temp1 data to TEMPDIR 73 | # Export the TEMPDIR to create a new .blackbox/pubkeyring-ascii.txt 74 | PATH_TO_BINARY is the path to .blackbox/pubring.gpg; if that's not found then pubring.kbx 75 | $GPG --keyring PATH_TO_BINARY --export -a --output .blackbox/pubkeyring-ascii.txt 76 | # Git add .blackbox/pubkeyring-ascii.txt and .blackbox/blackbox-admins.txt 77 | # Tell the user to git commit them. 78 | # Delete TEMPDIR 79 | 80 | # How to remove a key from the keyring? 81 | 82 | Old, binary format: 83 | # Remove key from the binary file 84 | $GPG --no-permission-warning --homedir="$KEYRINGDIR" --batch --yes --delete-key "$KEYNAME" || true 85 | # Git add any of these files if they exist: 86 | pubring.gpg pubring.kbx trustdb.gpg blackbox-admins.txt 87 | # Tell the user to git commit them. 88 | 89 | New, ascii format: 90 | # Make a tempdir called TEMPDIR 91 | # Import the pubkeyring-ascii.txt to TEMPDIR's keyring. (Skip if file not found) 92 | # Remove key from the ring file 93 | $GPG --no-permission-warning --homedir="$KEYRINGDIR" --batch --yes --delete-key "$KEYNAME" || true 94 | # Export the TEMPDIR to create a new .blackbox/pubkeyring-ascii.txt 95 | PATH_TO_BINARY is the path to .blackbox/pubring.gpg; if that's not found then pubring.kbx 96 | $GPG --keyring PATH_TO_BINARY --export -a --output .blackbox/pubkeyring-ascii.txt 97 | # Git add .blackbox/pubkeyring-ascii.txt and .blackbox/blackbox-admins.txt 98 | # Update the .blackbox copy of pubring.gpg, pubring.kbx, or trustdb.gpg (if they exist) 99 | # with copies from TEMPDIR (if they exist). Git add any files that are updated. 100 | # Tell the user to git commit them. 101 | # Delete TEMPDIR 102 | 103 | */ 104 | 105 | //func prepareUserKeychain() error { 106 | // return nil 107 | //} 108 | -------------------------------------------------------------------------------- /pkg/makesafe/makesafe_test.go: -------------------------------------------------------------------------------- 1 | package makesafe 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestRedact(t *testing.T) { 8 | for i, test := range []struct{ data, expected string }{ 9 | {"", `""`}, 10 | {"one", "one"}, 11 | {"has space.txt", `"has space.txt"`}, 12 | {"has\ttab.txt", `"hasXtab.txt"(redacted)`}, 13 | {"has\nnl.txt", `"hasXnl.txt"(redacted)`}, 14 | {"has\rret.txt", `"hasXret.txt"(redacted)`}, 15 | {"¡que!", `¡que!`}, 16 | {"thé", `thé`}, 17 | {"pound£", `pound£`}, 18 | {"*.go", `*.go`}, 19 | {"rm -rf / ; echo done", `"rm -rf / ; echo done"`}, 20 | {"smile\u263a", `smile☺`}, 21 | {"dub\U0001D4E6", `dub𝓦`}, 22 | {"four\U0010FFFF", `"fourX"(redacted)`}, 23 | } { 24 | g := Redact(test.data) 25 | if g == test.expected { 26 | t.Logf("%03d: PASSED", i) 27 | } else { 28 | t.Errorf("%03d: FAILED data=%q got=(%s) wanted=(%s)", i, test.data, g, test.expected) 29 | } 30 | } 31 | } 32 | 33 | func TestRedactMany(t *testing.T) { 34 | data := []string{ 35 | "", 36 | "one", 37 | "has space.txt", 38 | "has\ttab.txt", 39 | } 40 | g := RedactMany(data) 41 | if len(g) != 4 || g[0] != `""` || g[1] != `"has space.txt"` || g[2] != `"hasXtab.txt"(redacted)` { 42 | t.Logf("PASSED") 43 | } else { 44 | t.Errorf("FAILED got=(%q)", g) 45 | } 46 | } 47 | 48 | func TestShell(t *testing.T) { 49 | for i, test := range []struct{ data, expected string }{ 50 | {"", `""`}, 51 | {"one", "one"}, 52 | {"two\n", `$(printf '%q' 'two\n')`}, 53 | {"ta tab", `$(printf '%q' 'ta\ttab')`}, 54 | {"tab\ttab", `$(printf '%q' 'tab\ttab')`}, 55 | {"new\nline", `$(printf '%q' 'new\nline')`}, 56 | {"¡que!", `$(printf '%q' '\302\241que!')`}, 57 | {"thé", `$(printf '%q' 'th\303\251')`}, 58 | {"pound£", `$(printf '%q' 'pound\302\243')`}, 59 | {"*.go", `'*.go'`}, 60 | {"rm -rf / ; echo done", `'rm -rf / ; echo done'`}, 61 | {"smile\u263a", `$(printf '%q' 'smile\342\230\272')`}, 62 | {"dub\U0001D4E6", `$(printf '%q' 'dub\360\235\223\246')`}, 63 | {"four\U0010FFFF", `$(printf '%q' 'four\364\217\277\277')`}, 64 | } { 65 | g := Shell(test.data) 66 | if g == test.expected { 67 | t.Logf("%03d: PASSED", i) 68 | //t.Logf("%03d: PASSED go(%q) bash: %s", i, test.data, test.expected) 69 | } else { 70 | t.Errorf("%03d: FAILED data=%q got=`%s` wanted=`%s`", i, test.data, g, test.expected) 71 | } 72 | } 73 | } 74 | 75 | func TestEscapeRune(t *testing.T) { 76 | for i, test := range []struct { 77 | data rune 78 | expected string 79 | }{ 80 | {'a', `\141`}, 81 | {'é', `\303\251`}, 82 | {'☺', `\342\230\272`}, 83 | {'글', `\352\270\200`}, 84 | {'𩸽', `\360\251\270\275`}, 85 | //{"\U0010FEDC", `"'\U0010fedc'"`}, 86 | } { 87 | g := escapeRune(test.data) 88 | if g == test.expected { 89 | t.Logf("%03d: PASSED go=(%q) bash=(%s)", i, test.data, test.expected) 90 | } else { 91 | t.Errorf("%03d: FAILED data=%q got=(%s) wanted=(%s)", i, test.data, g, test.expected) 92 | } 93 | } 94 | } 95 | 96 | func TestShellMany(t *testing.T) { 97 | data := []string{ 98 | "", 99 | "one", 100 | "has space.txt", 101 | "¡que!", 102 | } 103 | g := ShellMany(data) 104 | if len(g) != 4 || g[0] != `""` || g[1] != "one" || g[2] != `"has space.txt"` || g[3] != `$(printf '%q' '\302\241que!')` { 105 | t.Logf("PASSED") 106 | } else { 107 | t.Errorf("FAILED got=(%q)", g) 108 | } 109 | } 110 | 111 | func TestFirstFewFlag(t *testing.T) { 112 | for i, test := range []struct { 113 | data []string 114 | expectedFlag bool 115 | expectedString string 116 | }{ 117 | {[]string{"", "one"}, false, ` one`}, 118 | {[]string{"one"}, false, `one`}, 119 | {[]string{"one", "two", "three", "longlonglong", "longlonglonglong", "manylonglonglog", "morelongonglonglong"}, true, ``}, 120 | } { 121 | gs, gf := FirstFewFlag(test.data) 122 | if test.expectedFlag { 123 | if gf == test.expectedFlag { 124 | t.Logf("%03d: PASSED", i) 125 | } else { 126 | t.Errorf("%03d: FAILED data=%q got=(%q) wanted=(%q)", i, test.data, gs, test.expectedString) 127 | } 128 | } else { 129 | if gf == test.expectedFlag && gs == test.expectedString { 130 | t.Logf("%03d: PASSED", i) 131 | } else { 132 | t.Errorf("%03d: FAILED data=%q got=(%q) wanted=(%q)", i, test.data, gs, test.expectedString) 133 | } 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /pkg/vcs/_all/all.go: -------------------------------------------------------------------------------- 1 | package all 2 | 3 | import ( 4 | _ "github.com/StackExchange/blackbox/v2/pkg/vcs/git" 5 | _ "github.com/StackExchange/blackbox/v2/pkg/vcs/none" 6 | ) 7 | -------------------------------------------------------------------------------- /pkg/vcs/git/git.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/StackExchange/blackbox/v2/pkg/bbutil" 9 | "github.com/StackExchange/blackbox/v2/pkg/commitlater" 10 | "github.com/StackExchange/blackbox/v2/pkg/makesafe" 11 | "github.com/StackExchange/blackbox/v2/pkg/vcs" 12 | ) 13 | 14 | var pluginName = "GIT" 15 | 16 | func init() { 17 | vcs.Register(pluginName, 100, newGit) 18 | } 19 | 20 | // VcsHandle is the handle 21 | type VcsHandle struct { 22 | commitTitle string 23 | commitHeaderPrinted bool // Has the "NEXT STEPS" header been printed? 24 | toCommit *commitlater.List // List of future commits 25 | } 26 | 27 | func newGit() (vcs.Vcs, error) { 28 | l := &commitlater.List{} 29 | return &VcsHandle{toCommit: l}, nil 30 | } 31 | 32 | // Name returns my name. 33 | func (v VcsHandle) Name() string { 34 | return pluginName 35 | } 36 | 37 | func ultimate(s string) int { return len(s) - 1 } 38 | 39 | // Discover returns true if we are a repo of this type; along with the Abs path to the repo root (or "" if we don't know). 40 | func (v VcsHandle) Discover() (bool, string) { 41 | out, err := bbutil.RunBashOutputSilent("git", "rev-parse", "--show-toplevel") 42 | if err != nil { 43 | return false, "" 44 | } 45 | if out == "" { 46 | fmt.Printf("WARNING: git rev-parse --show-toplevel has NO output??. Seems broken.") 47 | return false, "" 48 | } 49 | if out[ultimate(out)] == '\n' { 50 | out = out[0:ultimate(out)] 51 | } 52 | return err == nil, out 53 | } 54 | 55 | // SetFileTypeUnix informs the VCS that files should maintain unix-style line endings. 56 | func (v VcsHandle) SetFileTypeUnix(repobasedir string, files ...string) error { 57 | seen := make(map[string]bool) 58 | 59 | // Add to the .gitattributes in the same directory as the file. 60 | for _, file := range files { 61 | d, n := filepath.Split(file) 62 | af := filepath.Join(repobasedir, d, ".gitattributes") 63 | err := bbutil.Touch(af) 64 | if err != nil { 65 | return err 66 | } 67 | err = bbutil.AddLinesToFile(af, fmt.Sprintf("%q text eol=lf", n)) 68 | if err != nil { 69 | return err 70 | } 71 | seen[af] = true 72 | } 73 | 74 | var changedfiles []string 75 | for k := range seen { 76 | changedfiles = append(changedfiles, k) 77 | } 78 | 79 | v.NeedsCommit( 80 | "set gitattr=UNIX "+strings.Join(makesafe.RedactMany(files), " "), 81 | repobasedir, 82 | changedfiles, 83 | ) 84 | 85 | return nil 86 | } 87 | 88 | // IgnoreAnywhere tells the VCS to ignore these files anywhere rin the repo. 89 | func (v VcsHandle) IgnoreAnywhere(repobasedir string, files []string) error { 90 | // Add to the .gitignore file in the repobasedir. 91 | ignore := filepath.Join(repobasedir, ".gitignore") 92 | err := bbutil.Touch(ignore) 93 | if err != nil { 94 | return err 95 | } 96 | 97 | err = bbutil.AddLinesToFile(ignore, files...) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | v.NeedsCommit( 103 | "gitignore "+strings.Join(makesafe.RedactMany(files), " "), 104 | repobasedir, 105 | []string{".gitignore"}, 106 | ) 107 | return nil 108 | } 109 | 110 | func gitSafeFilename(name string) string { 111 | // TODO(tlim): Add unit tests. 112 | // TODO(tlim): Confirm that *?[] escaping works. 113 | if name == "" { 114 | return "ERROR" 115 | } 116 | var b strings.Builder 117 | b.Grow(len(name) + 2) 118 | for _, r := range name { 119 | if r == ' ' || r == '*' || r == '?' || r == '[' || r == ']' { 120 | b.WriteRune('\\') 121 | b.WriteRune(r) 122 | } else { 123 | b.WriteRune(r) 124 | } 125 | } 126 | if name[0] == '!' || name[0] == '#' { 127 | return `\` + b.String() 128 | } 129 | return b.String() 130 | } 131 | 132 | // IgnoreFiles tells the VCS to ignore these files, specified relative to RepoBaseDir. 133 | func (v VcsHandle) IgnoreFiles(repobasedir string, files []string) error { 134 | 135 | var lines []string 136 | for _, f := range files { 137 | lines = append(lines, "/"+gitSafeFilename(f)) 138 | } 139 | 140 | // Add to the .gitignore file in the repobasedir. 141 | ignore := filepath.Join(repobasedir, ".gitignore") 142 | err := bbutil.Touch(ignore) 143 | if err != nil { 144 | return err 145 | } 146 | err = bbutil.AddLinesToFile(ignore, lines...) 147 | if err != nil { 148 | return err 149 | } 150 | 151 | v.NeedsCommit( 152 | "gitignore "+strings.Join(makesafe.RedactMany(files), " "), 153 | repobasedir, 154 | []string{".gitignore"}, 155 | ) 156 | return nil 157 | } 158 | 159 | // Add makes a file visible to the VCS (like "git add"). 160 | func (v VcsHandle) Add(repobasedir string, files []string) error { 161 | 162 | if len(files) == 0 { 163 | return nil 164 | } 165 | 166 | // TODO(tlim): Make sure that files are within repobasedir. 167 | 168 | var gpgnames []string 169 | for _, n := range files { 170 | gpgnames = append(gpgnames, n+".gpg") 171 | } 172 | return bbutil.RunBash("git", append([]string{"add"}, gpgnames...)...) 173 | } 174 | 175 | // CommitTitle indicates what the next commit title will be. 176 | // This is used if a group of commits are merged into one. 177 | func (v *VcsHandle) CommitTitle(title string) { 178 | v.commitTitle = title 179 | } 180 | 181 | // NeedsCommit queues up commits for later execution. 182 | func (v *VcsHandle) NeedsCommit(message string, repobasedir string, names []string) { 183 | v.toCommit.Add(message, repobasedir, names) 184 | } 185 | 186 | // DebugCommits dumps the list of future commits. 187 | func (v VcsHandle) DebugCommits() commitlater.List { 188 | return *v.toCommit 189 | } 190 | 191 | // FlushCommits informs the VCS to do queued up commits. 192 | func (v VcsHandle) FlushCommits() error { 193 | return v.toCommit.Flush( 194 | v.commitTitle, 195 | func(files []string) error { 196 | return bbutil.RunBash("git", append([]string{"add"}, files...)...) 197 | }, 198 | v.suggestCommit, 199 | ) 200 | // TODO(tlim): Some day we can add a command line flag that indicates that commits are 201 | // to be done for real, not just suggested to the user. At that point, this function 202 | // can call v.toCommit.Flush() with a function that actually does the commits instead 203 | // of suggesting them. Flag could be called --commit=auto vs --commit=suggest. 204 | } 205 | 206 | // suggestCommit tells the user what commits are needed. 207 | func (v *VcsHandle) suggestCommit(messages []string, repobasedir string, files []string) error { 208 | if !v.commitHeaderPrinted { 209 | fmt.Printf("NEXT STEP: You need to manually check these in:\n") 210 | } 211 | v.commitHeaderPrinted = true 212 | 213 | fmt.Print(` git commit -m'`, strings.Join(messages, `' -m'`)+`'`) 214 | fmt.Print(" ") 215 | fmt.Print(strings.Join(makesafe.ShellMany(files), " ")) 216 | fmt.Println() 217 | return nil 218 | } 219 | 220 | // The following are "secret" functions only used by the integration testing system. 221 | 222 | // TestingInitRepo initializes a repo. 223 | func (v VcsHandle) TestingInitRepo() error { 224 | return bbutil.RunBash("git", "init") 225 | 226 | } 227 | -------------------------------------------------------------------------------- /pkg/vcs/none/none.go: -------------------------------------------------------------------------------- 1 | package none 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/StackExchange/blackbox/v2/pkg/commitlater" 7 | "github.com/StackExchange/blackbox/v2/pkg/vcs" 8 | ) 9 | 10 | var pluginName = "NONE" 11 | 12 | func init() { 13 | vcs.Register(pluginName, 0, newNone) 14 | } 15 | 16 | // VcsHandle is 17 | type VcsHandle struct { 18 | repoRoot string 19 | } 20 | 21 | func newNone() (vcs.Vcs, error) { 22 | return &VcsHandle{}, nil 23 | } 24 | 25 | // Name returns my name. 26 | func (v VcsHandle) Name() string { 27 | return pluginName 28 | } 29 | 30 | // Discover returns true if we are a repo of this type; along with the Abs path to the repo root (or "" if we don't know). 31 | func (v VcsHandle) Discover() (bool, string) { 32 | return true, "" // We don't know the root. 33 | } 34 | 35 | //// SetRepoRoot informs the Vcs of the VCS root. 36 | //func (v *VcsHandle) SetRepoRoot(dir string) { 37 | // v.repoRoot = dir 38 | //} 39 | 40 | // SetFileTypeUnix informs the VCS that files should maintain unix-style line endings. 41 | func (v VcsHandle) SetFileTypeUnix(repobasedir string, files ...string) error { 42 | return nil 43 | } 44 | 45 | // IgnoreAnywhere tells the VCS to ignore these files anywhere in the repo. 46 | func (v VcsHandle) IgnoreAnywhere(repobasedir string, files []string) error { 47 | return nil 48 | } 49 | 50 | // IgnoreFiles tells the VCS to ignore these files anywhere in the repo. 51 | func (v VcsHandle) IgnoreFiles(repobasedir string, files []string) error { 52 | return nil 53 | } 54 | 55 | // CommitTitle sets the title of the next commit. 56 | func (v VcsHandle) CommitTitle(title string) {} 57 | 58 | // NeedsCommit queues up commits for later execution. 59 | func (v VcsHandle) NeedsCommit(message string, repobasedir string, names []string) { 60 | return 61 | } 62 | 63 | // DebugCommits dumps a list of future commits. 64 | func (v VcsHandle) DebugCommits() commitlater.List { 65 | return commitlater.List{} 66 | } 67 | 68 | // FlushCommits informs the VCS to do queued up commits. 69 | func (v VcsHandle) FlushCommits() error { 70 | return nil 71 | } 72 | 73 | // The following are "secret" functions only used by the integration testing system. 74 | 75 | // TestingInitRepo initializes a repo. 76 | func (v VcsHandle) TestingInitRepo() error { 77 | fmt.Println("VCS=none, TestingInitRepo") 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /pkg/vcs/vcs.go: -------------------------------------------------------------------------------- 1 | package vcs 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "sort" 8 | "strings" 9 | 10 | "github.com/StackExchange/blackbox/v2/models" 11 | ) 12 | 13 | // Vcs is the handle 14 | type Vcs interface { 15 | models.Vcs 16 | } 17 | 18 | // NewFnSig function signature needed by reg. 19 | type NewFnSig func() (Vcs, error) 20 | 21 | // Item stores one item 22 | type Item struct { 23 | Name string 24 | New NewFnSig 25 | Priority int 26 | } 27 | 28 | // Catalog is the list of registered vcs's. 29 | var Catalog []*Item 30 | 31 | // Discover polls the VCS plug-ins to determine the VCS of directory. 32 | // The first to succeed is returned. 33 | // It never returns nil, since "NONE" is always valid. 34 | func Discover() (Vcs, string) { 35 | for _, v := range Catalog { 36 | h, err := v.New() 37 | if err != nil { 38 | return nil, "" // No idea how that would happen. 39 | } 40 | if b, repodir := h.Discover(); b { 41 | 42 | // Try to find the rel path from CWD to RepoBase 43 | wd, err := os.Getwd() 44 | if err != nil { 45 | fmt.Printf("ERROR: Can not determine cwd! Failing!\n") 46 | os.Exit(1) 47 | } 48 | //fmt.Printf("DISCCOVER: WD=%q REPO=%q\n", wd, repodir) 49 | if repodir != wd && strings.HasSuffix(repodir, wd) { 50 | // This is a terrible hack. We're basically guessing 51 | // at the filesystem layout. That said, it works on macOS. 52 | // TODO(tlim): Abstract this out into a separate function 53 | // so we can do integration tests on it (to know if it fails on 54 | // a particular operating system.) 55 | repodir = wd 56 | } 57 | r, err := filepath.Rel(wd, repodir) 58 | if err != nil { 59 | // Wait, we're not relative to each other? Give up and 60 | // just return the abs repodir. 61 | return h, repodir 62 | } 63 | return h, r 64 | } 65 | } 66 | // This can't happen. If it does, we'll panic and that's ok. 67 | return nil, "" 68 | } 69 | 70 | // Register a new VCS. 71 | func Register(name string, priority int, newfn NewFnSig) { 72 | //fmt.Printf("VCS registered: %v\n", name) 73 | item := &Item{ 74 | Name: name, 75 | New: newfn, 76 | Priority: priority, 77 | } 78 | Catalog = append(Catalog, item) 79 | 80 | // Keep the list sorted. 81 | sort.Slice(Catalog, func(i, j int) bool { return Catalog[j].Priority < Catalog[i].Priority }) 82 | } 83 | -------------------------------------------------------------------------------- /tools/Portfile.template: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4 2 | # $Id: Portfile 132962 2015-02-16 10:33:02Z ryandesign@macports.org $ 3 | 4 | PortSystem 1.0 5 | PortGroup github 1.0 6 | 7 | github.setup StackExchange blackbox @@VERSION@@ v 8 | name vcs_blackbox 9 | categories security 10 | platforms darwin 11 | maintainers whatexit.org:tal openmaintainer 12 | license BSD 13 | supported_archs noarch 14 | 15 | description Safely store secrets in git/hg/svn repos using GPG encryption 16 | 17 | long_description Storing secrets such as passwords, certificates and private keys \ 18 | in Git/Mercurial/SubVersion is dangerous. Blackbox makes it easy \ 19 | to store secrets safely using GPG encryption. They can be easily \ 20 | decrypted for editing or use in production. 21 | 22 | checksums rmd160 @@RMD160@@ \ 23 | sha256 @@SHA256@@ 24 | 25 | use_configure no 26 | 27 | build {} 28 | 29 | # This project's Makefile uses DESTDIR incorrectly. 30 | destroot.destdir DESTDIR=${destroot}${prefix} 31 | destroot.target packages-macports 32 | -------------------------------------------------------------------------------- /tools/auto_system_test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env expect 2 | 3 | # Run the confidence test non-interactively. Since the script 4 | # asks for passphrases, we use "expect" to simulate keyboard data entry. 5 | 6 | # Run the test: 7 | spawn tools/confidence_test.sh 8 | 9 | # As we run the confidence test, respond with the right password. 10 | # We do this for up to 300 times to prevent an infinite loop. 11 | 12 | set times 0; 13 | while { $times < 300 } { 14 | expect { 15 | # The script outputs what the password will be, and we save 16 | # that info in $pw any time we see the text. 17 | "my password is the lowercase letter a" { set pw "a\n" ; exp_continue } 18 | "my password is the lowercase letter b" { set pw "b\n" ; exp_continue } 19 | # If the passphrase is requested, send it. 20 | "Passphrase:" { send $pw ; exp_continue } 21 | # If we reach EOF, exit this loop. 22 | eof { break } 23 | } 24 | set times [ expr $times+1]; 25 | } 26 | -------------------------------------------------------------------------------- /tools/macports_report_upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Turn the Portfile.template into a Portfile. 4 | # Usage: 5 | # mk_portfile.sh TEMPLATE OUTPUTFILE VERSION 6 | 7 | set -e 8 | blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source ${blackbox_home}/../bin/_stack_lib.sh 10 | 11 | TEMPLATEFILE=tools/Portfile.template 12 | OUTPUTFILE=Portfile 13 | PORTVERSION=${1?"Arg 1 must be a version number like 1.20150222 (with no v)"} ; shift 14 | 15 | # Add the version number to the template. 16 | sed <"$TEMPLATEFILE" >"$OUTPUTFILE" -e 's/@@VERSION@@/'"$PORTVERSION"'/g' 17 | 18 | # Test it. Record the failure in $checksumout 19 | fgrep >/dev/null -x 'file:///var/tmp/ports' /opt/local/etc/macports/sources.conf || sudo sed -i -e '1s@^@file:///var/tmp/ports\'$'\n@' /opt/local/etc/macports/sources.conf 20 | rm -rf /var/tmp/ports 21 | mkdir -p /var/tmp/ports/security/vcs_blackbox 22 | cp Portfile /var/tmp/ports/security/vcs_blackbox 23 | ( cd /var/tmp/ports && sudo portindex ) 24 | make_self_deleting_tempfile checksumout 25 | set +e 26 | sudo port -v checksum vcs_blackbox > "$checksumout" 2>/dev/null 27 | ret=$? 28 | 29 | # If it failed, grab the checksums. Then re-process the template with them. 30 | if [[ $ret != 0 ]]; then 31 | RMD160=$(awk <"$checksumout" '/^Distfile checksum: .*rmd160/ { print $NF }') 32 | SHA256=$(awk <"$checksumout" '/^Distfile checksum: .*sha256/ { print $NF }') 33 | echo RMD160=$RMD160 34 | echo SHA256=$SHA256 35 | echo 36 | if [[ $RMD160 != '' && $SHA256 != '' ]]; then 37 | sed <"$TEMPLATEFILE" >"$OUTPUTFILE" -e 's/@@VERSION@@/'"$PORTVERSION"'/g' -e 's/@@RMD160@@/'"$RMD160"'/g' -e 's/@@SHA256@@/'"$SHA256"'/g' 38 | cp Portfile /var/tmp/ports/security/vcs_blackbox 39 | ( cd /var/tmp/ports && sudo portindex ) 40 | sudo port -v checksum vcs_blackbox 41 | fi 42 | fi 43 | 44 | # Generate the diff 45 | cp /opt/local/var/macports/sources/rsync.macports.org/release/tarballs/ports/security/vcs_blackbox/Portfile /var/tmp/ports/security/vcs_blackbox/Portfile.orig 46 | ( cd /var/tmp/ports/security/vcs_blackbox && diff --ignore-matching-lines='Id:' -u Portfile.orig Portfile ) > Portfile-vcs_blackbox.diff 47 | open -R Portfile-vcs_blackbox.diff 48 | 49 | echo 50 | echo 'portfile is in:' 51 | echo ' /var/tmp/ports/security/vcs_blackbox/Portfile' 52 | echo 'cleanup:' 53 | echo ' sudo vi /opt/local/etc/macports/sources.conf' 54 | 55 | echo " 56 | PLEASE OPEN A TICKET WITH THIS INFORMATION: 57 | https://trac.macports.org/newticket 58 | Summary: vcs_blackbox @$PORTVERSION Update to latest upstream 59 | Description: 60 | New upstream of vcs_blackbox. 61 | github.setup and checksums updated. 62 | Type: update 63 | Component: ports 64 | Port: vcs_blackbox 65 | Keywords: maintainer haspatch 66 | " 67 | echo 'Attach: Portfile-vcs_blackbox.diff' 68 | -------------------------------------------------------------------------------- /tools/mk_deb_fpmdir: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # Use fpm to package up files into a DEB . 4 | 5 | # Usage: 6 | # mk_deb_fpmdir PACKAGENAME MANIFEST1 MANIFEST2 ... 7 | 8 | # Example: 9 | # Make a package foopkg manifest.txt 10 | # Where "manifest.txt" contains: 11 | # exec /usr/bin/stack_makefqdn misc/stack_makefqdn.py 12 | # exec /usr/bin/bar bar/bar.sh 13 | # read /usr/man/man1/bar.1 bar/bar.1.man 14 | # 0444 /etc/foo.conf bar/foo.conf 15 | 16 | set -e 17 | 18 | # Parameters for this DEB: 19 | PACKAGENAME=${1?"First arg must be the package name."} 20 | shift 21 | 22 | # Defaults that can be overridden: 23 | # All packages are 1.0 unless otherwise specifed: 24 | : ${PKGVERSION:=1.0} ; 25 | # If there is no iteration set, default to use the number of commits in the repository: 26 | if [[ -z "${PKGRELEASE}" ]]; then 27 | PKGRELEASE=$(git rev-list HEAD --count) 28 | if [[ $? != 0 ]]; then 29 | # We're not in a git repo, fall back to 1 so we cope with being built from 30 | # a tarball 31 | PKGRELEASE=1 32 | fi 33 | fi 34 | 35 | # If there is no epoch, assume 0 36 | : ${PKGEPOCH:=0} 37 | 38 | # Allow us to set a different OUTPUTDIR if we're building in CI/CD 39 | if [[ -z "${OUTPUTDIR}" ]]; then 40 | # The DEB is output here: (should be a place that can be wiped) 41 | OUTPUTDIR="${HOME}/debbuild-${PACKAGENAME}" 42 | else 43 | echo "Using $OUTPUTDIR for OUTPUTDIR instead of ${HOME}/debbuild-${PACKAGENAME}" 44 | fi 45 | 46 | # The TeamCity templates expect to find the list of artifacts here: 47 | DEB_BIN_LIST="${OUTPUTDIR}/bin-packages.txt" 48 | 49 | # -- Now the real work can be done. 50 | 51 | # Clean the output dir. 52 | rm -rf "$OUTPUTDIR" 53 | 54 | mkdir -p "$OUTPUTDIR/installroot" 55 | 56 | # Copy the files into place: 57 | set -o pipefail # Error out if any manifest is not found. 58 | cat """$@""" | while read -a arr ; do 59 | PERM="${arr[0]}" 60 | case $PERM in 61 | \#*) continue ;; # Skip comments. 62 | exec) PERM=0755 ;; 63 | read) PERM=0744 ;; 64 | *) ;; 65 | esac 66 | DST="$OUTPUTDIR/installroot/${arr[1]}" 67 | SRC="${arr[2]}" 68 | if [[ $SRC == "cmd/"* || $SRC == *"/cmd/"* ]]; then 69 | ( cd $(dirname "$SRC" ) && go build -a -v ) 70 | fi 71 | install -D -T -b -m "$PERM" -T "$SRC" "$DST" 72 | done 73 | 74 | # Build the DEB: 75 | cd "$OUTPUTDIR" && fpm -s dir -t deb \ 76 | -a all \ 77 | -n "${PACKAGENAME}" \ 78 | --epoch "${PKGEPOCH}" \ 79 | --version "${PKGVERSION}" \ 80 | --iteration "${PKGRELEASE}" \ 81 | ${PKGDESCRIPTION:+ --description="${PKGDESCRIPTION}"} \ 82 | ${PKGVENDOR:+ --vendor="${PKGVENDOR}"} \ 83 | -C "$OUTPUTDIR/installroot" \ 84 | . 85 | 86 | # TeamCity templates for DEBS expect to find 87 | # the list of all packages created in bin-packages.txt. 88 | # Generate that list: 89 | find "$OUTPUTDIR" -maxdepth 1 -name '*.deb' >"$DEB_BIN_LIST" 90 | # Output it for debugging purposes: 91 | cat "$DEB_BIN_LIST" 92 | -------------------------------------------------------------------------------- /tools/mk_deb_fpmdir.stack_blackbox.txt: -------------------------------------------------------------------------------- 1 | exec /usr/bin/_blackbox_common.sh ../bin/_blackbox_common.sh 2 | exec /usr/bin/_stack_lib.sh ../bin/_stack_lib.sh 3 | exec /usr/bin/blackbox_addadmin ../bin/blackbox_addadmin 4 | exec /usr/bin/blackbox_cat ../bin/blackbox_cat 5 | exec /usr/bin/blackbox_decrypt_all_files ../bin/blackbox_decrypt_all_files 6 | exec /usr/bin/blackbox_decrypt_file ../bin/blackbox_decrypt_file 7 | exec /usr/bin/blackbox_deregister_file ../bin/blackbox_deregister_file 8 | exec /usr/bin/blackbox_diff ../bin/blackbox_diff 9 | exec /usr/bin/blackbox_edit ../bin/blackbox_edit 10 | exec /usr/bin/blackbox_edit_end ../bin/blackbox_edit_end 11 | exec /usr/bin/blackbox_edit_start ../bin/blackbox_edit_start 12 | exec /usr/bin/blackbox_initialize ../bin/blackbox_initialize 13 | exec /usr/bin/blackbox_listadmins ../bin/blackbox_listadmins 14 | exec /usr/bin/blackbox_list_files ../bin/blackbox_list_files 15 | exec /usr/bin/blackbox_list_admins ../bin/blackbox_list_admins 16 | exec /usr/bin/blackbox_postdeploy ../bin/blackbox_postdeploy 17 | exec /usr/bin/blackbox_recurse ../bin/blackbox_recurse 18 | exec /usr/bin/blackbox_register_new_file ../bin/blackbox_register_new_file 19 | exec /usr/bin/blackbox_removeadmin ../bin/blackbox_removeadmin 20 | exec /usr/bin/blackbox_shred_all_files ../bin/blackbox_shred_all_files 21 | exec /usr/bin/blackbox_update_all_files ../bin/blackbox_update_all_files 22 | exec /usr/bin/blackbox_view ../bin/blackbox_view 23 | exec /usr/bin/blackbox_whatsnew ../bin/blackbox_whatsnew 24 | -------------------------------------------------------------------------------- /tools/mk_macports: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Install files into MacPorts DESTDIR 4 | 5 | # Usage: 6 | # mk_macports MANIFEST MANIFEST1 ... 7 | 8 | # Where "manifest.txt" contains: 9 | # exec /usr/bin/stack_makefqdn misc/stack_makefqdn.py 10 | # exec /usr/bin/bar bar/bar.sh 11 | # read /usr/man/man1/bar.1 bar/bar.1.man 12 | # 0444 /etc/foo.conf bar/foo.conf 13 | # (NOTE: "exec" means 0755; "read" means 0744) 14 | 15 | set -e 16 | 17 | # Fail if DESTDIR is not set. 18 | DESTDIR="${DESTDIR?"Envvar DESTDIR must be set to destination dir."}" 19 | 20 | # Copy the files into place: 21 | cat """$@""" | while read -a arr ; do 22 | PERM="${arr[0]}" 23 | case $PERM in 24 | \#*) continue ;; # Skip comments. 25 | exec) PERM=0755 ;; 26 | read) PERM=0744 ;; 27 | *) ;; 28 | esac 29 | DST="$DESTDIR/${arr[1]}" 30 | SRC="${arr[2]}" 31 | install -m "$PERM" "$SRC" "$DST" 32 | done 33 | -------------------------------------------------------------------------------- /tools/mk_macports.vcs_blackbox.txt: -------------------------------------------------------------------------------- 1 | exec bin/_blackbox_common.sh ../bin/_blackbox_common.sh 2 | exec bin/_stack_lib.sh ../bin/_stack_lib.sh 3 | exec bin/blackbox_addadmin ../bin/blackbox_addadmin 4 | exec bin/blackbox_cat ../bin/blackbox_cat 5 | exec bin/blackbox_decrypt_all_files ../bin/blackbox_decrypt_all_files 6 | exec bin/blackbox_decrypt_file ../bin/blackbox_decrypt_file 7 | exec bin/blackbox_deregister_file ../bin/blackbox_deregister_file 8 | exec bin/blackbox_diff ../bin/blackbox_diff 9 | exec bin/blackbox_edit ../bin/blackbox_edit 10 | exec bin/blackbox_edit_end ../bin/blackbox_edit_end 11 | exec bin/blackbox_edit_start ../bin/blackbox_edit_start 12 | exec bin/blackbox_initialize ../bin/blackbox_initialize 13 | exec bin/blackbox_listadmins ../bin/blackbox_listadmins 14 | exec bin/blackbox_list_files ../bin/blackbox_list_files 15 | exec bin/blackbox_list_admins ../bin/blackbox_list_admins 16 | exec bin/blackbox_postdeploy ../bin/blackbox_postdeploy 17 | exec bin/blackbox_recurse ../bin/blackbox_recurse 18 | exec bin/blackbox_register_new_file ../bin/blackbox_register_new_file 19 | exec bin/blackbox_removeadmin ../bin/blackbox_removeadmin 20 | exec bin/blackbox_shred_all_files ../bin/blackbox_shred_all_files 21 | exec bin/blackbox_update_all_files ../bin/blackbox_update_all_files 22 | exec bin/blackbox_view ../bin/blackbox_view 23 | exec bin/blackbox_whatsnew ../bin/blackbox_whatsnew 24 | -------------------------------------------------------------------------------- /tools/mk_rpm_fpmdir: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # Use fpm to package up files into an RPM. 4 | 5 | # Usage: 6 | # mk_rpm_fpmdir PACKAGENAME MANIFEST1 MANIFEST2 ... 7 | 8 | # Example: 9 | # Make a package foopkg manifest.txt 10 | # Where "manifest.txt" contains: 11 | # exec /usr/bin/stack_makefqdn misc/stack_makefqdn.py 12 | # exec /usr/bin/bar bar/bar.sh 13 | # read /usr/man/man1/bar.1 bar/bar.1.man 14 | # 0444 /etc/foo.conf bar/foo.conf 15 | 16 | set -e 17 | 18 | # Parameters for this RPM: 19 | PACKAGENAME=${1?"First arg must be the package name."} 20 | shift 21 | 22 | # What is my name? 23 | CMDNAME=$(basename $0) 24 | 25 | # Defaults that can be overridden: 26 | # Packages are 1.0 unless otherwise specifed: 27 | : ${PKGVERSION:=1.0} ; 28 | # If there is no iteration set, default to use the number of commits in the repository: 29 | if [[ -z "${PKGRELEASE}" ]]; then 30 | PKGRELEASE=$(git rev-list HEAD --count) 31 | if [[ $? != 0 ]]; then 32 | # We're not in a git repo, fall back to 1 so we cope with being built from 33 | # a tarball 34 | PKGRELEASE=1 35 | fi 36 | fi 37 | 38 | # If there is no epoch, assume 0 39 | : ${PKGEPOCH:=0} 40 | 41 | # If no arch defined, assume any. Other good values include "native". 42 | : ${PKGARCH:=all} 43 | # NOTE: If we later compile code, we set this to "native", which 44 | # FPM will translate to the correct value for local conditions. 45 | 46 | # Allow us to set a different OUTPUTDIR if we're building in CI/CD 47 | if [[ -z "${OUTPUTDIR}" ]]; then 48 | # The RPM is output here: (should be a place that can be wiped) 49 | OUTPUTDIR="${HOME}/rpmbuild-${PACKAGENAME}" 50 | else 51 | echo "Using ${OUTPUTDIR} for OUTPUTDIR instead of ${HOME}/rpmbuild-${PACKAGENAME}" 52 | fi 53 | INSTALLROOT="$OUTPUTDIR/installroot" 54 | 55 | # StackOverflow's TeamCity templates expect to find the list of artifacts here: 56 | RPM_BIN_LIST="${OUTPUTDIR}/bin-packages.txt" 57 | 58 | # -- Now the real work can be done. 59 | 60 | # Clean the output dir. 61 | rm -rf "$OUTPUTDIR" 62 | mkdir -p "$INSTALLROOT" 63 | 64 | # If there is a build script, execute it. 65 | BUILDSCRIPTNAME="./build.${PACKAGENAME}.sh" 66 | if [[ -x $BUILDSCRIPTNAME ]]; then 67 | echo "========== $BUILDSCRIPTNAME FOUND. Running." 68 | if [[ $PKGARCH == "all" ]]; then 69 | echo 'WARNING: PKGARCH=all (which may not what you want)' 70 | # If you are using a build.*.sh script, you probably want to 71 | # set PKGARCH to "native" before you run mk_rpm_fpmdir. 72 | fi 73 | $BUILDSCRIPTNAME "$INSTALLROOT" "${PKGVERSION}" 74 | # If we used the build build.*.sh script, it must do all compilation. 75 | # Therefore, we disable the automagic GO build feature. 76 | GO_COMPILE=false 77 | else 78 | GO_COMPILE=true 79 | fi 80 | 81 | # If there are additional args for fpm, read them into a variable. There is 82 | # a chdir later, therefore we can't rely on the file path working at that time. 83 | FPM_OPTIONS_FILE="./fpm_opts.${PACKAGENAME}.sh" 84 | if [[ -f $FPM_OPTIONS_FILE ]]; then 85 | echo "========== $FPM_OPTIONS_FILE FOUND. Loading." 86 | FPM_OPTIONS=$(<$FPM_OPTIONS_FILE) 87 | fi 88 | # Warning: The contents of the file are evaluated therefore 89 | # quotes and special chars must be quoted. 90 | 91 | # Copy any static files into place: 92 | set -o pipefail # Error out if any manifest is not found. 93 | cat "$@" | while read -a arr ; do 94 | PERM="${arr[0]}" 95 | case $PERM in 96 | \#*) continue ;; # Skip comments. 97 | exec) PERM=0755 ;; 98 | read) PERM=0744 ;; 99 | *) ;; 100 | esac 101 | DST="$INSTALLROOT/${arr[1]}" 102 | SRC="${arr[2]}" 103 | if [[ ${#arr[@]} != 3 ]] ; then 104 | echo "ERROR: Line must contain 3 items." 105 | echo "DEBUG NUM=${#arr[@]} PERM=$PERM DST=$DST SRC=$SRC" 106 | exit 1 107 | fi 108 | if $GO_COMPILE && [[ $SRC == "cmd/"* || $SRC == *"/cmd/"* ]]; then 109 | echo "========== BUILD© $SRC" 110 | ( cd $(dirname "$SRC" ) && go get -d && go build ) 111 | PKGARCH=native 112 | else 113 | echo "========== COPY $SRC" 114 | fi 115 | if [[ ! -f "$SRC" ]]; then 116 | echo "${CMDNAME}: ERROR: File not found: $SRC" 117 | exit 1 118 | fi 119 | install -D -T -b -m "$PERM" -T "$SRC" "$DST" 120 | done 121 | 122 | set -x 123 | # Build the RPM out of what is found in $INSTALLROOT: 124 | cd "$OUTPUTDIR" && fpm -s dir -t rpm \ 125 | -a "${PKGARCH}" \ 126 | -n "${PACKAGENAME}" \ 127 | --epoch "${PKGEPOCH}" \ 128 | --version "${PKGVERSION}" \ 129 | --iteration "${PKGRELEASE}" \ 130 | ${PKGDESCRIPTION:+ --description="${PKGDESCRIPTION}"} \ 131 | ${PKGVENDOR:+ --vendor="${PKGVENDOR}"} \ 132 | ${FPM_OPTIONS:+ $FPM_OPTIONS} \ 133 | -C "$INSTALLROOT" \ 134 | . 135 | 136 | # TeamCity templates for RPMS expect to find 137 | # the list of all packages created in bin-packages.txt. 138 | # Generate that list: 139 | find "$OUTPUTDIR" -maxdepth 1 -name '*.rpm' >"$RPM_BIN_LIST" 140 | # Output it for debugging purposes: 141 | cat "$RPM_BIN_LIST" 142 | -------------------------------------------------------------------------------- /tools/mk_rpm_fpmdir.stack_blackbox.txt: -------------------------------------------------------------------------------- 1 | # Update tools/mk_rpm_fpmdir.stack_blackbox.txt. Other files generate from it. 2 | exec /etc/profile.d/usrblackbox.sh profile.d-usrblackbox.sh 3 | exec /usr/blackbox/bin/_blackbox_common.sh ../bin/_blackbox_common.sh 4 | exec /usr/blackbox/bin/_stack_lib.sh ../bin/_stack_lib.sh 5 | exec /usr/blackbox/bin/blackbox_addadmin ../bin/blackbox_addadmin 6 | exec /usr/blackbox/bin/blackbox_cat ../bin/blackbox_cat 7 | exec /usr/blackbox/bin/blackbox_decrypt_all_files ../bin/blackbox_decrypt_all_files 8 | exec /usr/blackbox/bin/blackbox_decrypt_file ../bin/blackbox_decrypt_file 9 | exec /usr/blackbox/bin/blackbox_deregister_file ../bin/blackbox_deregister_file 10 | exec /usr/blackbox/bin/blackbox_diff ../bin/blackbox_diff 11 | exec /usr/blackbox/bin/blackbox_edit ../bin/blackbox_edit 12 | exec /usr/blackbox/bin/blackbox_edit_end ../bin/blackbox_edit_end 13 | exec /usr/blackbox/bin/blackbox_edit_start ../bin/blackbox_edit_start 14 | exec /usr/blackbox/bin/blackbox_initialize ../bin/blackbox_initialize 15 | exec /usr/blackbox/bin/blackbox_listadmins ../bin/blackbox_listadmins 16 | exec /usr/blackbox/bin/blackbox_list_files ../bin/blackbox_list_files 17 | exec /usr/blackbox/bin/blackbox_list_admins ../bin/blackbox_list_admins 18 | exec /usr/blackbox/bin/blackbox_postdeploy ../bin/blackbox_postdeploy 19 | exec /usr/blackbox/bin/blackbox_recurse ../bin/blackbox_recurse 20 | exec /usr/blackbox/bin/blackbox_register_new_file ../bin/blackbox_register_new_file 21 | exec /usr/blackbox/bin/blackbox_removeadmin ../bin/blackbox_removeadmin 22 | exec /usr/blackbox/bin/blackbox_shred_all_files ../bin/blackbox_shred_all_files 23 | exec /usr/blackbox/bin/blackbox_update_all_files ../bin/blackbox_update_all_files 24 | exec /usr/blackbox/bin/blackbox_view ../bin/blackbox_view 25 | exec /usr/blackbox/bin/blackbox_whatsnew ../bin/blackbox_whatsnew 26 | -------------------------------------------------------------------------------- /tools/profile.d-usrblackbox-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Test profile.d-usrblackbox.sh 4 | 5 | # Make sure profile.d-usrblackbox.sh works. 6 | 7 | # Test variations including /usr/blackbox/bin is not in the path, is 8 | # already there in the front, middle, or end, and tests if the path has : 9 | # in weird places (front, middle, or both). 10 | 11 | # To run the test: 12 | # bash tools/profile.d-usrblackbox-test.sh | fgrep --color /usr/blackbox/bin 13 | # sh tools/profile.d-usrblackbox-test.sh | fgrep --color /usr/blackbox/bin 14 | 15 | # NOTE: profile.d-usrblackbox.sh is written to be so small that it fits as an "inline" file. 16 | # https://lwn.net/Articles/468678/ 17 | # To remove the last newline in the file: 18 | # perl -i -pe 'chomp if eof' profile.d-usrblackbox.sh 19 | 20 | for p in \ 21 | '/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin' \ 22 | '/usr/blackbox/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin' \ 23 | '/usr/local/bin:/bin:/usr/blackbox/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin' \ 24 | '/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/usr/blackbox/bin' \ 25 | '/Apple spaces/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin' \ 26 | ; do 27 | 28 | export PATH="$p" 29 | . tools/profile.d-usrblackbox.sh 30 | echo NEW: "$PATH" 31 | 32 | export PATH=":$p" 33 | . tools/profile.d-usrblackbox.sh 34 | echo NEW: "$PATH" 35 | 36 | export PATH="$p:" 37 | . tools/profile.d-usrblackbox.sh 38 | echo NEW: "$PATH" 39 | 40 | export PATH=":$p:" 41 | . tools/profile.d-usrblackbox.sh 42 | echo NEW: "$PATH" 43 | 44 | done 45 | -------------------------------------------------------------------------------- /tools/profile.d-usrblackbox.sh: -------------------------------------------------------------------------------- 1 | x=/usr/blackbox/bin 2 | case ":$PATH:" in *:$x:*);;*)PATH="$x:$PATH";;esac -------------------------------------------------------------------------------- /tools/test_functions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # NB: This is copied from _blackbox_common.sh 4 | function get_pubring_path() { 5 | : "${KEYRINGDIR:=keyrings/live}" ; 6 | if [[ -f "${KEYRINGDIR}/pubring.gpg" ]]; then 7 | echo "${KEYRINGDIR}/pubring.gpg" 8 | else 9 | echo "${KEYRINGDIR}/pubring.kbx" 10 | fi 11 | } 12 | 13 | function PHASE() { 14 | echo '********************' 15 | echo '********************' 16 | echo '*********' """$@""" 17 | echo '********************' 18 | echo '********************' 19 | } 20 | 21 | function md5sum_file() { 22 | # Portably generate the MD5 hash of file $1. 23 | case $(uname -s) in 24 | Darwin | FreeBSD ) 25 | md5 -r "$1" | awk '{ print $1 }' 26 | ;; 27 | NetBSD ) 28 | md5 -q "$1" 29 | ;; 30 | SunOS ) 31 | digest -a md5 "$1" 32 | ;; 33 | Linux ) 34 | md5sum "$1" | awk '{ print $1 }' 35 | ;; 36 | CYGWIN* ) 37 | md5sum "$1" | awk '{ print $1 }' 38 | ;; 39 | * ) 40 | echo 'ERROR: Unknown OS. Exiting.' 41 | exit 1 42 | ;; 43 | esac 44 | } 45 | 46 | function assert_file_missing() { 47 | if [[ -e "$1" ]]; then 48 | echo "ASSERT FAILED: ${1} should not exist." 49 | exit 1 50 | fi 51 | } 52 | 53 | function assert_file_exists() { 54 | if [[ ! -e "$1" ]]; then 55 | echo "ASSERT FAILED: ${1} should exist." 56 | echo "PWD=$(/usr/bin/env pwd -P)" 57 | #echo "LS START" 58 | #ls -la 59 | #echo "LS END" 60 | exit 1 61 | fi 62 | } 63 | function assert_file_md5hash() { 64 | local file="$1" 65 | local wanted="$2" 66 | assert_file_exists "$file" 67 | local found 68 | found=$(md5sum_file "$file") 69 | if [[ "$wanted" != "$found" ]]; then 70 | echo "ASSERT FAILED: $file hash wanted=$wanted found=$found" 71 | exit 1 72 | fi 73 | } 74 | function assert_file_group() { 75 | local file="$1" 76 | local wanted="$2" 77 | local found 78 | assert_file_exists "$file" 79 | 80 | case $(uname -s) in 81 | Darwin | FreeBSD | NetBSD ) 82 | found=$(stat -f '%Dg' "$file") 83 | ;; 84 | Linux | SunOS ) 85 | found=$(stat -c '%g' "$file") 86 | ;; 87 | CYGWIN* ) 88 | echo "ASSERT_FILE_GROUP: Running on Cygwin. Not being tested." 89 | return 0 90 | ;; 91 | * ) 92 | echo 'ERROR: Unknown OS. Exiting.' 93 | exit 1 94 | ;; 95 | esac 96 | 97 | echo "DEBUG: assert_file_group X${wanted}X vs. X${found}X" 98 | echo "DEBUG:" $(which stat) 99 | if [[ "$wanted" != "$found" ]]; then 100 | echo "ASSERT FAILED: $file chgrp group wanted=$wanted found=$found" 101 | exit 1 102 | fi 103 | } 104 | function assert_file_perm() { 105 | local wanted="$1" 106 | local file="$2" 107 | local found 108 | assert_file_exists "$file" 109 | 110 | case $(uname -s) in 111 | Darwin | FreeBSD | NetBSD ) 112 | found=$(stat -f '%Sp' "$file") 113 | ;; 114 | # NB(tlim): CYGWIN hasn't been tested. It might be more like Darwin. 115 | Linux | CYGWIN* | SunOS ) 116 | found=$(stat -c '%A' "$file") 117 | ;; 118 | * ) 119 | echo 'ERROR: Unknown OS. Exiting.' 120 | exit 1 121 | ;; 122 | esac 123 | 124 | echo "DEBUG: assert_file_perm X${wanted}X vs. X${found}X" 125 | echo "DEBUG:" $(which stat) 126 | if [[ "$wanted" != "$found" ]]; then 127 | echo "ASSERT FAILED: $file chgrp perm wanted=$wanted found=$found" 128 | exit 1 129 | fi 130 | } 131 | function assert_line_not_exists() { 132 | local target="$1" 133 | local file="$2" 134 | assert_file_exists "$file" 135 | if grep -F -x -s -q >/dev/null "$target" "$file" ; then 136 | echo "ASSERT FAILED: line '$target' should not exist in file $file" 137 | echo "==== file contents: START $file" 138 | cat "$file" 139 | echo "==== file contents: END $file" 140 | exit 1 141 | fi 142 | } 143 | function assert_line_exists() { 144 | local target="$1" 145 | local file="$2" 146 | assert_file_exists "$file" 147 | if ! grep -F -x -s -q >/dev/null "$target" "$file" ; then 148 | echo "ASSERT FAILED: line '$target' should exist in file $file" 149 | echo "==== file contents: START $file" 150 | cat "$file" 151 | echo "==== file contents: END $file" 152 | exit 1 153 | fi 154 | } 155 | --------------------------------------------------------------------------------