merge

- _blackbox_common.sh sets the default Blackbox directory
  for the new repositories using the first entry of the
  BLACKBOX_CANDIDATES array. This small change sets the
  first entry to the new .blackbox dir (instead of the keyring/live)

.circleci/config.yml

@@ -0,0 +1,56 @@
+version: 2
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - debian
+      - ubuntu
+
+jobs:
+
+  debian:
+    docker:
+      - image: debian:9.1
+    steps:
+      - checkout
+      - run:
+          name: 'Installing'
+          command: |
+            apt-get update -y
+            apt-get install -y build-essential expect git gnupg2 pinentry-tty procps rpm ruby-dev libffi-dev
+            gem install fpm
+      - run:
+          name: 'Cleaning'
+          command: |
+            rm -rf ~/.gpnupg
+            make clean
+      - run:
+          name: 'Testing'
+          command: |
+            GPG=gpg2 make test
+            make packages-deb
+            make packages-rpm
+
+  ubuntu:
+    docker:
+      - image: ubuntu:16.04
+    steps:
+      - checkout
+      - run:
+          name: 'Installing'
+          command: |
+            apt-get update -y
+            apt-get install -y build-essential expect git gnupg2 pinentry-tty procps rpm ruby-dev libffi-dev
+            gem install fpm
+      - run:
+          name: 'Cleaning'
+          command: |
+            rm -rf ~/.gpnupg
+            make clean
+      - run:
+          name: 'Testing'
+          command: |
+            GPG=gpg2 make test
+            make packages-deb
+            make packages-rpm

.gitattributes

@@ -0,0 +1,5 @@
+bin/** text eol=lf
+tools/** text eol=lf
+Makefile text eol=lf
+Portfile text eol=lf
+blackbox.plugin.zsh text eol=lf

CHANGELOG.md

@@ -1,7 +1,40 @@
-Release v1.20170127
+Release v1.20180618
 
-* Starting CHANGELOG.
+* Restore `make manual-install` with warning. (#258)
 
+Release v1.20180615
+
+* Standardize on .blackbox for config. Use keyrings/live for backwards compatibility.
+* Store keys in .blackbox directory (#218)
+* Suggest committing changes to pubring.gpg when running blackbox_removeadmin (#248)
+* Fix typo (#246)
+* Improve installation instructions (#244)
+* Fix replacing-expired-keys link in README (#241)
+* Fix problems when gpg2 is installed next to gpg (#237)
+* Many documentation corrections, updates, etc.
+* Exclude default keyring from import (#223)
+* .gitattributes not always updated (PR#146)
+* Fix bugs related to updating .gitattributes (PR#146)
+* Update readme with CircleCI link (#216)
+* Run the tests on a CI (#215)
+* Fixed Alpine compatibility (chmod) (#212)
+* direct repobase message to stderr (#204)
+* Improve Windows compatibility
+* NEW: .gitattributes Set Unix-only files to eol=lf
+* Silence 'not changed' output during keychain import (#200)
+* Improve FreeBSD compatibility
+* shred_file() outputs warning message to stderr. (#192)
+* Don't complain about GPG_AGENT_INFO if using newer gpg-agent (#189)
+* [FreeBSD] Fix use of chmod (#180)
+* Requiring a file to be entered to finish editing (#175)
+* Remove the key from the keyring when removing an admin (#173)
+* Add FreeBSD support (#172)
+* Add list admins commandline tool. (#170)
+ignore backup files and secring.gpg in $BLACKBOXDATA (#169)
+Allow parallel shredding of files (#167)
+* Add/improve Mingw support
+* Make "make confidence" less fragile
+* And a lot, lot more.
 
 Release v1.20170309
 
@@ -9,3 +42,22 @@ Release v1.20170309
 * macOS: make_tempdir must create shorter paths
 * Fix "make confidence" for newer version of Git
 * README.md: Add info about our new mailing list
+
+Release v1.20170611
+
+* confidence_test.sh verifies external tools exist
+* confidence_test.sh more reliable for non-UTF8 users
+* "make test" no longer prompts for passwords
+* blackbox works better when target directory lives in root (#194)
+* Add confidence_test.sh tests for admin operations
+* blackbox_list_admins fails (#193)
+* confidence_test.sh works better on FreeBSD
+* tools/confidence_test.sh: now works with gnupg-2.0 and gnupg-2.1
+* Blackbox now officially supports both gnupg-2.0 and gnupg-2.1
+* blackbox_shred_all_files: BUGFIX: Does not shred files with spaces
+* blackbox_removeadmin: disable gpg's confirmation
+* Sync mk_rpm_fpmdir from master
+
+Release v1.20170127
+
+* Starting CHANGELOG.

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014-2016 Stack Exchange, Inc.
+Copyright (c) 2014-2018 Stack Exchange, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -6,15 +6,18 @@ OUTPUTDIR?="$(BASEDIR)/debbuild-${PKGNAME}"
 
 all:
 	@echo 'Menu:'
-	@echo '  make update            Update any generated files'
-	@echo '  make packages          Make RPM packages'
-	@echo '  make packages-deb      Make DEB packages'
-	@echo '  make test              Run tests'
-	@echo '  make install           (incomplete)'
+	@echo '  make update             Update any generated files'
+	@echo '  make packages-rpm       Make RPM packages'
+	@echo '  make packages-deb       Make DEB packages'
+	@echo '  make symlinks-install   Make symlinks in /usr/local/bin/'
+	@echo '  make copy-install       Copy "bin" files to /usr/local/bin/'
+	@echo '  make usrlocal-uninstall Remove blackbox files from /usr/local/bin/'
+	@echo '  make test               Run tests'
 
 install:
 	@echo 'To install, copy the files from bin to somewhere in your PATH.'
-	@echo 'Or, if you use RPMs, "make packages" and install the result.'
+	@echo 'The README.md document gives more details.'
+	@echo 'Or run "make" (with no options) for more info.'
 
 # The default package type is RPM.
 packages: packages-rpm
@@ -51,12 +54,24 @@ unlock-rpm:
 #
 # Manual install
 #
-manual-install:
+symlinks-install:
 	@echo 'Symlinking files from ./bin to /usr/local/bin'
 	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do ln -fs `pwd`/$$f /usr/local/bin/$$f; done
 	@echo 'Done.'
 
-manual-uninstall:
+manual-install:
+	@echo '***************************************************************'
+	@echo '*                      DEPRECATED                             *'
+	@echo '* `make manual-install` is now called `make symlinks-install` *'
+	@echo '***************************************************************'
+	$(MAKE) symlinks-install
+
+copy-install:
+	@echo 'Copying files from ./bin to /usr/local/bin'
+	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do cp `pwd`/$$f /usr/local/bin/$$f; done
+	@echo 'Done.'
+
+usrlocal-uninstall:
 	@echo 'Removing blackbox files from /usr/local/bin'
 	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do rm /usr/local/bin/$$f; done
 	@echo 'Done.'
@@ -118,7 +133,7 @@ uninstall-stow:
 update: tools/mk_deb_fpmdir.stack_blackbox.txt tools/mk_macports.vcs_blackbox.txt
 
 clean:
-	rm tools/mk_deb_fpmdir.stack_blackbox.txt tools/mk_macports.vcs_blackbox.txt
+	rm -f tools/mk_deb_fpmdir.stack_blackbox.txt tools/mk_macports.vcs_blackbox.txt
 
 #
 # System Test:
@@ -127,7 +142,6 @@ test: confidence
 confidence:
 	@if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg should not exist. If it does, bugs may polute your .gnupg configuration. If the code has no bugs everything will be fine. Do you feel lucky?'; false ; fi
 	@if which >/dev/null gpg-agent ; then pkill gpg-agent ; rm -rf /tmp/tmp.* ; fi
-	@export PATH="$(PWD)/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:$(PATH)" ; tools/confidence_test.sh
-		tools/confidence_test.sh
+	@export PATH="$(PWD)/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:$(PATH)" ; tools/auto_system_test
 	@if which >/dev/null gpg-agent ; then pkill gpg-agent ; fi
 	@if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg was created which means the scripts might be poluting GnuPG configuration.  Fix this bug.'; false ; fi

README.md

@@ -1,4 +1,4 @@
-BlackBox
+BlackBox [![CircleCI](https://circleci.com/gh/StackExchange/blackbox.svg?style=shield)](https://circleci.com/gh/StackExchange/workflows/blackbox)
 ========
 
 Safely store secrets in a VCS repo (i.e. Git, Mercurial, Subversion or Perforce). These commands make it easy for you to Gnu Privacy Guard (GPG) encrypt specific files in a repo so they are "encrypted at rest" in your repository. However, the scripts make it easy to decrypt them when you need to view or edit them, and decrypt them for use in production. Originally written for Puppet, BlackBox now works with any Git or Mercurial repository.
@@ -14,30 +14,34 @@ Table of Contents
 - [Table of Contents](#table-of-contents)
 - [Overview](#overview)
 - [Why is this important?](#why-is-this-important)
-- [Installation Instructions:](#installation-instructions)
-- [Commands:](#commands)
-- [Compatibility:](#compatibility)
+- [Installation Instructions](#installation-instructions)
+- [Commands](#commands)
+- [Compatibility](#compatibility)
 - [How is the encryption done?](#how-is-the-encryption-done)
 - [What does this look like to the typical user?](#what-does-this-look-like-to-the-typical-user)
 - [How to use the secrets with Puppet?](#how-to-use-the-secrets-with-puppet)
-  - [Entire files:](#entire-files)
-  - [Small strings:](#small-strings)
-- [How to enroll a new file into the system?](#how-to-enroll-a-new-file-into-the-system)
-- [How to remove a file from the system?](#how-to-remove-a-file-from-the-system)
-- [How to indoctrinate a new user into the system?](#how-to-indoctrinate-a-new-user-into-the-system)
-- [How to remove a user from the system?](#how-to-remove-a-user-from-the-system)
-- [Enabling Blackbox For a Repo](#enabling-blackbox-for-a-repo)
+  - [Entire files](#entire-files)
+  - [Small strings](#small-strings)
+- File Management
+  - [How to enroll a new file into the system?](#how-to-enroll-a-new-file-into-the-system)
+  - [How to remove a file from the system?](#how-to-remove-a-file-from-the-system)
+- User Management
+  - [How to indoctrinate a new user into the system?](#how-to-indoctrinate-a-new-user-into-the-system)
+  - [How to remove a user from the system?](#how-to-remove-a-user-from-the-system)
+- Repo Management
+  - [Enabling BlackBox For a Repo](#enabling-blackbox-for-a-repo)
 - [Set up automated users or “role accounts”](#set-up-automated-users-or-role-accounts)
-- [Replace expired keys:](#replace-expired-keys)
-- [Some common errors:](#some-common-errors)
-- [Using Blackbox without a repo](#using-blackbox-without-a-repo)
-- [Some Subversion gotchas:](#some-subversion-gotchas)
+- [Replacing expired keys](#replacing-expired-keys)
+- [Some common errors](#some-common-errors)
+- [Using BlackBox on Windows](#using-blackbox-on-windows)
+- [Using BlackBox without a repo](#using-blackbox-without-a-repo)
+- [Some Subversion gotchas](#some-subversion-gotchas)
+- [Using Blackbox when gpg2 is installed next to gpg](#using-blackbox-when-gpg2-is-installed-next-to-gpg)
 - [How to submit bugs or ask questions?](#how-to-submit-bugs-or-ask-questions)
 - [Developer Info](#developer-info)
 - [Alternatives](#alternatives)
 - [License](#license)
 
-
 Overview
 ========
 
@@ -49,7 +53,22 @@ Rather than one GPG passphrase for all the files, each person with access has th
 
 Automated processes often need access to all the decrypted files. This is easy too. For example, suppose Git is being used for Puppet files. The master needs access to the decrypted version of all the files. Simply set up a GPG key for the Puppet master (or the role account that pushes new files to the Puppet master) and have that user run `blackbox_postdeploy` after any files are updated.
 
-Getting started is easy. Just `cd` into a Git, Mercurial, Subversion or Perforce repository and run `blackbox_initialize`. After that, if a file is to be encrypted, run `blackbox_register_new_file` and you are done. Add and remove keys with `blackbox_addadmin` and `blackbox_removeadmin`. To view and/or edit a file, run `blackbox_edit`; this will decrypt the file and open with whatever is specified by your $EDITOR environment variable. When you close the editor the file will automatically be encrypted again and the temporary plaintext file will be shredded. If you need to leave the file decrypted while you update you can use the`blackbox_edit_start` to decrypt the file and `blackbox_edit_end` when you want to "put it back in the box."
+Getting started is looks like this.
+First, if you don't have a GPG key, set it up using instructions
+such as:
+[Set up GPG key](https://help.github.com/articles/generating-a-new-gpg-key/).
+Now you are ready to go.
+`cd` into a Git, Mercurial, Subversion
+or Perforce repository and run `blackbox_initialize`. After that,
+if a file is to be encrypted, run `blackbox_register_new_file` and
+you are done. Add and remove keys with `blackbox_addadmin` and
+`blackbox_removeadmin`. To view and/or edit a file, run `blackbox_edit`;
+this will decrypt the file and open with whatever is specified by
+your $EDITOR environment variable. When you close the editor the
+file will automatically be encrypted again and the temporary plaintext
+file will be shredded. If you need to leave the file decrypted while
+you update you can use the`blackbox_edit_start` to decrypt the file
+and `blackbox_edit_end` when you want to "put it back in the box."
 
 Why is this important?
 ======================
@@ -58,22 +77,24 @@ OBVIOUSLY we don't want secret things like SSL private keys and passwords to be
 
 NOT SO OBVIOUSLY when we store "secrets" in a VCS repo like Git or Mercurial, suddenly we are less able to share our code with other people. Communication between subteams of an organization is hurt. You can't collaborate as well. Either you find yourself emailing individual files around (yuck!), making a special repo with just the files needed by your collaborators (yuck!!), or just deciding that collaboration isn't worth all that effort (yuck!!!).
 
-The ability to be open and transparent about our code, with the exception of a few specific files, is key to the kind of collaboration that DevOps and modern IT practitioniers need to do.
+The ability to be open and transparent about our code, with the exception of a few specific files, is key to the kind of collaboration that DevOps and modern IT practitioners need to do.
 
-Installation Instructions:
-==========================
+Installation Instructions
+=========================
 
--	*The MacPorts Way*: `sudo port install vcs_blackbox`
--	*The Homebrew Way*: `brew install blackbox`
--	*The RPM way*: Check out the repo and make an RPM via `make packages-rpm`; now you can distribute the RPM via local methods.
--	*The Debian/Ubuntu way*: Check out the repo and install [fpm](https://github.com/jordansissel/fpm). Now you can make a DEB `make packages-deb` that can be distributed via local methods.
--	*The hard way*: Copy all the files in "bin" to your "bin".
--	*The manual way*: `make manual-install` to install. `make manual-uninstall` to uninstall.
--	*The Antigen Way*: Add `antigen bundle StackExchange/blackbox` to your .zshrc
--	*The Zgen Way*: Add `zgen load StackExchange/blackbox` to your .zshrc where you're loading your other plugins.
+- *The hard way (manual*: Copy all the files in "bin" to your "bin".
+- *The hard way (automatic)*: `make copy-install` will copy the bin files into /usr/local/bin (uninstall with `make usrlocal-uninstall`).
+- *The symlinks way*: `make symlinks-install` will make symlinks of the bin files into /usr/local/bin (uninstall with `make usrlocal-uninstall`) (useful when doing development)
+- *The MacPorts Way*: `sudo port install vcs_blackbox`
+- *The Homebrew Way*: `brew install blackbox`
+- *The RPM way*: Check out the repo and make an RPM via `make packages-rpm`; now you can distribute the RPM via local methods. (Requires [fpm](https://github.com/jordansissel/fpm).)
+- *The Debian/Ubuntu way*: Check out the repo and make a DEB via `make packages-deb`; now you can distribute the DEB via local methods. (Requires [fpm](https://github.com/jordansissel/fpm).)
+- *The Antigen Way*: Add `antigen bundle StackExchange/blackbox` to your .zshrc
+- *The Zgen Way*: Add `zgen load StackExchange/blackbox` to your .zshrc where you're loading your other plugins.
+- *The Nix Way*: `nix-env -i blackbox`
 
-Commands:
-=========
+Commands
+========
 
 | Name:                               | Description:                                                            |
 |-------------------------------------|-------------------------------------------------------------------------|
@@ -81,12 +102,14 @@ Commands:
 | `blackbox_edit_start <file>`        | Decrypt a file so it can be updated                                     |
 | `blackbox_edit_end <file>`          | Encrypt a file after blackbox_edit_start was used                       |
 | `blackbox_cat <file>`               | Decrypt and view the contents of a file                                 |
+| `blackbox_view <file>`              | Like blackbox_cat but pipes to `less` or $PAGER                         |
 | `blackbox_diff`                     | Diff decrypted files against their original crypted version             |
 | `blackbox_initialize`               | Enable blackbox for a GIT or HG repo                                    |
 | `blackbox_register_new_file <file>` | Encrypt a file for the first time                                       |
 | `blackbox_deregister_file <file>`   | Remove a file from blackbox                                             |
 | `blackbox_list_files`               | List the files maintained by blackbox                                   |
 | `blackbox_list_admins`              | List admins currently authorized for blackbox                           |
+| `blackbox_decrypt_file <file>`      | Decrypt a file                                                          |
 | `blackbox_decrypt_all_files`        | Decrypt all managed files (INTERACTIVE)                                 |
 | `blackbox_postdeploy`               | Decrypt all managed files (batch)                                       |
 | `blackbox_addadmin <gpg-key>`       | Add someone to the list of people that can encrypt/decrypt secrets      |
@@ -95,52 +118,82 @@ Commands:
 | `blackbox_update_all_files`         | Decrypt then re-encrypt all files. Useful after keys are changed        |
 | `blackbox_whatsnew <file>`          | show what has changed in the last commit for a given file               |
 
-Compatibility:
-==============
+Compatibility
+=============
 
-Blackbox automatically determines which VCS you are using and does the right thing. It has a plug-in architecture to make it easy to extend to work with other systems. It has been tested to work with many operating systems.
+BlackBox automatically determines which VCS you are using and does the right thing. It has a plug-in architecture to make it easy to extend to work with other systems. It has been tested to work with many operating systems.
 
--	Version Control systems
-	-	`git` -- The Git
-	-	`hg` -- Mercurial
-	-	`svn` -- SubVersion (Thanks, Ben Drasin!)
-	-	`p4` -- Perforce
-	-	none -- The files can be decrypted outside of a repo if the keyrings directory is intact
--	Operating system
-	-	CentOS / RedHat
-	-	MacOS X
-	-	Cygwin (Thanks, Ben Drasin!)
-	-	MinGW (git bash on windows) **See Note Below**
+- Version Control systems
+  - `git` -- The Git
+  - `hg` -- Mercurial
+  - `svn` -- SubVersion (Thanks, Ben Drasin!)
+  - `p4` -- Perforce
+  - none -- The files can be decrypted outside of a repo if the `.blackbox` directory is intact
+- Operating system
+  - CentOS / RedHat
+  - MacOS X
+  - Cygwin (Thanks, Ben Drasin!) **See Note Below**
+  - MinGW (git bash on windows) **See Note Below**
 
 To add or fix support for a VCS system, look for code at the end of `bin/_blackbox_common.sh`
 
 To add or fix support for a new operating system, look for the case statements in `bin/_blackbox_common.sh` and `bin/_stack_lib.sh` and maybe `tools/confidence_test.sh`
 
-Note: Cywin support requires the following packages:
+Using BlackBox on Windows
+=========================
 
--	Normal operation:
-	-	gnupg
-	-	git or mercurial or subversion or perforce (as appropriate)
--	Development (if you will be adding code and want to run the confidence test)
-	-	procps
-	-	make
-	-	git (the confidence test currently only tests git)
+BlackBox can be used with Cygwin or MinGW.
 
-Note: MinGW (comes with Git for Windows) support requires the following additional installations
--	Normal operation:
-	-	[Git for Windows](https://git-scm.com/) (not tested with Mercurial)
-		-	Git Bash MINTTY returns a MinGW console.  So when you install make sure you pick `MINTTY` instead of windows console.  You'll be executing blackbox from the Git Bash prompt.
-		-	You need at least version 2.8.1 of Git for Windows.
-	-	[GnuWin32](https://sourceforge.net/projects/getgnuwin32/files/) - needed for various tools not least of which is mktemp which is used by blackbox
-		-	after downloading the install just provides you with some batch files.  Because of prior issues at sourceforge and to make sure you get the latest version of each package the batch files handle the brunt of the work of getting the correct packages and installing them for you.
-		-	from a **windows command prompt** run `download.bat`  once it has completed run `install.bat` then add the path for those tools to your PATH (ex: `PATH=%PATH%;c:\GnuWin32\bin`)
--	Development: 
-	-	unknown
+### Protect the line endings
+
+BlackBox assumes that `blackbox-admins.txt` and `blackbox-files.txt` will have
+LF line endings. Windows users should be careful to configure Git or other systems
+to not convert or "fix" those files.
+
+If you use Git, add the following lines to your `.gitattributes` file:
+
+    **/blackbox-admins.txt text eol=lf
+    **/blackbox-files.txt text eol=lf
+
+The latest version of `blackbox_initialize` will create a `.gitattributes` file in the `$BLACKBOXDATA`
+directory (usually `.blackbox`) for you.
+
+### Cygwin
+
+Cygwin support requires the following packages:
+
+Normal operation:
+
+- gnupg
+- git or mercurial or subversion or perforce (as appropriate)
+
+Development (if you will be adding code and want to run the confidence test)
+
+- procps
+- make
+- git (the confidence test currently only tests git)
+
+### MinGW
+
+MinGW (comes with Git for Windows) support requires the following:
+
+Normal operation:
+
+- [Git for Windows](https://git-scm.com/) (not tested with Mercurial)
+  - Git Bash MINTTY returns a MinGW console.  So when you install make sure you pick `MINTTY` instead of windows console.  You'll be executing blackbox from the Git Bash prompt.
+  - You need at least version 2.8.1 of Git for Windows.
+- [GnuWin32](https://sourceforge.net/projects/getgnuwin32/files/) - needed for various tools not least of which is mktemp which is used by blackbox
+  - after downloading the install just provides you with some batch files.  Because of prior issues at sourceforge and to make sure you get the latest version of each package the batch files handle the brunt of the work of getting the correct packages and installing them for you.
+  - from a **windows command prompt** run `download.bat`  once it has completed run `install.bat` then add the path for those tools to your PATH (ex: `PATH=%PATH%;c:\GnuWin32\bin`)
+
+Development:
+
+- unknown (if you develop Blackbox under MinGW, please let us know if any additional packages are required to run `make test`)
 
 How is the encryption done?
 ===========================
 
-GPG has many different ways to encrypt a file. BlackBox uses the mode that lets you specify a list of keys that can decrypt the messsage.
+GPG has many different ways to encrypt a file. BlackBox uses the mode that lets you specify a list of keys that can decrypt the message.
 
 If you have 5 people ("admins") that should be able to access the secrets, each creates a GPG key and adds their public key to the keychain. The GPG command used to encrypt the file lists all 5 key names, and therefore any 1 key can decrypt the file.
 
@@ -154,20 +207,20 @@ To remove someone's access, remove that admin's key name (i.e. email address) fr
 
 *If you use Puppet, why didn't you just use hiera-eyaml?* There are 4 reasons:
 
-1.	This works with any Git or Mercurial repo, even if you aren't using Puppet.
-2.	hiera-eyaml decrypts "on demand" which means your Puppet Master now uses a lot of CPU to decrypt keys every time it is contacted. It slows down your master, which, in my case, is already slow enough.
-3.	This works with binary files, without having to ASCIIify them and paste them into a YAML file. Have you tried to do this with a cert that is 10K long and changes every few weeks? Ick.
-4.	hiera-eyaml didn't exist when I wrote this.
+1. This works with any Git or Mercurial repo, even if you aren't using Puppet.
+2. hiera-eyaml decrypts "on demand" which means your Puppet Master now uses a lot of CPU to decrypt keys every time it is contacted. It slows down your master, which, in my case, is already slow enough.
+3. This works with binary files, without having to ASCIIify them and paste them into a YAML file. Have you tried to do this with a cert that is 10K long and changes every few weeks? Ick.
+4. hiera-eyaml didn't exist when I wrote this.
 
 What does this look like to the typical user?
 =============================================
 
--	If you need to, start the GPG Agent: `eval $(gpg-agent --daemon)`
--	Decrypt the file so it is editable: `blackbox_edit_start FILENAME`
--	(You will need to enter your GPG passphrase.)
--	Edit FILENAME as you desire: `vim FILENAME`
--	Re-encrypt the file: `blackbox_edit_end FILENAME`
--	Commit the changes. `git commit -a` or `hg commit`
+- If you need to, start the GPG Agent: `eval $(gpg-agent --daemon)`
+- Decrypt the file so it is editable: `blackbox_edit_start FILENAME`
+- (You will need to enter your GPG passphrase.)
+- Edit FILENAME as you desire: `vim FILENAME`
+- Re-encrypt the file: `blackbox_edit_end FILENAME`
+- Commit the changes. `git commit -a` or `hg commit`
 
 Wait... it can be even easier than that! Run `blackbox_edit FILENAME`, and it'll decrypt the file in a temp file and call `$EDITOR` on it, re-encrypting again after the editor is closed.
 
@@ -228,8 +281,8 @@ The variable `$the_password` will contain "my secret password" and can be used a
 How to enroll a new file into the system?
 =========================================
 
--	If you need to, start the GPG Agent: `eval $(gpg-agent --daemon)`
--	Add the file to the system:
+- If you need to, start the GPG Agent: `eval $(gpg-agent --daemon)`
+- Add the file to the system:
 
 ```
 blackbox_register_new_file path/to/file.name.key
@@ -261,7 +314,9 @@ blackbox_deregister_file path/to/file.name.key
 How to indoctrinate a new user into the system?
 ===============================================
 
-`keyrings/live/blackbox-admins.txt` is a file that lists which users are able to decrypt files. (More pedantically, it is a list of the GnuPG key names that the file is encrypted for.)
+FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?"
+
+`.blackbox/blackbox-admins.txt` is a file that lists which users are able to decrypt files. (More pedantically, it is a list of the GnuPG key names that the file is encrypted for.)
 
 To join the list of people that can edit the file requires three steps; You create a GPG key and add it to the key ring. Then, someone that already has access adds you to the system. Lastly, you should test your access.
 
@@ -273,7 +328,7 @@ If you don't already have a GPG key, here's how to generate one:
 gpg --gen-key
 ```
 
-Pick defaults for encryption settings, 0 expiration. Pick a VERY GOOD passphrase. Store the private key securely. Tip: Store it on a secure machine, or one with little or no internet access, with full-disk-encryption, etc. Your employer problably has rules about how to store such things.
+Pick defaults for encryption settings, 0 expiration. Pick a VERY GOOD passphrase. Store a backup of the private key someplace secure. For example, keep the backup copy on a USB drive that is locked in safe.  Or, at least put it on a machine secure machine with little or no internet access, full-disk-encryption, etc. Your employer probably has rules about how to store such things.
 
 Now that you have a GPG key, add yourself as an admin:
 
@@ -290,7 +345,7 @@ blackbox_addadmin tal@example.com
 When the command completes successfully, instructions on how to commit these changes will be output. Run the command as given to commit the changes. It will look like this:
 
 ```
-git commit -m'NEW ADMIN: tal@example.com' keyrings/live/pubring.gpg keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+git commit -m'NEW ADMIN: tal@example.com' .blackbox/pubring.gpg .blackbox/trustdb.gpg .blackbox/blackbox-admins.txt
 ```
 
 Then push it to the repo:
@@ -314,7 +369,8 @@ Ask someone that already has access to re-encrypt the data files. This gives you
 Pre-check: Verify the new keys look good.
 
 ```
-$ gpg --homedir=keyrings/live --list-keys 
+git pull    # Or whatever is required for your system
+gpg --homedir=.blackbox --list-keys
 ```
 
 For example, examine the key name (email address) to make sure it conforms to corporate standards.
@@ -322,7 +378,7 @@ For example, examine the key name (email address) to make sure it conforms to co
 Import the keychain into your personal keychain and reencrypt:
 
 ```
-gpg --import keyrings/live/pubring.gpg
+gpg --import .blackbox/pubring.gpg
 blackbox_update_all_files
 ```
 
@@ -358,27 +414,61 @@ When the command completes, you will be given a reminder to check in the change
 
 Note that their keys will still be in the key ring, but they will go unused. If you'd like to clean up the keyring, use the normal GPG commands and check in the file.
 
+FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?"
+
 ```
-gpg --homedir=keyrings/live --list-keys
-gpg --homedir=keyrings/live --delete-key olduser@example.com
-git commit -m'Cleaned olduser@example.com from keyring'  keyrings/live/*
+gpg --homedir=.blackbox --list-keys
+gpg --homedir=.blackbox --delete-key olduser@example.com
+git commit -m'Cleaned olduser@example.com from keyring'  .blackbox/*
 ```
 
+FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?"
+
 The key ring only has public keys. There are no secret keys to delete.
 
 Remember that this person did have access to all the secrets at one time. They could have made a copy. Therefore, to be completely secure, you should change all passwords, generate new SSL keys, and so on just like when anyone that had privileged access leaves an organization.
 
-Enabling Blackbox For a Repo
+Where is the configuration stored? .blackbox vs. keyrings/live
+==============================================================
+
+Blackbox stores its configuration data in the `.blackbox` subdirectory.  Older
+repos use `keyrings/live`.  For backwards compatibility either will work.
+
+All documentation refers to `.blackbox`.
+
+You can convert an old repo by simply renaming the directory:
+
+```
+mv keyrings/live .blackbox
+rmdir keyrings
+```
+
+There is no technical reason to convert old repos except that it is less
+confusing to users.
+
+This change was made in commit 60e782a0, release v1.20180615.
+
+The details:
+
+- First Blackbox checks `$BLACKBOXDATA`. If this environment variable is set, this is the directory that will be used. If it lists a directory that does not exist, Blackbox will print an error and exit.
+- If `$BLACKBOXDATA` is not set: (which is the typical use case)
+  - Blackbox will first try `keyrings/live` and use it if it exists.
+  - Otherwise the default `.blackbox` will be used.  If `.blackbox` does not exist, Blackbox will print an error and exit.
+
+
+Enabling BlackBox For a Repo
 ============================
 
 Overview:
 
 To add "blackbox" to a git or mercurial repo, you'll need to do the following:
 
-1.	Run the initialize script. This adds a few files to your repo in a directory called "keyrings".
-2.	For the first user, create a GPG key and add it to the key ring.
-3.	Encrypt the files you want to be "secret".
-4.	For any automated user (one that must be able to decrypt without a passphrase), create a GPG key and create a subkey with an empty passphrase.
+1. Run the initialize script. This adds a few files to your repo in a directory called ".blackbox".
+2. For the first user, create a GPG key and add it to the key ring.
+3. Encrypt the files you want to be "secret".
+4. For any automated user (one that must be able to decrypt without a passphrase), create a GPG key and create a subkey with an empty passphrase.
+
+FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?"
 
 ### Run the initialize script.
 
@@ -429,6 +519,8 @@ Set up automated users or "role accounts"
 
 i.e. This is how a Puppet Master can have access to the unencrypted data.
 
+FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?"
+
 An automated user (a "role account") is one that that must be able to decrypt without a passphrase. In general you'll want to do this for the user that pulls the files from the repo to the master. This may be automated with Jenkins CI or other CI system.
 
 GPG keys have to have a passphrase. However, passphrases are optional on subkeys. Therefore, we will create a key with a passphrase then create a subkey without a passphrase. Since the subkey is very powerful, it should be created on a very secure machine.
@@ -441,9 +533,9 @@ ProTip: If asked to generate entropy, consider running this on the same machine
 
 For the rest of this doc, you'll need to make the following substitutions:
 
--	ROLEUSER: svc_deployacct or whatever your role account's name is.
--	NEWMASTER: the machine this role account exists on.
--	SECUREHOST: The machine you use to create the keys.
+- ROLEUSER: svc_deployacct or whatever your role account's name is.
+- NEWMASTER: the machine this role account exists on.
+- SECUREHOST: The machine you use to create the keys.
 
 NOTE: This should be more automated/scripted. Patches welcome.
 
@@ -491,9 +583,9 @@ Command> save
 Now securely export this directory to NEWMASTER:
 
 ```
-$ gpg --homedir . --export -a svc_sadeploy >/tmp/NEWMASTER/pubkey.txt
-$ tar cvf /tmp/keys.tar .
-$ rsync -avP /tmp/keys.tar NEWMASTER:/tmp/.
+gpg --homedir . --export -a svc_sadeploy >/tmp/NEWMASTER/pubkey.txt
+tar cvf /tmp/keys.tar .
+rsync -avP /tmp/keys.tar NEWMASTER:/tmp/.
 ```
 
 On NEWMASTER, receive the new GnuPG config:
@@ -508,12 +600,12 @@ cd ~/.gnupg && tar xpvf /tmp/keys.tar
 Back on SECUREHOST, import the pubkey into the repository.
 
 ```
-$ cd keyrings/live
+$ cd .blackbox
 $ gpg --homedir . --import /tmp/NEWMASTER/pubkey.txt
 ```
 -->
 
-Back on SECUREHOST, add the new email address to keyrings/live/blackbox-admins.txt:
+Back on SECUREHOST, add the new email address to .blackbox/blackbox-admins.txt:
 
 ```
 cd /path/to/the/repo
@@ -523,14 +615,14 @@ blackbox_addadmin $KEYNAME /tmp/NEWMASTER
 Verify that secring.gpg is a zero-length file. If it isn't, you have somehow added a private key to the keyring. Start over.
 
 ```
-$ cd keyrings/live
-$ ls -l secring.gpg
+cd .blackbox
+ls -l secring.gpg
 ```
 
 Commit the recent changes:
 
 ```
-$ cd keyrings/live
+cd .blackbox
 git commit -m"Adding key for KEYNAME" pubring.gpg trustdb.gpg blackbox-admins.txt
 ```
 
@@ -547,7 +639,7 @@ On NEWMASTER, import the keys and decrypt the files:
 
 ```
 sudo -u svc_sadeploy bash   # Become the role account.
-gpg --import /etc/puppet/keyrings/live/pubring.gpg
+gpg --import /etc/puppet/.blackbox/pubring.gpg
 export PATH=$PATH:/path/to/blackbox/bin
 blackbox_postdeploy
 sudo -u puppet cat /etc/puppet/hieradata/blackbox.yaml # or any encrypted file.
@@ -555,7 +647,7 @@ sudo -u puppet cat /etc/puppet/hieradata/blackbox.yaml # or any encrypted file.
 
 ProTip: If you get "gpg: decryption failed: No secret key" then you forgot to re-encrypt blackbox.yaml with the new key.
 
-On SECUREHOST, securerly delete your files:
+On SECUREHOST, securely delete your files:
 
 ```
 cd /tmp/NEWMASTER
@@ -568,19 +660,30 @@ rm -rf /tmp/NEWMASTER
 
 Also shred any other temporary files you may have made.
 
-Replace expired keys:
-=====================
+Replacing expired keys
+======================
 
-If any one admin's key expires, you can no longer encrypt files. You will need to replace the key and re-encrypt.
-
--	Step 0: You see this error:
+If someone's key has already expired, blackbox will stop
+encrypting.  You see this error:
 
 ```
 $ blackbox_edit_end modified_file.txt
 --> Error: can't re-encrypt because a key has expired.
 ```
 
--	Step 1. Administrator removes expired user:
+FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?"
+
+You can also detect keys that are about to expire by issuing this command and manually reviewing the "expired:" dates:
+
+    gpg --homedir=.blackbox  --list-keys
+
+or... list UIDs that will expire within 1 month from today: (Warning: this also lists keys without an expiration date)
+
+    gpg --homedir=.blackbox --list-keys  --with-colons --fixed-list-mode  | grep ^uid | awk -F: '$6 < '$(( $(date +%s) + 2592000))
+
+Here's how to replace the key:
+
+- Step 1. Administrator removes expired user:
 
 Warning: This process will erase any unencrypted files that you were in the process of editing. Copy them elsewhere and restore the changes when done.
 
@@ -589,31 +692,31 @@ blackbox_removeadmin expired_user@example.com
 # This next command overwrites any changed unencrypted files. See warning above.
 blackbox_update_all_files
 git commit -m "Re-encrypt all files"
-gpg --homedir=keyrings/live --delete-key expired_user@example.com
-git commit -m 'Cleaned expired_user@example.com from keyring'  keyrings/live/*
+gpg --homedir=.blackbox --delete-key expired_user@example.com
+git commit -m 'Cleaned expired_user@example.com from keyring'  .blackbox/*
 git push
 ```
 
--	Step 2. Expired user adds an updated key:
+- Step 2. Expired user adds an updated key:
 
 ```
 git pull
 blackbox_addadmin updated_user@example.com
-git commit -m'NEW ADMIN: updated_user@example.com keyrings/live/pubring.gpg keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+git commit -m'NEW ADMIN: updated_user@example.com .blackbox/pubring.gpg .blackbox/trustdb.gpg .blackbox/blackbox-admins.txt
 git push
 ```
 
--	Step 3. Administrator re-encrypts all files with the updated key of the expired user:
+- Step 3. Administrator re-encrypts all files with the updated key of the expired user:
 
 ```
 git pull
-gpg --import keyrings/live/pubring.gpg 
-blackbox_update_all_files 
+gpg --import .blackbox/pubring.gpg
+blackbox_update_all_files
 git commit -m "Re-encrypt all files"
 git push
 ```
 
--	Step 4: Clean up:
+- Step 4: Clean up:
 
 Any files that were temporarily copied in the first step so as to not be overwritten can now be copied back and re-encrypted with the `blackbox_edit_end` command.
 
@@ -624,11 +727,13 @@ Any files that were temporarily copied in the first step so as to not be overwri
 It's possible to tell Git to decrypt versions of the file before running them through `git diff` or `git log`. To achieve this do:
 
 - Add the following to `.gitattributes` at the top of the git repository:
+
 ```
 *.gpg diff=blackbox
 ```
 
 - Add the following to `.git/config`:
+
 ```
 [diff "blackbox"]
     textconv = gpg --use-agent -q --batch --decrypt
@@ -636,33 +741,50 @@ It's possible to tell Git to decrypt versions of the file before running them th
 
 And now commands like `git log -p file.gpg` will show a nice log of the changes in the encrypted file.
 
-Some common errors:
-===================
+Some common errors
+==================
 
-`gpg: filename: skipped: No public key` -- Usually this means there is an item in `keyrings/live/blackbox-admins.txt` that is not the name of the key. Either something invalid was inserted (like a filename instead of a username) or a user has left the organization and their key was removed from the keychain, but their name wasn't removed from the blackbox-admins.txt file.
+`gpg: filename: skipped: No public key` -- Usually this means there is an item in `.blackbox/blackbox-admins.txt` that is not the name of the key. Either something invalid was inserted (like a filename instead of a username) or a user has left the organization and their key was removed from the keychain, but their name wasn't removed from the blackbox-admins.txt file.
 
 `gpg: decryption failed: No secret key` -- Usually means you forgot to re-encrypt the file with the new key.
 
-`Error: can't re-encrypt because a key has expired.` -- A user's key has expired and can't be used to encrypt any more. Follow the[Replace expired keys](#replace-expired-keys) tip.
+`Error: can't re-encrypt because a key has expired.` -- A user's key has expired and can't be used to encrypt any more. Follow the [Replace expired keys](#replace-expired-keys) tip.
+
+FYI: Your repo may use `keyrings/live` instead of `.blackbox`. See "Where is the configuration stored?"
 
 Using Blackbox without a repo
 =============================
 
-If the files are copied out of a repo they can still be decrypted and edited. Obviously edits, changes to keys, and such will be lost if they are made outside the repo. Also note that commands are most likely to only work if run from the base directory (i.e. the parent to the keyrings directory).
+If the files are copied out of a repo they can still be decrypted and edited. Obviously edits, changes to keys, and such will be lost if they are made outside the repo. Also note that commands are most likely to only work if run from the base directory (i.e. the parent to the .blackbox directory).
 
 The following commands have been tested outside a repo:
 
--	`blackbox_postdeploy`
--	`blackbox_edit_start`
--	`blackbox_edit_end`
+- `blackbox_postdeploy`
+- `blackbox_edit_start`
+- `blackbox_edit_end`
 
-Some Subversion gotchas:
-========================
+Some Subversion gotchas
+=======================
 
-The current implementation will store the blackbox in `/keyrings` at the root of the entire repo.  this will create an issue between environments that have different roots (ie, checking out `/` on development vs `/releases/foo` in production).  To get around this, you can `export BLACKBOX_REPOBASE=/path/to/repo` and set a specific base for your repo.
+The current implementation will store the blackbox in `/keyrings` at the root of the entire repo.  This will create an issue between environments that have different roots (i.e. checking out `/` on development vs `/releases/foo` in production). To get around this, you can `export BLACKBOX_REPOBASE=/path/to/repo` and set a specific base for your repo.
 
 This was originally written for git and supports a two-phase commit, in which `commit` is a local commit and "push" sends the change upstream to the version control server when something is registered or deregistered with the system.  The current implementation will immediately `commit` a file (to the upstream subversion server) when you execute a `blackbox_*` command.
 
+Using Blackbox when gpg2 is installed next to gpg
+=================================================
+
+In some situations, team members or automated roles need to install gpg
+2.x alongside the system gpg version 1.x to catch up with the team's gpg
+version. On Ubuntu 16, you can ```apt-get install gnupg2``` which
+installs the binary gpg2. If you want to use this gpg2 binary, run every
+blackbox command with GPG=gpg2.
+
+For example:
+
+```
+GPG=gpg2 blackbox_postdeploy
+```
+
 How to submit bugs or ask questions?
 ====================================
 
@@ -670,9 +792,7 @@ We welcome questions, bug reports and feedback!
 
 The best place to start is to join the [blackbox-project mailing list](https://groups.google.com/d/forum/blackbox-project) and ask there.
 
-Bugs are tracked here in Github. Please feel free to files bugs yourself:
-
--	https://github.com/StackExchange/blackbox/issues
+Bugs are tracked here in Github. Please feel free to [report bugs](https://github.com/StackExchange/blackbox/issues) yourself.
 
 Developer Info
 ==============
@@ -695,24 +815,26 @@ This runs through a number of system tests. It creates a repo, encrypts files, d
 
 Please submit tests with code changes:
 
-The best way to change Blackbox is via Test Driven Development. First add a test to `tools/confidence.sh`. This test should fail, and demonstrate the need for the change you are about to make. Then fix the bug or add the feature you want. When you are done, `make confidence` should pass all tests. The PR you submit should include your code as well as the new test. This way the confidence tests accumulate as the system grows as we know future changes don't break old features.
+The best way to change BlackBox is via Test Driven Development. First add a test to `tools/confidence.sh`. This test should fail, and demonstrate the need for the change you are about to make. Then fix the bug or add the feature you want. When you are done, `make confidence` should pass all tests. The PR you submit should include your code as well as the new test. This way the confidence tests accumulate as the system grows as we know future changes don't break old features.
 
 Note: The tests currently assume "git" and have been tested only on CentOS, Mac OS X, and Cygwin. Patches welcome!
 
 Alternatives
 ============
 
-Here are other open source packages that do something similar to Blackbox. If you like them better than Blackbox, please use them.
+Here are other open source packages that do something similar to BlackBox. If you like them better than BlackBox, please use them.
 
--	git-crypt: https://www.agwa.name/projects/git-crypt/
--	Pass: http://www.zx2c4.com/projects/password-store/
--	Transcrypt: https://github.com/elasticdog/transcrypt
--	Keyringer: https://keyringer.pw/
--	git-secret: https://github.com/sobolevn/git-secret
+- [git-crypt](https://www.agwa.name/projects/git-crypt/)
+- [Pass](http://www.zx2c4.com/projects/password-store/)
+- [Transcrypt](https://github.com/elasticdog/transcrypt)
+- [Keyringer](https://keyringer.pw/)
+- [git-secret](https://github.com/sobolevn/git-secret)
 
 git-crypt has the best git integration. Once set up it is nearly transparent to the users. However it only works with git.
 
+
 License
 =======
 
-This content is released under the MIT License. See the LICENSE.txt file.
+This content is released under the MIT License.
+See the [LICENSE.txt](LICENSE.txt) file.

RELEASE_ENGINEERING.md

@@ -17,7 +17,7 @@ There are 3 branches/tags:
 -	**tag stable:** Stable enough for use by most people.
 -	**tag production:** Burned in long enough that we are confident it can be widely adopted.
 
-If you are packaging Blackbox for distribution, you should track the *tag production*. You might also want to provide a separate package that tracks *tag stable:* for early adopters.
+If you are packaging BlackBox for distribution, you should track the *tag production*. You might also want to provide a separate package that tracks *tag stable:* for early adopters.
 
 Build Tasks
 ===========
@@ -99,6 +99,7 @@ Submit the diff file as a bug as instructed. The instructions should look like t
 
 Step 3: Watch for the update to happen.
 
+
 Updating MacPorts (manual)
 ==========================
 

Version2-Ideas.md

@@ -1,12 +1,12 @@
-# Ideas for blackbox Version 2
+# Ideas for BlackBox Version 2
 
 I'm writing this to solicit feedback and encourage discussion.
 
-Here are my thoughts on a "verison 2" of blackbox.  This is where
+Here are my thoughts on a "version 2" of BlackBox.  This is where
 I list ideas that would require major changes to the system. They
 might break backwards compatibility, though usually not.
 
-Blackbox grew from a few simple shell scripts used at StackOverflow.com
+BlackBox grew from a few simple shell scripts used at StackOverflow.com
 to a larger system used by dozens (hundreds?) of organizations. Not
 all the design decisions were "forward looking".
 
@@ -75,7 +75,7 @@ Backwards compatibility: This would add a "none" VCS, not remove any existing fu
 
 `bash` is fairly universal. It even exists on Windows.  However it is not the right language for large systems. Writing the acceptance tests is quite a bear.  Managing ".gitignore" files in bash is impossible and the current implementation fails in many cases.
 
-`python` is my second favorite langauge. It would make the code cleaner and more testable. However it is not installed everywhere.  I would also want to write it in Python3 (why start a new project in Python2?) but sadly Python3 is less common.  It is a chicken vs. egg situation.
+`python` is my second favorite language. It would make the code cleaner and more testable. However it is not installed everywhere.  I would also want to write it in Python3 (why start a new project in Python2?) but sadly Python3 is less common.  It is a chicken vs. egg situation.
 
 `go` is my favorite language. I could probably rewrite this in go in a weekend. However, now the code is compiled, not interpreted. Therefore we lose the ability to just "git clone" and have the tools you want.  Not everyone has a Go compiler installed on every machine.
 
@@ -94,4 +94,4 @@ However, I've never used it so I don't have any idea whether git-crypt is any be
 
 Of course, git-crypt doesn't work with SVN, HG, or any other VCS.  Is blackbox's strong point the fact that it support so many VCS systems?  To be honest, it originally only supported HG and GIT because I was at a company that used HG but then changed to GIT.  Supporting anything else was thanks to contributors. Heck, HG support hasn't even been tested recently (by me) since we've gone all git where I work.
 
-How important is this to blackbox users?
+How important is this to BlackBox users?

bin/_blackbox_common.sh

@@ -15,9 +15,15 @@ source "${0%/*}"/_stack_lib.sh
 # Where are we?
 : "${BLACKBOX_HOME:="$(cd "${0%/*}" ; pwd)"}" ;
 
-# Where in the VCS repo should the blackbox data be found?
-: "${BLACKBOXDATA:=keyrings/live}" ;   # If BLACKBOXDATA not set, set it.
-
+# What are the candidates for the blackbox data directory?
+#
+# The order of candidates matter. The first entry of the array
+# sets the default Blackbox directory for all new repositories.
+declare -a BLACKBOXDATA_CANDIDATES
+BLACKBOXDATA_CANDIDATES=(
+  '.blackbox'
+  'keyrings/live'
+)
 
 # If $EDITOR is not set, set it to "vi":
 : "${EDITOR:=vi}" ;
@@ -28,7 +34,7 @@ source "${0%/*}"/_stack_lib.sh
 function physical_directory_of() {
   local d=$(dirname "$1")
   local f=$(basename "$1")
-  (cd "$d" && echo "$(pwd -P)/$f" )
+  (cd "$d" && echo "$(pwd -P | sed 's/\/$//')/$f" )
 }
 
 # Set REPOBASE to the top of the repository
@@ -63,10 +69,20 @@ export REPOBASE=$(physical_directory_of "$REPOBASE")
 # after determining what we believe to be the answer.
 
 if [[ -n "$BLACKBOX_REPOBASE" ]]; then
-	echo "Using custom repobase: $BLACKBOX_REPOBASE"
+	echo "Using custom repobase: $BLACKBOX_REPOBASE" >&2
 	export REPOBASE="$BLACKBOX_REPOBASE"
 fi
 
+if [ -z "$BLACKBOXDATA" ] ; then
+  BLACKBOXDATA="${BLACKBOXDATA_CANDIDATES[0]}"
+  for candidate in ${BLACKBOXDATA_CANDIDATES[@]} ; do
+    if [ -d "$REPOBASE/$candidate" ] ; then
+      BLACKBOXDATA="$candidate"
+      break
+    fi
+  done
+fi
+
 KEYRINGDIR="$REPOBASE/$BLACKBOXDATA"
 BB_ADMINS_FILE="blackbox-admins.txt"
 BB_ADMINS="${KEYRINGDIR}/${BB_ADMINS_FILE}"
@@ -76,6 +92,10 @@ SECRING="${KEYRINGDIR}/secring.gpg"
 : "${DECRYPT_UMASK:=0022}" ;
 # : ${DECRYPT_UMASK:=o=} ;
 
+# $BB_FILES file format:
+# Filenames are listed one per line, relative to the base directory of the repo.
+# Each line is listed in "printf %q" format, which escapes special chars.
+
 # Checks if $1 is 0 bytes, and if $1/keyrings
 # is a directory
 function is_blackbox_repo() {
@@ -86,10 +106,18 @@ function is_blackbox_repo() {
   fi
 }
 
-# Return error if not on cryptlist.
+# is_on_cryptlist resturns an error if $1 not on cryptlist.
 function is_on_cryptlist() {
+  # $1: The filename.
   # Assumes $1 does NOT have the .gpg extension
-  file_contains_line "$BB_FILES" "$(vcs_relative_path "$1")"
+
+  # https://github.com/koalaman/shellcheck/wiki/SC2155
+  local name
+  name=$(vcs_relative_path "$1")
+  local encodedname
+  encodedname=$(printf "%q" "$name")
+
+  file_contains_line "$BB_FILES" "$encodedname"
 }
 
 # Exit with error if a file exists.
@@ -127,7 +155,7 @@ function fail_if_not_on_cryptlist() {
 
   if ! is_on_cryptlist "$name" ; then
     echo "ERROR: $name not found in $BB_FILES" >&2
-    echo "PWD=$(/bin/pwd)" >&2
+    echo "PWD=$(/usr/bin/env pwd)" >&2
     echo 'Exiting...' >&2
     exit 1
   fi
@@ -151,54 +179,92 @@ function get_pubring_path() {
   fi
 }
 
-# Output the unencrypted filename.
-function get_unencrypted_filename() {
-  echo "$(dirname "$1")/$(basename "$1" .gpg)" | sed -e 's#^\./##'
+# normalize_filename_arg takes a filename from the command line and
+# outputs the non-encrypted filename.
+function normalize_filename() {
+  # $1: the input from a user
+  # Use this if the user may have entered the encrypted or
+  # non-encrypted filename.
+  local name
+  name=$(vcs_relative_path "$1")
+  echo "$(dirname "$name")/$(basename "$name" .gpg)" | sed -e 's#^\./##'
 }
 
 # Output the encrypted filename.
-function get_encrypted_filename() {
-  echo "$(dirname "$1")/$(basename "$1" .gpg).gpg" | sed -e 's#^\./##'
+function get_gpg_filename() {
+  # $1: normalized file path
+  echo "$1".gpg
 }
 
+## Output the unencrypted filename.
+#function get_unencrypted_filename() {
+#  echo "$(dirname "$1")/$(basename "$1" .gpg)" | sed -e 's#^\./##'
+#}
+#
+## Output the encrypted filename.
+#function get_encrypted_filename() {
+#  echo "$(dirname "$1")/$(basename "$1" .gpg).gpg" | sed -e 's#^\./##'
+#}
+
 # Prepare keychain for use.
 function prepare_keychain() {
+  local keyringasc
   echo '========== Importing keychain: START' >&2
-  $GPG --import "$(get_pubring_path)" 2>&1 | egrep -v 'not changed$' >&2
+  # Works with gpg 2.0
+  #$GPG --import "$(get_pubring_path)" 2>&1 | egrep -v 'not changed$' >&2
+  # Works with gpg 2.0 and 2.1
+  # NB: We must export the keys to a format that can be imported.
+  make_self_deleting_tempfile keyringasc
+  export LANG="C.UTF-8"
+
+  #if gpg2 is installed next to gpg like on ubuntu 16
+  if [[ "$GPG" != "gpg2" ]]; then
+    $GPG --export --no-default-keyring --keyring "$(get_pubring_path)" >"$keyringasc"
+    $GPG --import "$keyringasc" 2>&1 | egrep -v 'not changed$' >&2
+  else
+    $GPG --keyring "$(get_pubring_path)" --export | $GPG --import
+  fi
+
   echo '========== Importing keychain: DONE' >&2
 }
 
-# Add file to list of encrypted files.
+# add_filename_to_cryptlist adds $1 to the list of encrypted files.
 function add_filename_to_cryptlist() {
+  # $1: The filename.
   # If the name is already on the list, this is a no-op.
-  # However no matter what the datestamp is updated.
-  
+
   # https://github.com/koalaman/shellcheck/wiki/SC2155
   local name
   name=$(vcs_relative_path "$1")
+  local encodedname
+  encodedname=$(printf "%q" "$name")
 
-  if file_contains_line "$BB_FILES" "$name" ; then
+
+  if file_contains_line "$BB_FILES" "$encodedname" ; then
     echo "========== File is registered. No need to add to list."
   else
     echo "========== Adding file to list."
     touch "$BB_FILES"
-    sort -u -o "$BB_FILES" <(echo "$name") "$BB_FILES"
+    sort -u -o "$BB_FILES" <(printf "%q\n" "$name") "$BB_FILES"
   fi
 }
 
-# Removes a file from the list of encrypted files
+# remove_filename_from_cryptlist removes $1 from the list of encrypted files.
 function remove_filename_from_cryptlist() {
+  # $1: The filename.
   # If the name is not already on the list, this is a no-op.
 
   # https://github.com/koalaman/shellcheck/wiki/SC2155
   local name
   name=$(vcs_relative_path "$1")
+  local encodedname
+  encodedname=$(printf "%q" "$name")
 
-  if ! file_contains_line "$BB_FILES" "$name" ; then
+  if ! file_contains_line "$BB_FILES" "$encodedname" ; then
     echo "========== File is not registered. No need to remove from list."
   else
     echo "========== Removing file from list."
-    remove_line "$BB_FILES" "$name"
+    remove_line "$BB_FILES" "$encodedname"
   fi
 }
 
@@ -277,8 +343,11 @@ function shred_file() {
     #NOTE: srm by default uses 35-pass Gutmann algorithm
     CMD=srm
     OPT=-f
+  elif _F=$(mktemp); rm -P "${_F}" >/dev/null 2>/dev/null ; then
+    CMD=rm
+    OPT=-Pf
   else
-    echo "shred_file: WARNING: No secure deletion utility (shred or srm) present; using insecure rm"
+    echo "shred_file: WARNING: No secure deletion utility (shred or srm) present; using insecure rm" >&2
     CMD=rm
     OPT=-f
   fi
@@ -397,8 +466,12 @@ function cp_permissions() {
       chmod $( stat -f '%p' "$1" | sed -e "s/^100//" ) "${@:2}"
       ;;
     Linux | CYGWIN* | MINGW* )
-      chmod --reference "$1" "${@:2}"
-      ;;
+      if [[ -e /etc/alpine-release ]]; then
+        chmod $( stat -c '%a' "$1" ) "${@:2}"
+      else
+        chmod --reference "$1" "${@:2}"
+      fi
+      ;; 
     * )
       echo 'ERROR: Unknown OS. Exiting. (cp_permissions)'
       exit 1
@@ -643,3 +716,20 @@ function vcs_notice_generic_file() {
     echo "WARNING:  If so, manually update the ignore file"
   fi
 }
+
+function gpg_agent_version_check() {
+  if ! hash 'gpg-agent' &> /dev/null; then
+    return 1
+  fi
+  local gpg_agent_version=$(gpg-agent --version | head -1 | awk '{ print $3 }' | tr -d '\n')
+  semverLT $gpg_agent_version "2.1.0"
+}
+
+function gpg_agent_notice() {
+  if [[ $(gpg_agent_version_check) == '0' && -z $GPG_AGENT_INFO ]];then
+    echo 'WARNING: You probably want to run gpg-agent as'
+    echo 'you will be asked for your passphrase many times.'
+    echo 'Example: $ eval $(gpg-agent --daemon)'
+    read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
+  fi
+}

bin/_blackbox_common_test.sh

@@ -6,7 +6,7 @@
 
 set -e
 . "${0%/*}/_blackbox_common.sh"
-. tools/test_functions.sh
+. /Users/tlimoncelli/gitwork/blackbox/tools/test_functions.sh
 
 PHASE 'Test cp-permissions: TestA'
 touch TestA TestB TestC TestD
@@ -22,4 +22,18 @@ assert_file_perm '--wxr--rwx' TestC
 assert_file_perm '----rwx---' TestD  # TestD doesn't change.
 rm -f TestA TestB TestC TestD
 
+PHASE 'Test vcs_relative_path: TestA'
+export REPOBASE='/Users/tlimoncelli/Applications (Parallels)/{fd3049c8-9fdd-48d5-aa16-d31daf3a6879} Applications.localized'
+FILE='Microsoft  Windows Fax and Scan.app/Contents'
+result=$(vcs_relative_path Contents)
+echo result=XXX${result}XXX
+if [[ $FILE != $result ]] ; then
+  echo FAIL
+fi
+
+unencrypted_file=$(get_unencrypted_filename "${result}.gpg")
+echo un=XXX${unencrypted_file}XXX
+encrypted_file=$(get_encrypted_filename "${result}")
+echo en=XXX${encrypted_file}XXX
+
 echo '========== DONE.'

bin/_stack_lib.sh

@@ -180,3 +180,109 @@ function fail_if_in_root_directory() {
       ;;
   esac
 }
+
+function semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+function semverEQ() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -ne $MAJOR_B ]; then
+        return 1
+    fi
+
+    if [ $MINOR_A -ne $MINOR_B ]; then
+        return 1
+    fi
+
+    if [ $PATCH_A -ne $PATCH_B ]; then
+        return 1
+    fi
+
+    if [[ "_$SPECIAL_A" != "_$SPECIAL_B" ]]; then
+        return 1
+    fi
+
+
+    return 0
+
+}
+
+function semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -lt $MINOR_B ]]; then
+        return 0
+    fi
+
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -le $MINOR_B && $PATCH_A -lt $PATCH_B ]]; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  != "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  != "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A" < "_$SPECIAL_B" ]]; then
+        return 0
+    fi
+
+    return 1
+
+}
+
+function semverGT() {
+    semverEQ $1 $2
+    local EQ=$?
+
+    semverLT $1 $2
+    local LT=$?
+
+    if [ $EQ -ne 0 ] && [ $LT -ne 0 ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+

bin/blackbox_cat

@@ -8,7 +8,7 @@ source "${0%/*}/_blackbox_common.sh"
 
 for param in "$@" ; do
   shreddable=0
-  unencrypted_file=$(get_unencrypted_filename "$param")
+  unencrypted_file=$(normalize_filename "$param")
   if [[ ! -e "$unencrypted_file" ]]; then
     "${BLACKBOX_HOME}/blackbox_edit_start" "$param"
     shreddable=1

bin/blackbox_decrypt_all_files

@@ -14,9 +14,7 @@
 export PATH=/usr/bin:/bin:"$PATH"
 
 set -e
+source "${0%/*}/_blackbox_common.sh"
 
-if [[ -z $GPG_AGENT_INFO ]]; then
-  eval $(gpg-agent --daemon)
-fi
-
+gpg_agent_notice
 exec blackbox_postdeploy "$@"

bin/blackbox_decrypt_file

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_decrypt_file -- Decrypt one or more blackbox files.
+#
+
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+if [ $# -eq 0 ]; then
+  echo >&2 "Please provide at least one file to decrypt"
+  exit 1
+fi
+
+"${BLACKBOX_HOME}/blackbox_edit_start" "$@"

bin/blackbox_diff

@@ -7,20 +7,16 @@
 set -e
 source "${0%/*}/_blackbox_common.sh"
 
-if [[ -z $GPG_AGENT_INFO ]]; then
-  echo 'WARNING: You probably want to run gpg-agent as'
-  echo 'you will be asked for your passphrase many times.'
-  echo 'Example: $ eval $(gpg-agent --daemon)'
-  read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
-fi
-
+gpg_agent_notice
 prepare_keychain
 
 modified_files=()
 modifications=()
 echo '========== DIFFING FILES: START'
-while IFS= read <&99 -r unencrypted_file; do
-  unencrypted_file=$(get_unencrypted_filename "$unencrypted_file")
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $encodedname)
+  unencrypted_file=$(get_unencrypted_filename "$name")
   encrypted_file=$(get_encrypted_filename "$unencrypted_file")
   fail_if_not_on_cryptlist "$unencrypted_file"
   if [[ -f "$unencrypted_file" ]]; then

bin/blackbox_initialize

@@ -35,6 +35,18 @@ vcs_add "${KEYRINGDIR}"
 touch "$BLACKBOXDATA/$BB_ADMINS_FILE" "$BLACKBOXDATA/$BB_FILES_FILE"
 vcs_add "$BLACKBOXDATA/$BB_ADMINS_FILE" "$BLACKBOXDATA/$BB_FILES_FILE"
 
+if [[ $VCS_TYPE = "git" ]]; then
+
+  # Set .gitattributes so that Windows users don't break the admin files.
+  FILE="$BLACKBOXDATA/.gitattributes"
+  touch "$FILE"
+  LINE='blackbox-admins.txt text eol=lf'
+  grep -qF "$LINE" "$FILE" || echo "$LINE" >> "$FILE"
+  LINE='blackbox-files.txt text eol=lf'
+  grep -qF "$LINE" "$FILE" || echo "$LINE" >> "$FILE"
+
+fi
+
 if [[ $VCS_TYPE = "svn" ]]; then
 	echo
 	echo

bin/blackbox_list_admins

@@ -5,4 +5,4 @@
 #
 set -e
 source "${0%/*}/_blackbox_common.sh"
-cat "$BB_ADMINS_FILE"
+cat "$BB_ADMINS"

bin/blackbox_list_files

@@ -5,4 +5,7 @@
 #
 set -e
 source "${0%/*}/_blackbox_common.sh"
-cat "$BB_FILES"
+
+while IFS= read <&99 -r encodedname; do
+  echo $encodedname
+done 99<"$BB_FILES"

bin/blackbox_postdeploy

@@ -27,8 +27,12 @@ prepare_keychain
 
 # Decrypt:
 echo '========== Decrypting new/changed files: START'
-while IFS= read <&99 -r unencrypted_file; do
-  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $name)
+
+  encrypted_file=$(get_encrypted_filename "$name")
+  unencrypted_file=$(get_unencrypted_filename "$name")
   decrypt_file_overwrite "$encrypted_file" "$unencrypted_file"
   cp_permissions "$encrypted_file" "$unencrypted_file"
   if [[ ! -z "$FILE_GROUP" ]]; then

bin/blackbox_register_new_file

@@ -44,7 +44,7 @@ function register_new_file() {
   vcs_ignore "$unencrypted_file"
   echo 'NOTE: "already tracked!" messages are safe to ignore.'
   vcs_add "$BB_FILES" "$encrypted_file"
-  vcs_commit "registered in blackbox: ${unencrypted_file}" "$BB_FILES" "$encrypted_file"
+  vcs_commit "registered in blackbox: ${unencrypted_file}" "$BB_FILES" "$encrypted_file" "$(vcs_ignore_file_path)"
 }
 
 for target in "$@"; do

bin/blackbox_removeadmin

@@ -22,7 +22,7 @@ remove_line "$BB_ADMINS" "$KEYNAME"
 
 
 # remove the admin key from the pubring
-$GPG --no-permission-warning --homedir="$KEYRINGDIR" --delete-key "$KEYNAME"
+$GPG --no-permission-warning --homedir="$KEYRINGDIR" --batch --yes --delete-key "$KEYNAME" || true
 pubring_path=$(get_pubring_path)
 vcs_add "$pubring_path" "$KEYRINGDIR/trustdb.gpg" "$BB_ADMINS"
 
@@ -31,4 +31,4 @@ vcs_add "$pubring_path" "$KEYRINGDIR/trustdb.gpg" "$BB_ADMINS"
 echo
 echo
 echo 'NEXT STEP: Check these into the repo.  Probably with a command like...'
-echo $VCS_TYPE commit -m\'REMOVED ADMIN: $KEYNAME\' "$BLACKBOXDATA/trustdb.gpg" "$BLACKBOXDATA/$BB_ADMINS_FILE"
+echo $VCS_TYPE commit -m\'REMOVED ADMIN: $KEYNAME\' "$BLACKBOXDATA/$(basename ${pubring_path})" "$BLACKBOXDATA/trustdb.gpg" "$BLACKBOXDATA/$BB_ADMINS_FILE"

bin/blackbox_shred_all_files

@@ -24,11 +24,13 @@ echo '========== FILES BEING SHREDDED:'
 
 exported_internal_shred_file() {
   source "$1/_blackbox_common.sh"
-  unencrypted_file=$(get_unencrypted_filename "$2")
-  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
+  #unencrypted_file=$(get_unencrypted_filename "$2")
+  unencrypted_file="$2"
   if [[ -f "$unencrypted_file" ]]; then
-    echo "    $unencrypted_file"
+    echo "    SHRED: $unencrypted_file"
     shred_file "$unencrypted_file"
+  else
+    echo "NOT FOUND: $unencrypted_file"
   fi
 }
 
@@ -38,6 +40,6 @@ DEREFERENCED_BIN_DIR="${0%/*}"
 MAX_PARALLEL_SHRED=10
 
 export IFS=
-xargs -I{} -n 1 -P $MAX_PARALLEL_SHRED bash -c "exported_internal_shred_file $DEREFERENCED_BIN_DIR {}" $DEREFERENCED_BIN_DIR/fake <"$BB_FILES"
+tr '\n' '\0' <"$BB_FILES" | xargs -0 -I{} -n 1 -P $MAX_PARALLEL_SHRED bash -c "exported_internal_shred_file $DEREFERENCED_BIN_DIR \"{}\"" $DEREFERENCED_BIN_DIR/fake
 
 echo '========== DONE.'

bin/blackbox_update_all_files

@@ -7,26 +7,24 @@
 set -e
 source "${0%/*}/_blackbox_common.sh"
 
-if [[ -z $GPG_AGENT_INFO ]]; then
-  echo 'WARNING: You probably want to run gpg-agent as'
-  echo 'you will be asked for your passphrase many times.'
-  echo 'Example: $ eval $(gpg-agent --daemon)'
-  read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
-fi
-
+gpg_agent_notice
 disclose_admins
 prepare_keychain
 
 echo '========== ENCRYPTED FILES TO BE RE-ENCRYPTED:'
-while IFS= read <&99 -r unencrypted_file; do
-    echo "    $unencrypted_file.gpg"
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $encodedname)
+  echo "    $name.gpg"
 done 99<"$BB_FILES"
 
 echo '========== FILES IN THE WAY:'
 need_warning=false
-while IFS= read <&99 -r unencrypted_file; do
-  unencrypted_file=$(get_unencrypted_filename "$unencrypted_file")
-  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $encodedname)
+  unencrypted_file=$(get_unencrypted_filename "$name")
+  encrypted_file=$(get_encrypted_filename "$name")
   if [[ -f "$unencrypted_file" ]]; then
     need_warning=true
     echo "    $unencrypted_file"
@@ -41,9 +39,11 @@ else
 fi
 
 echo '========== RE-ENCRYPTING FILES:'
-while IFS= read <&99 -r unencrypted_file; do
-  unencrypted_file=$(get_unencrypted_filename "$unencrypted_file")
-  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $encodedname)
+  unencrypted_file=$(get_unencrypted_filename "$name")
+  encrypted_file=$(get_encrypted_filename "$name")
   echo ========== PROCESSING '"'$unencrypted_file'"'
   fail_if_not_on_cryptlist "$unencrypted_file"
   decrypt_file_overwrite "$encrypted_file" "$unencrypted_file"

bin/blackbox_view

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_view -- Decrypt a file, view it, shred it
+#
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+for param in "$@" ; do
+  shreddable=0
+  unencrypted_file=$(get_unencrypted_filename "$param")
+  if [[ ! -e "$unencrypted_file" ]]; then
+    "${BLACKBOX_HOME}/blackbox_edit_start" "$param"
+    shreddable=1
+  fi
+  ${PAGER:-less} "$unencrypted_file"
+  if [[ $shreddable = 1 ]]; then
+    shred_file "$unencrypted_file"
+  fi
+done

bin/blackbox_whatsnew

@@ -14,13 +14,7 @@ then
 fi
 
 fail_if_not_in_repo
-
-if [[ -z $GPG_AGENT_INFO ]]; then
-  echo 'WARNING: You probably want to run gpg-agent as'
-  echo 'you will be asked for your passphrase many times.'
-  echo 'Example: $ eval $(gpg-agent --daemon)'
-  read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
-fi
+gpg_agent_notice
 
 COLUMNS=`tput cols`
 FILE=$1

blackbox.plugin.zsh

@@ -1,4 +1,4 @@
-#!/bin/zsh
+#!/usr/bin/env zsh
 # The MIT License (MIT)
 
 # Copyright (c) 2014 Stack Exchange, Inc.

tools/auto_system_test

@@ -0,0 +1,25 @@
+#!/usr/bin/env expect
+
+# Run the confidence test non-interactively.  Since the script
+# asks for passphrases, we use "expect" to simulate keyboard data entry.
+
+# Run the test:
+spawn tools/confidence_test.sh
+
+# As we run the confidence test, respond with the right password.
+# We do this for up to 300 times to prevent an infinite loop.
+
+set times 0;
+while { $times < 300 } {
+   expect {
+       # The script outputs what the password will be, and we save
+       # that info in $pw any time we see the text.
+       "my password is the lowercase letter a" { set pw "a\n" ; exp_continue }
+       "my password is the lowercase letter b" { set pw "b\n" ; exp_continue }
+       # If the passphrase is requested, send it.
+       "Passphrase:" { send $pw ; exp_continue }
+       # If we reach EOF, exit this loop.
+       eof { break }
+   }
+   set times [ expr $times+1];
+}

tools/confidence_test.sh

@@ -1,7 +1,30 @@
 #!/usr/bin/env bash
 
 blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../bin
-export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin"
+export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:${blackbox_home}"
+
+export LANG=C.UTF-8  # Required ro "gpg --export" to work properly.
+
+# TODO(tlim): The tests are hardcoded to this directory. This should be fixed.
+export BLACKBOXDATA=keyrings/live
+
+# This script requires many utilities, some are not
+# required by the usual blackbox scripts.  Test to make
+# sure we have them all.
+e=false
+for i in blackbox_addadmin blackbox_list_admins blackbox_register_new_file \
+  cat git gpg gpg-agent mkdir pinentry pinentry-tty rm tar which ; do
+  if ! which >/dev/null 2>&1  $i ; then
+    echo ERROR: Command not in PATH: $i
+    e=true
+  fi
+done
+if $e ; then
+  echo 'Exiting. Please install the above commands.'
+  echo 'This script requires many utilities not required by blackbox itself.'
+  echo PATH="$PATH"
+  exit 1
+fi
 
 set -e
 . _stack_lib.sh
@@ -19,10 +42,15 @@ cd "$test_repository"
 make_self_deleting_tempdir fake_alice_home
 make_self_deleting_tempdir fake_bob_home
 export GNUPGHOME="$fake_alice_home"
+#echo 'pinentry-program' "$(which pinentry-insecure-fake)" >> "$GNUPGHOME/gpg-agent.conf"
+echo 'pinentry-program' "$(which pinentry-tty)" >> "$GNUPGHOME/gpg-agent.conf"
+#pinentry-program "${blackbox_home}/pinentry-fake-insecure"
 eval "$(gpg-agent --homedir "$fake_alice_home" --daemon)"
 GPG_AGENT_INFO_ALICE="$GPG_AGENT_INFO"
 
 export GNUPGHOME="$fake_bob_home"
+#echo 'pinentry-program' "$(which pinentry-insecure-fake)" >> "$GNUPGHOME/gpg-agent.conf"
+echo 'pinentry-program' "$(which pinentry-tty)" >> "$GNUPGHOME/gpg-agent.conf"
 eval "$(gpg-agent --homedir "$fake_bob_home" --daemon)"
 GPG_AGENT_INFO_BOB="$GPG_AGENT_INFO"
 
@@ -82,7 +110,11 @@ git commit -m'INITIALIZE BLACKBOX' keyrings .gitignore
 PHASE 'and adds herself as an admin.'
 
 blackbox_addadmin alice@example.com
-git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.gpg keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.??? keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+
+make_self_deleting_tempfile adminlist1
+blackbox_list_admins >"$adminlist1"
+assert_file_md5hash "$adminlist1" "aa1db827772e1d51d453b844394b7617"
 
 
 PHASE 'Bob arrives.'
@@ -110,14 +142,18 @@ gpg --no-permission-warning --batch --gen-key "$gpgconfig"
 echo '========== Bob enrolls himself too.'
 
 blackbox_addadmin bob@example.com
-git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.gpg keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.??? keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+
+make_self_deleting_tempfile adminlist2
+blackbox_list_admins >"$adminlist2"
+assert_file_md5hash "$adminlist2" "0b82b0b3c96e6e5dd5faf60493fe5cf7"
 
 PHASE 'Alice does the second part to enroll bob.'
 become_alice
 
 PHASE 'She enrolls bob.'
-gpg --import keyrings/live/pubring.gpg
-# TODO(tlim) That --import can be eliminated... maybe?
+#gpg --import $(get_pubring_path)
+## TODO(tlim) That --import can be eliminated... maybe?
 
 PHASE 'She enrolls secrets.txt.'
 blackbox_register_new_file secret.txt
@@ -185,15 +221,14 @@ assert_file_md5hash secret.txt "d3e6bbdfc76fae7fd0a921f3408db1d1"
 PHASE 'appears.'
 become_bob
 
-PHASE 'Bob makes sure he has all new keys.'
-
-gpg --import keyrings/live/pubring.gpg
+#PHASE 'Bob makes sure he has all new keys.'
+#gpg --import keyrings/live/pubring.???
 
 # Pick a GID to use:
 # This users's default group:
 DEFAULT_GID_NUM=$(id -g)
 # Pick a group that is not the default group:
-TEST_GID_NUM=$(id -G | fmt -1 |  grep -xv "$(id -u)" | grep -xv "$(id -g)" | head -1)
+TEST_GID_NUM=$(grep -v "$DEFAULT_GID_NUM" /etc/group | cut -d: -f3 | sort -rn | head -1)
 echo "DEFAULT_GID_NUM=$DEFAULT_GID_NUM"
 echo "TEST_GID_NUM=$TEST_GID_NUM"
 
@@ -218,6 +253,10 @@ PHASE 'Bob removes Alice.'
 blackbox_removeadmin alice@example.com
 assert_line_not_exists 'alice@example.com' keyrings/live/blackbox-admins.txt
 
+make_self_deleting_tempfile adminlist3
+blackbox_list_admins >"$adminlist3"
+assert_file_md5hash "$adminlist3" "aadbfafd76ea66ff40dbfd239a69067f"
+
 PHASE 'Bob reencrypts files so alice can not access them.'
 blackbox_update_all_files
 
@@ -314,6 +353,7 @@ PHASE 'Bob shreds all exposed files.'
 assert_file_exists 'my/path/to/relsecrets.txt'
 assert_file_exists 'secret.txt'
 blackbox_shred_all_files
+which blackbox_shred_all_files
 assert_file_missing '!important!.txt'
 assert_file_missing '#andpounds.txt'
 assert_file_missing 'mistake.txt'
@@ -391,6 +431,10 @@ fi
 # Cleanup:
 blackbox_removeadmin abba@notarealuser.com
 
+make_self_deleting_tempfile adminlist4
+blackbox_list_admins >"$adminlist4"
+assert_file_md5hash "$adminlist4" "aadbfafd76ea66ff40dbfd239a69067f"
+
 
 # TODO: Create a new directory. "git clone" the repo into it.
 

tools/mk_rpm_fpmdir

@@ -107,7 +107,7 @@ cat "$@" | while read -a arr ; do
   fi
   if $GO_COMPILE && [[ $SRC == "cmd/"* || $SRC == *"/cmd/"* ]]; then
     echo "========== BUILD&COPY $SRC"
-    ( cd $(dirname "$SRC" ) && go get -d && go build -a ) 
+    ( cd $(dirname "$SRC" ) && go get -d && go build ) 
     PKGARCH=native
   else
     echo "==========       COPY $SRC"

tools/profile.d-usrblackbox-test.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env sh
 
 # Test profile.d-usrblackbox.sh
 

tools/test_functions.sh

@@ -1,5 +1,15 @@
 #!/usr/bin/env bash
 
+# NB: This is copied from _blackbox_common.sh
+function get_pubring_path() {
+  : "${KEYRINGDIR:=keyrings/live}" ;
+  if [[ -f "${KEYRINGDIR}/pubring.gpg" ]]; then
+    echo "${KEYRINGDIR}/pubring.gpg"
+  else
+    echo "${KEYRINGDIR}/pubring.kbx"
+  fi
+}
+
 function PHASE() {
   echo '********************'
   echo '********************'
@@ -11,7 +21,7 @@ function PHASE() {
 function md5sum_file() {
   # Portably generate the MD5 hash of file $1.
   case $(uname -s) in
-    Darwin )
+    Darwin | FreeBSD )
       md5 -r "$1" | awk '{ print $1 }'
       ;;
     Linux )
@@ -37,7 +47,7 @@ function assert_file_missing() {
 function assert_file_exists() {
   if [[ ! -e "$1" ]]; then
     echo "ASSERT FAILED: ${1} should exist."
-    echo "PWD=$(/bin/pwd -P)"
+    echo "PWD=$(/usr/bin/env pwd -P)"
     #echo "LS START"
     #ls -la
     #echo "LS END"