Update CHANGELOG.md

Add unit tests to tools/confidence_test.sh to verify admin operations work.

CHANGELOG.md

@@ -0,0 +1,26 @@
+Release v1.20170127
+
+* Starting CHANGELOG.
+
+
+Release v1.20170309
+
+* "make test" is an alias for "make confidence"
+* macOS: make_tempdir must create shorter paths
+* Fix "make confidence" for newer version of Git
+* README.md: Add info about our new mailing list
+
+Release v1.20170611
+
+* confidence_test.sh verifies external tools exist
+* confidence_test.sh more reliable for non-UTF8 users
+* "make test" no longer prompts for passwords
+* blackbox works better when target directory lives in root (#194)
+* Add confidence_test.sh tests for admin operations
+* blackbox_list_admins fails (#193)
+* confidence_test.sh works better on FreeBSD
+* tools/confidence_test.sh: now works with gnupg-2.0 and gnupg-2.1
+* Blackbox now officially supports both gnupg-2.0 and gnupg-2.1
+* blackbox_shred_all_files: BUGFIX: Does not shred files with spaces
+* blackbox_removeadmin: disable gpg's confirmation
+* Sync mk_rpm_fpmdir from master

Makefile

@@ -1,12 +1,15 @@
 SHELL=/bin/sh
 
 PKGNAME=stack_blackbox
+BASEDIR?=$(HOME)
+OUTPUTDIR?="$(BASEDIR)/debbuild-${PKGNAME}"
 
 all:
 	@echo 'Menu:'
 	@echo '  make update            Update any generated files'
 	@echo '  make packages          Make RPM packages'
 	@echo '  make packages-deb      Make DEB packages'
+	@echo '  make test              Run tests'
 	@echo '  make install           (incomplete)'
 
 install:
@@ -30,14 +33,14 @@ packages-rpm-debug:
 	@echo BUILD:
 	@PKGRELEASE=99 make packages
 	@echo ITEMS TO BE PACKAGED:
-	find ~/rpmbuild-$(PKGNAME)/installroot -type f
+	find $(BASEDIR)/rpmbuild-$(PKGNAME)/installroot -type f
 	@echo ITEMS ACTUALLY IN PACKAGE:
-	@rpm -qpl $$(cat ~/rpmbuild-$(PKGNAME)/bin-packages.txt)
+	@rpm -qpl $$(cat $(BASEDIR)/rpmbuild-$(PKGNAME)/bin-packages.txt)
 
 local-rpm:
 	@PKGRELEASE=1 make packages
 	-@sudo rpm -e $(PKGNAME)
-	sudo rpm -i $$(cat ~/rpmbuild-$(PKGNAME)/bin-packages.txt)
+	sudo rpm -i $$(cat $(BASEDIR)/rpmbuild-$(PKGNAME)/bin-packages.txt)
 
 lock-rpm:
 	sudo yum versionlock add $(PKGNAME)
@@ -63,7 +66,7 @@ manual-uninstall:
 #
 
 packages-deb:	tools/mk_deb_fpmdir.stack_blackbox.txt
-	cd tools && PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_deb_fpmdir stack_blackbox mk_deb_fpmdir.stack_blackbox.txt
+	cd tools && OUTPUTDIR=$(OUTPUTDIR) PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_deb_fpmdir stack_blackbox mk_deb_fpmdir.stack_blackbox.txt
 
 # Make mk_deb_fpmdir.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt:
 tools/mk_deb_fpmdir.stack_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt
@@ -75,12 +78,12 @@ packages-deb-debug:	tools/mk_deb_fpmdir.stack_blackbox.txt
 	@echo ITEMS TO BE PACKAGED:
 	find ~/debbuild-$(PKGNAME)/installroot -type f
 	@echo ITEMS ACTUALLY IN PACKAGE:
-	@dpkg --contents $$(cat ~/debbuild-$(PKGNAME)/bin-packages.txt)
+	@dpkg --contents $$(cat $(BASEDIR)/debbuild-$(PKGNAME)/bin-packages.txt)
 
 local-deb:
 	@PKGRELEASE=1 make packages
 	-@sudo dpkg -e $(PKGNAME)
-	sudo dpkg -i $$(cat ~/rpmbuild-$(PKGNAME)/bin-packages.txt)
+	sudo dpkg -i $$(cat $(BASEDIR)/rpmbuild-$(PKGNAME)/bin-packages.txt)
 
 #
 # MacPorts builds
@@ -120,10 +123,10 @@ clean:
 #
 # System Test:
 #
+test: confidence
 confidence:
 	@if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg should not exist. If it does, bugs may polute your .gnupg configuration. If the code has no bugs everything will be fine. Do you feel lucky?'; false ; fi
 	@if which >/dev/null gpg-agent ; then pkill gpg-agent ; rm -rf /tmp/tmp.* ; fi
-	@export PATH="$(PWD)/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:$(PATH)" ; tools/confidence_test.sh
-		tools/confidence_test.sh
+	@export PATH="$(PWD)/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:$(PATH)" ; tools/auto_system_test
 	@if which >/dev/null gpg-agent ; then pkill gpg-agent ; fi
 	@if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg was created which means the scripts might be poluting GnuPG configuration.  Fix this bug.'; false ; fi

README.md

@@ -5,6 +5,8 @@ Safely store secrets in a VCS repo (i.e. Git, Mercurial, Subversion or Perforce)
 
 A slide presentation about an older release [is on SlideShare](http://www.slideshare.net/TomLimoncelli/the-blackbox-project-sfae).
 
+Join our mailing list: [https://groups.google.com/d/forum/blackbox-project](https://groups.google.com/d/forum/blackbox-project)
+
 Table of Contents
 =================
 
@@ -29,6 +31,7 @@ Table of Contents
 - [Replace expired keys:](#replace-expired-keys)
 - [Some common errors:](#some-common-errors)
 - [Using Blackbox without a repo](#using-blackbox-without-a-repo)
+- [Some Subversion gotchas:](#some-subversion-gotchas)
 - [How to submit bugs or ask questions?](#how-to-submit-bugs-or-ask-questions)
 - [Developer Info](#developer-info)
 - [Alternatives](#alternatives)
@@ -55,12 +58,13 @@ OBVIOUSLY we don't want secret things like SSL private keys and passwords to be
 
 NOT SO OBVIOUSLY when we store "secrets" in a VCS repo like Git or Mercurial, suddenly we are less able to share our code with other people. Communication between subteams of an organization is hurt. You can't collaborate as well. Either you find yourself emailing individual files around (yuck!), making a special repo with just the files needed by your collaborators (yuck!!), or just deciding that collaboration isn't worth all that effort (yuck!!!).
 
-The ability to be open and transparent about our code, with the exception of a few specific files, is key to the kind of collaboration that DevOps and modern IT practitioniers need to do.
+The ability to be open and transparent about our code, with the exception of a few specific files, is key to the kind of collaboration that DevOps and modern IT practitioners need to do.
 
 Installation Instructions:
 ==========================
 
 -	*The MacPorts Way*: `sudo port install vcs_blackbox`
+-	*The Homebrew Way*: `brew install blackbox`
 -	*The RPM way*: Check out the repo and make an RPM via `make packages-rpm`; now you can distribute the RPM via local methods.
 -	*The Debian/Ubuntu way*: Check out the repo and install [fpm](https://github.com/jordansissel/fpm). Now you can make a DEB `make packages-deb` that can be distributed via local methods.
 -	*The hard way*: Copy all the files in "bin" to your "bin".
@@ -72,23 +76,24 @@ Commands:
 =========
 
 | Name:                               | Description:                                                            |
-|------------------------------|-------------------------------------------------------------------------|
-| `blackbox_edit`              | Decrypt, run $EDITOR, re-encrypt a file                                 |
-| `blackbox_edit_start`        | Decrypt a file so it can be updated                                     |
-| `blackbox_edit_end`          | Encrypt a file after blackbox_edit_start was used                       |
-| `blackbox_cat`               | Decrypt and view the contents of a file                                 |
+|-------------------------------------|-------------------------------------------------------------------------|
+| `blackbox_edit <file>`              | Decrypt, run $EDITOR, re-encrypt a file                                 |
+| `blackbox_edit_start <file>`        | Decrypt a file so it can be updated                                     |
+| `blackbox_edit_end <file>`          | Encrypt a file after blackbox_edit_start was used                       |
+| `blackbox_cat <file>`               | Decrypt and view the contents of a file                                 |
 | `blackbox_diff`                     | Diff decrypted files against their original crypted version             |
 | `blackbox_initialize`               | Enable blackbox for a GIT or HG repo                                    |
-| `blackbox_register_new_file` | Encrypt a file for the first time                                       |
-| `blackbox_deregister_file`   | Remove a file from blackbox                                             |
+| `blackbox_register_new_file <file>` | Encrypt a file for the first time                                       |
+| `blackbox_deregister_file <file>`   | Remove a file from blackbox                                             |
 | `blackbox_list_files`               | List the files maintained by blackbox                                   |
+| `blackbox_list_admins`              | List admins currently authorized for blackbox                           |
 | `blackbox_decrypt_all_files`        | Decrypt all managed files (INTERACTIVE)                                 |
 | `blackbox_postdeploy`               | Decrypt all managed files (batch)                                       |
-| `blackbox_addadmin`          | Add someone to the list of people that can encrypt/decrypt secrets      |
-| `blackbox_removeadmin`       | Remove someone from the list of people that can encrypt/decrypt secrets |
+| `blackbox_addadmin <gpg-key>`       | Add someone to the list of people that can encrypt/decrypt secrets      |
+| `blackbox_removeadmin <gpg-key>`    | Remove someone from the list of people that can encrypt/decrypt secrets |
 | `blackbox_shred_all_files`          | Safely delete any decrypted files                                       |
 | `blackbox_update_all_files`         | Decrypt then re-encrypt all files. Useful after keys are changed        |
-| `blackbox_whatsnew`          | show what has changed in the last commit for a given file               |
+| `blackbox_whatsnew <file>`          | show what has changed in the last commit for a given file               |
 
 Compatibility:
 ==============
@@ -105,6 +110,7 @@ Blackbox automatically determines which VCS you are using and does the right thi
 	-	CentOS / RedHat
 	-	MacOS X
 	-	Cygwin (Thanks, Ben Drasin!)
+	-	MinGW (git bash on windows) **See Note Below**
 
 To add or fix support for a VCS system, look for code at the end of `bin/_blackbox_common.sh`
 
@@ -120,6 +126,17 @@ Note: Cywin support requires the following packages:
 	-	make
 	-	git (the confidence test currently only tests git)
 
+Note: MinGW (comes with Git for Windows) support requires the following additional installations
+-	Normal operation:
+	-	[Git for Windows](https://git-scm.com/) (not tested with Mercurial)
+		-	Git Bash MINTTY returns a MinGW console.  So when you install make sure you pick `MINTTY` instead of windows console.  You'll be executing blackbox from the Git Bash prompt.
+		-	You need at least version 2.8.1 of Git for Windows.
+	-	[GnuWin32](https://sourceforge.net/projects/getgnuwin32/files/) - needed for various tools not least of which is mktemp which is used by blackbox
+		-	after downloading the install just provides you with some batch files.  Because of prior issues at sourceforge and to make sure you get the latest version of each package the batch files handle the brunt of the work of getting the correct packages and installing them for you.
+		-	from a **windows command prompt** run `download.bat`  once it has completed run `install.bat` then add the path for those tools to your PATH (ex: `PATH=%PATH%;c:\GnuWin32\bin`)
+-	Development: 
+	-	unknown
+
 How is the encryption done?
 ===========================
 
@@ -602,6 +619,23 @@ Any files that were temporarily copied in the first step so as to not be overwri
 
 (Thanks to @chishaku for finding a solution to this problem!)
 
+### Configure git to show diffs in encrypted files
+
+It's possible to tell Git to decrypt versions of the file before running them through `git diff` or `git log`. To achieve this do:
+
+- Add the following to `.gitattributes` at the top of the git repository:
+```
+*.gpg diff=blackbox
+```
+
+- Add the following to `.git/config`:
+```
+[diff "blackbox"]
+    textconv = gpg --use-agent -q --batch --decrypt
+````
+
+And now commands like `git log -p file.gpg` will show a nice log of the changes in the encrypted file.
+
 Some common errors:
 ===================
 
@@ -622,11 +656,22 @@ The following commands have been tested outside a repo:
 -	`blackbox_edit_start`
 -	`blackbox_edit_end`
 
+Some Subversion gotchas:
+========================
+
+The current implementation will store the blackbox in `/keyrings` at the root of the entire repo.  this will create an issue between environments that have different roots (ie, checking out `/` on development vs `/releases/foo` in production).  To get around this, you can `export BLACKBOX_REPOBASE=/path/to/repo` and set a specific base for your repo.
+
+This was originally written for git and supports a two-phase commit, in which `commit` is a local commit and "push" sends the change upstream to the version control server when something is registered or deregistered with the system.  The current implementation will immediately `commit` a file (to the upstream subversion server) when you execute a `blackbox_*` command.
+
 How to submit bugs or ask questions?
 ====================================
 
 We welcome questions, bug reports and feedback!
 
+The best place to start is to join the [blackbox-project mailing list](https://groups.google.com/d/forum/blackbox-project) and ask there.
+
+Bugs are tracked here in Github. Please feel free to files bugs yourself:
+
 -	https://github.com/StackExchange/blackbox/issues
 
 Developer Info
@@ -663,6 +708,7 @@ Here are other open source packages that do something similar to Blackbox. If yo
 -	Pass: http://www.zx2c4.com/projects/password-store/
 -	Transcrypt: https://github.com/elasticdog/transcrypt
 -	Keyringer: https://keyringer.pw/
+-	git-secret: https://github.com/sobolevn/git-secret
 
 git-crypt has the best git integration. Once set up it is nearly transparent to the users. However it only works with git.
 

RELEASE_ENGINEERING.md

@@ -27,7 +27,23 @@ Stable Releases
 
 Marking the software to be "stable":
 
-Step 1. Tag it.
+Step 1.  Update CHANGELOG.md
+
+Use "git log" to see what has changed and update CHANGELOG.md.
+
+For a new release, add:
+
+```
+   echo Release v1.$(date +%Y%m%d)
+```
+
+Commit with:
+
+```
+git commit -m'Update CHANGELOG.md' CHANGELOG.md
+```
+
+Step 2. Tag it.
 
 ```
 git pull
@@ -37,7 +53,7 @@ git tag stable
 git push origin tag stable
 ```
 
-Step 2. Mark your calendar 1 week from today to check to see if this should be promoted to production.
+Step 3. Mark your calendar 1 week from today to check to see if this should be promoted to production.
 
 Production Releases
 ===================

Version2-Ideas.md

@@ -0,0 +1,97 @@
+# Ideas for blackbox Version 2
+
+I'm writing this to solicit feedback and encourage discussion.
+
+Here are my thoughts on a "verison 2" of blackbox.  This is where
+I list ideas that would require major changes to the system. They
+might break backwards compatibility, though usually not.
+
+Blackbox grew from a few simple shell scripts used at StackOverflow.com
+to a larger system used by dozens (hundreds?) of organizations. Not
+all the design decisions were "forward looking".
+
+These are the things I'd like to change someday.
+
+[TOC]
+
+## Change the commmand names
+
+There should be one program, with subcommands that have names that make more sense:
+
+* `blackbox init`
+* `blackbox register <filename> <...>`
+* `blackbox deregister <filename> <...>`
+* `blackbox edit <filename> <...>`
+* `blackbox decrypt <filename> <...>`
+* `blackbox encrypt <filename> <...>`
+* `blackbox decrypt_all`
+* `blackbox addadmin <key>`
+* `blackbox removeadmin <key>`
+* `blackbox cat <filename> <...>`
+* `blackbox diff <filename> <...>`
+* `blackbox list_files`
+* `blackbox list_admins`
+* `blackbox shred_all`
+* `blackbox update_all`
+* `blackbox whatsnew`
+
+Backwards compatibility: The old commands would simply call the new commands.
+
+## Change the "keyrings" directory
+
+The name "keyrings" was unfortunate.  First, it should probably begin with a ".".  Second, it stores more than just keyrings.  Lastly, I'm finding that in most cases we want many repos to refer to the same keyring, which is not supported very well.
+
+A better system would be:
+
+1. If `$BLACKBOX_CONFIG` is set, use that directory.
+2. If the repo base directory has a file called ".blackbox_external", read that file as if you are reading `$BLACKBOX_CONFIG`
+3. If the repo base directory has a "keyrings" directory, use that.
+4. If the repo base directory has a ".blackboxconfig" directory, use that.
+
+Some thoughts on .blackbox_external:
+I'm not sure what the format should be, but I want it to be simple and expandable.  It should support support "../../dir/name" and "/long/path".  However some day we may want to include a Git URL and have the system automatically get the keychain from it. That means the format has to be something like directory:../dir/name so that later we can add git:the_url.
+
+
+Backwards compatibility: "keyrings" would be checked before .blackbox
+
+## Repo-less mode
+
+I can't imagine storing files that aren't in a repo. I just put everything in repos lately. I use it more than I use NFS.  That said, I have received feedback that people would like the ability to disable automatic committing of files.
+
+I prefer the file commits to be automatic because when they were manual, people often accidentally committed the plaintext file instead of the GPG file.  Fixing such mistakes is a PITA and, of yeah, a big security nightmare.
+
+That said, I'm willing to have a "repo-less" mode.
+
+When this mode is triggered, no add/commit/ignore tasks are done.  The search for the keyrings directory still uses `$BLACKBOX_CONFIG` but if that is unset it looks for .blackbox_config in the current directory, then recursively ".." until we hit "/".
+
+I think (but I'm not sure) this would benefit the entire system because it would force us to re-think what VCS actions are done when.
+
+I think (but I'm not sure) that a simple way to implement this would be to add an environment variable that overrides the automatic VCS detection. When set to "none", all VCS operations would basically become no-ops.  (This could be done by writing a plug-in that does nothing for all the vcs_* calls)
+
+Backwards compatibility: This would add a "none" VCS, not remove any existing functionality.
+
+
+## Is "bash" the right language?
+
+`bash` is fairly universal. It even exists on Windows.  However it is not the right language for large systems. Writing the acceptance tests is quite a bear.  Managing ".gitignore" files in bash is impossible and the current implementation fails in many cases.
+
+`python` is my second favorite langauge. It would make the code cleaner and more testable. However it is not installed everywhere.  I would also want to write it in Python3 (why start a new project in Python2?) but sadly Python3 is less common.  It is a chicken vs. egg situation.
+
+`go` is my favorite language. I could probably rewrite this in go in a weekend. However, now the code is compiled, not interpreted. Therefore we lose the ability to just "git clone" and have the tools you want.  Not everyone has a Go compiler installed on every machine.
+
+The system is basically unusable on Windows without Cygwin or MINGW.  A rewrite in python or go would make it work better on Windows, which currently requires Cygwin or MinGW (which is a bigger investment than installing Python). On the other hand, maybe Ubuntu-on-Windows makes that a non-issue.
+
+As long as the code is in `bash` the configuration files like `blackbox-files.txt` and `blackbox-admins.txt` have problems.  Filenames with carriage returns aren't supported.  If this was in Python/Go/etc. those files could be json or some format with decent quoting and we could handle funny file names better. On the other hand, maybe it is best that we don't support funny filenames... we shouldn't enable bad behavior.
+
+How important is itto blackbox users that the system is written in "bash"?
+
+
+## ditch the project and use git-crypt
+
+People tell me that git-crypt is better because, as a plug-in, automagically supports "git diff", "git log" and "git blame".
+
+However, I've never used it so I don't have any idea whether git-crypt is any better than blackbox.
+
+Of course, git-crypt doesn't work with SVN, HG, or any other VCS.  Is blackbox's strong point the fact that it support so many VCS systems?  To be honest, it originally only supported HG and GIT because I was at a company that used HG but then changed to GIT.  Supporting anything else was thanks to contributors. Heck, HG support hasn't even been tested recently (by me) since we've gone all git where I work.
+
+How important is this to blackbox users?

bin/_blackbox_common.sh

@@ -28,7 +28,7 @@ source "${0%/*}"/_stack_lib.sh
 function physical_directory_of() {
   local d=$(dirname "$1")
   local f=$(basename "$1")
-  (cd "$d" && echo "$(pwd -P)/$f" )
+  (cd "$d" && echo "$(pwd -P | sed 's/\/$//')/$f" )
 }
 
 # Set REPOBASE to the top of the repository
@@ -62,6 +62,11 @@ export REPOBASE=$(physical_directory_of "$REPOBASE")
 # FIXME: Verify this function by checking for .hg or .git
 # after determining what we believe to be the answer.
 
+if [[ -n "$BLACKBOX_REPOBASE" ]]; then
+	echo "Using custom repobase: $BLACKBOX_REPOBASE"
+	export REPOBASE="$BLACKBOX_REPOBASE"
+fi
+
 KEYRINGDIR="$REPOBASE/$BLACKBOXDATA"
 BB_ADMINS_FILE="blackbox-admins.txt"
 BB_ADMINS="${KEYRINGDIR}/${BB_ADMINS_FILE}"
@@ -158,8 +163,16 @@ function get_encrypted_filename() {
 
 # Prepare keychain for use.
 function prepare_keychain() {
+  local keyringasc
   echo '========== Importing keychain: START' >&2
-  $GPG --import "$(get_pubring_path)" 2>&1 | egrep -v 'not changed$' >&2
+  # Works with gpg 2.0
+  #$GPG --import "$(get_pubring_path)" 2>&1 | egrep -v 'not changed$' >&2
+  # Works with gpg 2.0 and 2.1
+  # NB: We must export the keys to a format that can be imported.
+  make_self_deleting_tempfile keyringasc
+  export LANG="C.UTF-8"
+  $GPG --export --keyring "$(get_pubring_path)" >"$keyringasc"
+  $GPG --import "$keyringasc"
   echo '========== Importing keychain: DONE' >&2
 }
 
@@ -272,7 +285,11 @@ function shred_file() {
     #NOTE: srm by default uses 35-pass Gutmann algorithm
     CMD=srm
     OPT=-f
+  elif _F=$(mktemp); rm -P "${_F}" >/dev/null 2>/dev/null ; then
+    CMD=rm
+    OPT=-Pf
   else
+    echo "shred_file: WARNING: No secure deletion utility (shred or srm) present; using insecure rm" >&2
     CMD=rm
     OPT=-f
   fi
@@ -368,7 +385,7 @@ function file_contains_line() {
 function md5sum_file() {
   # Portably generate the MD5 hash of file $1.
   case $(uname -s) in
-    Darwin )
+    Darwin | FreeBSD )
       md5 -r "$1" | awk '{ print $1 }'
       ;;
     Linux | CYGWIN* | MINGW* )
@@ -387,6 +404,9 @@ function cp_permissions() {
     Darwin )
       chmod $( stat -f '%p' "$1" ) "${@:2}"
       ;;
+    FreeBSD )
+      chmod $( stat -f '%p' "$1" | sed -e "s/^100//" ) "${@:2}"
+      ;;
     Linux | CYGWIN* | MINGW* )
       chmod --reference "$1" "${@:2}"
       ;;
@@ -566,7 +586,7 @@ function vcs_ignore_git() {
 }
 # Subversion
 function vcs_ignore_svn() {
-  svn propset svn:ignore "$(vcs_relative_path "$file")"
+  svn propset svn:ignore "$file" "$(vcs_relative_path)"
 }
 # Perforce
 function vcs_ignore_p4() {
@@ -634,3 +654,20 @@ function vcs_notice_generic_file() {
     echo "WARNING:  If so, manually update the ignore file"
   fi
 }
+
+function gpg_agent_version_check() {
+  if ! hash 'gpg-agent' &> /dev/null; then
+    return 1
+  fi
+  local gpg_agent_version=$(gpg-agent --version | head -1 | awk '{ print $3 }' | tr -d '\n')
+  semverLT $gpg_agent_version "2.1.0"
+}
+
+function gpg_agent_notice() {
+  if [[ $(gpg_agent_version_check) == '0' && -z $GPG_AGENT_INFO ]];then
+    echo 'WARNING: You probably want to run gpg-agent as'
+    echo 'you will be asked for your passphrase many times.'
+    echo 'Example: $ eval $(gpg-agent --daemon)'
+    read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
+  fi
+}

bin/_stack_lib.sh

@@ -53,8 +53,8 @@ function create_self_deleting_tempfile() {
   local filename
 
   case $(uname -s) in
-    Darwin )
-      : "${TMPDIR:=/tmp}"
+    Darwin | FreeBSD )
+      : "${TMPDIR:=/tmp}" ;
       filename=$(mktemp -t _stacklib_.XXXXXXXX )
       ;;
     Linux | CYGWIN* | MINGW* )
@@ -74,8 +74,8 @@ function create_self_deleting_tempdir() {
   local filename
 
   case $(uname -s) in
-    Darwin )
-      : "${TMPDIR:=/tmp}"
+    Darwin | FreeBSD )
+      : "${TMPDIR:=/tmp}" ;
       filename=$(mktemp -d -t _stacklib_.XXXXXXXX )
       ;;
     Linux | CYGWIN* | MINGW* )
@@ -98,8 +98,8 @@ function make_self_deleting_tempfile() {
   local name
 
   case $(uname -s) in
-    Darwin )
-      : "${TMPDIR:=/tmp}"
+    Darwin | FreeBSD )
+      : "${TMPDIR:=/tmp}" ;
       name=$(mktemp -t _stacklib_.XXXXXXXX )
       ;;
     Linux | CYGWIN* | MINGW* )
@@ -120,9 +120,12 @@ function make_tempdir() {
   local name
 
   case $(uname -s) in
-    Darwin )
-      : "${TMPDIR:=/tmp}"
-      name=$(mktemp -d -t _stacklib_.XXXXXXXX )
+    Darwin | FreeBSD )
+      : "${TMPDIR:=/tmp}" ;
+      # The full path to the temp directory must be short.
+      # This is used by blackbox's testing suite to make a fake GNUPGHOME,
+      # which needs to fit within sockaddr_un.sun_path (see unix(7)).
+      name=$(mktemp -d -t SO )
       ;;
     Linux | CYGWIN* | MINGW* )
       name=$(mktemp -d)
@@ -157,7 +160,7 @@ function fail_if_not_running_as_root() {
 function fail_if_in_root_directory() {
   # Verify nobody has tricked us into being in "/".
   case $(uname -s) in
-    Darwin )
+    Darwin | FreeBSD )
       if [[ $(stat -f'%i' / ) == $(stat -f'%i' . ) ]] ; then
         echo 'SECURITY ALERT: The current directory is the root directory.'
         echo 'Exiting...'
@@ -177,3 +180,109 @@ function fail_if_in_root_directory() {
       ;;
   esac
 }
+
+function semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+function semverEQ() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -ne $MAJOR_B ]; then
+        return 1
+    fi
+
+    if [ $MINOR_A -ne $MINOR_B ]; then
+        return 1
+    fi
+
+    if [ $PATCH_A -ne $PATCH_B ]; then
+        return 1
+    fi
+
+    if [[ "_$SPECIAL_A" != "_$SPECIAL_B" ]]; then
+        return 1
+    fi
+
+
+    return 0
+
+}
+
+function semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -lt $MINOR_B ]]; then
+        return 0
+    fi
+
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -le $MINOR_B && $PATCH_A -lt $PATCH_B ]]; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  != "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  != "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A" < "_$SPECIAL_B" ]]; then
+        return 0
+    fi
+
+    return 1
+
+}
+
+function semverGT() {
+    semverEQ $1 $2
+    local EQ=$?
+
+    semverLT $1 $2
+    local LT=$?
+
+    if [ $EQ -ne 0 ] && [ $LT -ne 0 ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+

bin/blackbox_decrypt_all_files

@@ -14,9 +14,7 @@
 export PATH=/usr/bin:/bin:"$PATH"
 
 set -e
+source "${0%/*}/_blackbox_common.sh"
 
-if [[ -z $GPG_AGENT_INFO ]]; then
-  eval $(gpg-agent --daemon)
-fi
-
+gpg_agent_notice
 exec blackbox_postdeploy "$@"

bin/blackbox_diff

@@ -7,13 +7,7 @@
 set -e
 source "${0%/*}/_blackbox_common.sh"
 
-if [[ -z $GPG_AGENT_INFO ]]; then
-  echo 'WARNING: You probably want to run gpg-agent as'
-  echo 'you will be asked for your passphrase many times.'
-  echo 'Example: $ eval $(gpg-agent --daemon)'
-  read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
-fi
-
+gpg_agent_notice
 prepare_keychain
 
 modified_files=()

bin/blackbox_edit_end

@@ -9,6 +9,11 @@ source "${0%/*}/_blackbox_common.sh"
 
 next_steps=()
 
+if [ $# -eq 0 ]; then
+  echo >&2 "Please provide at least one file for which editing has finished"
+  exit 1
+fi
+
 for param in "$@" ; do
 
   unencrypted_file=$(get_unencrypted_filename "$param")

bin/blackbox_edit_start

@@ -7,6 +7,11 @@
 set -e
 source "${0%/*}/_blackbox_common.sh"
 
+if [ $# -eq 0 ]; then
+  echo >&2 "Please provide at least one file to start editing"
+  exit 1
+fi
+
 for param in "$@" ; do
 
   unencrypted_file=$(get_unencrypted_filename "$param")

bin/blackbox_initialize

@@ -19,10 +19,15 @@ if [[ $1 != 'yes' ]]; then
   fi
 fi
 
+if [[ $VCS_TYPE = "unknown" ]]; then
+	echo 'Not in a known VCS directory'
+	exit 1
+fi
+
 change_to_vcs_root
 
 echo VCS_TYPE: $VCS_TYPE
-vcs_ignore keyrings/live/pubring.gpg~ keyrings/live/pubring.kbx~ keyrings/live/secring.gpg
+vcs_ignore "${BLACKBOXDATA}/pubring.gpg~" "${BLACKBOXDATA}/pubring.kbx~" "${BLACKBOXDATA}/secring.gpg"
 
 # Make directories
 mkdir -p "${KEYRINGDIR}"
@@ -30,11 +35,17 @@ vcs_add "${KEYRINGDIR}"
 touch "$BLACKBOXDATA/$BB_ADMINS_FILE" "$BLACKBOXDATA/$BB_FILES_FILE"
 vcs_add "$BLACKBOXDATA/$BB_ADMINS_FILE" "$BLACKBOXDATA/$BB_FILES_FILE"
 
-IGNOREFILE="$(vcs_ignore_file_path)"
-test -f "$IGNOREFILE" && vcs_add "$IGNOREFILE"
+if [[ $VCS_TYPE = "svn" ]]; then
+	echo
+	echo
+	echo '`subversion` automatically tracks the ignored files; you just need to commit.'
+else
+	IGNOREFILE="$(vcs_ignore_file_path)"
+	test -f "$IGNOREFILE" && vcs_add "$IGNOREFILE"
 
-# Make a suggestion:
-echo
-echo
-echo 'NEXT STEP: You need to manually check these in:'
-echo '     ' $VCS_TYPE commit -m\'INITIALIZE BLACKBOX\' keyrings "$IGNOREFILE"
+	# Make a suggestion:
+	echo
+	echo
+	echo 'NEXT STEP: You need to manually check these in:'
+	echo '     ' $VCS_TYPE commit -m\'INITIALIZE BLACKBOX\' keyrings "$IGNOREFILE"
+fi

bin/blackbox_list_admins

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_list_admins -- List authorized admins
+#
+set -e
+source "${0%/*}/_blackbox_common.sh"
+cat "$BB_ADMINS"

bin/blackbox_listadmins

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_listadmins -- List active admins for keyring
+#
+
+# Example:
+#    blackbox_listadmins
+#
+
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+fail_if_not_in_repo
+
+
+# simply display the contents of the admins file
+cat "$BB_ADMINS"

bin/blackbox_register_new_file

@@ -36,7 +36,7 @@ function register_new_file() {
   echo "========== UPDATING REPO:"
   shred_file "$unencrypted_file"
 
-  if "$SECRETSEXPOSED" ; then
+  if [[ "$SECRETSEXPOSED" == "true" ]] ; then
     vcs_remove "$unencrypted_file"
     vcs_add "$encrypted_file"
   fi
@@ -52,5 +52,9 @@ for target in "$@"; do
 done
 
 echo "========== UPDATING VCS: DONE"
-echo "Local repo updated.  Please push when ready."
-echo "    $VCS_TYPE push"
+if [[ $VCS_TYPE = "svn" ]]; then
+	echo "Local repo updated and file pushed to source control (unless an error was displayed)."
+else
+	echo "Local repo updated.  Please push when ready."
+	echo "    $VCS_TYPE push"
+fi

bin/blackbox_removeadmin

@@ -20,8 +20,15 @@ KEYNAME="$1"
 # Remove the email address from the BB_ADMINS file.
 remove_line "$BB_ADMINS" "$KEYNAME"
 
+
+# remove the admin key from the pubring
+$GPG --no-permission-warning --homedir="$KEYRINGDIR" --batch --yes --delete-key "$KEYNAME" || true
+pubring_path=$(get_pubring_path)
+vcs_add "$pubring_path" "$KEYRINGDIR/trustdb.gpg" "$BB_ADMINS"
+
+
 # Make a suggestion:
 echo
 echo
 echo 'NEXT STEP: Check these into the repo.  Probably with a command like...'
-echo $VCS_TYPE commit -m\'REMOVED ADMIN: $KEYNAME\' "$BLACKBOXDATA/$BB_ADMINS_FILE"
+echo $VCS_TYPE commit -m\'REMOVED ADMIN: $KEYNAME\' "$BLACKBOXDATA/trustdb.gpg" "$BLACKBOXDATA/$BB_ADMINS_FILE"

bin/blackbox_shred_all_files

@@ -21,13 +21,24 @@ source "${0%/*}/_blackbox_common.sh"
 change_to_vcs_root
 
 echo '========== FILES BEING SHREDDED:'
-while IFS= read <&99 -r unencrypted_file; do
-  unencrypted_file=$(get_unencrypted_filename "$unencrypted_file")
-  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
+
+exported_internal_shred_file() {
+  source "$1/_blackbox_common.sh"
+  unencrypted_file="$2"
   if [[ -f "$unencrypted_file" ]]; then
-    echo "    $unencrypted_file"
+    echo "    SHRED: $unencrypted_file"
     shred_file "$unencrypted_file"
+  else
+    echo "NOT FOUND: $unencrypted_file"
   fi
-done 99<"$BB_FILES"
+}
+
+export -f exported_internal_shred_file
+
+DEREFERENCED_BIN_DIR="${0%/*}"
+MAX_PARALLEL_SHRED=10
+
+export IFS=
+tr '\n' '\0' <"$BB_FILES" | xargs -0 -I{} -n 1 -P $MAX_PARALLEL_SHRED bash -c "exported_internal_shred_file $DEREFERENCED_BIN_DIR \"{}\"" $DEREFERENCED_BIN_DIR/fake
 
 echo '========== DONE.'

bin/blackbox_update_all_files

@@ -7,13 +7,7 @@
 set -e
 source "${0%/*}/_blackbox_common.sh"
 
-if [[ -z $GPG_AGENT_INFO ]]; then
-  echo 'WARNING: You probably want to run gpg-agent as'
-  echo 'you will be asked for your passphrase many times.'
-  echo 'Example: $ eval $(gpg-agent --daemon)'
-  read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
-fi
-
+gpg_agent_notice
 disclose_admins
 prepare_keychain
 

bin/blackbox_whatsnew

@@ -14,13 +14,7 @@ then
 fi
 
 fail_if_not_in_repo
-
-if [[ -z $GPG_AGENT_INFO ]]; then
-  echo 'WARNING: You probably want to run gpg-agent as'
-  echo 'you will be asked for your passphrase many times.'
-  echo 'Example: $ eval $(gpg-agent --daemon)'
-  read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
-fi
+gpg_agent_notice
 
 COLUMNS=`tput cols`
 FILE=$1

tools/auto_system_test

@@ -0,0 +1,25 @@
+#!/usr/bin/env expect
+
+# Run the confidence test non-interactively.  Since the script
+# asks for passphrases, we use "expect" to simulate keyboard data entry.
+
+# Run the test:
+spawn tools/confidence_test.sh
+
+# As we run the confidence test, respond with the right password.
+# We do this for up to 300 times to prevent an infinite loop.
+
+set times 0;
+while { $times < 300 } {
+   expect {
+       # The script outputs what the password will be, and we save
+       # that info in $pw any time we see the text.
+       "my password is the lowercase letter a" { set pw "a\n" ; exp_continue }
+       "my password is the lowercase letter b" { set pw "b\n" ; exp_continue }
+       # If the passphrase is requested, send it.
+       "Passphrase:" { send $pw ; exp_continue }
+       # If we reach EOF, exit this loop.
+       eof { break }
+   }
+   set times [ expr $times+1];
+}

tools/confidence_test.sh

@@ -1,7 +1,27 @@
 #!/usr/bin/env bash
 
 blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../bin
-export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin"
+export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:${blackbox_home}"
+
+export LANG=C.UTF-8  # Required ro "gpg --export" to work properly.
+
+# This script requires many utilities, some are not
+# required by the usual blackbox scripts.  Test to make
+# sure we have them all.
+e=false
+for i in blackbox_addadmin blackbox_list_admins blackbox_register_new_file \
+  cat git gpg gpg-agent mkdir pinentry pinentry-tty rm tar which ; do
+  if ! which >/dev/null 2>&1  $i ; then
+    echo ERROR: Command not in PATH: $i
+    e=true
+  fi
+done
+if $e ; then
+  echo 'Exiting. Please install the above commands.'
+  echo 'This script requires many utilities not required by blackbox itself.'
+  echo PATH="$PATH"
+  exit 1
+fi
 
 set -e
 . _stack_lib.sh
@@ -19,10 +39,15 @@ cd "$test_repository"
 make_self_deleting_tempdir fake_alice_home
 make_self_deleting_tempdir fake_bob_home
 export GNUPGHOME="$fake_alice_home"
+#echo 'pinentry-program' "$(which pinentry-insecure-fake)" >> "$GNUPGHOME/gpg-agent.conf"
+echo 'pinentry-program' "$(which pinentry-tty)" >> "$GNUPGHOME/gpg-agent.conf"
+#pinentry-program "${blackbox_home}/pinentry-fake-insecure"
 eval "$(gpg-agent --homedir "$fake_alice_home" --daemon)"
 GPG_AGENT_INFO_ALICE="$GPG_AGENT_INFO"
 
 export GNUPGHOME="$fake_bob_home"
+#echo 'pinentry-program' "$(which pinentry-insecure-fake)" >> "$GNUPGHOME/gpg-agent.conf"
+echo 'pinentry-program' "$(which pinentry-tty)" >> "$GNUPGHOME/gpg-agent.conf"
 eval "$(gpg-agent --homedir "$fake_bob_home" --daemon)"
 GPG_AGENT_INFO_BOB="$GPG_AGENT_INFO"
 
@@ -31,6 +56,7 @@ function become_alice() {
   export GPG_AGENT_INFO="$GPG_AGENT_INFO_ALICE"
   echo BECOMING ALICE: GNUPGHOME="$GNUPGHOME AGENT=$GPG_AGENT_INFO"
   mkdir -p .git ; touch .git/config
+  git init
   git config user.name "Alice Example"
   git config user.email alice@example.com
 }
@@ -46,7 +72,6 @@ function become_bob() {
 PHASE 'Alice creates a repo.  She creates secret.txt.'
 
 become_alice
-git init
 echo 'this is my secret' >secret.txt
 
 
@@ -82,7 +107,11 @@ git commit -m'INITIALIZE BLACKBOX' keyrings .gitignore
 PHASE 'and adds herself as an admin.'
 
 blackbox_addadmin alice@example.com
-git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.gpg keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.??? keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+
+make_self_deleting_tempfile adminlist1
+blackbox_list_admins >"$adminlist1"
+assert_file_md5hash "$adminlist1" "aa1db827772e1d51d453b844394b7617"
 
 
 PHASE 'Bob arrives.'
@@ -110,14 +139,18 @@ gpg --no-permission-warning --batch --gen-key "$gpgconfig"
 echo '========== Bob enrolls himself too.'
 
 blackbox_addadmin bob@example.com
-git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.gpg keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.??? keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+
+make_self_deleting_tempfile adminlist2
+blackbox_list_admins >"$adminlist2"
+assert_file_md5hash "$adminlist2" "0b82b0b3c96e6e5dd5faf60493fe5cf7"
 
 PHASE 'Alice does the second part to enroll bob.'
 become_alice
 
 PHASE 'She enrolls bob.'
-gpg --import keyrings/live/pubring.gpg
-# TODO(tlim) That --import can be eliminated... maybe?
+#gpg --import $(get_pubring_path)
+## TODO(tlim) That --import can be eliminated... maybe?
 
 PHASE 'She enrolls secrets.txt.'
 blackbox_register_new_file secret.txt
@@ -185,33 +218,30 @@ assert_file_md5hash secret.txt "d3e6bbdfc76fae7fd0a921f3408db1d1"
 PHASE 'appears.'
 become_bob
 
-PHASE 'Bob makes sure he has all new keys.'
-
-gpg --import keyrings/live/pubring.gpg
+#PHASE 'Bob makes sure he has all new keys.'
+#gpg --import keyrings/live/pubring.???
 
 # Pick a GID to use:
 # This users's default group:
-DEFAULT_GID_NAME=$(id -gn)
+DEFAULT_GID_NUM=$(id -g)
 # Pick a group that is not the default group:
-TEST_GID_NUM=$(id -G | fmt -1 | tail -n +2 | grep -xv "$(id -u)" | head -n 1)
-TEST_GID_NAME=$(python -c 'import grp; print grp.getgrgid('"$TEST_GID_NUM"').gr_name')
-echo "DEFAULT_GID_NAME=$DEFAULT_GID_NAME"
+TEST_GID_NUM=$(id -G | fmt -1 | sort -rn | grep -xv "$(id -u)" | grep -xv "$(id -g)" | head -1)
+echo "DEFAULT_GID_NUM=$DEFAULT_GID_NUM"
 echo "TEST_GID_NUM=$TEST_GID_NUM"
-echo "TEST_GID_NAME=$TEST_GID_NAME"
 
 PHASE 'Bob postdeploys... default.'
 blackbox_postdeploy
 assert_file_exists secret.txt
 assert_file_exists secret.txt.gpg
 assert_file_md5hash secret.txt "08a3fa763a05c018a38e9924363b97e7"
-assert_file_group secret.txt "$DEFAULT_GID_NAME"
+assert_file_group secret.txt "$DEFAULT_GID_NUM"
 
 PHASE 'Bob postdeploys... with a GID.'
 blackbox_postdeploy "$TEST_GID_NUM"
 assert_file_exists secret.txt
 assert_file_exists secret.txt.gpg
 assert_file_md5hash secret.txt "08a3fa763a05c018a38e9924363b97e7"
-assert_file_group secret.txt "$TEST_GID_NAME"
+assert_file_group secret.txt "$TEST_GID_NUM"
 
 PHASE 'Bob cleans up the secret.'
 rm secret.txt
@@ -220,6 +250,10 @@ PHASE 'Bob removes Alice.'
 blackbox_removeadmin alice@example.com
 assert_line_not_exists 'alice@example.com' keyrings/live/blackbox-admins.txt
 
+make_self_deleting_tempfile adminlist3
+blackbox_list_admins >"$adminlist3"
+assert_file_md5hash "$adminlist3" "aadbfafd76ea66ff40dbfd239a69067f"
+
 PHASE 'Bob reencrypts files so alice can not access them.'
 blackbox_update_all_files
 
@@ -316,6 +350,7 @@ PHASE 'Bob shreds all exposed files.'
 assert_file_exists 'my/path/to/relsecrets.txt'
 assert_file_exists 'secret.txt'
 blackbox_shred_all_files
+which blackbox_shred_all_files
 assert_file_missing '!important!.txt'
 assert_file_missing '#andpounds.txt'
 assert_file_missing 'mistake.txt'
@@ -393,6 +428,10 @@ fi
 # Cleanup:
 blackbox_removeadmin abba@notarealuser.com
 
+make_self_deleting_tempfile adminlist4
+blackbox_list_admins >"$adminlist4"
+assert_file_md5hash "$adminlist4" "aadbfafd76ea66ff40dbfd239a69067f"
+
 
 # TODO: Create a new directory. "git clone" the repo into it.
 

tools/mk_deb_fpmdir

@@ -22,13 +22,27 @@ shift
 # Defaults that can be overridden:
 # All packages are 1.0 unless otherwise specifed:
 : ${PKGVERSION:=1.0} ;
-# If there is no iteration setting, assume "1":
-: ${PKGRELEASE:=1}
+# If there is no iteration set, default to use the number of commits in the repository:
+if [[ -z "${PKGRELEASE}" ]]; then
+  PKGRELEASE=$(git rev-list HEAD --count)
+  if [[ $? != 0 ]]; then
+    # We're not in a git repo, fall back to 1 so we cope with being built from
+    # a tarball
+    PKGRELEASE=1
+  fi
+fi
+
 # If there is no epoch, assume 0
 : ${PKGEPOCH:=0}
 
-# The DEB is output here: (should be a place that can be wiped)
-OUTPUTDIR="${HOME}/debbuild-$PACKAGENAME"
+# Allow us to set a different OUTPUTDIR if we're building in CI/CD
+if [[ -z "${OUTPUTDIR}" ]]; then
+  # The DEB is output here: (should be a place that can be wiped)
+  OUTPUTDIR="${HOME}/debbuild-${PACKAGENAME}"
+else
+  echo "Using $OUTPUTDIR for OUTPUTDIR instead of ${HOME}/debbuild-${PACKAGENAME}"
+fi
+
 # The TeamCity templates expect to find the list of artifacts here:
 DEB_BIN_LIST="${OUTPUTDIR}/bin-packages.txt"
 
@@ -36,6 +50,7 @@ DEB_BIN_LIST="${OUTPUTDIR}/bin-packages.txt"
 
 # Clean the output dir.
 rm -rf "$OUTPUTDIR"
+
 mkdir -p "$OUTPUTDIR/installroot"
 
 # Copy the files into place:

tools/mk_deb_fpmdir.stack_blackbox.txt

@@ -9,7 +9,9 @@ exec /usr/bin/blackbox_edit             ../bin/blackbox_edit
 exec /usr/bin/blackbox_edit_end         ../bin/blackbox_edit_end
 exec /usr/bin/blackbox_edit_start       ../bin/blackbox_edit_start
 exec /usr/bin/blackbox_initialize       ../bin/blackbox_initialize
+exec /usr/bin/blackbox_listadmins       ../bin/blackbox_listadmins
 exec /usr/bin/blackbox_list_files       ../bin/blackbox_list_files
+exec /usr/bin/blackbox_list_admins      ../bin/blackbox_list_admins
 exec /usr/bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
 exec /usr/bin/blackbox_register_new_file ../bin/blackbox_register_new_file
 exec /usr/bin/blackbox_removeadmin      ../bin/blackbox_removeadmin

tools/mk_macports.vcs_blackbox.txt

@@ -9,7 +9,9 @@ exec bin/blackbox_edit             ../bin/blackbox_edit
 exec bin/blackbox_edit_end         ../bin/blackbox_edit_end
 exec bin/blackbox_edit_start       ../bin/blackbox_edit_start
 exec bin/blackbox_initialize       ../bin/blackbox_initialize
+exec bin/blackbox_listadmins       ../bin/blackbox_listadmins
 exec bin/blackbox_list_files       ../bin/blackbox_list_files
+exec bin/blackbox_list_admins      ../bin/blackbox_list_admins
 exec bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
 exec bin/blackbox_register_new_file ../bin/blackbox_register_new_file
 exec bin/blackbox_removeadmin      ../bin/blackbox_removeadmin

tools/mk_rpm_fpmdir

@@ -19,28 +19,78 @@ set -e
 PACKAGENAME=${1?"First arg must be the package name."}
 shift
 
+# What is my name?
+CMDNAME=$(basename $0)
+
 # Defaults that can be overridden:
-# All packages are 1.0 unless otherwise specifed:
+# Packages are 1.0 unless otherwise specifed:
 : ${PKGVERSION:=1.0} ;
-# If there is no iteration setting, assume "1":
-: ${PKGRELEASE:=1}
+# If there is no iteration set, default to use the number of commits in the repository:
+if [[ -z "${PKGRELEASE}" ]]; then
+  PKGRELEASE=$(git rev-list HEAD --count)
+  if [[ $? != 0 ]]; then
+    # We're not in a git repo, fall back to 1 so we cope with being built from
+    # a tarball
+    PKGRELEASE=1
+  fi
+fi
+
 # If there is no epoch, assume 0
 : ${PKGEPOCH:=0}
 
-# The RPM is output here: (should be a place that can be wiped)
-OUTPUTDIR="${HOME}/rpmbuild-$PACKAGENAME"
-# The TeamCity templates expect to find the list of artifacts here:
+# If no arch defined, assume any. Other good values include "native".
+: ${PKGARCH:=all}
+# NOTE: If we later compile code, we set this to "native", which
+# FPM will translate to the correct value for local conditions.
+
+# Allow us to set a different OUTPUTDIR if we're building in CI/CD
+if [[ -z "${OUTPUTDIR}" ]]; then
+  # The RPM is output here: (should be a place that can be wiped)
+  OUTPUTDIR="${HOME}/rpmbuild-${PACKAGENAME}"
+else
+  echo "Using ${OUTPUTDIR} for OUTPUTDIR instead of ${HOME}/rpmbuild-${PACKAGENAME}"
+fi
+INSTALLROOT="$OUTPUTDIR/installroot"
+
+# StackOverflow's TeamCity templates expect to find the list of artifacts here:
 RPM_BIN_LIST="${OUTPUTDIR}/bin-packages.txt"
 
 # -- Now the real work can be done.
 
 # Clean the output dir.
 rm -rf "$OUTPUTDIR"
-mkdir -p "$OUTPUTDIR/installroot"
+mkdir -p "$INSTALLROOT"
 
-# Copy the files into place:
+# If there is a build script, execute it.
+BUILDSCRIPTNAME="./build.${PACKAGENAME}.sh"
+if [[ -x $BUILDSCRIPTNAME ]]; then
+  echo "========== $BUILDSCRIPTNAME FOUND. Running."
+  if [[ $PKGARCH == "all" ]]; then
+    echo 'WARNING: PKGARCH=all (which may not what you want)'
+    # If you are using a build.*.sh script, you probably want to
+    # set PKGARCH to "native" before you run mk_rpm_fpmdir.
+  fi
+  $BUILDSCRIPTNAME "$INSTALLROOT" "${PKGVERSION}"
+  # If we used the build build.*.sh script, it must do all compilation.
+  # Therefore, we disable the automagic GO build feature.
+  GO_COMPILE=false
+else
+  GO_COMPILE=true
+fi
+
+# If there are additional args for fpm, read them into a variable.  There is
+# a chdir later, therefore we can't rely on the file path working at that time.
+FPM_OPTIONS_FILE="./fpm_opts.${PACKAGENAME}.sh"
+if [[ -f $FPM_OPTIONS_FILE ]]; then
+  echo "========== $FPM_OPTIONS_FILE FOUND. Loading."
+  FPM_OPTIONS=$(<$FPM_OPTIONS_FILE)
+fi
+# Warning: The contents of the file are evaluated therefore
+# quotes and special chars must be quoted.
+
+# Copy any static files into place:
 set -o pipefail  # Error out if any manifest is not found.
-cat """$@""" | while read -a arr ; do
+cat "$@" | while read -a arr ; do
   PERM="${arr[0]}"
   case $PERM in
     \#*)  continue ;;   # Skip comments.
@@ -48,24 +98,39 @@ cat """$@""" | while read -a arr ; do
     read) PERM=0744 ;;
     *) ;;
   esac
-  DST="$OUTPUTDIR/installroot/${arr[1]}"
+  DST="$INSTALLROOT/${arr[1]}"
   SRC="${arr[2]}"
-  if [[ $SRC == "cmd/"* || $SRC == *"/cmd/"* ]]; then
-    ( cd $(dirname "$SRC" ) && go build -a -v ) 
+  if [[ ${#arr[@]} != 3 ]] ; then
+    echo "ERROR: Line must contain 3 items."
+    echo "DEBUG NUM=${#arr[@]} PERM=$PERM DST=$DST SRC=$SRC"
+    exit 1
+  fi
+  if $GO_COMPILE && [[ $SRC == "cmd/"* || $SRC == *"/cmd/"* ]]; then
+    echo "========== BUILD&COPY $SRC"
+    ( cd $(dirname "$SRC" ) && go get -d && go build ) 
+    PKGARCH=native
+  else
+    echo "==========       COPY $SRC"
+  fi
+  if [[ ! -f "$SRC" ]]; then
+    echo "${CMDNAME}: ERROR: File not found: $SRC"
+    exit 1
   fi
   install -D -T -b -m "$PERM" -T "$SRC" "$DST"
 done
 
-# Build the RPM:
+set -x
+# Build the RPM out of what is found in $INSTALLROOT:
 cd "$OUTPUTDIR" && fpm -s dir -t rpm \
-  -a all \
+  -a "${PKGARCH}" \
   -n "${PACKAGENAME}" \
   --epoch "${PKGEPOCH}" \
   --version "${PKGVERSION}" \
   --iteration "${PKGRELEASE}" \
   ${PKGDESCRIPTION:+ --description="${PKGDESCRIPTION}"} \
   ${PKGVENDOR:+ --vendor="${PKGVENDOR}"} \
-  -C "$OUTPUTDIR/installroot" \
+  ${FPM_OPTIONS:+ $FPM_OPTIONS} \
+  -C "$INSTALLROOT" \
   .
 
 # TeamCity templates for RPMS expect to find

tools/mk_rpm_fpmdir.stack_blackbox.txt

@@ -11,7 +11,9 @@ exec /usr/blackbox/bin/blackbox_edit             ../bin/blackbox_edit
 exec /usr/blackbox/bin/blackbox_edit_end         ../bin/blackbox_edit_end
 exec /usr/blackbox/bin/blackbox_edit_start       ../bin/blackbox_edit_start
 exec /usr/blackbox/bin/blackbox_initialize       ../bin/blackbox_initialize
+exec /usr/blackbox/bin/blackbox_listadmins       ../bin/blackbox_listadmins
 exec /usr/blackbox/bin/blackbox_list_files       ../bin/blackbox_list_files
+exec /usr/blackbox/bin/blackbox_list_admins      ../bin/blackbox_list_admins
 exec /usr/blackbox/bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
 exec /usr/blackbox/bin/blackbox_register_new_file ../bin/blackbox_register_new_file
 exec /usr/blackbox/bin/blackbox_removeadmin      ../bin/blackbox_removeadmin

tools/test_functions.sh

@@ -1,5 +1,15 @@
 #!/usr/bin/env bash
 
+# NB: This is copied from _blackbox_common.sh
+function get_pubring_path() {
+  : "${KEYRINGDIR:=keyrings/live}" ;
+  if [[ -f "${KEYRINGDIR}/pubring.gpg" ]]; then
+    echo "${KEYRINGDIR}/pubring.gpg"
+  else
+    echo "${KEYRINGDIR}/pubring.kbx"
+  fi
+}
+
 function PHASE() {
   echo '********************'
   echo '********************'
@@ -11,7 +21,7 @@ function PHASE() {
 function md5sum_file() {
   # Portably generate the MD5 hash of file $1.
   case $(uname -s) in
-    Darwin )
+    Darwin | FreeBSD )
       md5 -r "$1" | awk '{ print $1 }'
       ;;
     Linux )
@@ -63,10 +73,10 @@ function assert_file_group() {
 
   case $(uname -s) in
     Darwin|FreeBSD )
-      found=$(stat -f '%Sg' "$file")
+      found=$(stat -f '%Dg' "$file")
       ;;
     Linux )
-      found=$(stat -c '%G' "$file")
+      found=$(stat -c '%g' "$file")
       ;;
     CYGWIN* )
       echo "ASSERT_FILE_GROUP: Running on Cygwin. Not being tested."
@@ -78,8 +88,10 @@ function assert_file_group() {
       ;;
   esac
 
+  echo "DEBUG: assert_file_group X${wanted}X vs. X${found}X"
+  echo "DEBUG:" $(which stat)
   if [[ "$wanted" != "$found" ]]; then
-    echo "ASSERT FAILED: $file chgrp wanted=$wanted found=$found"
+    echo "ASSERT FAILED: $file chgrp group wanted=$wanted found=$found"
     exit 1
   fi
 }
@@ -103,8 +115,10 @@ function assert_file_perm() {
       ;;
   esac
 
+  echo "DEBUG: assert_file_perm X${wanted}X vs. X${found}X"
+  echo "DEBUG:" $(which stat)
   if [[ "$wanted" != "$found" ]]; then
-    echo "ASSERT FAILED: $file chgrp wanted=$wanted found=$found"
+    echo "ASSERT FAILED: $file chgrp perm wanted=$wanted found=$found"
     exit 1
   fi
 }