merge

- _blackbox_common.sh sets the default Blackbox directory
  for the new repositories using the first entry of the
  BLACKBOX_CANDIDATES array. This small change sets the
  first entry to the new .blackbox dir (instead of the keyring/live)

.circleci/config.yml

@@ -0,0 +1,56 @@
+version: 2
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - debian
+      - ubuntu
+
+jobs:
+
+  debian:
+    docker:
+      - image: debian:9.1
+    steps:
+      - checkout
+      - run:
+          name: 'Installing'
+          command: |
+            apt-get update -y
+            apt-get install -y build-essential expect git gnupg2 pinentry-tty procps rpm ruby-dev libffi-dev
+            gem install fpm
+      - run:
+          name: 'Cleaning'
+          command: |
+            rm -rf ~/.gpnupg
+            make clean
+      - run:
+          name: 'Testing'
+          command: |
+            GPG=gpg2 make test
+            make packages-deb
+            make packages-rpm
+
+  ubuntu:
+    docker:
+      - image: ubuntu:16.04
+    steps:
+      - checkout
+      - run:
+          name: 'Installing'
+          command: |
+            apt-get update -y
+            apt-get install -y build-essential expect git gnupg2 pinentry-tty procps rpm ruby-dev libffi-dev
+            gem install fpm
+      - run:
+          name: 'Cleaning'
+          command: |
+            rm -rf ~/.gpnupg
+            make clean
+      - run:
+          name: 'Testing'
+          command: |
+            GPG=gpg2 make test
+            make packages-deb
+            make packages-rpm

.gitattributes

@@ -0,0 +1,5 @@
+bin/** text eol=lf
+tools/** text eol=lf
+Makefile text eol=lf
+Portfile text eol=lf
+blackbox.plugin.zsh text eol=lf

CHANGELOG.md

@@ -0,0 +1,63 @@
+Release v1.20180618
+
+* Restore `make manual-install` with warning. (#258)
+
+Release v1.20180615
+
+* Standardize on .blackbox for config. Use keyrings/live for backwards compatibility.
+* Store keys in .blackbox directory (#218)
+* Suggest committing changes to pubring.gpg when running blackbox_removeadmin (#248)
+* Fix typo (#246)
+* Improve installation instructions (#244)
+* Fix replacing-expired-keys link in README (#241)
+* Fix problems when gpg2 is installed next to gpg (#237)
+* Many documentation corrections, updates, etc.
+* Exclude default keyring from import (#223)
+* .gitattributes not always updated (PR#146)
+* Fix bugs related to updating .gitattributes (PR#146)
+* Update readme with CircleCI link (#216)
+* Run the tests on a CI (#215)
+* Fixed Alpine compatibility (chmod) (#212)
+* direct repobase message to stderr (#204)
+* Improve Windows compatibility
+* NEW: .gitattributes Set Unix-only files to eol=lf
+* Silence 'not changed' output during keychain import (#200)
+* Improve FreeBSD compatibility
+* shred_file() outputs warning message to stderr. (#192)
+* Don't complain about GPG_AGENT_INFO if using newer gpg-agent (#189)
+* [FreeBSD] Fix use of chmod (#180)
+* Requiring a file to be entered to finish editing (#175)
+* Remove the key from the keyring when removing an admin (#173)
+* Add FreeBSD support (#172)
+* Add list admins commandline tool. (#170)
+ignore backup files and secring.gpg in $BLACKBOXDATA (#169)
+Allow parallel shredding of files (#167)
+* Add/improve Mingw support
+* Make "make confidence" less fragile
+* And a lot, lot more.
+
+Release v1.20170309
+
+* "make test" is an alias for "make confidence"
+* macOS: make_tempdir must create shorter paths
+* Fix "make confidence" for newer version of Git
+* README.md: Add info about our new mailing list
+
+Release v1.20170611
+
+* confidence_test.sh verifies external tools exist
+* confidence_test.sh more reliable for non-UTF8 users
+* "make test" no longer prompts for passwords
+* blackbox works better when target directory lives in root (#194)
+* Add confidence_test.sh tests for admin operations
+* blackbox_list_admins fails (#193)
+* confidence_test.sh works better on FreeBSD
+* tools/confidence_test.sh: now works with gnupg-2.0 and gnupg-2.1
+* Blackbox now officially supports both gnupg-2.0 and gnupg-2.1
+* blackbox_shred_all_files: BUGFIX: Does not shred files with spaces
+* blackbox_removeadmin: disable gpg's confirmation
+* Sync mk_rpm_fpmdir from master
+
+Release v1.20170127
+
+* Starting CHANGELOG.

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014-2015 Stack Exchange, Inc.
+Copyright (c) 2014-2018 Stack Exchange, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,20 +1,105 @@
 SHELL=/bin/sh
 
 PKGNAME=stack_blackbox
+BASEDIR?=$(HOME)
+OUTPUTDIR?="$(BASEDIR)/debbuild-${PKGNAME}"
 
 all:
 	@echo 'Menu:'
-	@echo '  make packages          Make RPM packages'
-	@echo '  make packages-deb      Make DEB packages'
-	@echo '  make install           (incomplete)'
+	@echo '  make update             Update any generated files'
+	@echo '  make packages-rpm       Make RPM packages'
+	@echo '  make packages-deb       Make DEB packages'
+	@echo '  make symlinks-install   Make symlinks in /usr/local/bin/'
+	@echo '  make copy-install       Copy "bin" files to /usr/local/bin/'
+	@echo '  make usrlocal-uninstall Remove blackbox files from /usr/local/bin/'
+	@echo '  make test               Run tests'
 
 install:
 	@echo 'To install, copy the files from bin to somewhere in your PATH.'
-	@echo 'Or, if you use RPMs, "make packages" and install the result.'
+	@echo 'The README.md document gives more details.'
+	@echo 'Or run "make" (with no options) for more info.'
 
 # The default package type is RPM.
 packages: packages-rpm
 
+#
+# RPM builds
+#
+
+# NOTE: mk_rpm_fpmdir.stack_blackbox.txt is the master list of files.  All
+# other packages should generate their list from it.
+
+packages-rpm:
+	cd tools && PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_rpm_fpmdir stack_blackbox mk_rpm_fpmdir.stack_blackbox.txt
+
+packages-rpm-debug:
+	@echo BUILD:
+	@PKGRELEASE=99 make packages
+	@echo ITEMS TO BE PACKAGED:
+	find $(BASEDIR)/rpmbuild-$(PKGNAME)/installroot -type f
+	@echo ITEMS ACTUALLY IN PACKAGE:
+	@rpm -qpl $$(cat $(BASEDIR)/rpmbuild-$(PKGNAME)/bin-packages.txt)
+
+local-rpm:
+	@PKGRELEASE=1 make packages
+	-@sudo rpm -e $(PKGNAME)
+	sudo rpm -i $$(cat $(BASEDIR)/rpmbuild-$(PKGNAME)/bin-packages.txt)
+
+lock-rpm:
+	sudo yum versionlock add $(PKGNAME)
+
+unlock-rpm:
+	sudo yum versionlock clear
+
+#
+# Manual install
+#
+symlinks-install:
+	@echo 'Symlinking files from ./bin to /usr/local/bin'
+	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do ln -fs `pwd`/$$f /usr/local/bin/$$f; done
+	@echo 'Done.'
+
+manual-install:
+	@echo '***************************************************************'
+	@echo '*                      DEPRECATED                             *'
+	@echo '* `make manual-install` is now called `make symlinks-install` *'
+	@echo '***************************************************************'
+	$(MAKE) symlinks-install
+
+copy-install:
+	@echo 'Copying files from ./bin to /usr/local/bin'
+	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do cp `pwd`/$$f /usr/local/bin/$$f; done
+	@echo 'Done.'
+
+usrlocal-uninstall:
+	@echo 'Removing blackbox files from /usr/local/bin'
+	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do rm /usr/local/bin/$$f; done
+	@echo 'Done.'
+
+#
+# DEB builds
+#
+
+packages-deb:	tools/mk_deb_fpmdir.stack_blackbox.txt
+	cd tools && OUTPUTDIR=$(OUTPUTDIR) PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_deb_fpmdir stack_blackbox mk_deb_fpmdir.stack_blackbox.txt
+
+# Make mk_deb_fpmdir.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt:
+tools/mk_deb_fpmdir.stack_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt
+	sed -e '/^#/d' -e 's@/usr/blackbox/bin/@/usr/bin/@g' -e '/profile.d-usrblackbox.sh/d' <tools/mk_rpm_fpmdir.stack_blackbox.txt >$@
+
+packages-deb-debug:	tools/mk_deb_fpmdir.stack_blackbox.txt
+	@echo BUILD:
+	@PKGRELEASE=99 make packages-deb
+	@echo ITEMS TO BE PACKAGED:
+	find ~/debbuild-$(PKGNAME)/installroot -type f
+	@echo ITEMS ACTUALLY IN PACKAGE:
+	@dpkg --contents $$(cat $(BASEDIR)/debbuild-$(PKGNAME)/bin-packages.txt)
+
+local-deb:
+	@PKGRELEASE=1 make packages
+	-@sudo dpkg -e $(PKGNAME)
+	sudo dpkg -i $$(cat $(BASEDIR)/rpmbuild-$(PKGNAME)/bin-packages.txt)
+
 #
 # MacPorts builds
 #
@@ -23,72 +108,40 @@ packages: packages-rpm
 
 # Make mk_macports.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt:
 tools/mk_macports.vcs_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt
-	sed -e 's@/usr/blackbox/bin/@bin/@g' -e '/profile.d-usrblackbox.sh/d' <tools/mk_rpm_fpmdir.stack_blackbox.txt >$@
-
-# Make mk_deb_fpmdir.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt:
-tools/mk_deb_fpmdir.stack_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt
-	sed -e 's@/usr/blackbox/bin/@/usr/bin/@g' -e '/profile.d-usrblackbox.sh/d' <tools/mk_deb_fpmdir.stack_blackbox.txt >$@
+	sed  -e '/^#/d' -e 's@/usr/blackbox/bin/@bin/@g' -e '/profile.d-usrblackbox.sh/d' <tools/mk_rpm_fpmdir.stack_blackbox.txt >$@
 
 # MacPorts expects to run: make packages-macports DESTDIR=${destroot}
 packages-macports: tools/mk_macports.vcs_blackbox.txt
 	mkdir -p $(DESTDIR)/bin
 	cd tools && ./mk_macports mk_macports.vcs_blackbox.txt
 
-#
-# RPM builds
-#
-
-packages-rpm:
-	cd tools && PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_rpm_fpmdir stack_blackbox mk_rpm_fpmdir.stack_blackbox.txt
-
-packages-rpm-debug:
-	@echo BUILD:
-	@PKGRELEASE=99 make packages
-	@echo ITEMS TO BE PACKAGED:
-	find ~/rpmbuild-$(PKGNAME)/installroot -type f
-	@echo ITEMS ACTUALLY IN PACKAGE:
-	@rpm -qpl $$(cat ~/rpmbuild-$(PKGNAME)/bin-packages.txt)
-
-local-rpm:
-	@PKGRELEASE=1 make packages
-	-@sudo rpm -e $(PKGNAME)
-	sudo rpm -i $$(cat ~/rpmbuild-$(PKGNAME)/bin-packages.txt)
-
-lock-rpm:
-	sudo yum versionlock add $(PKGNAME)
-
-unlock-rpm:
-	sudo yum versionlock clear
-
-#
-# DEB builds
-#
-
-packages-deb:	tools/mk_deb_fpmdir.stack_blackbox.txt
-	cd tools && PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_deb_fpmdir stack_blackbox mk_deb_fpmdir.stack_blackbox.txt
-
-packages-deb-debug:	tools/mk_deb_fpmdir.stack_blackbox.txt
-	@echo BUILD:
-	@PKGRELEASE=99 make packages-deb
-	@echo ITEMS TO BE PACKAGED:
-	find ~/debbuild-$(PKGNAME)/installroot -type f
-	@echo ITEMS ACTUALLY IN PACKAGE:
-	@dpkg --contents $$(cat ~/debbuild-$(PKGNAME)/bin-packages.txt)
-
-local-deb:
-	@PKGRELEASE=1 make packages
-	-@sudo dpkg -e $(PKGNAME)
-	sudo dpkg -i $$(cat ~/rpmbuild-$(PKGNAME)/bin-packages.txt)
+# stow is a pretty easy way to manage simple local installs on GNU systems
+install-stow:
+	mkdir -p /usr/local/stow/blackbox/bin
+	cp bin/* /usr/local/stow/blackbox/bin
+	rm /usr/local/stow/blackbox/bin/Makefile
+	cd /usr/local/stow; stow -R blackbox
+uninstall-stow:
+	cd /usr/local/stow; stow -D blackbox
+	rm -rf /usr/local/stow/blackbox
 
 # Add other package types here.
 
+#
+# Updates
+#
+update: tools/mk_deb_fpmdir.stack_blackbox.txt tools/mk_macports.vcs_blackbox.txt
+
+clean:
+	rm -f tools/mk_deb_fpmdir.stack_blackbox.txt tools/mk_macports.vcs_blackbox.txt
+
 #
 # System Test:
 #
+test: confidence
 confidence:
 	@if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg should not exist. If it does, bugs may polute your .gnupg configuration. If the code has no bugs everything will be fine. Do you feel lucky?'; false ; fi
 	@if which >/dev/null gpg-agent ; then pkill gpg-agent ; rm -rf /tmp/tmp.* ; fi
-	@export PATH="$(PWD)/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:$(PATH)" ; tools/confidence_test.sh
-		tools/confidence_test.sh
+	@export PATH="$(PWD)/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:$(PATH)" ; tools/auto_system_test
 	@if which >/dev/null gpg-agent ; then pkill gpg-agent ; fi
 	@if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg was created which means the scripts might be poluting GnuPG configuration.  Fix this bug.'; false ; fi

RELEASE_ENGINEERING.md

@@ -0,0 +1,153 @@
+Table of Contents:
+==================
+
+-	[Branches and Tags:](#branches-and-tags)
+-	[Build Tasks](#build-tasks)
+-	[Stable Releases](#stable-releases)
+-	[Production Releases](#production-releases)
+-	[Updating MacPorts (automatic)](#updating-macports-automatic)
+-	[Updating MacPorts (manual)](#updating-macports-manual)
+
+Branches and Tags:
+==================
+
+There are 3 branches/tags:
+
+-	**HEAD:** The cutting edge of development.
+-	**tag stable:** Stable enough for use by most people.
+-	**tag production:** Burned in long enough that we are confident it can be widely adopted.
+
+If you are packaging BlackBox for distribution, you should track the *tag production*. You might also want to provide a separate package that tracks *tag stable:* for early adopters.
+
+Build Tasks
+===========
+
+Stable Releases
+===============
+
+Marking the software to be "stable":
+
+Step 1.  Update CHANGELOG.md
+
+Use "git log" to see what has changed and update CHANGELOG.md.
+
+For a new release, add:
+
+```
+   echo Release v1.$(date +%Y%m%d)
+```
+
+Commit with:
+
+```
+git commit -m'Update CHANGELOG.md' CHANGELOG.md
+```
+
+Step 2. Tag it.
+
+```
+git pull
+git tag -d stable
+git push origin :stable
+git tag stable
+git push origin tag stable
+```
+
+Step 3. Mark your calendar 1 week from today to check to see if this should be promoted to production.
+
+Production Releases
+===================
+
+If no bugs have been reported a full week after a stable tag has been pushed, mark the release to be "production".
+
+```
+git fetch
+git checkout stable
+git tag -d production
+git push origin :production
+git tag production
+git push origin tag production
+R="v1.$(date +%Y%m%d)"
+git tag "$R"
+git push origin tag "$R"
+```
+
+Updating MacPorts (automatic)
+=============================
+
+Step 1: Generate the Portfile
+
+```
+tools/macports_report_upgrade.sh  1.20150222
+```
+
+This script will generate a file called `Portfile-vcs_blackbox.diff` and instructions on how to submit it as a update request.
+
+Step 2: Submit the update request.
+
+Submit the diff file as a bug as instructed. The instructions should look like this:
+
+-	PLEASE OPEN A TICKET WITH THIS INFORMATION: https://trac.macports.org/newticket
+-	Summary: `vcs_blackbox @1.20150222 Update to latest upstream`
+-	Description: `New upstream of vcs_blackbox.
+	github.setup and checksums updated.`
+-	Type: `update`
+-	Component: `ports`
+-	Port: `vcs_blackbox`
+-	Keywords: `maintainer haspatch`
+-	Attach this file: `Portfile-vcs_blackbox.diff`
+
+Step 3: Watch for the update to happen.
+
+
+Updating MacPorts (manual)
+==========================
+
+This is the old, manual, procedure. If the automated procedure fails to work, these notes may or may not be helpful.
+
+The ultimate result of the script should be the output of `diff -u Portfile.orig Portfile` which is sent as an attachment to MacPorts. The new `Portfile` should have these changes:
+
+1.	The `github.setup` line should have a new version number.
+2.	The `checksums` line(s) should have updated checksums.
+
+How to generate the checksums?
+
+The easiest way is to to make a Portfile with incorrect checksums, then run `sudo port -v checksum vcs_blackbox` to see what they should have been. Fix the file, and try again until the checksum command works.
+
+Next run `port lint vcs_blackbox` and make sure it has no errors.
+
+Some useful commands:
+
+Change repos in sources.conf:
+
+```
+sudo vi /opt/local/etc/macports/sources.conf
+  Add this line early in the file:
+  file:///var/tmp/ports
+```
+
+Add a local repo:
+
+```
+fgrep >/dev/null -x 'file:///var/tmp/ports' /opt/local/etc/macports/sources.conf || sudo sed -i -e '1s@^@file:///var/tmp/ports\'$'\n@' /opt/local/etc/macports/sources.conf
+```
+
+Remove the local repo:
+
+```
+sudo sed -i -e '\@^file:///var/tmp/ports@d' /opt/local/etc/macports/sources.conf
+```
+
+Test a Portfile:
+
+```
+sudo port uninstall vcs_blackbox
+sudo port clean --all vcs_blackbox
+rm -rf ~/.macports/opt/local/var/macports/sources/rsync.macports.org/release/tarballs/ports/security/vcs_blackbox/
+rm -rf /var/tmp/ports
+mkdir -p /var/tmp/ports/security/vcs_blackbox
+cp Portfile /var/tmp/ports/security/vcs_blackbox
+cd /var/tmp/ports && portindex
+sudo port -v checksum vcs_blackbox
+sudo port install vcs_blackbox
+```

RELEASE_ENGINEERING.txt

@@ -1,119 +0,0 @@
-# Branches and Tags:
-
-There are 3 branches/tags:
-
-* **HEAD:** The cutting edge of development.
-* **tag stable:** Stable enough for use by most people.
-* **tag production:** Burned in long enough that we are confident it can be widely adopted.
-
-If you are packaging Blackbox for distribution, you should track the *tag production*.  You might also want to provide a separate package that tracks *tag stable:* for early adopters.
-
-# Build Tasks
-
-# Stable Releases
-
-Marking the software to be "stable":
-
-Step 1. Tag it.
-
-```
-git pull
-git tag -d stable
-git push origin :stable
-git tag stable
-git push origin tag stable
-```
-
-Step 2. Mark your calendar 1 week from today to check
-to see if this should be promoted to production.
-
-
-# Production Releases
-
-If no bugs have been reported a full week after a stable tag has been pushed, mark the release to be "production".
-
-```
-git fetch
-git checkout stable
-git tag -d production
-git push origin :production
-git tag production
-git push origin tag production
-R="v1.$(date +%Y%m%d)"
-git tag "$R"
-git push origin tag "$R"
-```
-
-# Updating MacPorts (automatic)
-
-Step 1: Generate the Portfile
-
-```
-tools/macports_report_upgrade.sh  1.20150222
-```
-
-This script will generate a file called `Portfile-vcs_blackbox.diff` and instructions on how to submit it as a update request.
-
-Step 2: Submit the update request.
-
-Submit the diff file as a bug as instructed. The instructions should look like this:
-
-* PLEASE OPEN A TICKET WITH THIS INFORMATION:
-    https://trac.macports.org/newticket
-* Summary: `vcs_blackbox @1.20150222 Update to latest upstream`
-* Description: ```New upstream of vcs_blackbox.
-github.setup and checksums updated.```
-* Type: `update`
-* Component: `ports`
-* Port: `vcs_blackbox`
-* Keywords: `maintainer`
-* Attach this file: `Portfile-vcs_blackbox.diff`
-
-Step 3: Watch for the update to happen.
-
-# Updating MacPorts (manual)
-
-This is the old, manual, procedure.  If the automated procedure work, these notes may or may not be helpful.
-
-The ultimate result of the script should be a `diff -u Portfile.orig Portfile`.  The new `Portfile` should have these changes:
-
-1. The `github.setup` line should have a new version number.
-2. The `checksums` line(s) should have updated checksums.
-
-How to generate the checksums?
-
-The easiest way is to to make a Portfile with incorrect checksums, then run `sudo port -v checksum vcs_blackbox` to see what they should have been.  Fix the file, and try again.
-
-When the the checksum command works, run `port lint vcs_blackbox` and make sure it has no errors.
-
-Some useful commands:
-
-Change repos in sources.conf:
-```
-sudo vi /opt/local/etc/macports/sources.conf
-  Add this line early in the file:
-  file:///var/tmp/ports
-```
-
-Add a local repo in an automated manner:
-```
-fgrep >/dev/null -x 'file:///var/tmp/ports' /opt/local/etc/macports/sources.conf || sudo sed -i -e '1s@^@file:///var/tmp/ports\'$'\n@' /opt/local/etc/macports/sources.conf
-```
-
-Remove the local repo in an automated manner:
-```
-sudo sed -i -e '\@^file:///var/tmp/ports@d' /opt/local/etc/macports/sources.conf
-```
-
-Test a Portfile:
-``` 
-sudo port uninstall vcs_blackbox
-sudo port clean --all vcs_blackbox
-rm -rf ~/.macports/opt/local/var/macports/sources/rsync.macports.org/release/tarballs/ports/security/vcs_blackbox/
-rm -rf /var/tmp/ports
-mkdir -p /var/tmp/ports/security/vcs_blackbox
-cp Portfile /var/tmp/ports/security/vcs_blackbox
-cd /var/tmp/ports && portindex
-sudo port -v checksum vcs_blackbox
-sudo port install vcs_blackbox
-```

Version2-Ideas.md

@@ -0,0 +1,97 @@
+# Ideas for BlackBox Version 2
+
+I'm writing this to solicit feedback and encourage discussion.
+
+Here are my thoughts on a "version 2" of BlackBox.  This is where
+I list ideas that would require major changes to the system. They
+might break backwards compatibility, though usually not.
+
+BlackBox grew from a few simple shell scripts used at StackOverflow.com
+to a larger system used by dozens (hundreds?) of organizations. Not
+all the design decisions were "forward looking".
+
+These are the things I'd like to change someday.
+
+[TOC]
+
+## Change the commmand names
+
+There should be one program, with subcommands that have names that make more sense:
+
+* `blackbox init`
+* `blackbox register <filename> <...>`
+* `blackbox deregister <filename> <...>`
+* `blackbox edit <filename> <...>`
+* `blackbox decrypt <filename> <...>`
+* `blackbox encrypt <filename> <...>`
+* `blackbox decrypt_all`
+* `blackbox addadmin <key>`
+* `blackbox removeadmin <key>`
+* `blackbox cat <filename> <...>`
+* `blackbox diff <filename> <...>`
+* `blackbox list_files`
+* `blackbox list_admins`
+* `blackbox shred_all`
+* `blackbox update_all`
+* `blackbox whatsnew`
+
+Backwards compatibility: The old commands would simply call the new commands.
+
+## Change the "keyrings" directory
+
+The name "keyrings" was unfortunate.  First, it should probably begin with a ".".  Second, it stores more than just keyrings.  Lastly, I'm finding that in most cases we want many repos to refer to the same keyring, which is not supported very well.
+
+A better system would be:
+
+1. If `$BLACKBOX_CONFIG` is set, use that directory.
+2. If the repo base directory has a file called ".blackbox_external", read that file as if you are reading `$BLACKBOX_CONFIG`
+3. If the repo base directory has a "keyrings" directory, use that.
+4. If the repo base directory has a ".blackboxconfig" directory, use that.
+
+Some thoughts on .blackbox_external:
+I'm not sure what the format should be, but I want it to be simple and expandable.  It should support support "../../dir/name" and "/long/path".  However some day we may want to include a Git URL and have the system automatically get the keychain from it. That means the format has to be something like directory:../dir/name so that later we can add git:the_url.
+
+
+Backwards compatibility: "keyrings" would be checked before .blackbox
+
+## Repo-less mode
+
+I can't imagine storing files that aren't in a repo. I just put everything in repos lately. I use it more than I use NFS.  That said, I have received feedback that people would like the ability to disable automatic committing of files.
+
+I prefer the file commits to be automatic because when they were manual, people often accidentally committed the plaintext file instead of the GPG file.  Fixing such mistakes is a PITA and, of yeah, a big security nightmare.
+
+That said, I'm willing to have a "repo-less" mode.
+
+When this mode is triggered, no add/commit/ignore tasks are done.  The search for the keyrings directory still uses `$BLACKBOX_CONFIG` but if that is unset it looks for .blackbox_config in the current directory, then recursively ".." until we hit "/".
+
+I think (but I'm not sure) this would benefit the entire system because it would force us to re-think what VCS actions are done when.
+
+I think (but I'm not sure) that a simple way to implement this would be to add an environment variable that overrides the automatic VCS detection. When set to "none", all VCS operations would basically become no-ops.  (This could be done by writing a plug-in that does nothing for all the vcs_* calls)
+
+Backwards compatibility: This would add a "none" VCS, not remove any existing functionality.
+
+
+## Is "bash" the right language?
+
+`bash` is fairly universal. It even exists on Windows.  However it is not the right language for large systems. Writing the acceptance tests is quite a bear.  Managing ".gitignore" files in bash is impossible and the current implementation fails in many cases.
+
+`python` is my second favorite language. It would make the code cleaner and more testable. However it is not installed everywhere.  I would also want to write it in Python3 (why start a new project in Python2?) but sadly Python3 is less common.  It is a chicken vs. egg situation.
+
+`go` is my favorite language. I could probably rewrite this in go in a weekend. However, now the code is compiled, not interpreted. Therefore we lose the ability to just "git clone" and have the tools you want.  Not everyone has a Go compiler installed on every machine.
+
+The system is basically unusable on Windows without Cygwin or MINGW.  A rewrite in python or go would make it work better on Windows, which currently requires Cygwin or MinGW (which is a bigger investment than installing Python). On the other hand, maybe Ubuntu-on-Windows makes that a non-issue.
+
+As long as the code is in `bash` the configuration files like `blackbox-files.txt` and `blackbox-admins.txt` have problems.  Filenames with carriage returns aren't supported.  If this was in Python/Go/etc. those files could be json or some format with decent quoting and we could handle funny file names better. On the other hand, maybe it is best that we don't support funny filenames... we shouldn't enable bad behavior.
+
+How important is itto blackbox users that the system is written in "bash"?
+
+
+## ditch the project and use git-crypt
+
+People tell me that git-crypt is better because, as a plug-in, automagically supports "git diff", "git log" and "git blame".
+
+However, I've never used it so I don't have any idea whether git-crypt is any better than blackbox.
+
+Of course, git-crypt doesn't work with SVN, HG, or any other VCS.  Is blackbox's strong point the fact that it support so many VCS systems?  To be honest, it originally only supported HG and GIT because I was at a company that used HG but then changed to GIT.  Supporting anything else was thanks to contributors. Heck, HG support hasn't even been tested recently (by me) since we've gone all git where I work.
+
+How important is this to BlackBox users?

bin/_blackbox_common.sh

@@ -7,75 +7,124 @@
 # Usage:
 #
 #   set -e
-#   . _blackbox_common.sh
+#   source "${0%/*}/_blackbox_common.sh"
 
-# Where in the VCS repo should the blackbox data be found?
-: "${BLACKBOXDATA:=keyrings/live}" ;   # If BLACKBOXDATA not set, set it.
+# Load additional useful functions
+source "${0%/*}"/_stack_lib.sh
 
+# Where are we?
+: "${BLACKBOX_HOME:="$(cd "${0%/*}" ; pwd)"}" ;
+
+# What are the candidates for the blackbox data directory?
+#
+# The order of candidates matter. The first entry of the array
+# sets the default Blackbox directory for all new repositories.
+declare -a BLACKBOXDATA_CANDIDATES
+BLACKBOXDATA_CANDIDATES=(
+  '.blackbox'
+  'keyrings/live'
+)
 
 # If $EDITOR is not set, set it to "vi":
 : "${EDITOR:=vi}" ;
 
+# Allow overriding gpg command
+: "${GPG:=gpg}" ;
 
-# Outputs a string that is the base directory of this VCS repo.
-# By side-effect, sets the variable VCS_TYPE to either 'git', 'hg',
-# 'svn' or 'unknown'.
-function _determine_vcs_base_and_type() {
-  if git rev-parse --show-toplevel 2>/dev/null ; then
-    VCS_TYPE=git
-  elif [ -d ".svn" ] ; then
-    #find topmost dir with .svn sub-dir
-    parent=""
-    grandparent="."
-    mydir="$(pwd)"
-    while [ -d "$grandparent/.svn" ]; do
-      parent=$grandparent
-      grandparent="$parent/.."
-    done
-
-    if [ ! -z "$parent" ]; then
-      cd "$parent"
-      pwd
-    else
-      exit 1
-    fi
-    cd "$mydir"
-    VCS_TYPE=svn
-  elif hg root 2>/dev/null ; then
-    # NOTE: hg has to be tested last because it always "succeeds".
-    VCS_TYPE=hg
-  else
-    echo /dev/null
-    VCS_TYPE=unknown
-  fi
-  export VCS_TYPE
-  # FIXME: Verify this function by checking for .hg or .git
-  # after determining what we believe to be the answer.
+function physical_directory_of() {
+  local d=$(dirname "$1")
+  local f=$(basename "$1")
+  (cd "$d" && echo "$(pwd -P | sed 's/\/$//')/$f" )
 }
 
-REPOBASE=$(_determine_vcs_base_and_type)
+# Set REPOBASE to the top of the repository
+# Set VCS_TYPE to 'git', 'hg', 'svn' or 'unknown'
+if which >/dev/null 2>/dev/null git && git rev-parse --show-toplevel >/dev/null 2>&1 ; then
+  VCS_TYPE=git
+  REPOBASE=$(git rev-parse --show-toplevel)
+elif [ -d ".svn" ] ; then
+  # Find topmost dir with .svn sub-dir
+  parent=""
+  grandparent="."
+  while [ -d "$grandparent/.svn" ]; do
+    parent=$grandparent
+    grandparent="$parent/.."
+  done
+
+  REPOBASE=$(cd "$parent" ; pwd)
+  VCS_TYPE=svn
+elif which >/dev/null 2>/dev/null hg && hg root >/dev/null 2>&1 ; then
+  # NOTE: hg has to be tested last because it always "succeeds".
+  VCS_TYPE=hg
+  REPOBASE=$(hg root 2>/dev/null)
+else
+  # We aren't in a repo at all.  Assume the cwd is the root
+  # of the tree.
+  VCS_TYPE=unknown
+  REPOBASE="$(pwd)"
+fi
+export VCS_TYPE
+export REPOBASE=$(physical_directory_of "$REPOBASE")
+# FIXME: Verify this function by checking for .hg or .git
+# after determining what we believe to be the answer.
+
+if [[ -n "$BLACKBOX_REPOBASE" ]]; then
+	echo "Using custom repobase: $BLACKBOX_REPOBASE" >&2
+	export REPOBASE="$BLACKBOX_REPOBASE"
+fi
+
+if [ -z "$BLACKBOXDATA" ] ; then
+  BLACKBOXDATA="${BLACKBOXDATA_CANDIDATES[0]}"
+  for candidate in ${BLACKBOXDATA_CANDIDATES[@]} ; do
+    if [ -d "$REPOBASE/$candidate" ] ; then
+      BLACKBOXDATA="$candidate"
+      break
+    fi
+  done
+fi
+
 KEYRINGDIR="$REPOBASE/$BLACKBOXDATA"
 BB_ADMINS_FILE="blackbox-admins.txt"
 BB_ADMINS="${KEYRINGDIR}/${BB_ADMINS_FILE}"
 BB_FILES_FILE="blackbox-files.txt"
 BB_FILES="${KEYRINGDIR}/${BB_FILES_FILE}"
 SECRING="${KEYRINGDIR}/secring.gpg"
-PUBRING="${KEYRINGDIR}/pubring.gpg"
 : "${DECRYPT_UMASK:=0022}" ;
 # : ${DECRYPT_UMASK:=o=} ;
 
-# Return error if not on cryptlist.
+# $BB_FILES file format:
+# Filenames are listed one per line, relative to the base directory of the repo.
+# Each line is listed in "printf %q" format, which escapes special chars.
+
+# Checks if $1 is 0 bytes, and if $1/keyrings
+# is a directory
+function is_blackbox_repo() {
+  if [[ -n "$1" ]] && [[ -d "$1/keyrings" ]]; then
+    return 0 # Yep, its a repo
+  else
+    return 1
+  fi
+}
+
+# is_on_cryptlist resturns an error if $1 not on cryptlist.
 function is_on_cryptlist() {
+  # $1: The filename.
   # Assumes $1 does NOT have the .gpg extension
-  local rname=$(vcs_relative_path "$1")
-  grep -F -x -s -q "$rname" "$BB_FILES"
+
+  # https://github.com/koalaman/shellcheck/wiki/SC2155
+  local name
+  name=$(vcs_relative_path "$1")
+  local encodedname
+  encodedname=$(printf "%q" "$name")
+
+  file_contains_line "$BB_FILES" "$encodedname"
 }
 
 # Exit with error if a file exists.
 function fail_if_exists() {
   if [[ -f "$1" ]]; then
-    echo ERROR: "$1" exists.  "$2"
-    echo Exiting...
+    echo ERROR: "$1" exists.  "$2" >&2
+    echo Exiting... >&2
     exit 1
   fi
 }
@@ -83,18 +132,17 @@ function fail_if_exists() {
 # Exit with error if a file is missing.
 function fail_if_not_exists() {
   if [[ ! -f "$1" ]]; then
-    echo ERROR: "$1" not found.  "$2"
-    echo Exiting...
+    echo ERROR: "$1" not found.  "$2" >&2
+    echo Exiting... >&2
     exit 1
   fi
 }
 
 # Exit we we aren't in a VCS repo.
 function fail_if_not_in_repo() {
-  _determine_vcs_base_and_type
   if [[ $VCS_TYPE = "unknown" ]]; then
-    echo "ERROR: This must be run in a VCS repo: git, hg, or svn."
-    echo Exiting...
+    echo "ERROR: This must be run in a VCS repo: git, hg, or svn." >&2
+    echo Exiting... >&2
     exit 1
   fi
 }
@@ -106,9 +154,9 @@ function fail_if_not_on_cryptlist() {
   local name="$1"
 
   if ! is_on_cryptlist "$name" ; then
-    echo "ERROR: $name not found in $BB_FILES"
-    echo "PWD="$(/bin/pwd)
-    echo 'Exiting...'
+    echo "ERROR: $name not found in $BB_FILES" >&2
+    echo "PWD=$(/usr/bin/env pwd)" >&2
+    echo 'Exiting...' >&2
     exit 1
   fi
 }
@@ -116,48 +164,113 @@ function fail_if_not_on_cryptlist() {
 # Exit with error if keychain contains secret keys.
 function fail_if_keychain_has_secrets() {
   if [[ -s ${SECRING} ]]; then
-    echo 'ERROR: The file' "$SECRING" 'should be empty.'
-    echo 'Did someone accidentally add this private key to the ring?'
-    echo 'Exiting...'
+    echo 'ERROR: The file' "$SECRING" 'should be empty.' >&2
+    echo 'Did someone accidentally add this private key to the ring?' >&2
+    echo 'Exiting...' >&2
     exit 1
   fi
 }
 
-# Output the unencrypted filename.
-function get_unencrypted_filename() {
-  echo $(dirname "$1")/$(basename "$1" .gpg) | sed -e 's#^\./##'
+function get_pubring_path() {
+  if [[ -f "${KEYRINGDIR}/pubring.gpg" ]]; then
+    echo "${KEYRINGDIR}/pubring.gpg"
+  else
+    echo "${KEYRINGDIR}/pubring.kbx"
+  fi
+}
+
+# normalize_filename_arg takes a filename from the command line and
+# outputs the non-encrypted filename.
+function normalize_filename() {
+  # $1: the input from a user
+  # Use this if the user may have entered the encrypted or
+  # non-encrypted filename.
+  local name
+  name=$(vcs_relative_path "$1")
+  echo "$(dirname "$name")/$(basename "$name" .gpg)" | sed -e 's#^\./##'
 }
 
 # Output the encrypted filename.
-function get_encrypted_filename() {
-  echo $(dirname "$1")/$(basename "$1" .gpg).gpg | sed -e 's#^\./##'
+function get_gpg_filename() {
+  # $1: normalized file path
+  echo "$1".gpg
 }
 
+## Output the unencrypted filename.
+#function get_unencrypted_filename() {
+#  echo "$(dirname "$1")/$(basename "$1" .gpg)" | sed -e 's#^\./##'
+#}
+#
+## Output the encrypted filename.
+#function get_encrypted_filename() {
+#  echo "$(dirname "$1")/$(basename "$1" .gpg).gpg" | sed -e 's#^\./##'
+#}
+
 # Prepare keychain for use.
 function prepare_keychain() {
-  echo '========== Importing keychain: START'
-  gpg --import "${PUBRING}" 2>&1 | egrep -v 'not changed$'
-  echo '========== Importing keychain: DONE'
+  local keyringasc
+  echo '========== Importing keychain: START' >&2
+  # Works with gpg 2.0
+  #$GPG --import "$(get_pubring_path)" 2>&1 | egrep -v 'not changed$' >&2
+  # Works with gpg 2.0 and 2.1
+  # NB: We must export the keys to a format that can be imported.
+  make_self_deleting_tempfile keyringasc
+  export LANG="C.UTF-8"
+
+  #if gpg2 is installed next to gpg like on ubuntu 16
+  if [[ "$GPG" != "gpg2" ]]; then
+    $GPG --export --no-default-keyring --keyring "$(get_pubring_path)" >"$keyringasc"
+    $GPG --import "$keyringasc" 2>&1 | egrep -v 'not changed$' >&2
+  else
+    $GPG --keyring "$(get_pubring_path)" --export | $GPG --import
+  fi
+
+  echo '========== Importing keychain: DONE' >&2
 }
 
-# Add file to list of encrypted files.
+# add_filename_to_cryptlist adds $1 to the list of encrypted files.
 function add_filename_to_cryptlist() {
+  # $1: The filename.
   # If the name is already on the list, this is a no-op.
-  # However no matter what the datestamp is updated.
-  local name=$(vcs_relative_path "$1")
 
-  if grep -s -q "$name" "$BB_FILES" ; then
-    echo ========== File is registered. No need to add to list.
+  # https://github.com/koalaman/shellcheck/wiki/SC2155
+  local name
+  name=$(vcs_relative_path "$1")
+  local encodedname
+  encodedname=$(printf "%q" "$name")
+
+
+  if file_contains_line "$BB_FILES" "$encodedname" ; then
+    echo "========== File is registered. No need to add to list."
   else
-    echo ========== Adding file to list.
+    echo "========== Adding file to list."
     touch "$BB_FILES"
-    sort -u -o "$BB_FILES" <(echo "$name") "$BB_FILES"
+    sort -u -o "$BB_FILES" <(printf "%q\n" "$name") "$BB_FILES"
+  fi
+}
+
+# remove_filename_from_cryptlist removes $1 from the list of encrypted files.
+function remove_filename_from_cryptlist() {
+  # $1: The filename.
+  # If the name is not already on the list, this is a no-op.
+
+  # https://github.com/koalaman/shellcheck/wiki/SC2155
+  local name
+  name=$(vcs_relative_path "$1")
+  local encodedname
+  encodedname=$(printf "%q" "$name")
+
+  if ! file_contains_line "$BB_FILES" "$encodedname" ; then
+    echo "========== File is not registered. No need to remove from list."
+  else
+    echo "========== Removing file from list."
+    remove_line "$BB_FILES" "$encodedname"
   fi
 }
 
 # Print out who the current BB ADMINS are:
 function disclose_admins() {
-  echo ========== blackbox administrators are:
+  echo "========== blackbox administrators are:"
   cat "$BB_ADMINS"
 }
 
@@ -168,9 +281,9 @@ function encrypt_file() {
   unencrypted="$1"
   encrypted="$2"
 
-  echo "========== Encrypting: $unencrypted"
-  gpg --yes --trust-model=always --encrypt -o "$encrypted"  $(awk '{ print "-r" $1 }' < "$BB_ADMINS") "$unencrypted"
-  echo '========== Encrypting: DONE'
+  echo "========== Encrypting: $unencrypted" >&2
+  $GPG --use-agent --yes --trust-model=always --encrypt -o "$encrypted"  $(awk '{ print "-r" $1 }' < "$BB_ADMINS") "$unencrypted" >&2
+  echo '========== Encrypting: DONE' >&2
 }
 
 # Decrypt .gpg file, asking "yes/no" before overwriting unencrypted file.
@@ -181,11 +294,11 @@ function decrypt_file() {
   encrypted="$1"
   unencrypted="$2"
 
-  echo "========== EXTRACTING $unencrypted"
+  echo "========== EXTRACTING $unencrypted" >&2
 
   old_umask=$(umask)
   umask "$DECRYPT_UMASK"
-  gpg -q --decrypt -o "$unencrypted" "$encrypted"
+  $GPG --use-agent -q --decrypt -o "$unencrypted" "$encrypted" >&2
   umask "$old_umask"
 }
 
@@ -207,12 +320,12 @@ function decrypt_file_overwrite() {
 
   old_umask=$(umask)
   umask "$DECRYPT_UMASK"
-  gpg --yes -q --decrypt -o "$unencrypted" "$encrypted"
+  $GPG --use-agent --yes -q --decrypt -o "$unencrypted" "$encrypted" >&2
   umask "$old_umask"
 
   new_hash=$(md5sum_file "$unencrypted")
   if [[ "$old_hash" != "$new_hash" ]]; then
-    echo "========== EXTRACTED $unencrypted"
+    echo "========== EXTRACTED $unencrypted" >&2
   fi
 }
 
@@ -223,14 +336,18 @@ function shred_file() {
   local OPT
   name="$1"
 
-  if which shred >/dev/null ; then
+  if which shred >/dev/null 2>/dev/null ; then
     CMD=shred
     OPT=-u
-  elif which srm >/dev/null ; then
+  elif which srm >/dev/null 2>/dev/null ; then
     #NOTE: srm by default uses 35-pass Gutmann algorithm
     CMD=srm
     OPT=-f
+  elif _F=$(mktemp); rm -P "${_F}" >/dev/null 2>/dev/null ; then
+    CMD=rm
+    OPT=-Pf
   else
+    echo "shred_file: WARNING: No secure deletion utility (shred or srm) present; using insecure rm" >&2
     CMD=rm
     OPT=-f
   fi
@@ -255,21 +372,64 @@ function enumerate_subdirs() {
     done
   done <"$listfile" | sort -u
 }
+ 
 
 # chdir to the base of the repo.
 function change_to_vcs_root() {
-  if [[ $REPOBASE = '' ]]; then
-    echo 'ERROR: _determine_vcs_base_and_type failed to set REPOBASE.'
+  # if vcs_root not explicitly defined, use $REPOBASE
+
+  local rbase=${1:-$REPOBASE} # use $1 but if unset use $REPOBASE
+
+  cd "$rbase"
+
+}
+
+# $1 is a string pointing to a directory.  Outputs a
+# list of  valid blackbox repos,relative to $1
+function enumerate_blackbox_repos() {
+  if [[ -z "$1" ]]; then
+    echo "enumerate_blackbox_repos: ERROR: No Repo provided to Enumerate"
     exit 1
   fi
-  cd "$REPOBASE"
+
+  # https://github.com/koalaman/shellcheck/wiki/Sc2045
+  for dir in $1*/; do
+    if is_blackbox_repo "$dir"; then
+      echo "$dir"
+    fi
+  done
 }
 
 # Output the path of a file relative to the repo base
 function vcs_relative_path() {
   # Usage: vcs_relative_path file
   local name="$1"
-  python -c 'import os ; print(os.path.relpath("'"$(pwd -P)"'/'"$name"'", "'"$REPOBASE"'"))'
+  #python -c 'import os ; print(os.path.relpath("'"$(pwd -P)"'/'"$name"'", "'"$REPOBASE"'"))'
+  local p=$( printf "%s" "$( pwd -P )/${1}" | sed 's#//*#/#g' )
+  local name="${p#$REPOBASE}"
+  name=$( printf "%s" "$name" | sed 's#^/##g' | sed 's#/$##g' )
+  printf "%s" "$name"
+}
+
+# Removes a line from a text file
+function remove_line() {
+  local tempfile
+
+  make_self_deleting_tempfile tempfile
+
+  # Ensure source file exists
+  touch "$1"
+  grep -Fsxv "$2" "$1" > "$tempfile" || true
+
+  # Using cat+rm instead of cp will preserve permissions/ownership
+  cat "$tempfile" > "$1"
+}
+
+# Determine if a file contains a given line
+function file_contains_line() {
+  # $1: the file
+  # $2: the line
+  grep -xsqF "$2" "$1"
 }
 
 #
@@ -283,38 +443,50 @@ function vcs_relative_path() {
 function md5sum_file() {
   # Portably generate the MD5 hash of file $1.
   case $(uname -s) in
-    Darwin )
+    Darwin | FreeBSD )
       md5 -r "$1" | awk '{ print $1 }'
       ;;
-    Linux )
-      md5sum "$1" | awk '{ print $1 }'
-      ;;
-    CYGWIN* )
+    Linux | CYGWIN* | MINGW* )
       md5sum "$1" | awk '{ print $1 }'
       ;;
     * )
-      echo 'ERROR: Unknown OS. Exiting.'
+      echo 'ERROR: Unknown OS. Exiting. (md5sum_file)'
       exit 1
       ;;
   esac
 }
 
+function cp_permissions() {
+  # Copy the perms of $1 onto $2 .. end.
+  case $(uname -s) in
+    Darwin )
+      chmod $( stat -f '%p' "$1" ) "${@:2}"
+      ;;
+    FreeBSD )
+      chmod $( stat -f '%p' "$1" | sed -e "s/^100//" ) "${@:2}"
+      ;;
+    Linux | CYGWIN* | MINGW* )
+      if [[ -e /etc/alpine-release ]]; then
+        chmod $( stat -c '%a' "$1" ) "${@:2}"
+      else
+        chmod --reference "$1" "${@:2}"
+      fi
+      ;; 
+    * )
+      echo 'ERROR: Unknown OS. Exiting. (cp_permissions)'
+      exit 1
+      ;;
+  esac
+}
+
+
 #
 # Abstract the difference between git and hg:
 #
 
-# Are we in git, hg, or unknown repo?
-function which_vcs() {
-  if [[ $VCS_TYPE = '' ]]; then
-    _determine_vcs_base_and_type >/dev/null
-  fi
-  echo "$VCS_TYPE"
-}
-
-
 # Is this file in the current repo?
 function is_in_vcs() {
-  is_in_$(which_vcs) """$@"""
+  is_in_$VCS_TYPE "$@"
 }
 # Mercurial
 function is_in_hg() {
@@ -349,58 +521,215 @@ function is_in_svn() {
     echo false
   fi
 }
+# Perforce
+function is_in_p4() {
+  local filename
+  filename="$1"
+
+  if p4 list "$filename" ; then
+    echo true
+  else
+    echo false
+  fi
+}
+# No repo
+function is_in_unknown() {
+  echo true
+}
 
 
 # Add a file to the repo (but don't commit it).
 function vcs_add() {
-  vcs_add_$(which_vcs) """$@"""
+  vcs_add_$VCS_TYPE "$@"
 }
 # Mercurial
 function vcs_add_hg() {
-  hg add """$@"""
+  hg add "$@"
 }
 # Git
 function vcs_add_git() {
-  git add """$@"""
+  git add "$@"
 }
 # Subversion
 function vcs_add_svn() {
-  svn add --parents """$@"""
+  svn add --parents "$@"
+}
+# Perfoce
+function vcs_add_p4() {
+  p4 add "$@"
+}
+# No repo
+function vcs_add_unknown() {
+  :
 }
 
 
 # Commit a file to the repo
 function vcs_commit() {
-  vcs_commit_$(which_vcs) """$@"""
+  vcs_commit_$VCS_TYPE "$@"
 }
 # Mercurial
 function vcs_commit_hg() {
-  hg commit -m"""$@"""
+  hg commit -m "$@"
 }
 # Git
 function vcs_commit_git() {
-  git commit -m"""$@"""
+  git commit -m "$@"
 }
 # Subversion
 function vcs_commit_svn() {
-  svn commit -m"""$@"""
+  svn commit -m "$@"
+}
+# Perforce
+function vcs_commit_p4() {
+  p4 submit -d "$@"
+}
+# No repo
+function vcs_commit_unknown() {
+  :
 }
 
 
 # Remove file from repo, even if it was deleted locally already.
 # If it doesn't exist yet in the repo, it should be a no-op.
 function vcs_remove() {
-  vcs_remove_$(which_vcs) """$@"""
+  vcs_remove_$VCS_TYPE "$@"
 }
 # Mercurial
 function vcs_remove_hg() {
-  hg rm -A -- """$@"""
+  hg rm -A -- "$@"
 }
 # Git
 function vcs_remove_git() {
-  git rm --ignore-unmatch -f -- """$@"""
+  git rm --ignore-unmatch -f -- "$@"
 }
 # Subversion
 function vcs_remove_svn() {
-  svn delete """$@"""
+  svn delete "$@"
+}
+# Perforce
+function vcs_remove_p4() {
+  p4 delete "$@"
+}
+# No repo
+function vcs_remove_unknown() {
+  :
+}
+
+# Get a path for the ignore file if possible in current vcs
+function vcs_ignore_file_path() {
+  vcs_ignore_file_path_$VCS_TYPE
+}
+# Mercurial
+function vcs_ignore_file_path_hg() {
+  echo "$REPOBASE/.hgignore"
+}
+# Git
+function vcs_ignore_file_path_git() {
+  echo "$REPOBASE/.gitignore"
+}
+
+
+# Ignore a file in a repo.  If it was already ignored, this is a no-op.
+function vcs_ignore() {
+  local file
+  for file in "$@"; do
+    vcs_ignore_$VCS_TYPE "$file"
+  done
+}
+# Mercurial
+function vcs_ignore_hg() {
+  vcs_ignore_generic_file "$(vcs_ignore_file_path)" "$file"
+}
+# Git
+function vcs_ignore_git() {
+  vcs_ignore_generic_file "$(vcs_ignore_file_path)" "$file"
+  git add "$REPOBASE/.gitignore"
+}
+# Subversion
+function vcs_ignore_svn() {
+  svn propset svn:ignore "$file" "$(vcs_relative_path)"
+}
+# Perforce
+function vcs_ignore_p4() {
+  :
+}
+# No repo
+function vcs_ignore_unknown() {
+  :
+}
+# Generic - add line to file
+function vcs_ignore_generic_file() {
+  local file
+  file="$(vcs_relative_path "$2")"
+  file="${file/\$\//}"
+  file="$(echo "/$file" | sed 's/\([\*\?]\)/\\\1/g')"
+  if ! file_contains_line "$1" "$file" ; then
+    echo "$file" >> "$1"
+    vcs_add "$1"
+  fi
+}
+
+
+# Notice (un-ignore) a file in a repo.  If it was not ignored, this is
+# a no-op
+function vcs_notice() {
+  local file
+  for file in "$@"; do
+    vcs_notice_$VCS_TYPE "$file"
+  done
+}
+# Mercurial
+function vcs_notice_hg() {
+  vcs_notice_generic_file "$REPOBASE/.hgignore" "$file"
+}
+# Git
+function vcs_notice_git() {
+  vcs_notice_generic_file "$REPOBASE/.gitignore" "$file"
+  git add "$REPOBASE/.gitignore"
+}
+# Subversion
+function vcs_notice_svn() {
+  svn propdel svn:ignore "$(vcs_relative_path "$file")"
+}
+# Perforce
+function vcs_notice_p4() {
+  :
+}
+# No repo
+function vcs_notice_unknown() {
+  :
+}
+# Generic - remove line to file
+function vcs_notice_generic_file() {
+  local file
+  file="$(vcs_relative_path "$2")"
+  file="${file/\$\//}"
+  file="$(echo "/$file" | sed 's/\([\*\?]\)/\\\1/g')"
+  if file_contains_line "$1" "$file" ; then
+    remove_line "$1" "$file"
+    vcs_add "$1"
+  fi
+  if file_contains_line "$1" "${file:1}" ; then
+    echo "WARNING:  Found a non-absolute ignore match in $1"
+    echo "WARNING:  Confirm the pattern is intended to only exclude $file"
+    echo "WARNING:  If so, manually update the ignore file"
+  fi
+}
+
+function gpg_agent_version_check() {
+  if ! hash 'gpg-agent' &> /dev/null; then
+    return 1
+  fi
+  local gpg_agent_version=$(gpg-agent --version | head -1 | awk '{ print $3 }' | tr -d '\n')
+  semverLT $gpg_agent_version "2.1.0"
+}
+
+function gpg_agent_notice() {
+  if [[ $(gpg_agent_version_check) == '0' && -z $GPG_AGENT_INFO ]];then
+    echo 'WARNING: You probably want to run gpg-agent as'
+    echo 'you will be asked for your passphrase many times.'
+    echo 'Example: $ eval $(gpg-agent --daemon)'
+    read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
+  fi
 }

bin/_blackbox_common_test.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+#
+# _blackbox_common_test.sh -- Unit tests of functions from _blackbox_common.sh
+#
+
+set -e
+. "${0%/*}/_blackbox_common.sh"
+. /Users/tlimoncelli/gitwork/blackbox/tools/test_functions.sh
+
+PHASE 'Test cp-permissions: TestA'
+touch TestA TestB TestC TestD
+chmod 0347 TestA
+chmod 0700 TestB
+chmod 0070 TestC
+chmod 0070 TestD
+cp_permissions TestA TestB TestC
+# NOTE: cp_permissions is not touching TestD.
+assert_file_perm '--wxr--rwx' TestA
+assert_file_perm '--wxr--rwx' TestB
+assert_file_perm '--wxr--rwx' TestC
+assert_file_perm '----rwx---' TestD  # TestD doesn't change.
+rm -f TestA TestB TestC TestD
+
+PHASE 'Test vcs_relative_path: TestA'
+export REPOBASE='/Users/tlimoncelli/Applications (Parallels)/{fd3049c8-9fdd-48d5-aa16-d31daf3a6879} Applications.localized'
+FILE='Microsoft  Windows Fax and Scan.app/Contents'
+result=$(vcs_relative_path Contents)
+echo result=XXX${result}XXX
+if [[ $FILE != $result ]] ; then
+  echo FAIL
+fi
+
+unencrypted_file=$(get_unencrypted_filename "${result}.gpg")
+echo un=XXX${unencrypted_file}XXX
+encrypted_file=$(get_encrypted_filename "${result}")
+echo en=XXX${encrypted_file}XXX
+
+echo '========== DONE.'

bin/_stack_lib.sh

@@ -1,6 +1,9 @@
 # Library functions for bash scripts at Stack Exchange.
 
+# NOTE: This file is open sourced. Do not put Stack-proprietary code here.
+
 # Usage:
+#
 #   set -e
 #   . _stack_lib.sh
 
@@ -8,13 +11,13 @@
 
 function debugmsg() {
   # Log to stderr.
-  echo 1>&2 LOG: """$@"""
+  echo 1>&2 LOG: "$@"
   :
 }
 
 function logit() {
   # Log to stderr.
-  echo 1>&2 LOG: """$@"""
+  echo 1>&2 LOG: "$@"
 }
 
 function fail_out() {
@@ -46,6 +49,48 @@ function add_on_exit()
     fi
 }
 
+function create_self_deleting_tempfile() {
+  local filename
+
+  case $(uname -s) in
+    Darwin | FreeBSD )
+      : "${TMPDIR:=/tmp}" ;
+      filename=$(mktemp -t _stacklib_.XXXXXXXX )
+      ;;
+    Linux | CYGWIN* | MINGW* )
+      filename=$(mktemp)
+      ;;
+    * )
+      echo 'ERROR: Unknown OS. Exiting. (create_self_deleting_tempfile)'
+      exit 1
+      ;;
+  esac
+
+  add_on_exit rm -f "$filename"
+  echo "$filename"
+}
+
+function create_self_deleting_tempdir() {
+  local filename
+
+  case $(uname -s) in
+    Darwin | FreeBSD )
+      : "${TMPDIR:=/tmp}" ;
+      filename=$(mktemp -d -t _stacklib_.XXXXXXXX )
+      ;;
+    Linux | CYGWIN* | MINGW* )
+      filename=$(mktemp -d)
+      ;;
+    * )
+      echo 'ERROR: Unknown OS. Exiting. (create_self_deleting_tempdir)'
+      exit 1
+      ;;
+  esac
+
+  add_on_exit rm -rf "$filename"
+  echo "$filename"
+}
+
 # Securely and portably create a temporary file that will be deleted
 # on EXIT.  $1 is the variable name to store the result.
 function make_self_deleting_tempfile() {
@@ -53,18 +98,15 @@ function make_self_deleting_tempfile() {
   local name
 
   case $(uname -s) in
-    Darwin )
-      : ${TMPDIR:=/tmp} ;
-      name=$(mktemp -t _stacklib_ )
+    Darwin | FreeBSD )
+      : "${TMPDIR:=/tmp}" ;
+      name=$(mktemp -t _stacklib_.XXXXXXXX )
       ;;
-    Linux )
-      name=$(mktemp)
-      ;;
-    CYGWIN* )
+    Linux | CYGWIN* | MINGW* )
       name=$(mktemp)
       ;;
     * )
-      echo 'ERROR: Unknown OS. Exiting.'
+      echo 'ERROR: Unknown OS. Exiting. (make_self_deleting_tempfile)'
       exit 1
       ;;
   esac
@@ -78,18 +120,18 @@ function make_tempdir() {
   local name
 
   case $(uname -s) in
-    Darwin )
+    Darwin | FreeBSD )
       : "${TMPDIR:=/tmp}" ;
-      name=$(mktemp -d -t _stacklib_ )
+      # The full path to the temp directory must be short.
+      # This is used by blackbox's testing suite to make a fake GNUPGHOME,
+      # which needs to fit within sockaddr_un.sun_path (see unix(7)).
+      name=$(mktemp -d -t SO )
       ;;
-    Linux )
-      name=$(mktemp -d)
-      ;;
-    CYGWIN* )
+    Linux | CYGWIN* | MINGW* )
       name=$(mktemp -d)
       ;;
     * )
-      echo 'ERROR: Unknown OS. Exiting.'
+      echo 'ERROR: Unknown OS. Exiting. (make_tempdir)'
       exit 1
       ;;
   esac
@@ -99,12 +141,12 @@ function make_tempdir() {
 
 function make_self_deleting_tempdir() {
   local __resultvar="$1"
-  local dirname
+  local dname
 
-  make_tempdir dirname
+  make_tempdir dname
 
-  add_on_exit rm -rf "$dirname"
-  eval $__resultvar="$dirname"
+  add_on_exit rm -rf "$dname"
+  eval $__resultvar="$dname"
 }
 
 function fail_if_not_running_as_root() {
@@ -118,21 +160,14 @@ function fail_if_not_running_as_root() {
 function fail_if_in_root_directory() {
   # Verify nobody has tricked us into being in "/".
   case $(uname -s) in
-    Darwin )
+    Darwin | FreeBSD )
       if [[ $(stat -f'%i' / ) == $(stat -f'%i' . ) ]] ; then
         echo 'SECURITY ALERT: The current directory is the root directory.'
         echo 'Exiting...'
         exit 1
       fi
       ;;
-    Linux )
-      if [[ $(stat -c'%i' / ) == $(stat -c'%i' . ) ]] ; then
-        echo 'SECURITY ALERT: The current directory is the root directory.'
-        echo 'Exiting...'
-        exit 1
-      fi
-      ;;
-    CYGWIN* )
+    Linux | CYGWIN* | MINGW* )
       if [[ $(stat -c'%i' / ) == $(stat -c'%i' . ) ]] ; then
         echo 'SECURITY ALERT: The current directory is the root directory.'
         echo 'Exiting...'
@@ -140,8 +175,114 @@ function fail_if_in_root_directory() {
       fi
       ;;
     * )
-      echo 'ERROR: Unknown OS. Exiting.'
+      echo 'ERROR: Unknown OS. Exiting. (fail_if_in_root_directory)'
       exit 1
       ;;
   esac
 }
+
+function semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+function semverEQ() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -ne $MAJOR_B ]; then
+        return 1
+    fi
+
+    if [ $MINOR_A -ne $MINOR_B ]; then
+        return 1
+    fi
+
+    if [ $PATCH_A -ne $PATCH_B ]; then
+        return 1
+    fi
+
+    if [[ "_$SPECIAL_A" != "_$SPECIAL_B" ]]; then
+        return 1
+    fi
+
+
+    return 0
+
+}
+
+function semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -lt $MINOR_B ]]; then
+        return 0
+    fi
+
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -le $MINOR_B && $PATCH_A -lt $PATCH_B ]]; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  != "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  != "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A" < "_$SPECIAL_B" ]]; then
+        return 0
+    fi
+
+    return 1
+
+}
+
+function semverGT() {
+    semverEQ $1 $2
+    local EQ=$?
+
+    semverLT $1 $2
+    local LT=$?
+
+    if [ $EQ -ne 0 ] && [ $LT -ne 0 ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+

bin/blackbox_addadmin

@@ -9,9 +9,7 @@
 #
 
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source ${blackbox_home}/_blackbox_common.sh
-source ${blackbox_home}/_stack_lib.sh
+source "${0%/*}/_blackbox_common.sh"
 
 fail_if_not_in_repo
 
@@ -31,24 +29,25 @@ make_self_deleting_tempfile pubkeyfile
 
 # The second argument, if present, is the directory to find the GPG keys to be imported.
 if [[ -z $2 ]]; then
-  gpg --export -a "$KEYNAME" >"$pubkeyfile"
+  $GPG --export -a "$KEYNAME" >"$pubkeyfile"
 else
   # TODO(tlim): This could probably be done with GNUPGHOME
   # but that affects all commands; we just want it to affect the key export.
-  gpg --homedir="$2" --export -a "$KEYNAME" >"$pubkeyfile"
+  $GPG --homedir="$2" --export -a "$KEYNAME" >"$pubkeyfile"
 fi
 
 if [[ $(wc -l < "$pubkeyfile") = 0 ]]; then
-  fail_out "GPG key '$KEYNAME' not found.  Please create it with: gpg --gen-key"
+  fail_out "GPG key '$KEYNAME' not found.  Please create it with: $GPG --gen-key"
   exit 1
 fi
 
 # Import it:
-gpg --no-permission-warning --homedir="$KEYRINGDIR" --import "$pubkeyfile"
-vcs_add "$KEYRINGDIR/pubring.gpg" "$KEYRINGDIR/trustdb.gpg" "$BB_ADMINS"
+$GPG --no-permission-warning --homedir="$KEYRINGDIR" --import "$pubkeyfile"
+pubring_path=$(get_pubring_path)
+vcs_add "$pubring_path" "$KEYRINGDIR/trustdb.gpg" "$BB_ADMINS"
 
 # Make a suggestion:
 echo
 echo
 echo 'NEXT STEP: You need to manually check these in:'
-echo '     ' $VCS_TYPE commit -m\'NEW ADMIN: $KEYNAME\' "$BLACKBOXDATA/pubring.gpg" "$BLACKBOXDATA/trustdb.gpg" "$BLACKBOXDATA/$BB_ADMINS_FILE"
+echo '     ' $VCS_TYPE commit -m\'NEW ADMIN: $KEYNAME\' "$BLACKBOXDATA/$(basename ${pubring_path})" "$BLACKBOXDATA/trustdb.gpg" "$BLACKBOXDATA/$BB_ADMINS_FILE"

bin/blackbox_cat

@@ -1,17 +1,16 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_cat.sh -- Decrypt a file, cat it, shred it
+# blackbox_cat -- Decrypt a file, cat it, shred it
 #
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
+source "${0%/*}/_blackbox_common.sh"
 
-for param in """$@""" ; do
+for param in "$@" ; do
   shreddable=0
-  unencrypted_file=$(get_unencrypted_filename "$param")
+  unencrypted_file=$(normalize_filename "$param")
   if [[ ! -e "$unencrypted_file" ]]; then
-    "${blackbox_home}/blackbox_edit_start" "$param"
+    "${BLACKBOX_HOME}/blackbox_edit_start" "$param"
     shreddable=1
   fi
   cat "$unencrypted_file"

bin/blackbox_decrypt_all_files

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_decrypt_all_files -- Decrypt all blackbox files (INTERACTIVE).
+#
+
+# Usage:
+#   blackbox_decrypt_all_files [GROUP]
+#       GROUP is optional.  If supplied, the resulting files
+#       are chgrp'ed to that group.
+
+# Since this is often run in a security-critical situation, we
+# force /usr/bin and /bin to the front of the PATH.
+export PATH=/usr/bin:/bin:"$PATH"
+
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+gpg_agent_notice
+exec blackbox_postdeploy "$@"

bin/blackbox_decrypt_file

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_decrypt_file -- Decrypt one or more blackbox files.
+#
+
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+if [ $# -eq 0 ]; then
+  echo >&2 "Please provide at least one file to decrypt"
+  exit 1
+fi
+
+"${BLACKBOX_HOME}/blackbox_edit_start" "$@"

bin/blackbox_deregister_file

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_deregister_file -- Remove a file from the blackbox system.
+#
+# Takes an encrypted file and removes it from the blackbox system.
+# The encrypted file will also be removed from the filesystem.
+# The unencrypted file, if it exists, will be left alone.
+
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+unencrypted_file=$(get_unencrypted_filename "$1")
+encrypted_file=$(get_encrypted_filename "$1")
+
+if [[ "$1" == "$unencrypted_file" ]]; then
+  echo ERROR: Please only deregister encrypted files.
+  exit 1
+fi
+
+echo ========== PLAINFILE "$unencrypted_file"
+echo ========== ENCRYPTED "$encrypted_file"
+
+fail_if_not_exists "$encrypted_file" "Please specify an existing file."
+
+prepare_keychain
+remove_filename_from_cryptlist "$unencrypted_file"
+vcs_remove "$encrypted_file"
+vcs_notice "$unencrypted_file"
+vcs_add "$BB_FILES" 
+
+vcs_commit "Removing from blackbox: ${unencrypted_file}" "$BB_FILES" "$encrypted_file"
+echo "========== UPDATING VCS: DONE"
+echo "Local repo updated.  Please push when ready."
+echo "    $VCS_TYPE push"

bin/blackbox_diff

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_diff -- Show all differences.
+#
+
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+gpg_agent_notice
+prepare_keychain
+
+modified_files=()
+modifications=()
+echo '========== DIFFING FILES: START'
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $encodedname)
+  unencrypted_file=$(get_unencrypted_filename "$name")
+  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
+  fail_if_not_on_cryptlist "$unencrypted_file"
+  if [[ -f "$unencrypted_file" ]]; then
+    out=$(diff -u <($GPG --yes -q --decrypt "$encrypted_file") "$unencrypted_file" || true)
+    if [ "$out" != "" ]; then
+      modified_files+=("$unencrypted_file")
+      modifications+=("$out")
+    fi
+  fi
+done 99<"$BB_FILES"
+modified_files_number=${#modified_files[@]}
+for (( i=0; i<${modified_files_number}; i++ )); do
+  echo ========== PROCESSING '"'${modified_files[$i]}'"'
+  echo -e "${modifications[$i]}\n"
+done
+echo '========== DIFFING FILES: DONE'
+
+fail_if_keychain_has_secrets
+
+echo '========== DONE.'
+
+if [ ${#modified_files[@]} -eq 0 ] ; then
+  exit 0
+fi
+
+echo 'Likely next steps:'
+for f in "${modified_files[@]}" ; do
+  echo "    blackbox_edit_end" '"'$f'"'
+done

bin/blackbox_edit

@@ -1,27 +1,33 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_edit.sh -- Decrypt a file temporarily for edition, then re-encrypts it again
+# blackbox_edit -- Decrypt a file temporarily for edition, then re-encrypts it again
 #
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
+source "${0%/*}/_blackbox_common.sh"
+
+for param in "$@" ; do
 
-for param in """$@""" ; do
   unencrypted_file=$(get_unencrypted_filename "$param")
-  if  ! is_on_cryptlist "$param" && ! is_on_cryptlist "$unencrypted_file" ; then
+  encrypted_file=$(get_encrypted_filename "$param")
+
+  echo >&2 ========== PLAINFILE '"'$unencrypted_file'"'
+  echo >&2 ========== ENCRYPTED '"'$encrypted_file'"'
+
+  if  ! is_on_cryptlist "$encrypted_file" && ! is_on_cryptlist "$unencrypted_file" ; then
     read -r -p "Encrypt file $param? (y/n) " ans
     case "$ans" in
       y* | Y*)
-        "${blackbox_home}/blackbox_register_new_file" "$param"
+        "${BLACKBOX_HOME}/blackbox_register_new_file" "$unencrypted_file"
         ;;
       *)
-        echo 'Skipping...'
+        echo >&2 'Skipping...'
         continue
         ;;
     esac
   fi
-  "${blackbox_home}/blackbox_edit_start" "$param"
-  "$EDITOR" "$(get_unencrypted_filename "$param")"
-  "${blackbox_home}/blackbox_edit_end" "$param"
+  "${BLACKBOX_HOME}/blackbox_edit_start" "$unencrypted_file"
+  $EDITOR "$(get_unencrypted_filename "$unencrypted_file")"
+  "${BLACKBOX_HOME}/blackbox_edit_end" "$unencrypted_file"
+
 done

bin/blackbox_edit_end

@@ -5,23 +5,36 @@
 #
 
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
+source "${0%/*}/_blackbox_common.sh"
 
-unencrypted_file=$(get_unencrypted_filename "$1")
-encrypted_file=$(get_encrypted_filename "$1")
-echo ========== PLAINFILE "$unencrypted_file"
-echo ========== ENCRYPTED "$encrypted_file"
+next_steps=()
 
-fail_if_not_on_cryptlist "$unencrypted_file"
-fail_if_not_exists "$unencrypted_file" "No unencrypted version to encrypt!"
-fail_if_keychain_has_secrets
+if [ $# -eq 0 ]; then
+  echo >&2 "Please provide at least one file for which editing has finished"
+  exit 1
+fi
 
-encrypt_file "$unencrypted_file" "$encrypted_file"
-shred_file "$unencrypted_file"
+for param in "$@" ; do
 
-_determine_vcs_base_and_type
+  unencrypted_file=$(get_unencrypted_filename "$param")
+  encrypted_file=$(get_encrypted_filename "$param")
 
-echo "========== UPDATED ${encrypted_file}"
-echo "Likely next step:"
-echo "    $VCS_TYPE commit -m\"${encrypted_file} updated\" $encrypted_file"
+  echo >&2 ========== PLAINFILE '"'$unencrypted_file'"'
+  echo >&2 ========== ENCRYPTED '"'$encrypted_file'"'
+
+  fail_if_not_on_cryptlist "$unencrypted_file"
+  fail_if_not_exists "$unencrypted_file" "No unencrypted version to encrypt!"
+  fail_if_keychain_has_secrets
+
+  encrypt_file "$unencrypted_file" "$encrypted_file"
+  shred_file "$unencrypted_file"
+  echo >&2 ========== UPDATED '"'$encrypted_file'"'
+  next_steps+=( "    $VCS_TYPE commit -m\"${encrypted_file} updated\" \"$encrypted_file\"" )
+
+done
+
+echo >&2 "Likely next step:"
+for x in "${next_steps[@]}"
+do
+  echo >&2 "$x"
+done

bin/blackbox_edit_start

@@ -1,17 +1,23 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_edit_start.sh -- Decrypt a file for editing.
+# blackbox_edit_start -- Decrypt a file for editing.
 #
 
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
+source "${0%/*}/_blackbox_common.sh"
+
+if [ $# -eq 0 ]; then
+  echo >&2 "Please provide at least one file to start editing"
+  exit 1
+fi
+
+for param in "$@" ; do
 
-for param in """$@""" ; do
   unencrypted_file=$(get_unencrypted_filename "$param")
   encrypted_file=$(get_encrypted_filename "$param")
-  echo ========== PLAINFILE "$unencrypted_file"
+
+  echo >&2 ========== PLAINFILE '"'$unencrypted_file'"'
 
   fail_if_not_on_cryptlist "$unencrypted_file"
   fail_if_not_exists "$encrypted_file" "This should not happen."
@@ -19,10 +25,12 @@ for param in """$@""" ; do
     rm -f "$unencrypted_file"
   fi
   if [[ -f "$unencrypted_file" ]]; then
-    echo SKIPPING: "$1" "Will not overwrite non-empty files."
+    echo >&2 SKIPPING: "$1" "Will not overwrite non-empty files."
     continue
   fi
 
   prepare_keychain
+  # FIXME(tlim): prepare_keychain only needs to run once, outside of the loop.
   decrypt_file "$encrypted_file" "$unencrypted_file"
+
 done

bin/blackbox_initialize

@@ -9,10 +9,7 @@
 #
 
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
-
-_determine_vcs_base_and_type  # Sets VCS_TYPE
+source "${0%/*}/_blackbox_common.sh"
 
 if [[ $1 != 'yes' ]]; then
   read -r -p "Enable blackbox for this $VCS_TYPE repo? (yes/no) " ans
@@ -22,36 +19,45 @@ if [[ $1 != 'yes' ]]; then
   fi
 fi
 
+if [[ $VCS_TYPE = "unknown" ]]; then
+	echo 'Not in a known VCS directory'
+	exit 1
+fi
+
 change_to_vcs_root
 
 echo VCS_TYPE: $VCS_TYPE
-
-if [[ $VCS_TYPE = "git" || $VCS_TYPE = "hg" ]]; then
-  # Update .gitignore or .hgignore
-
-  IGNOREFILE="${REPOBASE}/.${VCS_TYPE}ignore"
-  if ! grep -sx >/dev/null 'pubring.gpg~' "$IGNOREFILE" ; then
-    echo 'pubring.gpg~' >>"$IGNOREFILE"
-  fi
-  if ! grep -sx >/dev/null 'secring.gpg' "$IGNOREFILE" ; then
-    echo 'secring.gpg' >>"$IGNOREFILE"
-  fi
-elif [[ $VCS_TYPE = "svn" ]]; then
-  # add file to svn ignore propset
-  IGNOREFILE="";
-  svn propset svn:ignore 'pubring.gpg~
-secring.gpg' .
-  svn commit -m "ignore file list"
-fi
+vcs_ignore "${BLACKBOXDATA}/pubring.gpg~" "${BLACKBOXDATA}/pubring.kbx~" "${BLACKBOXDATA}/secring.gpg"
 
 # Make directories
 mkdir -p "${KEYRINGDIR}"
 vcs_add "${KEYRINGDIR}"
 touch "$BLACKBOXDATA/$BB_ADMINS_FILE" "$BLACKBOXDATA/$BB_FILES_FILE"
-vcs_add "$IGNOREFILE" "$BLACKBOXDATA/$BB_ADMINS_FILE" "$BLACKBOXDATA/$BB_FILES_FILE"
+vcs_add "$BLACKBOXDATA/$BB_ADMINS_FILE" "$BLACKBOXDATA/$BB_FILES_FILE"
 
-# Make a suggestion:
-echo
-echo
-echo 'NEXT STEP: You need to manually check these in:'
-echo '     ' $VCS_TYPE commit -m\'INITIALIZE BLACKBOX\' keyrings "$IGNOREFILE"
+if [[ $VCS_TYPE = "git" ]]; then
+
+  # Set .gitattributes so that Windows users don't break the admin files.
+  FILE="$BLACKBOXDATA/.gitattributes"
+  touch "$FILE"
+  LINE='blackbox-admins.txt text eol=lf'
+  grep -qF "$LINE" "$FILE" || echo "$LINE" >> "$FILE"
+  LINE='blackbox-files.txt text eol=lf'
+  grep -qF "$LINE" "$FILE" || echo "$LINE" >> "$FILE"
+
+fi
+
+if [[ $VCS_TYPE = "svn" ]]; then
+	echo
+	echo
+	echo '`subversion` automatically tracks the ignored files; you just need to commit.'
+else
+	IGNOREFILE="$(vcs_ignore_file_path)"
+	test -f "$IGNOREFILE" && vcs_add "$IGNOREFILE"
+
+	# Make a suggestion:
+	echo
+	echo
+	echo 'NEXT STEP: You need to manually check these in:'
+	echo '     ' $VCS_TYPE commit -m\'INITIALIZE BLACKBOX\' keyrings "$IGNOREFILE"
+fi

bin/blackbox_list_admins

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_list_admins -- List authorized admins
+#
+set -e
+source "${0%/*}/_blackbox_common.sh"
+cat "$BB_ADMINS"

bin/blackbox_list_files

@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_list_files.sh -- List files that black box is tracking
+# blackbox_list_files -- List files that black box is tracking
 #
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
-cat "$BB_FILES"
+source "${0%/*}/_blackbox_common.sh"
+
+while IFS= read <&99 -r encodedname; do
+  echo $encodedname
+done 99<"$BB_FILES"

bin/blackbox_listadmins

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_listadmins -- List active admins for keyring
+#
+
+# Example:
+#    blackbox_listadmins
+#
+
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+fail_if_not_in_repo
+
+
+# simply display the contents of the admins file
+cat "$BB_ADMINS"

bin/blackbox_postdeploy

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_postdeploy.sh -- Decrypt all blackbox files.
+# blackbox_postdeploy -- Decrypt all blackbox files.
 #
 
 # Usage:
@@ -14,8 +14,7 @@
 export PATH=/usr/bin:/bin:"$PATH"
 
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
+source "${0%/*}/_blackbox_common.sh"
 
 if [[ "$1" == "" ]]; then
   FILE_GROUP=""
@@ -28,12 +27,18 @@ prepare_keychain
 
 # Decrypt:
 echo '========== Decrypting new/changed files: START'
-while IFS= read <&99 -r unencrypted_file; do
-  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $name)
+
+  encrypted_file=$(get_encrypted_filename "$name")
+  unencrypted_file=$(get_unencrypted_filename "$name")
   decrypt_file_overwrite "$encrypted_file" "$unencrypted_file"
-  chmod g+r "$unencrypted_file"
+  cp_permissions "$encrypted_file" "$unencrypted_file"
   if [[ ! -z "$FILE_GROUP" ]]; then
+    chmod g+r "$unencrypted_file"
     chgrp "$FILE_GROUP" "$unencrypted_file"
   fi
 done 99<"$BB_FILES"
+
 echo '========== Decrypting new/changed files: DONE'

bin/blackbox_recurse

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# proposed space for blackbox recurion...coming soon
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+echo "$REBOBASE"

bin/blackbox_register_new_file

@@ -1,70 +1,60 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_register_new_file.sh -- Enroll a new file in the blackbox system.
+# blackbox_register_new_file -- Enroll new file(s) in the blackbox system.
 #
-# Takes a previously unencrypted file and enrolls it into the blackbox
-# system.  It will be kept in the repo as an encrypted file.  On deployment
+# Takes previously unencrypted file(s) and enrolls them into the blackbox
+# system.  Each file will be kept in the repo as an encrypted file.  On deployment
 # to systems that need the plaintext (unencrypted) versions, run
 # blackbox_postdeploy.sh to decrypt all the files.
 
-# TODO(tlim): Add the unencrypted file to .hgignore
-
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
-_determine_vcs_base_and_type
+source "${0%/*}/_blackbox_common.sh"
 
-unencrypted_file=$(get_unencrypted_filename "$1")
-encrypted_file=$(get_encrypted_filename "$1")
+function register_new_file() {
+  unencrypted_file=$(get_unencrypted_filename "$1")
+  encrypted_file=$(get_encrypted_filename "$1")
 
-if [[ "$1" == "$encrypted_file" ]]; then
-  echo ERROR: Please only register unencrypted files.
-  exit 1
-fi
-
-echo ========== PLAINFILE "$unencrypted_file"
-echo ========== ENCRYPTED "$encrypted_file"
-
-fail_if_not_exists "$unencrypted_file" "Please specify an existing file."
-fail_if_exists "$encrypted_file" "Will not overwrite."
-
-prepare_keychain
-encrypt_file "$unencrypted_file" "$encrypted_file"
-add_filename_to_cryptlist "$unencrypted_file"
-
-# Is the unencrypted file already in HG? (ie. are we correcting a bad situation)
-SECRETSEXPOSED=$(is_in_vcs "${unencrypted_file}")
-echo "========== CREATED: ${encrypted_file}"
-echo "========== UPDATING REPO:"
-shred_file "$unencrypted_file"
-
-VCSCMD=$(which_vcs)
-if "$SECRETSEXPOSED" ; then
-  vcs_remove "$unencrypted_file"
-  vcs_add "$encrypted_file"
-  COMMIT_FILES=("$BB_FILES" "$encrypted_file" "$unencrypted_file")
-else
-  COMMIT_FILES=("$BB_FILES" "$encrypted_file")
-fi
-
-# TODO(tlim): This should be moved to _blackbox_common.sh in a
-# VCS-independent way.
-IGNOREFILE="${REPOBASE}/.${VCS_TYPE}ignore"
-if [[ $VCS_TYPE = 'git' ]]; then
-  relfile="$(vcs_relative_path "$unencrypted_file")"
-  relfileb="${relfile/\$\//}"
-  ignored_file="$(echo "${relfileb}" |  sed 's/\([\*\?]\)/\\\1/g' | sed 's/^\([!#]\)/\\\1/')"
-  if ! grep -Fsx >/dev/null "$ignored_file" "$IGNOREFILE"; then
-    echo "$ignored_file" >>"$IGNOREFILE"
-    COMMIT_FILES+=("$IGNOREFILE")
+  if [[ "$1" == "$encrypted_file" ]]; then
+    echo ERROR: Please only register unencrypted files.
+    exit 1
   fi
-  vcs_add "$IGNOREFILE"
-fi
 
-echo 'NOTE: "already tracked!" messages are safe to ignore.'
-vcs_add "$BB_FILES" "$encrypted_file"
-vcs_commit "registered in blackbox: ${unencrypted_file}" "${COMMIT_FILES[@]}"
+  echo "========== PLAINFILE $unencrypted_file"
+  echo "========== ENCRYPTED $encrypted_file"
+
+  fail_if_not_exists "$unencrypted_file" "Please specify an existing file."
+  fail_if_exists "$encrypted_file" "Will not overwrite."
+
+  prepare_keychain
+  encrypt_file "$unencrypted_file" "$encrypted_file"
+  add_filename_to_cryptlist "$unencrypted_file"
+
+  # Is the unencrypted file already in HG? (ie. are we correcting a bad situation)
+  SECRETSEXPOSED=$(is_in_vcs "${unencrypted_file}")
+  echo "========== CREATED: ${encrypted_file}"
+  echo "========== UPDATING REPO:"
+  shred_file "$unencrypted_file"
+
+  if [[ "$SECRETSEXPOSED" == "true" ]] ; then
+    vcs_remove "$unencrypted_file"
+    vcs_add "$encrypted_file"
+  fi
+
+  vcs_ignore "$unencrypted_file"
+  echo 'NOTE: "already tracked!" messages are safe to ignore.'
+  vcs_add "$BB_FILES" "$encrypted_file"
+  vcs_commit "registered in blackbox: ${unencrypted_file}" "$BB_FILES" "$encrypted_file" "$(vcs_ignore_file_path)"
+}
+
+for target in "$@"; do
+  register_new_file "$target"
+done
+
 echo "========== UPDATING VCS: DONE"
-echo "Local repo updated.  Please push when ready."
-echo "    $VCSCMD push"
+if [[ $VCS_TYPE = "svn" ]]; then
+	echo "Local repo updated and file pushed to source control (unless an error was displayed)."
+else
+	echo "Local repo updated.  Please push when ready."
+	echo "    $VCS_TYPE push"
+fi

bin/blackbox_removeadmin

@@ -10,9 +10,7 @@
 #
 
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
-source "${blackbox_home}/_stack_lib.sh"
+source "${0%/*}/_blackbox_common.sh"
 
 fail_if_not_in_repo
 
@@ -20,12 +18,17 @@ KEYNAME="$1"
 : "${KEYNAME:?ERROR: First argument must be a keyname (email address)}" ;
 
 # Remove the email address from the BB_ADMINS file.
-make_self_deleting_tempfile bbtemp
-cp "$BB_ADMINS" "$bbtemp"
-fgrep -v -x "$KEYNAME" <"$bbtemp" >"$BB_ADMINS"
+remove_line "$BB_ADMINS" "$KEYNAME"
+
+
+# remove the admin key from the pubring
+$GPG --no-permission-warning --homedir="$KEYRINGDIR" --batch --yes --delete-key "$KEYNAME" || true
+pubring_path=$(get_pubring_path)
+vcs_add "$pubring_path" "$KEYRINGDIR/trustdb.gpg" "$BB_ADMINS"
+
 
 # Make a suggestion:
 echo
 echo
 echo 'NEXT STEP: Check these into the repo.  Probably with a command like...'
-echo $VCS_TYPE commit -m\'REMOVED ADMIN: $KEYNAME\' "$BLACKBOXDATA/$BB_ADMINS_FILE"
+echo $VCS_TYPE commit -m\'REMOVED ADMIN: $KEYNAME\' "$BLACKBOXDATA/$(basename ${pubring_path})" "$BLACKBOXDATA/trustdb.gpg" "$BLACKBOXDATA/$BB_ADMINS_FILE"

bin/blackbox_shred_all_files

@@ -16,19 +16,30 @@
 # have been decrypted for editing, you will see an empty list.
 
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
+source "${0%/*}/_blackbox_common.sh"
 
 change_to_vcs_root
 
 echo '========== FILES BEING SHREDDED:'
-for i in $(<"$BB_FILES") ; do
-  unencrypted_file=$(get_unencrypted_filename "$i")
-  encrypted_file=$(get_encrypted_filename "$i")
+
+exported_internal_shred_file() {
+  source "$1/_blackbox_common.sh"
+  #unencrypted_file=$(get_unencrypted_filename "$2")
+  unencrypted_file="$2"
   if [[ -f "$unencrypted_file" ]]; then
-    echo "    $unencrypted_file"
+    echo "    SHRED: $unencrypted_file"
     shred_file "$unencrypted_file"
+  else
+    echo "NOT FOUND: $unencrypted_file"
   fi
-done
+}
+
+export -f exported_internal_shred_file
+
+DEREFERENCED_BIN_DIR="${0%/*}"
+MAX_PARALLEL_SHRED=10
+
+export IFS=
+tr '\n' '\0' <"$BB_FILES" | xargs -0 -I{} -n 1 -P $MAX_PARALLEL_SHRED bash -c "exported_internal_shred_file $DEREFERENCED_BIN_DIR \"{}\"" $DEREFERENCED_BIN_DIR/fake
 
 echo '========== DONE.'

bin/blackbox_update_all_files

@@ -5,32 +5,31 @@
 #
 
 set -e
-blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "${blackbox_home}/_blackbox_common.sh"
-
-if [[ -z $GPG_AGENT_INFO ]]; then
-  echo 'WARNING: You probably want to run gpg-agent as'
-  echo 'you will be asked for your passphrase many times.'
-  echo 'Example: $ eval $(gpg-agent --daemon)'
-  read -r -p 'Press CTRL-C now to stop. ENTER to continue: '
-fi
+source "${0%/*}/_blackbox_common.sh"
 
+gpg_agent_notice
 disclose_admins
 prepare_keychain
 
 echo '========== ENCRYPTED FILES TO BE RE-ENCRYPTED:'
-awk <"$BB_FILES" '{ print "    " $1 ".gpg" }'
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $encodedname)
+  echo "    $name.gpg"
+done 99<"$BB_FILES"
 
 echo '========== FILES IN THE WAY:'
 need_warning=false
-for i in $(<"$BB_FILES") ; do
-  unencrypted_file=$(get_unencrypted_filename "$i")
-  encrypted_file=$(get_encrypted_filename "$i")
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $encodedname)
+  unencrypted_file=$(get_unencrypted_filename "$name")
+  encrypted_file=$(get_encrypted_filename "$name")
   if [[ -f "$unencrypted_file" ]]; then
     need_warning=true
     echo "    $unencrypted_file"
   fi
-done
+done 99<"$BB_FILES"
 if "$need_warning" ; then
   echo
   echo 'WARNING: This will overwrite any unencrypted files laying about.'
@@ -40,22 +39,26 @@ else
 fi
 
 echo '========== RE-ENCRYPTING FILES:'
-for i in $(<"$BB_FILES") ; do
-  unencrypted_file=$(get_unencrypted_filename "$i")
-  encrypted_file=$(get_encrypted_filename "$i")
-  echo ========== PROCESSING "$unencrypted_file"
+while IFS= read <&99 -r encodedname; do
+  local name
+  name=$(echo $encodedname)
+  unencrypted_file=$(get_unencrypted_filename "$name")
+  encrypted_file=$(get_encrypted_filename "$name")
+  echo ========== PROCESSING '"'$unencrypted_file'"'
   fail_if_not_on_cryptlist "$unencrypted_file"
   decrypt_file_overwrite "$encrypted_file" "$unencrypted_file"
   encrypt_file "$unencrypted_file" "$encrypted_file"
   shred_file "$unencrypted_file"
-done
+done 99<"$BB_FILES"
 
 fail_if_keychain_has_secrets
 
 echo '========== COMMITING TO VCS:'
-vcs_commit 'Re-encrypted keys' $(awk <"$BB_FILES" '{ print $1 ".gpg" }' )
+while IFS= read <&99 -r unencrypted_file; do
+    vcs_add "$unencrypted_file.gpg"
+done 99<"$BB_FILES"
+vcs_commit 'Re-encrypted keys'
 
-VCSCMD=$(which_vcs)
 echo '========== DONE.'
 echo 'Likely next step:'
-echo "    ${VCSCMD} push"
+echo "    $VCS_TYPE push"

bin/blackbox_view

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_view -- Decrypt a file, view it, shred it
+#
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+for param in "$@" ; do
+  shreddable=0
+  unencrypted_file=$(get_unencrypted_filename "$param")
+  if [[ ! -e "$unencrypted_file" ]]; then
+    "${BLACKBOX_HOME}/blackbox_edit_start" "$param"
+    shreddable=1
+  fi
+  ${PAGER:-less} "$unencrypted_file"
+  if [[ $shreddable = 1 ]]; then
+    shred_file "$unencrypted_file"
+  fi
+done

bin/blackbox_whatsnew

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_whatsnew - show what has changed in the last commit for a given file
+#
+
+set -e
+source "${0%/*}/_blackbox_common.sh"
+
+if [[ $# -ne 1 ]]
+then
+  echo "Pass only 1 file at a time"
+  exit 1
+fi
+
+fail_if_not_in_repo
+gpg_agent_notice
+
+COLUMNS=`tput cols`
+FILE=$1
+GIT="git log --abbrev-commit --pretty=oneline"
+CURR_COMMIT=`$GIT                  $FILE | head -1 | awk '{print $1}'`
+PREV_COMMIT=`$GIT ${CURR_COMMIT}~1 $FILE | head -1 | awk '{print $1}'`
+# Use colordiff if available
+if which colordiff > /dev/null 2>&1
+  then DIFF="colordiff"
+  else DIFF="diff"
+fi
+
+cat_commit()
+{
+  COMMIT=$1
+  git checkout $COMMIT $FILE
+  echo "[$COMMIT] $FILE"
+  echo "---------------------"
+  "${BLACKBOX_HOME}/blackbox_cat" $FILE | sed '/========== PLAINFILE/,/========== EXTRACTING/d'
+}
+
+CURR_CONTENT=`cat_commit $CURR_COMMIT`
+PREV_CONTENT=`cat_commit $PREV_COMMIT`
+clear
+
+# For some unknown reason this command executes fine but return exit code 1
+$DIFF -y --width $COLUMNS \
+  <(echo "CURRENT"  "$CURR_CONTENT" | fold -w $(( $COLUMNS / 2 - 4 ))  ) \
+  <(echo "PREVIOUS" "$PREV_CONTENT" | fold -w $(( $COLUMNS / 2 - 4 ))  )
+
+git checkout $CURR_COMMIT $FILE
+echo

blackbox.plugin.zsh

@@ -1,4 +1,4 @@
-#!/bin/zsh
+#!/usr/bin/env zsh
 # The MIT License (MIT)
 
 # Copyright (c) 2014 Stack Exchange, Inc.

tools/auto_system_test

@@ -0,0 +1,25 @@
+#!/usr/bin/env expect
+
+# Run the confidence test non-interactively.  Since the script
+# asks for passphrases, we use "expect" to simulate keyboard data entry.
+
+# Run the test:
+spawn tools/confidence_test.sh
+
+# As we run the confidence test, respond with the right password.
+# We do this for up to 300 times to prevent an infinite loop.
+
+set times 0;
+while { $times < 300 } {
+   expect {
+       # The script outputs what the password will be, and we save
+       # that info in $pw any time we see the text.
+       "my password is the lowercase letter a" { set pw "a\n" ; exp_continue }
+       "my password is the lowercase letter b" { set pw "b\n" ; exp_continue }
+       # If the passphrase is requested, send it.
+       "Passphrase:" { send $pw ; exp_continue }
+       # If we reach EOF, exit this loop.
+       eof { break }
+   }
+   set times [ expr $times+1];
+}

tools/confidence_test.sh

@@ -1,109 +1,40 @@
 #!/usr/bin/env bash
 
 blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../bin
-export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin"
+export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:${blackbox_home}"
 
-. _stack_lib.sh
+export LANG=C.UTF-8  # Required ro "gpg --export" to work properly.
+
+# TODO(tlim): The tests are hardcoded to this directory. This should be fixed.
+export BLACKBOXDATA=keyrings/live
+
+# This script requires many utilities, some are not
+# required by the usual blackbox scripts.  Test to make
+# sure we have them all.
+e=false
+for i in blackbox_addadmin blackbox_list_admins blackbox_register_new_file \
+  cat git gpg gpg-agent mkdir pinentry pinentry-tty rm tar which ; do
+  if ! which >/dev/null 2>&1  $i ; then
+    echo ERROR: Command not in PATH: $i
+    e=true
+  fi
+done
+if $e ; then
+  echo 'Exiting. Please install the above commands.'
+  echo 'This script requires many utilities not required by blackbox itself.'
+  echo PATH="$PATH"
+  exit 1
+fi
 
 set -e
+. _stack_lib.sh
+. tools/test_functions.sh
 
-function PHASE() {
-  echo '********************'
-  echo '********************'
-  echo '*********' """$@"""
-  echo '********************'
-  echo '********************'
-}
+PHASE 'UNIT TESTS'
 
-function md5sum_file() {
-  # Portably generate the MD5 hash of file $1.
-  case $(uname -s) in
-    Darwin )
-      md5 -r "$1" | awk '{ print $1 }'
-      ;;
-    Linux )
-      md5sum "$1" | awk '{ print $1 }'
-      ;;
-    CYGWIN* )
-      md5sum "$1" | awk '{ print $1 }'
-      ;;
-    * )
-      echo 'ERROR: Unknown OS. Exiting.'
-      exit 1
-      ;;
-  esac
-}
+_blackbox_common_test.sh
 
-function assert_file_missing() {
-  if [[ -e "$1" ]]; then
-    echo "ASSERT FAILED: ${1} should not exist."
-    exit 1
-  fi
-}
-
-function assert_file_exists() {
-  if [[ ! -e "$1" ]]; then
-    echo "ASSERT FAILED: ${1} should exist."
-    echo "PWD="$(/bin/pwd -P)
-    #echo "LS START"
-    #ls -la
-    #echo "LS END"
-    exit 1
-  fi
-}
-function assert_file_md5hash() {
-  local file="$1"
-  local wanted="$2"
-  assert_file_exists "$file"
-  local found=$(md5sum_file "$file")
-  if [[ "$wanted" != "$found" ]]; then
-    echo "ASSERT FAILED: $file hash wanted=$wanted found=$found"
-    exit 1
-  fi
-}
-function assert_file_group() {
-  local file="$1"
-  local wanted="$2"
-  assert_file_exists "$file"
-
-  case $(uname -s) in
-    CYGWIN* )
-      echo "ASSERT_FILE_GROUP: Running on Cygwin. Not being tested."
-      return 0
-      ;;
-  esac
-
-  local found=$(ls -lg "$file" | awk '{ print $3 }')
-  # NB(tlim): We could do this with 'stat' but it would break on BSD-style OSs.
-  if [[ "$wanted" != "$found" ]]; then
-    echo "ASSERT FAILED: $file chgrp wanted=$wanted found=$found"
-    exit 1
-  fi
-}
-function assert_line_not_exists() {
-  local target="$1"
-  local file="$2"
-  assert_file_exists "$file"
-  if grep -F -x -s -q >/dev/null "$target" "$file" ; then
-    echo "ASSERT FAILED: line '$target' should not exist in file $file"
-    echo ==== file contents: START "$file"
-    cat "$file"
-    echo ==== file contents: END "$file"
-    exit 1
-  fi
-}
-function assert_line_exists() {
-  local target="$1"
-  local file="$2"
-  assert_file_exists "$file"
-  if ! grep -F -x -s -q >/dev/null "$target" "$file" ; then
-    echo "ASSERT FAILED: line '$target' should not exist in file $file"
-    echo ==== file contents: START "$file"
-    cat "$file"
-    echo ==== file contents: END "$file"
-    exit 1
-  fi
-}
+PHASE 'SYSTEM TESTS'
 
 make_tempdir test_repository
 cd "$test_repository"
@@ -111,33 +42,39 @@ cd "$test_repository"
 make_self_deleting_tempdir fake_alice_home
 make_self_deleting_tempdir fake_bob_home
 export GNUPGHOME="$fake_alice_home"
-eval $(gpg-agent --homedir "$fake_alice_home" --daemon)
+#echo 'pinentry-program' "$(which pinentry-insecure-fake)" >> "$GNUPGHOME/gpg-agent.conf"
+echo 'pinentry-program' "$(which pinentry-tty)" >> "$GNUPGHOME/gpg-agent.conf"
+#pinentry-program "${blackbox_home}/pinentry-fake-insecure"
+eval "$(gpg-agent --homedir "$fake_alice_home" --daemon)"
 GPG_AGENT_INFO_ALICE="$GPG_AGENT_INFO"
 
 export GNUPGHOME="$fake_bob_home"
-eval $(gpg-agent --homedir "$fake_bob_home" --daemon)
+#echo 'pinentry-program' "$(which pinentry-insecure-fake)" >> "$GNUPGHOME/gpg-agent.conf"
+echo 'pinentry-program' "$(which pinentry-tty)" >> "$GNUPGHOME/gpg-agent.conf"
+eval "$(gpg-agent --homedir "$fake_bob_home" --daemon)"
 GPG_AGENT_INFO_BOB="$GPG_AGENT_INFO"
 
 function become_alice() {
   export GNUPGHOME="$fake_alice_home"
   export GPG_AGENT_INFO="$GPG_AGENT_INFO_ALICE"
-  echo BECOMING ALICE: GNUPGHOME=$GNUPGHOME AGENT=$GPG_AGENT_INFO
-  git config --global user.name "Alice Example"
-  git config --global user.email alice@example.com
+  echo BECOMING ALICE: GNUPGHOME="$GNUPGHOME AGENT=$GPG_AGENT_INFO"
+  mkdir -p .git ; touch .git/config
+  git init
+  git config user.name "Alice Example"
+  git config user.email alice@example.com
 }
 
 function become_bob() {
   export GNUPGHOME="$fake_bob_home"
   export GPG_AGENT_INFO="$GPG_AGENT_INFO_BOB"
-  git config --global user.name "Bob Example"
-  git config --global user.email bob@example.com
+  mkdir -p .git ; touch .git/config
+  git config user.name "Bob Example"
+  git config user.email bob@example.com
 }
 
-
 PHASE 'Alice creates a repo.  She creates secret.txt.'
 
 become_alice
-git init
 echo 'this is my secret' >secret.txt
 
 
@@ -173,7 +110,11 @@ git commit -m'INITIALIZE BLACKBOX' keyrings .gitignore
 PHASE 'and adds herself as an admin.'
 
 blackbox_addadmin alice@example.com
-git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.gpg keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.??? keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+
+make_self_deleting_tempfile adminlist1
+blackbox_list_admins >"$adminlist1"
+assert_file_md5hash "$adminlist1" "aa1db827772e1d51d453b844394b7617"
 
 
 PHASE 'Bob arrives.'
@@ -201,20 +142,24 @@ gpg --no-permission-warning --batch --gen-key "$gpgconfig"
 echo '========== Bob enrolls himself too.'
 
 blackbox_addadmin bob@example.com
-git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.gpg keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+git commit -m'NEW ADMIN: alice@example.com' keyrings/live/pubring.??? keyrings/live/trustdb.gpg keyrings/live/blackbox-admins.txt
+
+make_self_deleting_tempfile adminlist2
+blackbox_list_admins >"$adminlist2"
+assert_file_md5hash "$adminlist2" "0b82b0b3c96e6e5dd5faf60493fe5cf7"
 
 PHASE 'Alice does the second part to enroll bob.'
 become_alice
 
 PHASE 'She enrolls bob.'
-gpg --import keyrings/live/pubring.gpg
-# TODO(tlim) That --import can be eliminated... maybe?
+#gpg --import $(get_pubring_path)
+## TODO(tlim) That --import can be eliminated... maybe?
 
 PHASE 'She enrolls secrets.txt.'
 blackbox_register_new_file secret.txt
 assert_file_missing secret.txt
 assert_file_exists secret.txt.gpg
-assert_line_exists 'secret.txt' .gitignore
+assert_line_exists '/secret.txt' .gitignore
 
 PHASE 'She decrypts secrets.txt.'
 blackbox_edit_start secret.txt
@@ -229,44 +174,89 @@ assert_file_missing secret.txt
 assert_file_exists secret.txt.gpg
 
 
-PHASE 'Bob appears.'
+PHASE 'Alice copies files to a non-repo directory. (NO REPO)'
+
+# Copy the repo entirely:
+make_self_deleting_tempdir fake_alice_filedir
+tar cf - . | ( cd "$fake_alice_filedir" && tar xpvf - )
+# Remove the .git directory
+rm -rf "$fake_alice_filedir/.git"
+(
+cd "$fake_alice_filedir"
+assert_file_missing '.git'
+assert_file_exists 'secret.txt.gpg'
+assert_file_missing 'secret.txt'
+blackbox_postdeploy
+assert_file_missing '.git'
+assert_file_exists 'secret.txt.gpg'
+assert_file_exists 'secret.txt'
+assert_file_md5hash secret.txt "08a3fa763a05c018a38e9924363b97e7"
+
+PHASE 'Alice shreds these non-repo files. (NO REPO)'
+blackbox_shred_all_files
+assert_file_missing '.git'
+assert_file_exists 'secret.txt.gpg'
+assert_file_missing 'secret.txt'
+
+PHASE 'Alice decrypts secrets.txt (NO REPO).'
+blackbox_edit_start secret.txt
+assert_file_exists secret.txt
+assert_file_exists secret.txt.gpg
+assert_file_md5hash secret.txt "08a3fa763a05c018a38e9924363b97e7"
+
+PHASE 'Alice edits secrets.txt. (NO REPO EDIT)'
+echo 'NOREPO EDIT' >secret.txt
+assert_file_md5hash secret.txt "d3e6bbdfc76fae7fd0a921f3408db1d1"
+blackbox_edit_end secret.txt
+assert_file_missing secret.txt
+assert_file_exists secret.txt.gpg
+
+PHASE 'Alice decrypts secrets.txt (NO REPO EDIT).'
+blackbox_edit_start secret.txt
+assert_file_exists secret.txt
+assert_file_exists secret.txt.gpg
+assert_file_md5hash secret.txt "d3e6bbdfc76fae7fd0a921f3408db1d1"
+)
+
+PHASE 'appears.'
 become_bob
 
-PHASE 'Bob makes sure he has all new keys.'
-
-gpg --import keyrings/live/pubring.gpg
+#PHASE 'Bob makes sure he has all new keys.'
+#gpg --import keyrings/live/pubring.???
 
 # Pick a GID to use:
 # This users's default group:
-DEFAULT_GID_NAME=$(id -gn)
+DEFAULT_GID_NUM=$(id -g)
 # Pick a group that is not the default group:
-TEST_GID_NUM=$(id -G | fmt -1 | tail -n +2 | grep -xv $(id -u) | head -n 1)
-TEST_GID_NAME=$(python -c 'import grp; print grp.getgrgid('"$TEST_GID_NUM"').gr_name')
-echo DEFAULT_GID_NAME=$DEFAULT_GID_NAME
-echo TEST_GID_NUM=$TEST_GID_NUM
-echo TEST_GID_NAME=$TEST_GID_NAME
+TEST_GID_NUM=$(grep -v "$DEFAULT_GID_NUM" /etc/group | cut -d: -f3 | sort -rn | head -1)
+echo "DEFAULT_GID_NUM=$DEFAULT_GID_NUM"
+echo "TEST_GID_NUM=$TEST_GID_NUM"
 
 PHASE 'Bob postdeploys... default.'
 blackbox_postdeploy
 assert_file_exists secret.txt
 assert_file_exists secret.txt.gpg
 assert_file_md5hash secret.txt "08a3fa763a05c018a38e9924363b97e7"
-assert_file_group secret.txt "$DEFAULT_GID_NAME"
+assert_file_group secret.txt "$DEFAULT_GID_NUM"
 
 PHASE 'Bob postdeploys... with a GID.'
-blackbox_postdeploy $TEST_GID_NUM
+blackbox_postdeploy "$TEST_GID_NUM"
 assert_file_exists secret.txt
 assert_file_exists secret.txt.gpg
 assert_file_md5hash secret.txt "08a3fa763a05c018a38e9924363b97e7"
-assert_file_group secret.txt "$TEST_GID_NAME"
+assert_file_group secret.txt "$TEST_GID_NUM"
 
 PHASE 'Bob cleans up the secret.'
 rm secret.txt
 
-PHASE 'Bob removes alice.'
+PHASE 'Bob removes Alice.'
 blackbox_removeadmin alice@example.com
 assert_line_not_exists 'alice@example.com' keyrings/live/blackbox-admins.txt
 
+make_self_deleting_tempfile adminlist3
+blackbox_list_admins >"$adminlist3"
+assert_file_md5hash "$adminlist3" "aadbfafd76ea66ff40dbfd239a69067f"
+
 PHASE 'Bob reencrypts files so alice can not access them.'
 blackbox_update_all_files
 
@@ -298,7 +288,7 @@ blackbox_register_new_file mistake.txt
 assert_file_missing mistake.txt
 assert_file_exists mistake.txt.gpg
 # NOTE: It is still in the history. That should be corrected someday.
-assert_line_exists 'mistake.txt' .gitignore
+assert_line_exists '/mistake.txt' .gitignore
 
 PHASE 'Bob enrolls my/path/to/relsecrets.txt.'
 mkdir my my/path my/path/to
@@ -309,7 +299,7 @@ assert_file_missing relsecrets.txt
 assert_file_exists relsecrets.txt.gpg
 assert_file_missing .gitignore
 assert_file_exists ../../../.gitignore
-assert_line_exists 'my/path/to/relsecrets.txt' ../../../.gitignore
+assert_line_exists '/my/path/to/relsecrets.txt' ../../../.gitignore
 
 PHASE 'Bob decrypts relsecrets.txt.'
 cd ..
@@ -324,41 +314,144 @@ echo A very important file >'!important!.txt'
 blackbox_register_new_file '!important!.txt'
 assert_file_missing '!important!.txt'
 assert_file_exists '!important!.txt'.gpg
-assert_line_exists '\!important!.txt' .gitignore
+assert_line_exists '/!important!.txt' .gitignore
 
 PHASE 'Bob enrolls #andpounds.txt'
 echo A very commented file >'#andpounds.txt'
 blackbox_register_new_file '#andpounds.txt'
 assert_file_missing '#andpounds.txt'
 assert_file_exists '#andpounds.txt'.gpg
-assert_line_exists '\#andpounds.txt' .gitignore
+assert_line_exists '/#andpounds.txt' .gitignore
 
 PHASE 'Bob enrolls stars*bars?.txt'
-echo A very commented file >'stars*bars?.txt'
+echo A very wild and questioned file >'stars*bars?.txt'
 blackbox_register_new_file 'stars*bars?.txt'
 assert_file_missing 'stars*bars?.txt'
 assert_file_exists 'stars*bars?.txt'.gpg
-assert_line_exists 'stars\*bars\?.txt' .gitignore
+assert_line_exists '/stars\*bars\?.txt' .gitignore
 
-PHASE 'Bob enrolls stars bars.txt'
-echo A very commented file >'stars bars.txt'
-blackbox_register_new_file 'stars bars.txt'
-assert_file_missing 'stars bars.txt'
-assert_file_exists 'stars bars.txt'.gpg
-assert_line_exists 'stars bars.txt' .gitignore
+PHASE 'Bob enrolls space space.txt'
+echo A very spacey file >'space space.txt'
+blackbox_register_new_file 'space space.txt'
+assert_file_missing 'space space.txt'
+assert_file_exists 'space space.txt'.gpg
+assert_line_exists '/space space.txt' .gitignore
 
-# TODO(tlim): Add test to make sure that now alice can NOT decrypt.
+PHASE 'Bob checks out stars*bars?.txt.'
+blackbox_edit_start 'stars*bars?.txt'
+assert_file_exists 'stars*bars?.txt'
+assert_file_exists 'stars*bars?.txt'
+assert_file_md5hash 'stars*bars?.txt' "448e018faade28cede2bf6f33c3c2dfb"
+
+PHASE 'Bob checks out space space.txt.'
+blackbox_edit_start 'space space.txt'
+assert_file_exists 'space space.txt'
+assert_file_exists 'space space.txt'
+assert_file_md5hash 'space space.txt' "de1d4e4a07046f81af5d3c0194b78742"
+
+PHASE 'Bob shreds all exposed files.'
+assert_file_exists 'my/path/to/relsecrets.txt'
+assert_file_exists 'secret.txt'
+blackbox_shred_all_files
+which blackbox_shred_all_files
+assert_file_missing '!important!.txt'
+assert_file_missing '#andpounds.txt'
+assert_file_missing 'mistake.txt'
+assert_file_missing 'my/path/to/relsecrets.txt'
+assert_file_missing 'secret.txt'
+assert_file_missing 'space space.txt'
+assert_file_missing 'stars*bars?.txt'
+assert_file_exists '!important!.txt.gpg'
+assert_file_exists '#andpounds.txt.gpg'
+assert_file_exists 'mistake.txt.gpg'
+assert_file_exists 'my/path/to/relsecrets.txt.gpg'
+assert_file_exists 'secret.txt.gpg'
+assert_file_exists 'space space.txt.gpg'
+assert_file_exists 'stars*bars?.txt.gpg'
+
+PHASE 'Bob updates all files.'
+blackbox_update_all_files
+assert_file_missing '!important!.txt'
+assert_file_missing '#andpounds.txt'
+assert_file_missing 'mistake.txt'
+assert_file_missing 'my/path/to/relsecrets.txt'
+assert_file_missing 'secret.txt'
+assert_file_missing 'space space.txt'
+assert_file_missing 'stars*bars?.txt'
+assert_file_exists '!important!.txt.gpg'
+assert_file_exists '#andpounds.txt.gpg'
+assert_file_exists 'mistake.txt.gpg'
+assert_file_exists 'my/path/to/relsecrets.txt.gpg'
+assert_file_exists 'secret.txt.gpg'
+assert_file_exists 'space space.txt.gpg'
+assert_file_exists 'stars*bars?.txt.gpg'
+
+PHASE 'Bob DEregisters mistake.txt'
+touch 'mistake.txt'
+blackbox_deregister_file 'mistake.txt.gpg'
+assert_file_exists 'keyrings/live/blackbox-admins.txt'
+assert_file_exists 'keyrings/live/blackbox-files.txt'
+assert_line_not_exists 'mistake.txt' 'keyrings/live/blackbox-files.txt'
+assert_file_missing 'mistake.txt.gpg'
+assert_file_exists 'mistake.txt'
+# Now remove 'mistake.txt' to leave the area clean.
+rm 'mistake.txt'
+
+PHASE 'Bob enrolls multiple files: multi1.txt and multi2.txt'
+echo 'One singular sensation.' >'multi1.txt'
+echo 'Another singular sensation.' >'multi2.txt'
+blackbox_register_new_file 'multi1.txt' 'multi2.txt'
+assert_file_missing 'multi1.txt'
+assert_file_exists 'multi1.txt'.gpg
+assert_line_exists '/multi1.txt' .gitignore
+assert_file_missing 'multi2.txt'
+assert_file_exists 'multi2.txt'.gpg
+assert_line_exists '/multi2.txt' .gitignore
+
+PHASE 'Alice returns. She should be locked out'
+assert_file_missing 'secret.txt'
+become_alice
+PHASE 'Alice tries to decrypt secret.txt. Is blocked.'
+if blackbox_edit_start secret.txt ; then
+  echo 'ERROR: Alice was able to decrypt secret.txt!  She should have been blocked.'
+  exit 1
+else
+  echo 'NOTE: Alice was not able to decrypt secret.txt as expected.'
+fi
+
+PHASE 'Bob returns.  Tries to update all files with a corrupt blackbox-admins.txt'
+become_bob
+# Corrupt the blackbox-admins.txt list:
+echo 'abba@notarealuser.com' >> keyrings/live/blackbox-admins.txt
+# Make sure it fails.
+if blackbox_update_all_files; then
+  echo '!!!!! blackbox_update_all_files should have failed and it did NOT.'
+  exit 1
+fi
+# Cleanup:
+blackbox_removeadmin abba@notarealuser.com
+
+make_self_deleting_tempfile adminlist4
+blackbox_list_admins >"$adminlist4"
+assert_file_md5hash "$adminlist4" "aadbfafd76ea66ff40dbfd239a69067f"
+
+
+# TODO: Create a new directory. "git clone" the repo into it.
 
 #
 # ASSERTIONS
 #
 
+echo '========== Verifying .gnupg was not accidentally created.'
+
 if [[ -e $HOME/.gnupg ]]; then
   echo "ASSERT FAILED: $HOME/.gnupg should not exist."
   exit 1
 fi
 
+echo '========== DONE with tests.  Outputing some diagnostics:'
+
 find .git?* * -type f -ls
 echo cd "$test_repository"
 echo rm -rf "$test_repository"
-echo DONE.
+echo 'SUCCESS! Doing final clean-up then exiting.'

tools/macports_report_upgrade.sh

@@ -43,7 +43,7 @@ fi
 
 # Generate the diff
 cp /opt/local/var/macports/sources/rsync.macports.org/release/tarballs/ports/security/vcs_blackbox/Portfile /var/tmp/ports/security/vcs_blackbox/Portfile.orig
-( cd /var/tmp/ports/security/vcs_blackbox && diff -u Portfile.orig Portfile ) > Portfile-vcs_blackbox.diff
+( cd /var/tmp/ports/security/vcs_blackbox && diff --ignore-matching-lines='Id:' -u Portfile.orig Portfile ) > Portfile-vcs_blackbox.diff
 open -R Portfile-vcs_blackbox.diff
 
 echo
@@ -62,6 +62,6 @@ github.setup and checksums updated.
     Type: update
     Component: ports
     Port: vcs_blackbox
-    Keywords: maintainer
+    Keywords: maintainer haspatch
 "
 echo 'Attach: Portfile-vcs_blackbox.diff'

tools/mk_deb_fpmdir

@@ -22,13 +22,27 @@ shift
 # Defaults that can be overridden:
 # All packages are 1.0 unless otherwise specifed:
 : ${PKGVERSION:=1.0} ;
-# If there is no iteration setting, assume "1":
-: ${PKGRELEASE:=1}
+# If there is no iteration set, default to use the number of commits in the repository:
+if [[ -z "${PKGRELEASE}" ]]; then
+  PKGRELEASE=$(git rev-list HEAD --count)
+  if [[ $? != 0 ]]; then
+    # We're not in a git repo, fall back to 1 so we cope with being built from
+    # a tarball
+    PKGRELEASE=1
+  fi
+fi
+
 # If there is no epoch, assume 0
 : ${PKGEPOCH:=0}
 
-# The DEB is output here: (should be a place that can be wiped)
-OUTPUTDIR="${HOME}/debbuild-$PACKAGENAME"
+# Allow us to set a different OUTPUTDIR if we're building in CI/CD
+if [[ -z "${OUTPUTDIR}" ]]; then
+  # The DEB is output here: (should be a place that can be wiped)
+  OUTPUTDIR="${HOME}/debbuild-${PACKAGENAME}"
+else
+  echo "Using $OUTPUTDIR for OUTPUTDIR instead of ${HOME}/debbuild-${PACKAGENAME}"
+fi
+
 # The TeamCity templates expect to find the list of artifacts here:
 DEB_BIN_LIST="${OUTPUTDIR}/bin-packages.txt"
 
@@ -36,6 +50,7 @@ DEB_BIN_LIST="${OUTPUTDIR}/bin-packages.txt"
 
 # Clean the output dir.
 rm -rf "$OUTPUTDIR"
+
 mkdir -p "$OUTPUTDIR/installroot"
 
 # Copy the files into place:
@@ -51,14 +66,14 @@ cat """$@""" | while read -a arr ; do
   DST="$OUTPUTDIR/installroot/${arr[1]}"
   SRC="${arr[2]}"
   if [[ $SRC == "cmd/"* || $SRC == *"/cmd/"* ]]; then
-    ( cd $(dirname "$SRC" ) && go build -a -v ) 
+    ( cd $(dirname "$SRC" ) && go build -a -v )
   fi
   install -D -T -b -m "$PERM" -T "$SRC" "$DST"
 done
 
 # Build the DEB:
 cd "$OUTPUTDIR" && fpm -s dir -t deb \
-  -a x86_64 \
+  -a all \
   -n "${PACKAGENAME}" \
   --epoch "${PKGEPOCH}" \
   --version "${PKGVERSION}" \

tools/mk_deb_fpmdir.stack_blackbox.txt

@@ -2,13 +2,19 @@ exec /usr/bin/_blackbox_common.sh       ../bin/_blackbox_common.sh
 exec /usr/bin/_stack_lib.sh             ../bin/_stack_lib.sh
 exec /usr/bin/blackbox_addadmin         ../bin/blackbox_addadmin
 exec /usr/bin/blackbox_cat              ../bin/blackbox_cat
+exec /usr/bin/blackbox_decrypt_all_files  ../bin/blackbox_decrypt_all_files
+exec /usr/bin/blackbox_deregister_file  ../bin/blackbox_deregister_file
+exec /usr/bin/blackbox_diff             ../bin/blackbox_diff
 exec /usr/bin/blackbox_edit             ../bin/blackbox_edit
 exec /usr/bin/blackbox_edit_end         ../bin/blackbox_edit_end
 exec /usr/bin/blackbox_edit_start       ../bin/blackbox_edit_start
 exec /usr/bin/blackbox_initialize       ../bin/blackbox_initialize
+exec /usr/bin/blackbox_listadmins       ../bin/blackbox_listadmins
+exec /usr/bin/blackbox_list_files       ../bin/blackbox_list_files
+exec /usr/bin/blackbox_list_admins      ../bin/blackbox_list_admins
 exec /usr/bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
 exec /usr/bin/blackbox_register_new_file ../bin/blackbox_register_new_file
 exec /usr/bin/blackbox_removeadmin      ../bin/blackbox_removeadmin
 exec /usr/bin/blackbox_shred_all_files  ../bin/blackbox_shred_all_files
 exec /usr/bin/blackbox_update_all_files ../bin/blackbox_update_all_files
-exec /usr/bin/blackbox_list_files       ../bin/blackbox_list_files
+exec /usr/bin/blackbox_whatsnew         ../bin/blackbox_whatsnew

tools/mk_macports.vcs_blackbox.txt

@@ -2,13 +2,19 @@ exec bin/_blackbox_common.sh       ../bin/_blackbox_common.sh
 exec bin/_stack_lib.sh             ../bin/_stack_lib.sh
 exec bin/blackbox_addadmin         ../bin/blackbox_addadmin
 exec bin/blackbox_cat              ../bin/blackbox_cat
+exec bin/blackbox_decrypt_all_files  ../bin/blackbox_decrypt_all_files
+exec bin/blackbox_deregister_file  ../bin/blackbox_deregister_file
+exec bin/blackbox_diff             ../bin/blackbox_diff
 exec bin/blackbox_edit             ../bin/blackbox_edit
 exec bin/blackbox_edit_end         ../bin/blackbox_edit_end
 exec bin/blackbox_edit_start       ../bin/blackbox_edit_start
 exec bin/blackbox_initialize       ../bin/blackbox_initialize
+exec bin/blackbox_listadmins       ../bin/blackbox_listadmins
+exec bin/blackbox_list_files       ../bin/blackbox_list_files
+exec bin/blackbox_list_admins      ../bin/blackbox_list_admins
 exec bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
 exec bin/blackbox_register_new_file ../bin/blackbox_register_new_file
 exec bin/blackbox_removeadmin      ../bin/blackbox_removeadmin
 exec bin/blackbox_shred_all_files  ../bin/blackbox_shred_all_files
 exec bin/blackbox_update_all_files ../bin/blackbox_update_all_files
-exec bin/blackbox_list_files       ../bin/blackbox_list_files
+exec bin/blackbox_whatsnew         ../bin/blackbox_whatsnew

tools/mk_rpm_fpmdir

@@ -19,28 +19,78 @@ set -e
 PACKAGENAME=${1?"First arg must be the package name."}
 shift
 
+# What is my name?
+CMDNAME=$(basename $0)
+
 # Defaults that can be overridden:
-# All packages are 1.0 unless otherwise specifed:
+# Packages are 1.0 unless otherwise specifed:
 : ${PKGVERSION:=1.0} ;
-# If there is no iteration setting, assume "1":
-: ${PKGRELEASE:=1}
+# If there is no iteration set, default to use the number of commits in the repository:
+if [[ -z "${PKGRELEASE}" ]]; then
+  PKGRELEASE=$(git rev-list HEAD --count)
+  if [[ $? != 0 ]]; then
+    # We're not in a git repo, fall back to 1 so we cope with being built from
+    # a tarball
+    PKGRELEASE=1
+  fi
+fi
+
 # If there is no epoch, assume 0
 : ${PKGEPOCH:=0}
 
-# The RPM is output here: (should be a place that can be wiped)
-OUTPUTDIR="${HOME}/rpmbuild-$PACKAGENAME"
-# The TeamCity templates expect to find the list of artifacts here:
+# If no arch defined, assume any. Other good values include "native".
+: ${PKGARCH:=all}
+# NOTE: If we later compile code, we set this to "native", which
+# FPM will translate to the correct value for local conditions.
+
+# Allow us to set a different OUTPUTDIR if we're building in CI/CD
+if [[ -z "${OUTPUTDIR}" ]]; then
+  # The RPM is output here: (should be a place that can be wiped)
+  OUTPUTDIR="${HOME}/rpmbuild-${PACKAGENAME}"
+else
+  echo "Using ${OUTPUTDIR} for OUTPUTDIR instead of ${HOME}/rpmbuild-${PACKAGENAME}"
+fi
+INSTALLROOT="$OUTPUTDIR/installroot"
+
+# StackOverflow's TeamCity templates expect to find the list of artifacts here:
 RPM_BIN_LIST="${OUTPUTDIR}/bin-packages.txt"
 
 # -- Now the real work can be done.
 
 # Clean the output dir.
 rm -rf "$OUTPUTDIR"
-mkdir -p "$OUTPUTDIR/installroot"
+mkdir -p "$INSTALLROOT"
 
-# Copy the files into place:
+# If there is a build script, execute it.
+BUILDSCRIPTNAME="./build.${PACKAGENAME}.sh"
+if [[ -x $BUILDSCRIPTNAME ]]; then
+  echo "========== $BUILDSCRIPTNAME FOUND. Running."
+  if [[ $PKGARCH == "all" ]]; then
+    echo 'WARNING: PKGARCH=all (which may not what you want)'
+    # If you are using a build.*.sh script, you probably want to
+    # set PKGARCH to "native" before you run mk_rpm_fpmdir.
+  fi
+  $BUILDSCRIPTNAME "$INSTALLROOT" "${PKGVERSION}"
+  # If we used the build build.*.sh script, it must do all compilation.
+  # Therefore, we disable the automagic GO build feature.
+  GO_COMPILE=false
+else
+  GO_COMPILE=true
+fi
+
+# If there are additional args for fpm, read them into a variable.  There is
+# a chdir later, therefore we can't rely on the file path working at that time.
+FPM_OPTIONS_FILE="./fpm_opts.${PACKAGENAME}.sh"
+if [[ -f $FPM_OPTIONS_FILE ]]; then
+  echo "========== $FPM_OPTIONS_FILE FOUND. Loading."
+  FPM_OPTIONS=$(<$FPM_OPTIONS_FILE)
+fi
+# Warning: The contents of the file are evaluated therefore
+# quotes and special chars must be quoted.
+
+# Copy any static files into place:
 set -o pipefail  # Error out if any manifest is not found.
-cat """$@""" | while read -a arr ; do
+cat "$@" | while read -a arr ; do
   PERM="${arr[0]}"
   case $PERM in
     \#*)  continue ;;   # Skip comments.
@@ -48,24 +98,39 @@ cat """$@""" | while read -a arr ; do
     read) PERM=0744 ;;
     *) ;;
   esac
-  DST="$OUTPUTDIR/installroot/${arr[1]}"
+  DST="$INSTALLROOT/${arr[1]}"
   SRC="${arr[2]}"
-  if [[ $SRC == "cmd/"* || $SRC == *"/cmd/"* ]]; then
-    ( cd $(dirname "$SRC" ) && go build -a -v ) 
+  if [[ ${#arr[@]} != 3 ]] ; then
+    echo "ERROR: Line must contain 3 items."
+    echo "DEBUG NUM=${#arr[@]} PERM=$PERM DST=$DST SRC=$SRC"
+    exit 1
+  fi
+  if $GO_COMPILE && [[ $SRC == "cmd/"* || $SRC == *"/cmd/"* ]]; then
+    echo "========== BUILD&COPY $SRC"
+    ( cd $(dirname "$SRC" ) && go get -d && go build ) 
+    PKGARCH=native
+  else
+    echo "==========       COPY $SRC"
+  fi
+  if [[ ! -f "$SRC" ]]; then
+    echo "${CMDNAME}: ERROR: File not found: $SRC"
+    exit 1
   fi
   install -D -T -b -m "$PERM" -T "$SRC" "$DST"
 done
 
-# Build the RPM:
+set -x
+# Build the RPM out of what is found in $INSTALLROOT:
 cd "$OUTPUTDIR" && fpm -s dir -t rpm \
-  -a x86_64 \
+  -a "${PKGARCH}" \
   -n "${PACKAGENAME}" \
   --epoch "${PKGEPOCH}" \
   --version "${PKGVERSION}" \
   --iteration "${PKGRELEASE}" \
   ${PKGDESCRIPTION:+ --description="${PKGDESCRIPTION}"} \
   ${PKGVENDOR:+ --vendor="${PKGVENDOR}"} \
-  -C "$OUTPUTDIR/installroot" \
+  ${FPM_OPTIONS:+ $FPM_OPTIONS} \
+  -C "$INSTALLROOT" \
   .
 
 # TeamCity templates for RPMS expect to find

tools/mk_rpm_fpmdir.stack_blackbox.txt

@@ -1,15 +1,22 @@
+# Update tools/mk_rpm_fpmdir.stack_blackbox.txt.  Other files generate from it.
 read /etc/profile.d/usrblackbox.sh               profile.d-usrblackbox.sh
 exec /usr/blackbox/bin/_blackbox_common.sh       ../bin/_blackbox_common.sh
 exec /usr/blackbox/bin/_stack_lib.sh             ../bin/_stack_lib.sh
 exec /usr/blackbox/bin/blackbox_addadmin         ../bin/blackbox_addadmin
 exec /usr/blackbox/bin/blackbox_cat              ../bin/blackbox_cat
+exec /usr/blackbox/bin/blackbox_decrypt_all_files  ../bin/blackbox_decrypt_all_files
+exec /usr/blackbox/bin/blackbox_deregister_file  ../bin/blackbox_deregister_file
+exec /usr/blackbox/bin/blackbox_diff             ../bin/blackbox_diff
 exec /usr/blackbox/bin/blackbox_edit             ../bin/blackbox_edit
 exec /usr/blackbox/bin/blackbox_edit_end         ../bin/blackbox_edit_end
 exec /usr/blackbox/bin/blackbox_edit_start       ../bin/blackbox_edit_start
 exec /usr/blackbox/bin/blackbox_initialize       ../bin/blackbox_initialize
+exec /usr/blackbox/bin/blackbox_listadmins       ../bin/blackbox_listadmins
+exec /usr/blackbox/bin/blackbox_list_files       ../bin/blackbox_list_files
+exec /usr/blackbox/bin/blackbox_list_admins      ../bin/blackbox_list_admins
 exec /usr/blackbox/bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
 exec /usr/blackbox/bin/blackbox_register_new_file ../bin/blackbox_register_new_file
 exec /usr/blackbox/bin/blackbox_removeadmin      ../bin/blackbox_removeadmin
 exec /usr/blackbox/bin/blackbox_shred_all_files  ../bin/blackbox_shred_all_files
 exec /usr/blackbox/bin/blackbox_update_all_files ../bin/blackbox_update_all_files
-exec /usr/blackbox/bin/blackbox_list_files       ../bin/blackbox_list_files
+exec /usr/blackbox/bin/blackbox_whatsnew         ../bin/blackbox_whatsnew

tools/profile.d-usrblackbox-test.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env sh
 
 # Test profile.d-usrblackbox.sh
 

tools/test_functions.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+# NB: This is copied from _blackbox_common.sh
+function get_pubring_path() {
+  : "${KEYRINGDIR:=keyrings/live}" ;
+  if [[ -f "${KEYRINGDIR}/pubring.gpg" ]]; then
+    echo "${KEYRINGDIR}/pubring.gpg"
+  else
+    echo "${KEYRINGDIR}/pubring.kbx"
+  fi
+}
+
+function PHASE() {
+  echo '********************'
+  echo '********************'
+  echo '*********' """$@"""
+  echo '********************'
+  echo '********************'
+}
+
+function md5sum_file() {
+  # Portably generate the MD5 hash of file $1.
+  case $(uname -s) in
+    Darwin | FreeBSD )
+      md5 -r "$1" | awk '{ print $1 }'
+      ;;
+    Linux )
+      md5sum "$1" | awk '{ print $1 }'
+      ;;
+    CYGWIN* )
+      md5sum "$1" | awk '{ print $1 }'
+      ;;
+    * )
+      echo 'ERROR: Unknown OS. Exiting.'
+      exit 1
+      ;;
+  esac
+}
+
+function assert_file_missing() {
+  if [[ -e "$1" ]]; then
+    echo "ASSERT FAILED: ${1} should not exist."
+    exit 1
+  fi
+}
+
+function assert_file_exists() {
+  if [[ ! -e "$1" ]]; then
+    echo "ASSERT FAILED: ${1} should exist."
+    echo "PWD=$(/usr/bin/env pwd -P)"
+    #echo "LS START"
+    #ls -la
+    #echo "LS END"
+    exit 1
+  fi
+}
+function assert_file_md5hash() {
+  local file="$1"
+  local wanted="$2"
+  assert_file_exists "$file"
+  local found
+  found=$(md5sum_file "$file")
+  if [[ "$wanted" != "$found" ]]; then
+    echo "ASSERT FAILED: $file hash wanted=$wanted found=$found"
+    exit 1
+  fi
+}
+function assert_file_group() {
+  local file="$1"
+  local wanted="$2"
+  local found
+  assert_file_exists "$file"
+
+  case $(uname -s) in
+    Darwin|FreeBSD )
+      found=$(stat -f '%Dg' "$file")
+      ;;
+    Linux )
+      found=$(stat -c '%g' "$file")
+      ;;
+    CYGWIN* )
+      echo "ASSERT_FILE_GROUP: Running on Cygwin. Not being tested."
+      return 0
+      ;;
+    * )
+      echo 'ERROR: Unknown OS. Exiting.'
+      exit 1
+      ;;
+  esac
+
+  echo "DEBUG: assert_file_group X${wanted}X vs. X${found}X"
+  echo "DEBUG:" $(which stat)
+  if [[ "$wanted" != "$found" ]]; then
+    echo "ASSERT FAILED: $file chgrp group wanted=$wanted found=$found"
+    exit 1
+  fi
+}
+function assert_file_perm() {
+  local wanted="$1"
+  local file="$2"
+  local found
+  assert_file_exists "$file"
+
+  case $(uname -s) in
+    Darwin|FreeBSD )
+      found=$(stat -f '%Sp' "$file")
+      ;;
+    # NB(tlim): CYGWIN hasn't been tested. It might be more like Darwin.
+    Linux | CYGWIN* )
+      found=$(stat -c '%A' "$file")
+      ;;
+    * )
+      echo 'ERROR: Unknown OS. Exiting.'
+      exit 1
+      ;;
+  esac
+
+  echo "DEBUG: assert_file_perm X${wanted}X vs. X${found}X"
+  echo "DEBUG:" $(which stat)
+  if [[ "$wanted" != "$found" ]]; then
+    echo "ASSERT FAILED: $file chgrp perm wanted=$wanted found=$found"
+    exit 1
+  fi
+}
+function assert_line_not_exists() {
+  local target="$1"
+  local file="$2"
+  assert_file_exists "$file"
+  if grep -F -x -s -q >/dev/null "$target" "$file" ; then
+    echo "ASSERT FAILED: line '$target' should not exist in file $file"
+    echo "==== file contents: START $file"
+    cat "$file"
+    echo "==== file contents: END $file"
+    exit 1
+  fi
+}
+function assert_line_exists() {
+  local target="$1"
+  local file="$2"
+  assert_file_exists "$file"
+  if ! grep -F -x -s -q >/dev/null "$target" "$file" ; then
+    echo "ASSERT FAILED: line '$target' should exist in file $file"
+    echo "==== file contents: START $file"
+    cat "$file"
+    echo "==== file contents: END $file"
+    exit 1
+  fi
+}