Merge remote-tracking branch 'upstream/master' into keyring_new_format

Makefile

@@ -15,29 +15,13 @@ install:
 # The default package type is RPM.
 packages: packages-rpm
 
-#
-# MacPorts builds
-#
-# To test:
-# rm -rf /tmp/foo ; mkdir -p /tmp/foo;make packages-macports DESTDIR=/tmp/foo;find /tmp/foo -ls
-
-# Make mk_macports.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt:
-tools/mk_macports.vcs_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt
-	sed -e 's@/usr/blackbox/bin/@bin/@g' -e '/profile.d-usrblackbox.sh/d' <tools/mk_rpm_fpmdir.stack_blackbox.txt >$@
-
-# Make mk_deb_fpmdir.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt:
-tools/mk_deb_fpmdir.stack_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt
-	sed -e 's@/usr/blackbox/bin/@/usr/bin/@g' -e '/profile.d-usrblackbox.sh/d' <tools/mk_deb_fpmdir.stack_blackbox.txt >$@
-
-# MacPorts expects to run: make packages-macports DESTDIR=${destroot}
-packages-macports: tools/mk_macports.vcs_blackbox.txt
-	mkdir -p $(DESTDIR)/bin
-	cd tools && ./mk_macports mk_macports.vcs_blackbox.txt
-
 #
 # RPM builds
 #
 
+# NOTE: mk_rpm_fpmdir.stack_blackbox.txt is the master list of files.  All
+# other packages should generate their list from it.
+
 packages-rpm:
 	cd tools && PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_rpm_fpmdir stack_blackbox mk_rpm_fpmdir.stack_blackbox.txt
 
@@ -67,6 +51,10 @@ unlock-rpm:
 packages-deb:	tools/mk_deb_fpmdir.stack_blackbox.txt
 	cd tools && PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_deb_fpmdir stack_blackbox mk_deb_fpmdir.stack_blackbox.txt
 
+# Make mk_deb_fpmdir.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt:
+tools/mk_deb_fpmdir.stack_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt
+	sed -e 's@/usr/blackbox/bin/@/usr/bin/@g' -e '/profile.d-usrblackbox.sh/d' <tools/mk_deb_fpmdir.stack_blackbox.txt >$@
+
 packages-deb-debug:	tools/mk_deb_fpmdir.stack_blackbox.txt
 	@echo BUILD:
 	@PKGRELEASE=99 make packages-deb
@@ -80,6 +68,21 @@ local-deb:
 	-@sudo dpkg -e $(PKGNAME)
 	sudo dpkg -i $$(cat ~/rpmbuild-$(PKGNAME)/bin-packages.txt)
 
+#
+# MacPorts builds
+#
+# To test:
+# rm -rf /tmp/foo ; mkdir -p /tmp/foo;make packages-macports DESTDIR=/tmp/foo;find /tmp/foo -ls
+
+# Make mk_macports.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt:
+tools/mk_macports.vcs_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt
+	sed -e 's@/usr/blackbox/bin/@bin/@g' -e '/profile.d-usrblackbox.sh/d' <tools/mk_rpm_fpmdir.stack_blackbox.txt >$@
+
+# MacPorts expects to run: make packages-macports DESTDIR=${destroot}
+packages-macports: tools/mk_macports.vcs_blackbox.txt
+	mkdir -p $(DESTDIR)/bin
+	cd tools && ./mk_macports mk_macports.vcs_blackbox.txt
+
 # Add other package types here.
 
 #

README.md

@@ -91,20 +91,23 @@ Commands:
 
 | Name: | Description: |
 | --- | --- |
-| `blackbox_addadmin` | Add someone to the list of people that can encrypt/decrypt secrets |
-| `blackbox_cat` | Decrypt and view the contents of a file |
-| `blackbox_diff` | Diff decrypted files against their original crypted version |
 | `blackbox_edit` | Decrypt, run $EDITOR, re-encrypt a file |
 | `blackbox_edit_start` | Decrypt a file so it can be updated |
 | `blackbox_edit_end` | Encrypt a file after blackbox_edit_start was used |
+| `blackbox_cat` | Decrypt and view the contents of a file |
+| `blackbox_diff` | Diff decrypted files against their original crypted version |
 | `blackbox_initialize` | Enable blackbox for a GIT or HG repo |
-| `blackbox_postdeploy` | Decrypt all managed files |
 | `blackbox_register_new_file` | Encrypt a file for the first time |
+| `blackbox_list_files` | List the files maintained by blackbox |
+| `blackbox_decrypt_all_files` | Decrypt all managed files (INTERACTIVE) |
+| `blackbox_postdeploy` | Decrypt all managed files (batch) |
+| `blackbox_addadmin` | Add someone to the list of people that can encrypt/decrypt secrets |
 | `blackbox_removeadmin` | Remove someone from the list of people that can encrypt/decrypt secrets |
 | `blackbox_shred_all_files` | Safely delete any decrypted files |
 | `blackbox_update_all_files` | Decrypt then re-encrypt all files. Useful after keys are changed |
 | `blackbox_whatsnew` | show what has changed in the last commit for a given file |
 
+
 Compatibility:
 ============================
 

bin/_blackbox_common.sh

@@ -75,8 +75,8 @@ function is_on_cryptlist() {
 # Exit with error if a file exists.
 function fail_if_exists() {
   if [[ -f "$1" ]]; then
-    echo ERROR: "$1" exists.  "$2"
+    echo ERROR: "$1" exists.  "$2" >&2
-    echo Exiting...
+    echo Exiting... >&2
     exit 1
   fi
 }
@@ -84,8 +84,8 @@ function fail_if_exists() {
 # Exit with error if a file is missing.
 function fail_if_not_exists() {
   if [[ ! -f "$1" ]]; then
-    echo ERROR: "$1" not found.  "$2"
+    echo ERROR: "$1" not found.  "$2" >&2
-    echo Exiting...
+    echo Exiting... >&2
     exit 1
   fi
 }
@@ -94,8 +94,8 @@ function fail_if_not_exists() {
 function fail_if_not_in_repo() {
   _determine_vcs_base_and_type
   if [[ $VCS_TYPE = "unknown" ]]; then
-    echo "ERROR: This must be run in a VCS repo: git, hg, or svn."
+    echo "ERROR: This must be run in a VCS repo: git, hg, or svn." >&2
-    echo Exiting...
+    echo Exiting... >&2
     exit 1
   fi
 }
@@ -107,9 +107,9 @@ function fail_if_not_on_cryptlist() {
   local name="$1"
 
   if ! is_on_cryptlist "$name" ; then
-    echo "ERROR: $name not found in $BB_FILES"
+    echo "ERROR: $name not found in $BB_FILES" >&2
-    echo "PWD="$(/bin/pwd)
+    echo "PWD="$(/bin/pwd) >&2
-    echo 'Exiting...'
+    echo 'Exiting...' >&2
     exit 1
   fi
 }
@@ -117,9 +117,9 @@ function fail_if_not_on_cryptlist() {
 # Exit with error if keychain contains secret keys.
 function fail_if_keychain_has_secrets() {
   if [[ -s ${SECRING} ]]; then
-    echo 'ERROR: The file' "$SECRING" 'should be empty.'
+    echo 'ERROR: The file' "$SECRING" 'should be empty.' >&2
-    echo 'Did someone accidentally add this private key to the ring?'
+    echo 'Did someone accidentally add this private key to the ring?' >&2
-    echo 'Exiting...'
+    echo 'Exiting...' >&2
     exit 1
   fi
 }
@@ -144,9 +144,9 @@ function get_encrypted_filename() {
 
 # Prepare keychain for use.
 function prepare_keychain() {
-  echo '========== Importing keychain: START'
+  echo '========== Importing keychain: START' >&2
-  gpg --import "$(get_pubring_path)" 2>&1 | egrep -v 'not changed$'
+  gpg --import "$(get_pubring_path)" 2>&1 | egrep -v 'not changed$' >&2
-  echo '========== Importing keychain: DONE'
+  echo '========== Importing keychain: DONE' >&2
 }
 
 # Add file to list of encrypted files.
@@ -177,9 +177,9 @@ function encrypt_file() {
   unencrypted="$1"
   encrypted="$2"
 
-  echo "========== Encrypting: $unencrypted"
+  echo "========== Encrypting: $unencrypted" >&2
-  gpg --use-agent --yes --trust-model=always --encrypt -o "$encrypted"  $(awk '{ print "-r" $1 }' < "$BB_ADMINS") "$unencrypted"
+  gpg --use-agent --no-tty --yes --trust-model=always --encrypt -o "$encrypted"  $(awk '{ print "-r" $1 }' < "$BB_ADMINS") "$unencrypted" >&2
-  echo '========== Encrypting: DONE'
+  echo '========== Encrypting: DONE' >&2
 }
 
 # Decrypt .gpg file, asking "yes/no" before overwriting unencrypted file.
@@ -190,11 +190,11 @@ function decrypt_file() {
   encrypted="$1"
   unencrypted="$2"
 
-  echo '========== EXTRACTING ''"'$unencrypted'"'
+  echo '========== EXTRACTING ''"'$unencrypted'"' >&2
 
   old_umask=$(umask)
   umask "$DECRYPT_UMASK"
-  gpg --use-agent -q --decrypt -o "$unencrypted" "$encrypted"
+  gpg --use-agent --no-tty -q --decrypt -o "$unencrypted" "$encrypted" >&2
   umask "$old_umask"
 }
 
@@ -216,12 +216,12 @@ function decrypt_file_overwrite() {
 
   old_umask=$(umask)
   umask "$DECRYPT_UMASK"
-  gpg --use-agent --yes -q --decrypt -o "$unencrypted" "$encrypted"
+  gpg --use-agent --no-tty --yes -q --decrypt -o "$unencrypted" "$encrypted" >&2
   umask "$old_umask"
 
   new_hash=$(md5sum_file "$unencrypted")
   if [[ "$old_hash" != "$new_hash" ]]; then
-    echo "========== EXTRACTED $unencrypted"
+    echo "========== EXTRACTED $unencrypted" >&2
   fi
 }
 

bin/_stack_lib.sh

@@ -1,6 +1,9 @@
 # Library functions for bash scripts at Stack Exchange.
 
+# NOTE: This file is open sourced. Do not put Stack-proprietary code here.
+
 # Usage:
+#
 #   set -e
 #   . _stack_lib.sh
 
@@ -46,6 +49,48 @@ function add_on_exit()
     fi
 }
 
+function create_self_deleting_tempfile() {
+  local filename
+
+  case $(uname -s) in
+    Darwin )
+      : ${TMPDIR:=/tmp} ;
+      filename=$(mktemp -t _stacklib_ )
+      ;;
+    Linux )
+      filename=$(mktemp)
+      ;;
+    * )
+      echo 'ERROR: Unknown OS. Exiting.'
+      exit 1
+      ;;
+  esac
+
+  add_on_exit rm -f "$filename"
+  echo "$filename"
+}
+
+function create_self_deleting_tempdir() {
+  local filename
+
+  case $(uname -s) in
+    Darwin )
+      : ${TMPDIR:=/tmp} ;
+      filename=$(mktemp -d -t _stacklib_ )
+      ;;
+    Linux )
+      filename=$(mktemp -d)
+      ;;
+    * )
+      echo 'ERROR: Unknown OS. Exiting.'
+      exit 1
+      ;;
+  esac
+
+  add_on_exit rm -rf "$filename"
+  echo "$filename"
+}
+
 # Securely and portably create a temporary file that will be deleted
 # on EXIT.  $1 is the variable name to store the result.
 function make_self_deleting_tempfile() {
@@ -55,14 +100,11 @@ function make_self_deleting_tempfile() {
   case $(uname -s) in
     Darwin )
       : ${TMPDIR:=/tmp} ;
-      name=$(mktemp -t _stacklib_.XXXXXXX )
+      name=$(mktemp -t _stacklib_ )
       ;;
     Linux )
       name=$(mktemp)
       ;;
-    CYGWIN* )
-      name=$(mktemp)
-      ;;
     * )
       echo 'ERROR: Unknown OS. Exiting.'
       exit 1
@@ -79,15 +121,12 @@ function make_tempdir() {
 
   case $(uname -s) in
     Darwin )
-      : "${TMPDIR:=/tmp}" ;
+      : ${TMPDIR:=/tmp} ;
       name=$(mktemp -d -t _stacklib_ )
       ;;
     Linux )
       name=$(mktemp -d)
       ;;
-    CYGWIN* )
-      name=$(mktemp -d)
-      ;;
     * )
       echo 'ERROR: Unknown OS. Exiting.'
       exit 1
@@ -99,12 +138,12 @@ function make_tempdir() {
 
 function make_self_deleting_tempdir() {
   local __resultvar="$1"
-  local dirname
+  local dname
 
-  make_tempdir dirname
+  make_tempdir dname
 
-  add_on_exit rm -rf "$dirname"
+  add_on_exit rm -rf "$dname"
-  eval $__resultvar="$dirname"
+  eval $__resultvar="$dname"
 }
 
 function fail_if_not_running_as_root() {

bin/blackbox_cat

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_cat.sh -- Decrypt a file, cat it, shred it
+# blackbox_cat -- Decrypt a file, cat it, shred it
 #
 set -e
 blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

bin/blackbox_decrypt_all_files

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+#
+# blacbox_decrypt_all_files -- Decrypt all blackbox files (INTERACTIVE).
+#
+
+# Usage:
+#   blacbox_decrypt_all_files [GROUP]
+#       GROUP is optional.  If supplied, the resulting files
+#       are chgrp'ed to that group.
+
+# Since this is often run in a security-critical situation, we
+# force /usr/bin and /bin to the front of the PATH.
+export PATH=/usr/bin:/bin:"$PATH"
+
+set -e
+
+if [[ -z $GPG_AGENT_INFO ]]; then
+  eval $(gpg-agent --daemon)
+fi
+
+exec blackbox_postdeploy.sh "$@"

bin/blackbox_edit

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_edit.sh -- Decrypt a file temporarily for edition, then re-encrypts it again
+# blackbox_edit -- Decrypt a file temporarily for edition, then re-encrypts it again
 #
 set -e
 blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

bin/blackbox_edit_start

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_edit_start.sh -- Decrypt a file for editing.
+# blackbox_edit_start -- Decrypt a file for editing.
 #
 
 set -e
@@ -11,7 +11,7 @@ source "${blackbox_home}/_blackbox_common.sh"
 for param in """$@""" ; do
   unencrypted_file=$(get_unencrypted_filename "$param")
   encrypted_file=$(get_encrypted_filename "$param")
-  echo ========== PLAINFILE '"'$unencrypted_file'"'
+  echo >&2 ========== PLAINFILE '"'$unencrypted_file'"'
 
   fail_if_not_on_cryptlist "$unencrypted_file"
   fail_if_not_exists "$encrypted_file" "This should not happen."
@@ -19,7 +19,7 @@ for param in """$@""" ; do
     rm -f "$unencrypted_file"
   fi
   if [[ -f "$unencrypted_file" ]]; then
-    echo SKIPPING: "$1" "Will not overwrite non-empty files."
+    echo >&2 SKIPPING: "$1" "Will not overwrite non-empty files."
     continue
   fi
 

bin/blackbox_list_files

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_list_files.sh -- List files that black box is tracking
+# blackbox_list_files -- List files that black box is tracking
 #
 set -e
 blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

bin/blackbox_postdeploy

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_postdeploy.sh -- Decrypt all blackbox files.
+# blackbox_postdeploy -- Decrypt all blackbox files.
 #
 
 # Usage:

bin/blackbox_register_new_file

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_register_new_file.sh -- Enroll a new file in the blackbox system.
+# blackbox_register_new_file -- Enroll a new file in the blackbox system.
 #
 # Takes a previously unencrypted file and enrolls it into the blackbox
 # system.  It will be kept in the repo as an encrypted file.  On deployment

tools/mk_deb_fpmdir.stack_blackbox.txt

@@ -2,13 +2,16 @@ exec /usr/bin/_blackbox_common.sh       ../bin/_blackbox_common.sh
 exec /usr/bin/_stack_lib.sh             ../bin/_stack_lib.sh
 exec /usr/bin/blackbox_addadmin         ../bin/blackbox_addadmin
 exec /usr/bin/blackbox_cat              ../bin/blackbox_cat
+exec /usr/bin/blackbox_decrypt_all_files ../bin/blackbox_decrypt_all_files
+exec /usr/bin/blackbox_diff              ../bin/blackbox_diff
 exec /usr/bin/blackbox_edit             ../bin/blackbox_edit
 exec /usr/bin/blackbox_edit_end         ../bin/blackbox_edit_end
 exec /usr/bin/blackbox_edit_start       ../bin/blackbox_edit_start
 exec /usr/bin/blackbox_initialize       ../bin/blackbox_initialize
+exec /usr/bin/blackbox_list_files       ../bin/blackbox_list_files
 exec /usr/bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
 exec /usr/bin/blackbox_register_new_file ../bin/blackbox_register_new_file
 exec /usr/bin/blackbox_removeadmin      ../bin/blackbox_removeadmin
 exec /usr/bin/blackbox_shred_all_files  ../bin/blackbox_shred_all_files
 exec /usr/bin/blackbox_update_all_files ../bin/blackbox_update_all_files
-exec /usr/bin/blackbox_list_files       ../bin/blackbox_list_files
+exec /usr/bin/blackbox_whatsnew         ../bin/blackbox_whatsnew

tools/mk_macports.vcs_blackbox.txt

@@ -2,13 +2,16 @@ exec bin/_blackbox_common.sh       ../bin/_blackbox_common.sh
 exec bin/_stack_lib.sh             ../bin/_stack_lib.sh
 exec bin/blackbox_addadmin         ../bin/blackbox_addadmin
 exec bin/blackbox_cat              ../bin/blackbox_cat
+exec bin/blackbox_decrypt_all_files  ../bin/blackbox_decrypt_all_files
+exec bin/blackbox_diff             ../bin/blackbox_diff
 exec bin/blackbox_edit             ../bin/blackbox_edit
 exec bin/blackbox_edit_end         ../bin/blackbox_edit_end
 exec bin/blackbox_edit_start       ../bin/blackbox_edit_start
 exec bin/blackbox_initialize       ../bin/blackbox_initialize
+exec bin/blackbox_list_files       ../bin/blackbox_list_files
 exec bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
 exec bin/blackbox_register_new_file ../bin/blackbox_register_new_file
 exec bin/blackbox_removeadmin      ../bin/blackbox_removeadmin
 exec bin/blackbox_shred_all_files  ../bin/blackbox_shred_all_files
 exec bin/blackbox_update_all_files ../bin/blackbox_update_all_files
-exec bin/blackbox_list_files       ../bin/blackbox_list_files
+exec bin/blackbox_whatsnew         ../bin/blackbox_whatsnew

tools/mk_rpm_fpmdir.stack_blackbox.txt

@@ -3,13 +3,16 @@ exec /usr/blackbox/bin/_blackbox_common.sh       ../bin/_blackbox_common.sh
 exec /usr/blackbox/bin/_stack_lib.sh             ../bin/_stack_lib.sh
 exec /usr/blackbox/bin/blackbox_addadmin         ../bin/blackbox_addadmin
 exec /usr/blackbox/bin/blackbox_cat              ../bin/blackbox_cat
+exec /usr/blackbox/bin/blackbox_decrypt_all_files  ../bin/blackbox_decrypt_all_files
+exec /usr/blackbox/bin/blackbox_diff             ../bin/blackbox_diff
 exec /usr/blackbox/bin/blackbox_edit             ../bin/blackbox_edit
 exec /usr/blackbox/bin/blackbox_edit_end         ../bin/blackbox_edit_end
 exec /usr/blackbox/bin/blackbox_edit_start       ../bin/blackbox_edit_start
 exec /usr/blackbox/bin/blackbox_initialize       ../bin/blackbox_initialize
+exec /usr/blackbox/bin/blackbox_list_files       ../bin/blackbox_list_files
 exec /usr/blackbox/bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
 exec /usr/blackbox/bin/blackbox_register_new_file ../bin/blackbox_register_new_file
 exec /usr/blackbox/bin/blackbox_removeadmin      ../bin/blackbox_removeadmin
 exec /usr/blackbox/bin/blackbox_shred_all_files  ../bin/blackbox_shred_all_files
 exec /usr/blackbox/bin/blackbox_update_all_files ../bin/blackbox_update_all_files
-exec /usr/blackbox/bin/blackbox_list_files       ../bin/blackbox_list_files
+exec /usr/blackbox/bin/blackbox_whatsnew         ../bin/blackbox_whatsnew