- Add blackbox_list_files to RPM and MacPorts packages.

- Makefile should not require DESTDIR variable.

Makefile

@@ -1,12 +1,11 @@
 SHELL=/bin/sh
-BIN=tools
 
 PKGNAME=stack_blackbox
 
 all:
 	@echo 'Menu:'
 	@echo '  make packages          Make RPM packages'
-	@echo '  make install           (incomplete)
+	@echo '  make install           (incomplete)'
 
 install:
 	@echo 'To install, copy the files from bin to somewhere in your PATH.'
@@ -15,12 +14,32 @@ install:
 # The default package type is RPM.
 packages: packages-rpm
 
+#
+# MacPorts builds
+#
+# To test:
+# rm -rf /tmp/foo ; mkdir -p /tmp/foo;make packages-macports DESTDIR=/tmp/foo;find /tmp/foo -ls
+
+# Make mk_macports.vcs_blackbox.txt from mk_rpm_fpmdir.stack_blackbox.txt:
+tools/mk_macports.vcs_blackbox.txt: tools/mk_rpm_fpmdir.stack_blackbox.txt
+	sed -e 's@/usr/blackbox/bin/@bin/@g' -e '/profile.d-usrblackbox.sh/d' <tools/mk_rpm_fpmdir.stack_blackbox.txt >$@
+
+check-destdir:
+	ifndef DESTDIR
+	  $(error DESTDIR is undefined)
+	endif
+
+# MacPorts expects to run: make packages-macports DESTDIR=${destroot}
+packages-macports: tools/mk_macports.vcs_blackbox.txt check-destdir
+	mkdir -p $(DESTDIR)/bin
+	cd tools && ./mk_macports mk_macports.vcs_blackbox.txt
+
 #
 # RPM builds
 #
 
 packages-rpm:
-	PKGRELEASE="$${PKGRELEASE}" $(BIN)/build_rpm.sh stack_blackbox tools/rpm_filelist.txt
+	cd tools && PKGRELEASE="$${PKGRELEASE}" PKGDESCRIPTION="Safely store secrets in git/hg/svn repos using GPG encryption" ./mk_rpm_fpmdir stack_blackbox mk_rpm_fpmdir.stack_blackbox.txt
 
 packages-rpm-debug:
 	@echo BUILD:

README.md

@@ -74,6 +74,24 @@ exception of a few specific files, is key to the kind of
 collaboration that DevOps and modern IT practitioniers
 need to do.
 
+Commands:
+============================
+
+
+| Name: | Description: |
+| --- | --- |
+| `blackbox_addadmin` | Add someone to the list of people that can encrypt/decrypt secrets |
+| `blackbox_cat` | Decrypt and view the contents of a file |
+| `blackbox_edit` | Decrypt, run $EDITOR, re-encrypt a file |
+| `blackbox_edit_start` | Decrypt a file so it can be updated |
+| `blackbox_edit_end` | Encrypt a file after blackbox_edit_start was used |
+| `blackbox_initialize` | Enable blackbox for a GIT or HG repo |
+| `blackbox_postdeploy` | Decrypt all managed files |
+| `blackbox_register_new_file` | Encrypt a file for the first time |
+| `blackbox_removeadmin` | Remove someone from the list of people that can encrypt/decrypt secrets |
+| `blackbox_shred_all_files` | Safely delete any decrypted files |
+| `blackbox_update_all_file` | Decrypt then re-encrypt all files. Useful after keys are changed |
+
 Compatibility:
 ============================
 
@@ -87,7 +105,7 @@ It has been tested to work with many operating systems.
   * `hg` -- Mercurial
   * `svn` -- SubVersion (Thanks, Ben Drasin!)
 * Operating system
-  * CentOS
+  * CentOS / RedHat
   * MacOS X
   * Cygwin (Thanks, Ben Drasin!)
 

RELEASE_ENGINEERING.TXT

@@ -0,0 +1,45 @@
+# Branches and Tags:
+
+There are 3 branches/tags:
+
+* **HEAD:** The cutting edge of development.
+* **tag stable:** Stable enough for use by most people.
+* **tag production:** Burned in long enough that we are confident it can be widely adopted.
+
+If you are packaging Blackbox for distribution, you should track the *tag production*.  You might also want to provide a separate package that tracks *tag stable:* for early adopters.
+
+# Build Tasks
+
+# Stable Releases
+
+Marking the software to be "stable":
+
+Step 1. Tag it.
+
+```
+git pull
+git tag -d stable
+git push origin :stable
+git tag stable
+git push origin tag stable
+```
+
+Step 2. Mark your calendar 1 week from today to check
+to see if this should be promoted to production.
+
+
+# Production Releases
+
+If no bugs have been reported a full week after a stable tag has been pushed, mark the release to be "production".
+
+```
+git fetch
+git checkout stable
+git tag -d production
+git push origin :production
+git tag production
+git push origin tag production
+R="v1.$(date +%Y%m%d)"
+git tag "$R"
+git push origin tag "$R"
+```

bin/_blackbox_common.sh

@@ -11,6 +11,11 @@
 
 # Where in the VCS repo should the blackbox data be found?
 : ${BLACKBOXDATA:=keyrings/live} ;   # If BLACKBOXDATA not set, set it.
+
+
+# If $EDITOR is not set, set it to "vi":
+: ${EDITOR:=vi} ;
+
  
 # Outputs a string that is the base directory of this VCS repo.
 # By side-effect, sets the variable VCS_TYPE to either 'git', 'hg',
@@ -56,7 +61,8 @@ BB_FILES_FILE="blackbox-files.txt"
 BB_FILES="${KEYRINGDIR}/${BB_FILES_FILE}"
 SECRING="${KEYRINGDIR}/secring.gpg"
 PUBRING="${KEYRINGDIR}/pubring.gpg"
-: ${DECRYPT_UMASK:=o=} ;
+: ${DECRYPT_UMASK:=0022} ;
+# : ${DECRYPT_UMASK:=o=} ;
 
 # Return error if not on cryptlist.
 function is_on_cryptlist() {

bin/blackbox_addadmin

@@ -9,8 +9,9 @@
 #
 
 set -e
-. _blackbox_common.sh
-. _stack_lib.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
+source ${blackbox_home}/_stack_lib.sh
 
 fail_if_not_in_repo
 

bin/blackbox_cat

@@ -4,7 +4,8 @@
 # blackbox_cat.sh -- Decrypt a file, cat it, shred it
 #
 set -e
-. _blackbox_common.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
 
 for param in """$@""" ; do
   shreddable=0

bin/blackbox_edit

@@ -4,11 +4,12 @@
 # blackbox_edit.sh -- Decrypt a file temporarily for edition, then re-encrypts it again
 #
 set -e
-. _blackbox_common.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
 
 for param in """$@""" ; do
   unencrypted_file=$(get_unencrypted_filename "$param")
-  if  [[! is_on_cryptlist "$param" ]] && [[! is_on_cryptlist "$unencrypted_file" ]] ; then
+  if  ! is_on_cryptlist "$param" && ! is_on_cryptlist "$unencrypted_file" ; then
     read -r -p "Encrypt file $param? (y/n) " ans
     case "$ans" in
       y* | Y*)

bin/blackbox_edit_end

@@ -5,7 +5,8 @@
 #
 
 set -e
-. _blackbox_common.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
 
 unencrypted_file=$(get_unencrypted_filename "$1")
 encrypted_file=$(get_encrypted_filename "$1")

bin/blackbox_edit_start

@@ -5,7 +5,8 @@
 #
 
 set -e
-. _blackbox_common.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
 
 for param in """$@""" ; do
   unencrypted_file=$(get_unencrypted_filename "$param")

bin/blackbox_initialize

@@ -9,7 +9,8 @@
 #
 
 set -e
-. _blackbox_common.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
 
 _determine_vcs_base_and_type  # Sets VCS_TYPE
 

bin/blackbox_list_files

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+#
+# blackbox_list_files.sh -- List files that black box is tracking
+#
+set -e
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
+cat "$BB_FILES"

bin/blackbox_postdeploy

@@ -14,7 +14,8 @@
 export PATH=/usr/bin:/bin:"$PATH"
 
 set -e
-. _blackbox_common.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
 
 if [[ "$1" == "" ]]; then
   FILE_GROUP=""

bin/blackbox_register_new_file

@@ -11,7 +11,8 @@
 # TODO(tlim): Add the unencrypted file to .hgignore
 
 set -e
-. _blackbox_common.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
 _determine_vcs_base_and_type
 
 unencrypted_file=$(get_unencrypted_filename "$1")
@@ -45,6 +46,8 @@ if $SECRETSEXPOSED ; then
   COMMIT_FILES="$BB_FILES $encrypted_file $unencrypted_file"
 else
   COMMIT_FILES="$BB_FILES $encrypted_file"
+  # FIXME(tal): This should be an array so that filenames with
+  # spaces aren't a problem.
 fi
 
 # TODO(tlim): This should be moved to _blackbox_common.sh in a
@@ -60,7 +63,7 @@ if [[ $VCS_TYPE = 'git' ]]; then
 fi
 
 echo 'NOTE: "already tracked!" messages are safe to ignore.'
-vcs_add $BB_FILES $encrypted_file
+vcs_add "$BB_FILES" $encrypted_file
 vcs_commit "registered in blackbox: ${unencrypted_file}" $COMMIT_FILES
 echo "========== UPDATING VCS: DONE"
 echo "Local repo updated.  Please push when ready."

bin/blackbox_removeadmin

@@ -10,8 +10,9 @@
 #
 
 set -e
-. _blackbox_common.sh
-. _stack_lib.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
+source ${blackbox_home}/_stack_lib.sh
 
 fail_if_not_in_repo
 

bin/blackbox_shred_all_files

@@ -16,12 +16,13 @@
 # have been decrypted for editing, you will see an empty list.
 
 set -e
-. _blackbox_common.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
 
 change_to_root
 
 echo '========== FILES BEING SHREDDED:'
-for i in $(<$BB_FILES) ; do
+for i in $(<"$BB_FILES") ; do
   unencrypted_file=$(get_unencrypted_filename "$i")
   encrypted_file=$(get_encrypted_filename "$i")
   if [[ -f "$unencrypted_file" ]]; then

bin/blackbox_update_all_files

@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 
 #
-# blackbox_update_all_files -- Re-encrypt file after edits.
+# blackbox_update_all_files -- Decrypt then re-encrypt all files. Useful after keys are changed.
 #
 
 set -e
-. _blackbox_common.sh
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${blackbox_home}/_blackbox_common.sh
 
 if [[ -z $GPG_AGENT_INFO ]]; then
   echo 'WARNING: You probably want to run gpg-agent as'
@@ -22,7 +23,7 @@ awk <"$BB_FILES" '{ print "    " $1 ".gpg" }'
 
 echo '========== FILES IN THE WAY:'
 need_warning=false
-for i in $(<$BB_FILES) ; do
+for i in $(<"$BB_FILES") ; do
   unencrypted_file=$(get_unencrypted_filename "$i")
   encrypted_file=$(get_encrypted_filename "$i")
   if [[ -f "$unencrypted_file" ]]; then
@@ -39,7 +40,7 @@ else
 fi
 
 echo '========== RE-ENCRYPTING FILES:'
-for i in $(<$BB_FILES) ; do
+for i in $(<"$BB_FILES") ; do
   unencrypted_file=$(get_unencrypted_filename "$i")
   encrypted_file=$(get_encrypted_filename "$i")
   echo ========== PROCESSING "$unencrypted_file"
@@ -52,7 +53,7 @@ done
 fail_if_keychain_has_secrets
 
 echo '========== COMMITING TO VCS:'
-vcs_commit 'Re-encrypted keys' $(awk <$BB_FILES '{ print $1 ".gpg" }' )
+vcs_commit 'Re-encrypted keys' $(awk <"$BB_FILES" '{ print $1 ".gpg" }' )
 
 VCSCMD=$(which_vcs)
 echo '========== DONE.'

tools/confidence_test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
-export PATH="$HOME/gitwork/blackbox/bin":/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin
+blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../bin
+export PATH=${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin
 
 . _stack_lib.sh
 

tools/mk_macports

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Install files into MacPorts DESTDIR
+
+# Usage:
+#   mk_macports MANIFEST MANIFEST1 ...
+
+# Where "manifest.txt" contains:
+#   exec /usr/bin/stack_makefqdn  misc/stack_makefqdn.py
+#   exec /usr/bin/bar             bar/bar.sh
+#   read /usr/man/man1/bar.1      bar/bar.1.man
+#   0444 /etc/foo.conf            bar/foo.conf
+# (NOTE: "exec" means 0755; "read" means 0744)
+
+set -e
+
+# Fail if DESTDIR is not set.
+DESTDIR="${DESTDIR?"Envvar DESTDIR must be set to destination dir."}"
+
+# Copy the files into place:
+cat """$@""" | while read -a arr ; do
+  PERM="${arr[0]}"
+  case $PERM in
+    \#*)  continue ;;   # Skip comments.
+    exec) PERM=0755 ;;
+    read) PERM=0744 ;;
+    *) ;;
+  esac
+  DST="$DESTDIR/${arr[1]}"
+  SRC="${arr[2]}"
+  install -m "$PERM" "$SRC" "$DST"
+done

tools/mk_macports.vcs_blackbox.txt

@@ -0,0 +1,14 @@
+exec bin/_blackbox_common.sh       ../bin/_blackbox_common.sh
+exec bin/_stack_lib.sh             ../bin/_stack_lib.sh
+exec bin/blackbox_addadmin         ../bin/blackbox_addadmin
+exec bin/blackbox_cat              ../bin/blackbox_cat
+exec bin/blackbox_edit             ../bin/blackbox_edit
+exec bin/blackbox_edit_end         ../bin/blackbox_edit_end
+exec bin/blackbox_edit_start       ../bin/blackbox_edit_start
+exec bin/blackbox_initialize       ../bin/blackbox_initialize
+exec bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
+exec bin/blackbox_register_new_file ../bin/blackbox_register_new_file
+exec bin/blackbox_removeadmin      ../bin/blackbox_removeadmin
+exec bin/blackbox_shred_all_files  ../bin/blackbox_shred_all_files
+exec bin/blackbox_update_all_files ../bin/blackbox_update_all_files
+exec bin/blackbox_list_files       ../bin/blackbox_list_files

tools/mk_rpm_fpmdir

@@ -1,21 +1,17 @@
-#!/bin/bash
+#! /usr/bin/env bash
 
-# build_rpm.sh - Build an RPM of these files.  (uses FPM)
+# Use fpm to package up files into an RPM.
 
 # Usage:
-#   make_rpm.sh PACKAGENAME MANIFEST1 MANIFEST2 ...
+#   mk_rpm_fpmdir PACKAGENAME MANIFEST1 MANIFEST2 ...
 
 # Example:
 #   Make a package foopkg manifest.txt
 # Where "manifest.txt" contains:
-#   exec /usr/bin/foo        foo/foo
-#   exec /usr/bin/bar        bar/bar.sh
-#   read /usr/man/man1/bar.1 bar/bar.1.man
-#   0444 /etc/foo.conf       bar/foo.conf
-#
-# Col1  chmod-style permissions or "exec" for 0755, "read" for 0744.
-# Col2  Installation location.
-# Col3  Source of the file.
+#   exec /usr/bin/stack_makefqdn  misc/stack_makefqdn.py
+#   exec /usr/bin/bar             bar/bar.sh
+#   read /usr/man/man1/bar.1      bar/bar.1.man
+#   0444 /etc/foo.conf            bar/foo.conf
 
 set -e
 
@@ -23,15 +19,17 @@ set -e
 PACKAGENAME=${1?"First arg must be the package name."}
 shift
 
-# Defaults that can be overridden via env variables:
+# Defaults that can be overridden:
 # All packages are 1.0 unless otherwise specifed:
 : ${PKGVERSION:=1.0} ;
 # If there is no iteration setting, assume "1":
 : ${PKGRELEASE:=1}
+# If there is no epoch, assume 0
+: ${PKGEPOCH:=0}
 
 # The RPM is output here: (should be a place that can be wiped)
 OUTPUTDIR="${HOME}/rpmbuild-$PACKAGENAME"
-# Our build system expects to find the list of artifacts here:
+# The TeamCity templates expect to find the list of artifacts here:
 RPM_BIN_LIST="${OUTPUTDIR}/bin-packages.txt"
 
 # -- Now the real work can be done.
@@ -41,35 +39,38 @@ rm -rf "$OUTPUTDIR"
 mkdir -p "$OUTPUTDIR/installroot"
 
 # Copy the files into place:
-cat """$@""" | grep -v '^$' | while read -a arr ; do
+set -o pipefail  # Error out if any manifest is not found.
+cat """$@""" | while read -a arr ; do
   PERM="${arr[0]}"
-  DEST="${arr[1]}"
-  SRC="${arr[2]}"
-  echo ========== "$PERM $DEST"
   case $PERM in
     \#*)  continue ;;   # Skip comments.
     exec) PERM=0755 ;;
     read) PERM=0744 ;;
     *) ;;
   esac
-  FULLDEST="$OUTPUTDIR/installroot/${arr[1]}"
-  install -D -T -b -m "$PERM" -T "$SRC" "$FULLDEST"
+  DST="$OUTPUTDIR/installroot/${arr[1]}"
+  SRC="${arr[2]}"
+  if [[ $SRC == "cmd/"* || $SRC == *"/cmd/"* ]]; then
+    ( cd $(dirname "$SRC" ) && go build -a -v ) 
+  fi
+  install -D -T -b -m "$PERM" -T "$SRC" "$DST"
 done
 
 # Build the RPM:
 cd "$OUTPUTDIR" && fpm -s dir -t rpm \
   -a x86_64 \
-  --epoch '0' \
   -n "${PACKAGENAME}" \
+  --epoch "${PKGEPOCH}" \
   --version "${PKGVERSION}" \
   --iteration "${PKGRELEASE}" \
-  --description 'Safely store secrets in Git/Hg repos using GPG encryption' \
+  ${PKGDESCRIPTION:+ --description="${PKGDESCRIPTION}"} \
+  ${PKGVENDOR:+ --vendor="${PKGVENDOR}"} \
   -C "$OUTPUTDIR/installroot" \
   .
 
-# Our build system expects to find the list of all packages created
-# in bin-packages.txt.  Generate that list:
+# TeamCity templates for RPMS expect to find
+# the list of all packages created in bin-packages.txt.
+# Generate that list:
 find "$OUTPUTDIR" -maxdepth 1 -name '*.rpm' >"$RPM_BIN_LIST"
-# Output the list for debugging purposes:
-echo ========== "$RPM_BIN_LIST"
+# Output it for debugging purposes:
 cat "$RPM_BIN_LIST"

tools/mk_rpm_fpmdir.stack_blackbox.txt

@@ -0,0 +1,15 @@
+read /etc/profile.d/usrblackbox.sh               profile.d-usrblackbox.sh
+exec /usr/blackbox/bin/_blackbox_common.sh       ../bin/_blackbox_common.sh
+exec /usr/blackbox/bin/_stack_lib.sh             ../bin/_stack_lib.sh
+exec /usr/blackbox/bin/blackbox_addadmin         ../bin/blackbox_addadmin
+exec /usr/blackbox/bin/blackbox_cat              ../bin/blackbox_cat
+exec /usr/blackbox/bin/blackbox_edit             ../bin/blackbox_edit
+exec /usr/blackbox/bin/blackbox_edit_end         ../bin/blackbox_edit_end
+exec /usr/blackbox/bin/blackbox_edit_start       ../bin/blackbox_edit_start
+exec /usr/blackbox/bin/blackbox_initialize       ../bin/blackbox_initialize
+exec /usr/blackbox/bin/blackbox_postdeploy       ../bin/blackbox_postdeploy
+exec /usr/blackbox/bin/blackbox_register_new_file ../bin/blackbox_register_new_file
+exec /usr/blackbox/bin/blackbox_removeadmin      ../bin/blackbox_removeadmin
+exec /usr/blackbox/bin/blackbox_shred_all_files  ../bin/blackbox_shred_all_files
+exec /usr/blackbox/bin/blackbox_update_all_files ../bin/blackbox_update_all_files
+exec /usr/blackbox/bin/blackbox_list_files       ../bin/blackbox_list_files

tools/rpm_filelist.txt

@@ -1,14 +0,0 @@
-read /etc/profile.d/usrblackbox.sh                tools/profile.d-usrblackbox.sh
-exec /usr/blackbox/bin/_blackbox_common.sh        bin/_blackbox_common.sh
-exec /usr/blackbox/bin/_stack_lib.sh              bin/_stack_lib.sh
-exec /usr/blackbox/bin/blackbox_addadmin          bin/blackbox_addadmin
-exec /usr/blackbox/bin/blackbox_cat             bin/blackbox_cat
-exec /usr/blackbox/bin/blackbox_edit              bin/blackbox_edit
-exec /usr/blackbox/bin/blackbox_edit_end          bin/blackbox_edit_end
-exec /usr/blackbox/bin/blackbox_edit_start        bin/blackbox_edit_start
-exec /usr/blackbox/bin/blackbox_initialize        bin/blackbox_initialize
-exec /usr/blackbox/bin/blackbox_postdeploy        bin/blackbox_postdeploy
-exec /usr/blackbox/bin/blackbox_register_new_file bin/blackbox_register_new_file
-exec /usr/blackbox/bin/blackbox_removeadmin       bin/blackbox_removeadmin
-exec /usr/blackbox/bin/blackbox_shred_all_files   bin/blackbox_shred_all_files
-exec /usr/blackbox/bin/blackbox_update_all_files  bin/blackbox_update_all_files