Update RELEASE_ENGINEERING.md

This reverts commit 87b7cd0eae.

CHANGELOG.md

@@ -1,3 +1,27 @@
+Release v1.20200429
+
+NOTE: While there is now support for NetBSD and SunOS/SmartOS, the
+release process only tests on macOS and CentOS7 because that's all I
+have access to.
+
+* Fix tools that break when ".." or "." are used in a path (#304)
+* Respect PREFIX variable for copy-install (#294)
+* Documentation: Add pkgsrc install instructions (#292)
+* Improve support for Windows (#291)
+* Clarify gpg version usage (#290)
+* Many documentation fixes
+* DOCUMENTATION: Promote 'getting started' to a section, enumerate steps (#283)
+* Commit changes to gitignore when deregistering (#282)
+* Add support for NetBSD and SunOS (SmartOS)
+* Defend against ShellShock
+* .blackbox is now the default config directory for new repos. (#272)
+* Add blackbox_decrypt_file (#270)
+* Improved compatibility: change"/bin/[x]" to "/usr/bin/env [x]" (#265)
+* Add blackbox_less. (#263)
+* add nix method of install (#261)
+* Linked setting up of GPG key (#260)
+
+
 Release v1.20180618
 
 * Restore `make manual-install` with warning. (#258)

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014-2018 Stack Exchange, Inc.
+Copyright (c) 2014-2019 Stack Exchange, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,5 +1,5 @@
 SHELL=/bin/sh
-
+PREFIX?=/usr/local
 PKGNAME=stack_blackbox
 BASEDIR?=$(HOME)
 OUTPUTDIR?="$(BASEDIR)/debbuild-${PKGNAME}"
@@ -9,9 +9,9 @@ all:
 	@echo '  make update             Update any generated files'
 	@echo '  make packages-rpm       Make RPM packages'
 	@echo '  make packages-deb       Make DEB packages'
-	@echo '  make symlinks-install   Make symlinks in /usr/local/bin/'
-	@echo '  make copy-install       Copy "bin" files to /usr/local/bin/'
-	@echo '  make usrlocal-uninstall Remove blackbox files from /usr/local/bin/'
+	@echo '  make symlinks-install   Make symlinks in ${PREFIX}/bin/'
+	@echo '  make copy-install       Copy "bin" files to ${PREFIX}/bin/'
+	@echo '  make copy-uninstall     Remove blackbox files from ${PREFIX}/bin/'
 	@echo '  make test               Run tests'
 
 install:
@@ -55,8 +55,8 @@ unlock-rpm:
 # Manual install
 #
 symlinks-install:
-	@echo 'Symlinking files from ./bin to /usr/local/bin'
-	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do ln -fs `pwd`/$$f /usr/local/bin/$$f; done
+	@echo "Symlinking files from ./bin to ${PREFIX}/bin"
+	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do ln -fs `pwd`/$$f $(PREFIX)/bin/$$f; done
 	@echo 'Done.'
 
 manual-install:
@@ -67,13 +67,13 @@ manual-install:
 	$(MAKE) symlinks-install
 
 copy-install:
-	@echo 'Copying files from ./bin to /usr/local/bin'
-	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do cp `pwd`/$$f /usr/local/bin/$$f; done
+	@echo "Copying files from ./bin to ${PREFIX}/bin"
+	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do cp `pwd`/$$f $(PREFIX)/bin/$$f; done
 	@echo 'Done.'
 
-usrlocal-uninstall:
-	@echo 'Removing blackbox files from /usr/local/bin'
-	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do rm /usr/local/bin/$$f; done
+copy-uninstall:
+	@echo "Removing blackbox files from ${PREFIX}/bin"
+	@cd bin && for f in `find . -type f -iname "*" ! -iname "Makefile"`; do rm $(PREFIX)/bin/$$f; done
 	@echo 'Done.'
 
 #
@@ -142,6 +142,6 @@ test: confidence
 confidence:
 	@if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg should not exist. If it does, bugs may polute your .gnupg configuration. If the code has no bugs everything will be fine. Do you feel lucky?'; false ; fi
 	@if which >/dev/null gpg-agent ; then pkill gpg-agent ; rm -rf /tmp/tmp.* ; fi
-	@export PATH="$(PWD)/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:$(PATH)" ; tools/auto_system_test
+	@export PATH="$(PWD)/bin:$(PREFIX)/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:$(PATH)" ; tools/auto_system_test
 	@if which >/dev/null gpg-agent ; then pkill gpg-agent ; fi
 	@if [ -e ~/.gnupg ]; then echo ERROR: '~/.gnupg was created which means the scripts might be poluting GnuPG configuration.  Fix this bug.'; false ; fi

README.md

@@ -53,20 +53,26 @@ Rather than one GPG passphrase for all the files, each person with access has th
 
 Automated processes often need access to all the decrypted files. This is easy too. For example, suppose Git is being used for Puppet files. The master needs access to the decrypted version of all the files. Simply set up a GPG key for the Puppet master (or the role account that pushes new files to the Puppet master) and have that user run `blackbox_postdeploy` after any files are updated.
 
-Getting started is looks like this.
-First, if you don't have a GPG key, set it up using instructions
-such as:
-[Set up GPG key](https://help.github.com/articles/generating-a-new-gpg-key/).
+Getting started
+---------------
+
+1. If you don't have a GPG key, set it up using instructions such as:
+[Set up GPG key](https://help.github.com/articles/generating-a-new-gpg-key/). \
 Now you are ready to go.
-`cd` into a Git, Mercurial, Subversion
-or Perforce repository and run `blackbox_initialize`. After that,
-if a file is to be encrypted, run `blackbox_register_new_file` and
-you are done. Add and remove keys with `blackbox_addadmin` and
-`blackbox_removeadmin`. To view and/or edit a file, run `blackbox_edit`;
+
+1. `cd` into a Git, Mercurial, Subversion or Perforce repository and run `blackbox_initialize`.
+
+1. If a file is to be encrypted, run `blackbox_register_new_file` and you are done.
+
+1. Add and remove keys with `blackbox_addadmin` and `blackbox_removeadmin`.
+
+1. To view and/or edit a file, run `blackbox_edit`;
 this will decrypt the file and open with whatever is specified by
-your $EDITOR environment variable. When you close the editor the
+your $EDITOR environment variable. \
+When you close the editor the
 file will automatically be encrypted again and the temporary plaintext
-file will be shredded. If you need to leave the file decrypted while
+file will be shredded. \
+If you need to leave the file decrypted while
 you update you can use the`blackbox_edit_start` to decrypt the file
 and `blackbox_edit_end` when you want to "put it back in the box."
 
@@ -82,9 +88,9 @@ The ability to be open and transparent about our code, with the exception of a f
 Installation Instructions
 =========================
 
-- *The hard way (manual*: Copy all the files in "bin" to your "bin".
-- *The hard way (automatic)*: `make copy-install` will copy the bin files into /usr/local/bin (uninstall with `make usrlocal-uninstall`).
-- *The symlinks way*: `make symlinks-install` will make symlinks of the bin files into /usr/local/bin (uninstall with `make usrlocal-uninstall`) (useful when doing development)
+- *The hard way (manual)*: Copy all the files in "bin" to your "bin".
+- *The hard way (automatic)*: `make copy-install` will copy the bin files into $PREFIX/bin, default is /usr/local (uninstall with `make copy-uninstall`).
+- *The symlinks way*: `make symlinks-install` will make symlinks of the bin files into $PREFIX/bin, default is /usr/local (uninstall with `make copy-uninstall`) (useful when doing development)
 - *The MacPorts Way*: `sudo port install vcs_blackbox`
 - *The Homebrew Way*: `brew install blackbox`
 - *The RPM way*: Check out the repo and make an RPM via `make packages-rpm`; now you can distribute the RPM via local methods. (Requires [fpm](https://github.com/jordansissel/fpm).)
@@ -92,6 +98,7 @@ Installation Instructions
 - *The Antigen Way*: Add `antigen bundle StackExchange/blackbox` to your .zshrc
 - *The Zgen Way*: Add `zgen load StackExchange/blackbox` to your .zshrc where you're loading your other plugins.
 - *The Nix Way*: `nix-env -i blackbox`
+- *The Pkgsrc Way*: `pkgin in scm-blackbox`
 
 Commands
 ========
@@ -134,6 +141,8 @@ BlackBox automatically determines which VCS you are using and does the right thi
   - MacOS X
   - Cygwin (Thanks, Ben Drasin!) **See Note Below**
   - MinGW (git bash on windows) **See Note Below**
+  - NetBSD
+  - SmartOS
 
 To add or fix support for a VCS system, look for code at the end of `bin/_blackbox_common.sh`
 
@@ -328,7 +337,19 @@ If you don't already have a GPG key, here's how to generate one:
 gpg --gen-key
 ```
 
-Pick defaults for encryption settings, 0 expiration. Pick a VERY GOOD passphrase. Store a backup of the private key someplace secure. For example, keep the backup copy on a USB drive that is locked in safe.  Or, at least put it on a machine secure machine with little or no internet access, full-disk-encryption, etc. Your employer probably has rules about how to store such things.
+WARNING: New versions of GPG generate keys which are not understood by
+old versions of GPG.  If you generate a key with a new version of GPG,
+this will cause problems for users of older versions of GPG.
+Therefore it is recommended that you either assure that everyone using
+Blackbox have the exact same version of GPG, or generate GPG keys
+using a version of GPG as old as the oldest version of GPG used by
+everyone using Blackbox.
+
+Pick defaults for encryption settings, 0 expiration. Pick a VERY GOOD passphrase. Store a backup of the private key someplace secure. For example, keep the backup copy on a USB drive that is locked in safe.  Or, at least put it on a secure machine with little or no internet access, full-disk-encryption, etc. Your employer probably has rules about how to store such things.
+
+FYI: If generating the key is slow, this is usually because the system
+isn't generating enough entropy.  Tip: Open another window on that
+machine and run this command: `ls -R /`
 
 Now that you have a GPG key, add yourself as an admin:
 

RELEASE_ENGINEERING.md

@@ -2,6 +2,7 @@ Table of Contents:
 ==================
 
 -	[Branches and Tags:](#branches-and-tags)
+-	[Testing:](#testing)
 -	[Build Tasks](#build-tasks)
 -	[Stable Releases](#stable-releases)
 -	[Production Releases](#production-releases)
@@ -19,6 +20,25 @@ There are 3 branches/tags:
 
 If you are packaging BlackBox for distribution, you should track the *tag production*. You might also want to provide a separate package that tracks *tag stable:* for early adopters.
 
+Testing
+=======
+
+Tips:
+
+* macOS: `brew install gpg pinentry`
+* FreeBSD: `pkg install gpg gmake`
+* CentOS7: `yum install gpg`
+
+To run a suite of tests:
+
+```
+cd ~/src/github.com/StackExchange/blackbox
+make test
+```
+
+FYI: For FreeBSD, use `gmake test`
+
+
 Build Tasks
 ===========
 
@@ -72,6 +92,12 @@ git tag "$R"
 git push origin tag "$R"
 ```
 
+Step 4. Get credit!
+
+Record the fact that you did this release in your weekly accomplishments file.
+
+
+
 Updating MacPorts (automatic)
 =============================
 

Version2-Ideas.md

@@ -86,7 +86,7 @@ As long as the code is in `bash` the configuration files like `blackbox-files.tx
 How important is itto blackbox users that the system is written in "bash"?
 
 
-## ditch the project and use git-crypt
+## Ditch the project and use git-crypt
 
 People tell me that git-crypt is better because, as a plug-in, automagically supports "git diff", "git log" and "git blame".
 

bin/_blackbox_common.sh

@@ -213,7 +213,8 @@ function add_filename_to_cryptlist() {
   else
     echo "========== Adding file to list."
     touch "$BB_FILES"
-    sort -u -o "$BB_FILES" <(echo "$name") "$BB_FILES"
+    echo "$name" >> "$BB_FILES"
+    sort -u -o "$BB_FILES" "$BB_FILES"
   fi
 }
 
@@ -411,6 +412,12 @@ function md5sum_file() {
     Darwin | FreeBSD )
       md5 -r "$1" | awk '{ print $1 }'
       ;;
+    NetBSD )
+      md5 -q "$1"
+      ;;
+    SunOS )
+      digest -a md5 "$1"
+      ;;
     Linux | CYGWIN* | MINGW* )
       md5sum "$1" | awk '{ print $1 }'
       ;;
@@ -427,10 +434,13 @@ function cp_permissions() {
     Darwin )
       chmod $( stat -f '%p' "$1" ) "${@:2}"
       ;;
-    FreeBSD )
+    FreeBSD | NetBSD )
       chmod $( stat -f '%p' "$1" | sed -e "s/^100//" ) "${@:2}"
       ;;
-    Linux | CYGWIN* | MINGW* )
+    SunOS )
+      chmod $( stat -c '%a' "$1" ) "${@:2}"
+      ;;
+    Linux | CYGWIN* | MINGW* | SunOS )
       if [[ -e /etc/alpine-release ]]; then
         chmod $( stat -c '%a' "$1" ) "${@:2}"
       else

bin/_stack_lib.sh

@@ -57,7 +57,7 @@ function create_self_deleting_tempfile() {
       : "${TMPDIR:=/tmp}" ;
       filename=$(mktemp -t _stacklib_.XXXXXXXX )
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | NetBSD | SunOS )
       filename=$(mktemp)
       ;;
     * )
@@ -78,7 +78,7 @@ function create_self_deleting_tempdir() {
       : "${TMPDIR:=/tmp}" ;
       filename=$(mktemp -d -t _stacklib_.XXXXXXXX )
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | NetBSD | SunOS )
       filename=$(mktemp -d)
       ;;
     * )
@@ -102,7 +102,7 @@ function make_self_deleting_tempfile() {
       : "${TMPDIR:=/tmp}" ;
       name=$(mktemp -t _stacklib_.XXXXXXXX )
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | NetBSD | SunOS )
       name=$(mktemp)
       ;;
     * )
@@ -127,7 +127,7 @@ function make_tempdir() {
       # which needs to fit within sockaddr_un.sun_path (see unix(7)).
       name=$(mktemp -d -t SO )
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | NetBSD | SunOS )
       name=$(mktemp -d)
       ;;
     * )
@@ -160,14 +160,14 @@ function fail_if_not_running_as_root() {
 function fail_if_in_root_directory() {
   # Verify nobody has tricked us into being in "/".
   case $(uname -s) in
-    Darwin | FreeBSD )
+    Darwin | FreeBSD | NetBSD )
       if [[ $(stat -f'%i' / ) == $(stat -f'%i' . ) ]] ; then
         echo 'SECURITY ALERT: The current directory is the root directory.'
         echo 'Exiting...'
         exit 1
       fi
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | SunOS )
       if [[ $(stat -c'%i' / ) == $(stat -c'%i' . ) ]] ; then
         echo 'SECURITY ALERT: The current directory is the root directory.'
         echo 'Exiting...'

bin/blackbox_addadmin

@@ -19,7 +19,8 @@ KEYNAME="$1"
 # Add the email address to the BB_ADMINS file.  Remove any duplicates.
 # The file must exist for sort to act as we expect.
 touch "$BB_ADMINS"
-sort  -fdu -o "$BB_ADMINS" <(echo "$1") "$BB_ADMINS"
+echo "$1" >> "$BB_ADMINS"
+sort -fdu -o "$BB_ADMINS" "$BB_ADMINS"
 
 
 # Add the user's key to the keychain.

bin/blackbox_deregister_file

@@ -29,7 +29,7 @@ vcs_remove "$encrypted_file"
 vcs_notice "$unencrypted_file"
 vcs_add "$BB_FILES" 
 
-vcs_commit "Removing from blackbox: ${unencrypted_file}" "$BB_FILES" "$encrypted_file"
+vcs_commit "Removing from blackbox: ${unencrypted_file}" "$BB_FILES" "$encrypted_file" "$(vcs_ignore_file_path)"
 echo "========== UPDATING VCS: DONE"
 echo "Local repo updated.  Please push when ready."
 echo "    $VCS_TYPE push"

bin/blackbox_initialize

@@ -59,5 +59,5 @@ else
 	echo
 	echo
 	echo 'NEXT STEP: You need to manually check these in:'
-	echo '     ' $VCS_TYPE commit -m\'INITIALIZE BLACKBOX\' keyrings "$IGNOREFILE"
+	echo '     ' $VCS_TYPE commit -m\'INITIALIZE BLACKBOX\' "$BLACKBOXDATA" "$IGNOREFILE"
 fi

bin/blackbox_shred_all_files

@@ -39,7 +39,12 @@ export -f exported_internal_shred_file
 DEREFERENCED_BIN_DIR="${0%/*}"
 MAX_PARALLEL_SHRED=10
 
+bash_args=
+if bash --help | grep import-functions >/dev/null 2>/dev/null; then
+  bash_args=--import-functions
+fi
+
 export IFS=
-tr '\n' '\0' <"$BB_FILES" | xargs -0 -I{} -n 1 -P $MAX_PARALLEL_SHRED bash -c "exported_internal_shred_file $DEREFERENCED_BIN_DIR \"{}\"" $DEREFERENCED_BIN_DIR/fake
+tr '\n' '\0' <"$BB_FILES" | xargs -0 -I{} -P $MAX_PARALLEL_SHRED bash $bash_args -c "exported_internal_shred_file $DEREFERENCED_BIN_DIR \"{}\"" $DEREFERENCED_BIN_DIR/fake
 
 echo '========== DONE.'

tools/confidence_test.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../bin
-export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:${blackbox_home}"
+export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:/usr/pkg/bin:/usr/pkg/gnu/bin:${blackbox_home}"
 
 export LANG=C.UTF-8  # Required ro "gpg --export" to work properly.
 

tools/test_functions.sh

@@ -24,6 +24,12 @@ function md5sum_file() {
     Darwin | FreeBSD )
       md5 -r "$1" | awk '{ print $1 }'
       ;;
+    NetBSD )
+      md5 -q "$1"
+      ;;
+    SunOS )
+      digest -a md5 "$1"
+      ;;
     Linux )
       md5sum "$1" | awk '{ print $1 }'
       ;;
@@ -72,10 +78,10 @@ function assert_file_group() {
   assert_file_exists "$file"
 
   case $(uname -s) in
-    Darwin|FreeBSD )
+    Darwin | FreeBSD | NetBSD )
       found=$(stat -f '%Dg' "$file")
       ;;
-    Linux )
+    Linux | SunOS )
       found=$(stat -c '%g' "$file")
       ;;
     CYGWIN* )
@@ -102,11 +108,11 @@ function assert_file_perm() {
   assert_file_exists "$file"
 
   case $(uname -s) in
-    Darwin|FreeBSD )
+    Darwin | FreeBSD | NetBSD )
       found=$(stat -f '%Sp' "$file")
       ;;
     # NB(tlim): CYGWIN hasn't been tested. It might be more like Darwin.
-    Linux | CYGWIN* )
+    Linux | CYGWIN* | SunOS )
       found=$(stat -c '%A' "$file")
       ;;
     * )