Update RELEASE_ENGINEERING.md

Add support for NetBSD and SunOS (SmartOS)

CHANGELOG.md

@@ -1,3 +1,15 @@
+Release v1.20181219
+
+* New OS support: Add support for NetBSD and SunOS (SmartOS)
+* Testing: Improve confidence test.
+* .blackbox is now the default config directory for new repos. (#272)
+* Add blackbox_decrypt_file (#270)
+* Improved compatibility: change"/bin/[x]" to "/usr/bin/env [x]" (#265)
+* Add blackbox_less. (#263)
+* add nix method of install (#261)
+* Linked setting up of GPG key (#260)
+
+
 Release v1.20180618
 
 * Restore `make manual-install` with warning. (#258)

README.md

@@ -134,6 +134,8 @@ BlackBox automatically determines which VCS you are using and does the right thi
   - MacOS X
   - Cygwin (Thanks, Ben Drasin!) **See Note Below**
   - MinGW (git bash on windows) **See Note Below**
+  - NetBSD
+  - SmartOS
 
 To add or fix support for a VCS system, look for code at the end of `bin/_blackbox_common.sh`
 

RELEASE_ENGINEERING.md

@@ -25,6 +25,17 @@ Build Tasks
 Stable Releases
 ===============
 
+Step 0. Test the software
+
+Run this command to run the unit and system tests:
+
+```
+make test
+```
+
+NOTE: The tests require pinentry-tty. On macOS with NIX this
+can be installed via: `nix-env -i pinentry`
+
 Marking the software to be "stable":
 
 Step 1.  Update CHANGELOG.md

bin/_blackbox_common.sh

@@ -92,10 +92,6 @@ SECRING="${KEYRINGDIR}/secring.gpg"
 : "${DECRYPT_UMASK:=0022}" ;
 # : ${DECRYPT_UMASK:=o=} ;
 
-# $BB_FILES file format:
-# Filenames are listed one per line, relative to the base directory of the repo.
-# Each line is listed in "printf %q" format, which escapes special chars.
-
 # Checks if $1 is 0 bytes, and if $1/keyrings
 # is a directory
 function is_blackbox_repo() {
@@ -106,18 +102,10 @@ function is_blackbox_repo() {
   fi
 }
 
-# is_on_cryptlist resturns an error if $1 not on cryptlist.
+# Return error if not on cryptlist.
 function is_on_cryptlist() {
-  # $1: The filename.
   # Assumes $1 does NOT have the .gpg extension
-
-  # https://github.com/koalaman/shellcheck/wiki/SC2155
-  local name
-  name=$(vcs_relative_path "$1")
-  local encodedname
-  encodedname=$(printf "%q" "$name")
-
-  file_contains_line "$BB_FILES" "$encodedname"
+  file_contains_line "$BB_FILES" "$(vcs_relative_path "$1")"
 }
 
 # Exit with error if a file exists.
@@ -179,33 +167,16 @@ function get_pubring_path() {
   fi
 }
 
-# normalize_filename_arg takes a filename from the command line and
-# outputs the non-encrypted filename.
-function normalize_filename() {
-  # $1: the input from a user
-  # Use this if the user may have entered the encrypted or
-  # non-encrypted filename.
-  local name
-  name=$(vcs_relative_path "$1")
-  echo "$(dirname "$name")/$(basename "$name" .gpg)" | sed -e 's#^\./##'
+# Output the unencrypted filename.
+function get_unencrypted_filename() {
+  echo "$(dirname "$1")/$(basename "$1" .gpg)" | sed -e 's#^\./##'
 }
 
 # Output the encrypted filename.
-function get_gpg_filename() {
-  # $1: normalized file path
-  echo "$1".gpg
+function get_encrypted_filename() {
+  echo "$(dirname "$1")/$(basename "$1" .gpg).gpg" | sed -e 's#^\./##'
 }
 
-## Output the unencrypted filename.
-#function get_unencrypted_filename() {
-#  echo "$(dirname "$1")/$(basename "$1" .gpg)" | sed -e 's#^\./##'
-#}
-#
-## Output the encrypted filename.
-#function get_encrypted_filename() {
-#  echo "$(dirname "$1")/$(basename "$1" .gpg).gpg" | sed -e 's#^\./##'
-#}
-
 # Prepare keychain for use.
 function prepare_keychain() {
   local keyringasc
@@ -228,43 +199,37 @@ function prepare_keychain() {
   echo '========== Importing keychain: DONE' >&2
 }
 
-# add_filename_to_cryptlist adds $1 to the list of encrypted files.
+# Add file to list of encrypted files.
 function add_filename_to_cryptlist() {
-  # $1: The filename.
   # If the name is already on the list, this is a no-op.
-
+  # However no matter what the datestamp is updated.
+  
   # https://github.com/koalaman/shellcheck/wiki/SC2155
   local name
   name=$(vcs_relative_path "$1")
-  local encodedname
-  encodedname=$(printf "%q" "$name")
 
-
-  if file_contains_line "$BB_FILES" "$encodedname" ; then
+  if file_contains_line "$BB_FILES" "$name" ; then
     echo "========== File is registered. No need to add to list."
   else
     echo "========== Adding file to list."
     touch "$BB_FILES"
-    sort -u -o "$BB_FILES" <(printf "%q\n" "$name") "$BB_FILES"
+    sort -u -o "$BB_FILES" <(echo "$name") "$BB_FILES"
   fi
 }
 
-# remove_filename_from_cryptlist removes $1 from the list of encrypted files.
+# Removes a file from the list of encrypted files
 function remove_filename_from_cryptlist() {
-  # $1: The filename.
   # If the name is not already on the list, this is a no-op.
 
   # https://github.com/koalaman/shellcheck/wiki/SC2155
   local name
   name=$(vcs_relative_path "$1")
-  local encodedname
-  encodedname=$(printf "%q" "$name")
 
-  if ! file_contains_line "$BB_FILES" "$encodedname" ; then
+  if ! file_contains_line "$BB_FILES" "$name" ; then
     echo "========== File is not registered. No need to remove from list."
   else
     echo "========== Removing file from list."
-    remove_line "$BB_FILES" "$encodedname"
+    remove_line "$BB_FILES" "$name"
   fi
 }
 
@@ -446,6 +411,12 @@ function md5sum_file() {
     Darwin | FreeBSD )
       md5 -r "$1" | awk '{ print $1 }'
       ;;
+    NetBSD )
+      md5 -q "$1"
+      ;;
+    SunOS )
+      digest -a md5 "$1"
+      ;;
     Linux | CYGWIN* | MINGW* )
       md5sum "$1" | awk '{ print $1 }'
       ;;
@@ -462,10 +433,13 @@ function cp_permissions() {
     Darwin )
       chmod $( stat -f '%p' "$1" ) "${@:2}"
       ;;
-    FreeBSD )
+    FreeBSD | NetBSD )
       chmod $( stat -f '%p' "$1" | sed -e "s/^100//" ) "${@:2}"
       ;;
-    Linux | CYGWIN* | MINGW* )
+    SunOS )
+      chmod $( stat -c '%a' "$1" ) "${@:2}"
+      ;;
+    Linux | CYGWIN* | MINGW* | SunOS )
       if [[ -e /etc/alpine-release ]]; then
         chmod $( stat -c '%a' "$1" ) "${@:2}"
       else

bin/_blackbox_common_test.sh

@@ -6,7 +6,7 @@
 
 set -e
 . "${0%/*}/_blackbox_common.sh"
-. /Users/tlimoncelli/gitwork/blackbox/tools/test_functions.sh
+. tools/test_functions.sh
 
 PHASE 'Test cp-permissions: TestA'
 touch TestA TestB TestC TestD
@@ -22,18 +22,4 @@ assert_file_perm '--wxr--rwx' TestC
 assert_file_perm '----rwx---' TestD  # TestD doesn't change.
 rm -f TestA TestB TestC TestD
 
-PHASE 'Test vcs_relative_path: TestA'
-export REPOBASE='/Users/tlimoncelli/Applications (Parallels)/{fd3049c8-9fdd-48d5-aa16-d31daf3a6879} Applications.localized'
-FILE='Microsoft  Windows Fax and Scan.app/Contents'
-result=$(vcs_relative_path Contents)
-echo result=XXX${result}XXX
-if [[ $FILE != $result ]] ; then
-  echo FAIL
-fi
-
-unencrypted_file=$(get_unencrypted_filename "${result}.gpg")
-echo un=XXX${unencrypted_file}XXX
-encrypted_file=$(get_encrypted_filename "${result}")
-echo en=XXX${encrypted_file}XXX
-
 echo '========== DONE.'

bin/_stack_lib.sh

@@ -57,7 +57,7 @@ function create_self_deleting_tempfile() {
       : "${TMPDIR:=/tmp}" ;
       filename=$(mktemp -t _stacklib_.XXXXXXXX )
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | NetBSD | SunOS )
       filename=$(mktemp)
       ;;
     * )
@@ -78,7 +78,7 @@ function create_self_deleting_tempdir() {
       : "${TMPDIR:=/tmp}" ;
       filename=$(mktemp -d -t _stacklib_.XXXXXXXX )
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | NetBSD | SunOS )
       filename=$(mktemp -d)
       ;;
     * )
@@ -102,7 +102,7 @@ function make_self_deleting_tempfile() {
       : "${TMPDIR:=/tmp}" ;
       name=$(mktemp -t _stacklib_.XXXXXXXX )
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | NetBSD | SunOS )
       name=$(mktemp)
       ;;
     * )
@@ -127,7 +127,7 @@ function make_tempdir() {
       # which needs to fit within sockaddr_un.sun_path (see unix(7)).
       name=$(mktemp -d -t SO )
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | NetBSD | SunOS )
       name=$(mktemp -d)
       ;;
     * )
@@ -160,14 +160,14 @@ function fail_if_not_running_as_root() {
 function fail_if_in_root_directory() {
   # Verify nobody has tricked us into being in "/".
   case $(uname -s) in
-    Darwin | FreeBSD )
+    Darwin | FreeBSD | NetBSD )
       if [[ $(stat -f'%i' / ) == $(stat -f'%i' . ) ]] ; then
         echo 'SECURITY ALERT: The current directory is the root directory.'
         echo 'Exiting...'
         exit 1
       fi
       ;;
-    Linux | CYGWIN* | MINGW* )
+    Linux | CYGWIN* | MINGW* | SunOS )
       if [[ $(stat -c'%i' / ) == $(stat -c'%i' . ) ]] ; then
         echo 'SECURITY ALERT: The current directory is the root directory.'
         echo 'Exiting...'

bin/blackbox_cat

@@ -8,7 +8,7 @@ source "${0%/*}/_blackbox_common.sh"
 
 for param in "$@" ; do
   shreddable=0
-  unencrypted_file=$(normalize_filename "$param")
+  unencrypted_file=$(get_unencrypted_filename "$param")
   if [[ ! -e "$unencrypted_file" ]]; then
     "${BLACKBOX_HOME}/blackbox_edit_start" "$param"
     shreddable=1

bin/blackbox_diff

@@ -13,10 +13,8 @@ prepare_keychain
 modified_files=()
 modifications=()
 echo '========== DIFFING FILES: START'
-while IFS= read <&99 -r encodedname; do
-  local name
-  name=$(echo $encodedname)
-  unencrypted_file=$(get_unencrypted_filename "$name")
+while IFS= read <&99 -r unencrypted_file; do
+  unencrypted_file=$(get_unencrypted_filename "$unencrypted_file")
   encrypted_file=$(get_encrypted_filename "$unencrypted_file")
   fail_if_not_on_cryptlist "$unencrypted_file"
   if [[ -f "$unencrypted_file" ]]; then

bin/blackbox_list_files

@@ -5,7 +5,4 @@
 #
 set -e
 source "${0%/*}/_blackbox_common.sh"
-
-while IFS= read <&99 -r encodedname; do
-  echo $encodedname
-done 99<"$BB_FILES"
+cat "$BB_FILES"

bin/blackbox_postdeploy

@@ -27,12 +27,8 @@ prepare_keychain
 
 # Decrypt:
 echo '========== Decrypting new/changed files: START'
-while IFS= read <&99 -r encodedname; do
-  local name
-  name=$(echo $name)
-
-  encrypted_file=$(get_encrypted_filename "$name")
-  unencrypted_file=$(get_unencrypted_filename "$name")
+while IFS= read <&99 -r unencrypted_file; do
+  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
   decrypt_file_overwrite "$encrypted_file" "$unencrypted_file"
   cp_permissions "$encrypted_file" "$unencrypted_file"
   if [[ ! -z "$FILE_GROUP" ]]; then

bin/blackbox_shred_all_files

@@ -39,7 +39,12 @@ export -f exported_internal_shred_file
 DEREFERENCED_BIN_DIR="${0%/*}"
 MAX_PARALLEL_SHRED=10
 
+bash_args=
+if bash --help | grep import-functions >/dev/null 2>/dev/null; then
+  bash_args=--import-functions
+fi
+
 export IFS=
-tr '\n' '\0' <"$BB_FILES" | xargs -0 -I{} -n 1 -P $MAX_PARALLEL_SHRED bash -c "exported_internal_shred_file $DEREFERENCED_BIN_DIR \"{}\"" $DEREFERENCED_BIN_DIR/fake
+tr '\n' '\0' <"$BB_FILES" | xargs -0 -I{} -P $MAX_PARALLEL_SHRED bash $bash_args -c "exported_internal_shred_file $DEREFERENCED_BIN_DIR \"{}\"" $DEREFERENCED_BIN_DIR/fake
 
 echo '========== DONE.'

bin/blackbox_update_all_files

@@ -12,19 +12,15 @@ disclose_admins
 prepare_keychain
 
 echo '========== ENCRYPTED FILES TO BE RE-ENCRYPTED:'
-while IFS= read <&99 -r encodedname; do
-  local name
-  name=$(echo $encodedname)
-  echo "    $name.gpg"
+while IFS= read <&99 -r unencrypted_file; do
+    echo "    $unencrypted_file.gpg"
 done 99<"$BB_FILES"
 
 echo '========== FILES IN THE WAY:'
 need_warning=false
-while IFS= read <&99 -r encodedname; do
-  local name
-  name=$(echo $encodedname)
-  unencrypted_file=$(get_unencrypted_filename "$name")
-  encrypted_file=$(get_encrypted_filename "$name")
+while IFS= read <&99 -r unencrypted_file; do
+  unencrypted_file=$(get_unencrypted_filename "$unencrypted_file")
+  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
   if [[ -f "$unencrypted_file" ]]; then
     need_warning=true
     echo "    $unencrypted_file"
@@ -39,11 +35,9 @@ else
 fi
 
 echo '========== RE-ENCRYPTING FILES:'
-while IFS= read <&99 -r encodedname; do
-  local name
-  name=$(echo $encodedname)
-  unencrypted_file=$(get_unencrypted_filename "$name")
-  encrypted_file=$(get_encrypted_filename "$name")
+while IFS= read <&99 -r unencrypted_file; do
+  unencrypted_file=$(get_unencrypted_filename "$unencrypted_file")
+  encrypted_file=$(get_encrypted_filename "$unencrypted_file")
   echo ========== PROCESSING '"'$unencrypted_file'"'
   fail_if_not_on_cryptlist "$unencrypted_file"
   decrypt_file_overwrite "$encrypted_file" "$unencrypted_file"

tools/confidence_test.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../bin
-export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:${blackbox_home}"
+export PATH="${blackbox_home}:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/local/bin:/usr/pkg/bin:/usr/pkg/gnu/bin:${blackbox_home}"
 
 export LANG=C.UTF-8  # Required ro "gpg --export" to work properly.
 

tools/test_functions.sh

@@ -24,6 +24,12 @@ function md5sum_file() {
     Darwin | FreeBSD )
       md5 -r "$1" | awk '{ print $1 }'
       ;;
+    NetBSD )
+      md5 -q "$1"
+      ;;
+    SunOS )
+      digest -a md5 "$1"
+      ;;
     Linux )
       md5sum "$1" | awk '{ print $1 }'
       ;;
@@ -72,10 +78,10 @@ function assert_file_group() {
   assert_file_exists "$file"
 
   case $(uname -s) in
-    Darwin|FreeBSD )
+    Darwin | FreeBSD | NetBSD )
       found=$(stat -f '%Dg' "$file")
       ;;
-    Linux )
+    Linux | SunOS )
       found=$(stat -c '%g' "$file")
       ;;
     CYGWIN* )
@@ -102,11 +108,11 @@ function assert_file_perm() {
   assert_file_exists "$file"
 
   case $(uname -s) in
-    Darwin|FreeBSD )
+    Darwin | FreeBSD | NetBSD )
       found=$(stat -f '%Sp' "$file")
       ;;
     # NB(tlim): CYGWIN hasn't been tested. It might be more like Darwin.
-    Linux | CYGWIN* )
+    Linux | CYGWIN* | SunOS )
       found=$(stat -c '%A' "$file")
       ;;
     * )