From 501c09ccd5af3ef9da5a89864ea11aed264afe82 Mon Sep 17 00:00:00 2001 From: Tyler Akins Date: Fri, 12 Jun 2015 10:20:00 -0500 Subject: [PATCH 1/8] Removing TODO - it was completed earlier --- bin/blackbox_register_new_file | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/blackbox_register_new_file b/bin/blackbox_register_new_file index 77b997d..19ad7cc 100755 --- a/bin/blackbox_register_new_file +++ b/bin/blackbox_register_new_file @@ -8,8 +8,6 @@ # to systems that need the plaintext (unencrypted) versions, run # blackbox_postdeploy.sh to decrypt all the files. -# TODO(tlim): Add the unencrypted file to .hgignore - set -e blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source "${blackbox_home}/_blackbox_common.sh" From 01e681035dc3cb874215e6cff947246f644f200c Mon Sep 17 00:00:00 2001 From: Tyler Akins Date: Fri, 12 Jun 2015 13:23:45 -0500 Subject: [PATCH 2/8] Adding file deregister tool + extrapolating code This solves some TODOs by moving shared code out into `_blackbox_common.sh`. New VCS commands were added, `vcs_ignore` and `vcs_notice` (the opposite of ignore). Made some utility functions * `remove_filename_from_cryptlist` - The opposite of `add_file_to_cryptlist` * `remove_line` - Removes a single line from a text file --- README.md | 8 +-- bin/_blackbox_common.sh | 114 +++++++++++++++++++++++++++++++++ bin/blackbox_deregister_file | 36 +++++++++++ bin/blackbox_initialize | 23 +------ bin/blackbox_register_new_file | 19 +----- 5 files changed, 157 insertions(+), 43 deletions(-) create mode 100755 bin/blackbox_deregister_file diff --git a/README.md b/README.md index 72f4cb4..84d53bb 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ Commands: | `blackbox_diff` | Diff decrypted files against their original crypted version | | `blackbox_initialize` | Enable blackbox for a GIT or HG repo | | `blackbox_register_new_file` | Encrypt a file for the first time | +| blackbox_deregister_file | Remove a file from blackbox | | `blackbox_list_files` | List the files maintained by blackbox | | `blackbox_decrypt_all_files` | Decrypt all managed files (INTERACTIVE) | | `blackbox_postdeploy` | Decrypt all managed files (batch) | @@ -290,10 +291,9 @@ How to remove a file from the system? ============================ This is a manual process. It happens quite rarely. - -1. Remove the file ``keyrings/live/blackbox-files.txt`` -2. Remove references from ``.gitignore`` or ``.hgignore`` -3. Use ``git rm`` or ``hg rm`` as expected. +``` +blackbox_deregister_file path/to/file.name.key +``` How to indoctrinate a new user into the system? ============================ diff --git a/bin/_blackbox_common.sh b/bin/_blackbox_common.sh index 275ef59..3f8e65e 100755 --- a/bin/_blackbox_common.sh +++ b/bin/_blackbox_common.sh @@ -164,6 +164,19 @@ function add_filename_to_cryptlist() { fi } +# Removes a file from the list of encrypted files +function remove_filename_from_cryptlist() { + # If the name is not already on the list, this is a no-op. + local name=$(vcs_relative_path "$1") + + if ! grep -s -q "$name" "$BB_FILES" ; then + echo ========== File is not registered. No need to remove from list. + else + echo ========== Removing file from list. + remove_line "$BB_FILES" "$name" + fi +} + # Print out who the current BB ADMINS are: function disclose_admins() { echo ========== blackbox administrators are: @@ -281,6 +294,21 @@ function vcs_relative_path() { python -c 'import os ; print(os.path.relpath("'"$(pwd -P)"'/'"$name"'", "'"$REPOBASE"'"))' } +# Removes a line from a text file +function remove_line() { + local tempfile + + tempfile="${1}.blackbox-temp.${RANDOM}.$$" + + # Ensure source file exists + touch "$1" + grep -Fsxv "$2" "$1" > "$tempfile" || true + + # Using cat+rm instead of cp will preserve permissions/ownership + cat "$tempfile" > "$1" + rm "$tempfile" +} + # # Portability Section: # @@ -452,3 +480,89 @@ function vcs_remove_p4() { function vcs_remove_unknown() { : } + + +# Ignore a file in a repo. If it was already ignored, this is a no-op. +function vcs_ignore() { + local file + for file in "$@"; do + vcs_ignore_$(which_vcs) "$file" + done +} +# Mercurial +function vcs_ignore_hg() { + vcs_ignore_generic_file "$REPOBASE/.hgignore" "$file" +} +# Git +function vcs_ignore_git() { + vcs_ignore_generic_file "$REPOBASE/.gitignore" "$file" +} +# Subversion +function vcs_ignore_svn() { + svn propset svn:ignore "$(vcs_relative_path "$file")" +} +# Perforce +function vcs_ignore_p4() { + : +} +# No repo +function vcs_ignore_unknown() { + : +} +# Generic - add line to file +function vcs_ignore_generic_file() { + local file + file="$(vcs_relative_path "$2")" + file="${file/\$\//}" + file="$(echo "/$file" | sed 's/\([\*\?]\)/\\\1/g')" + if ! grep -Fsx "$file" "$1" > /dev/null; then + echo "$file" >> "$1" + vcs_add "$1" + fi +} + + +# Notice (un-ignore) a file in a repo. If it was not ignored, this is +# a no-op +function vcs_notice() { + local file + for file in "$@"; do + vcs_notice_$(which_vcs) "$file" + done +} +# Mercurial +function vcs_notice_hg() { + vcs_notice_generic_file "$REPOBASE/.hgignore" "$file" +} +# Git +function vcs_notice_git() { + vcs_notice_generic_file "$REPOBASE/.gitignore" "$file" +} +# Subversion +function vcs_notice_svn() { + svn propdel svn:ignore "$(vcs_relative_path "$file")" +} +# Perforce +function vcs_notice_p4() { + : +} +# No repo +function vcs_notice_unknown() { + : +} +# Generic - remove line to file +function vcs_notice_generic_file() { + local file + file="$(vcs_relative_path "$2")" + file="${file/\$\//}" + file="$(echo "/$file" | sed 's/\([\*\?]\)/\\\1/g')" + if grep -Fsx "$file" "$1" > /dev/null; then + remove_line "$1" "$file" + vcs_add "$1" + fi + if grep -Fsx "${file:1}" "$1" > /dev/null; then + echo "WARNING: Found a non-absolute ignore match in $1" + echo "WARNING: Confirm the pattern is intended to only exclude $file" + echo "WARNING: If so, manually update the ignore file" + fi +} diff --git a/bin/blackbox_deregister_file b/bin/blackbox_deregister_file new file mode 100755 index 0000000..b5756bd --- /dev/null +++ b/bin/blackbox_deregister_file @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# +# blackbox_deregister_file -- Remove a file from the blackbox system. +# +# Takes an encrypted file and removes it from the blackbox system. The +# encrypted file will also be removed from the filesystem. + +set -e +blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source "${blackbox_home}/_blackbox_common.sh" +_determine_vcs_base_and_type + +unencrypted_file=$(get_unencrypted_filename "$1") +encrypted_file=$(get_encrypted_filename "$1") + +if [[ "$1" == "$unencrypted_file" ]]; then + echo ERROR: Please only deregister encrypted files. + exit 1 +fi + +echo ========== PLAINFILE "$unencrypted_file" +echo ========== ENCRYPTED "$encrypted_file" + +fail_if_not_exists "$encrypted_file" "Please specify an existing file." + +prepare_keychain +remove_filename_from_cryptlist "$unencrypted_file" +vcs_notice "$unencrypted_file" +git rm "$encrypted_file" +vcs_remove "$BB_FILES" + +vcs_commit "Removing from blackbox: ${unencrypted_file}" +echo "========== UPDATING VCS: DONE" +echo "Local repo updated. Please push when ready." +echo " $(which_vcs) push" diff --git a/bin/blackbox_initialize b/bin/blackbox_initialize index 2c55956..e85e8d9 100755 --- a/bin/blackbox_initialize +++ b/bin/blackbox_initialize @@ -25,28 +25,7 @@ fi change_to_vcs_root echo VCS_TYPE: $VCS_TYPE - -if [[ $VCS_TYPE = "git" || $VCS_TYPE = "hg" ]]; then - # Update .gitignore or .hgignore - - IGNOREFILE="${REPOBASE}/.${VCS_TYPE}ignore" - if ! grep -sx >/dev/null 'pubring.gpg~' "$IGNOREFILE" ; then - echo 'pubring.gpg~' >>"$IGNOREFILE" - fi - if ! grep -sx >/dev/null 'pubring.kbx~' "$IGNOREFILE" ; then - echo 'pubring.kbx~' >>"$IGNOREFILE" - fi - if ! grep -sx >/dev/null 'secring.gpg' "$IGNOREFILE" ; then - echo 'secring.gpg' >>"$IGNOREFILE" - fi -elif [[ $VCS_TYPE = "svn" ]]; then - # add file to svn ignore propset - IGNOREFILE=""; - svn propset svn:ignore 'pubring.gpg~ -pubring.kbx~ -secring.gpg' . - svn commit -m "ignore file list" -fi +vcs_ignore keyrings/live/pubring.gpg~ keyrings/live/pubring.kbx~ keyrings/live/secring.gpg # Make directories mkdir -p "${KEYRINGDIR}" diff --git a/bin/blackbox_register_new_file b/bin/blackbox_register_new_file index 19ad7cc..4a7120d 100755 --- a/bin/blackbox_register_new_file +++ b/bin/blackbox_register_new_file @@ -11,7 +11,6 @@ set -e blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source "${blackbox_home}/_blackbox_common.sh" -_determine_vcs_base_and_type unencrypted_file=$(get_unencrypted_filename "$1") encrypted_file=$(get_encrypted_filename "$1") @@ -37,7 +36,6 @@ echo "========== CREATED: ${encrypted_file}" echo "========== UPDATING REPO:" shred_file "$unencrypted_file" -VCSCMD=$(which_vcs) if "$SECRETSEXPOSED" ; then vcs_remove "$unencrypted_file" vcs_add "$encrypted_file" @@ -46,23 +44,10 @@ else COMMIT_FILES=("$BB_FILES" "$encrypted_file") fi -# TODO(tlim): This should be moved to _blackbox_common.sh in a -# VCS-independent way. -IGNOREFILE="${REPOBASE}/.${VCS_TYPE}ignore" -if [[ $VCS_TYPE = 'git' ]]; then - relfile="$(vcs_relative_path "$unencrypted_file")" - relfileb="${relfile/\$\//}" - ignored_file="$(echo "${relfileb}" | sed 's/\([\*\?]\)/\\\1/g' | sed 's/^\([!#]\)/\\\1/')" - if ! grep -Fsx >/dev/null "$ignored_file" "$IGNOREFILE"; then - echo "$ignored_file" >>"$IGNOREFILE" - COMMIT_FILES+=("$IGNOREFILE") - fi - vcs_add "$IGNOREFILE" -fi - +vcs_ignore "$unencrypted_file" echo 'NOTE: "already tracked!" messages are safe to ignore.' vcs_add "$BB_FILES" "$encrypted_file" vcs_commit "registered in blackbox: ${unencrypted_file}" "${COMMIT_FILES[@]}" echo "========== UPDATING VCS: DONE" echo "Local repo updated. Please push when ready." -echo " $VCSCMD push" +echo " $(which_vcs) push" From db9b2eef606c0cf705ed3a2362ef1e3ac3a3b190 Mon Sep 17 00:00:00 2001 From: Tyler Akins Date: Fri, 12 Jun 2015 13:27:42 -0500 Subject: [PATCH 3/8] Using """x""" is identical to "x" --- bin/_blackbox_common.sh | 32 ++++++++++++++++---------------- bin/_stack_lib.sh | 4 ++-- bin/blackbox_cat | 2 +- bin/blackbox_edit | 2 +- bin/blackbox_edit_start | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/bin/_blackbox_common.sh b/bin/_blackbox_common.sh index 3f8e65e..0cd31ae 100755 --- a/bin/_blackbox_common.sh +++ b/bin/_blackbox_common.sh @@ -351,7 +351,7 @@ function which_vcs() { # Is this file in the current repo? function is_in_vcs() { - is_in_$(which_vcs) """$@""" + is_in_$(which_vcs) "$@" } # Mercurial function is_in_hg() { @@ -405,23 +405,23 @@ function is_in_unknown() { # Add a file to the repo (but don't commit it). function vcs_add() { - vcs_add_$(which_vcs) """$@""" + vcs_add_$(which_vcs) "$@" } # Mercurial function vcs_add_hg() { - hg add """$@""" + hg add "$@" } # Git function vcs_add_git() { - git add """$@""" + git add "$@" } # Subversion function vcs_add_svn() { - svn add --parents """$@""" + svn add --parents "$@" } # Perfoce function vcs_add_p4() { - p4 add """$@""" + p4 add "$@" } # No repo function vcs_add_unknown() { @@ -431,23 +431,23 @@ function vcs_add_unknown() { # Commit a file to the repo function vcs_commit() { - vcs_commit_$(which_vcs) """$@""" + vcs_commit_$(which_vcs) "$@" } # Mercurial function vcs_commit_hg() { - hg commit -m"""$@""" + hg commit -m"$@" } # Git function vcs_commit_git() { - git commit -m"""$@""" + git commit -m"$@" } # Subversion function vcs_commit_svn() { - svn commit -m"""$@""" + svn commit -m"$@" } # Perforce function vcs_commit_p4() { - p4 submit -d """$@""" + p4 submit -d "$@" } # No repo function vcs_commit_unknown() { @@ -458,23 +458,23 @@ function vcs_commit_unknown() { # Remove file from repo, even if it was deleted locally already. # If it doesn't exist yet in the repo, it should be a no-op. function vcs_remove() { - vcs_remove_$(which_vcs) """$@""" + vcs_remove_$(which_vcs) "$@" } # Mercurial function vcs_remove_hg() { - hg rm -A -- """$@""" + hg rm -A -- "$@" } # Git function vcs_remove_git() { - git rm --ignore-unmatch -f -- """$@""" + git rm --ignore-unmatch -f -- "$@" } # Subversion function vcs_remove_svn() { - svn delete """$@""" + svn delete "$@" } # Perforce function vcs_remove_p4() { - p4 delete """$@""" + p4 delete "$@" } # No repo function vcs_remove_unknown() { diff --git a/bin/_stack_lib.sh b/bin/_stack_lib.sh index e3c3abb..2064fd4 100755 --- a/bin/_stack_lib.sh +++ b/bin/_stack_lib.sh @@ -11,13 +11,13 @@ function debugmsg() { # Log to stderr. - echo 1>&2 LOG: """$@""" + echo 1>&2 LOG: "$@" : } function logit() { # Log to stderr. - echo 1>&2 LOG: """$@""" + echo 1>&2 LOG: "$@" } function fail_out() { diff --git a/bin/blackbox_cat b/bin/blackbox_cat index 604eee7..2e2a908 100755 --- a/bin/blackbox_cat +++ b/bin/blackbox_cat @@ -7,7 +7,7 @@ set -e blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source "${blackbox_home}/_blackbox_common.sh" -for param in """$@""" ; do +for param in "$@" ; do shreddable=0 unencrypted_file=$(get_unencrypted_filename "$param") if [[ ! -e "$unencrypted_file" ]]; then diff --git a/bin/blackbox_edit b/bin/blackbox_edit index b72f957..11c01ff 100755 --- a/bin/blackbox_edit +++ b/bin/blackbox_edit @@ -7,7 +7,7 @@ set -e blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source "${blackbox_home}/_blackbox_common.sh" -for param in """$@""" ; do +for param in "$@" ; do unencrypted_file=$(get_unencrypted_filename "$param") if ! is_on_cryptlist "$param" && ! is_on_cryptlist "$unencrypted_file" ; then read -r -p "Encrypt file $param? (y/n) " ans diff --git a/bin/blackbox_edit_start b/bin/blackbox_edit_start index cf327b7..39a683f 100755 --- a/bin/blackbox_edit_start +++ b/bin/blackbox_edit_start @@ -8,7 +8,7 @@ set -e blackbox_home=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source "${blackbox_home}/_blackbox_common.sh" -for param in """$@""" ; do +for param in "$@" ; do unencrypted_file=$(get_unencrypted_filename "$param") encrypted_file=$(get_encrypted_filename "$param") echo >&2 ========== PLAINFILE '"'$unencrypted_file'"' From 83249517a52ec8379fc94a6e8f0bb1cd5e1df2ac Mon Sep 17 00:00:00 2001 From: Tyler Akins Date: Fri, 12 Jun 2015 13:31:26 -0500 Subject: [PATCH 4/8] Updating wording to remove the "manual" bit. Not sure how this should get phrased, thus it's a separate commit. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 84d53bb..b6f8747 100644 --- a/README.md +++ b/README.md @@ -290,7 +290,7 @@ blackbox_register_new_file path/to/file.name.key How to remove a file from the system? ============================ -This is a manual process. It happens quite rarely. +This happens quite rarely, but we've got it covered: ``` blackbox_deregister_file path/to/file.name.key ``` From 6e04e42f4ac9d85dab3e6447f14a784d1e3e11a4 Mon Sep 17 00:00:00 2001 From: Tyler Akins Date: Mon, 15 Jun 2015 12:22:39 -0500 Subject: [PATCH 5/8] Updated to use remove_line I didn't notice this earlier --- bin/blackbox_removeadmin | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/blackbox_removeadmin b/bin/blackbox_removeadmin index b2e068e..e0d6c4b 100755 --- a/bin/blackbox_removeadmin +++ b/bin/blackbox_removeadmin @@ -20,9 +20,7 @@ KEYNAME="$1" : "${KEYNAME:?ERROR: First argument must be a keyname (email address)}" ; # Remove the email address from the BB_ADMINS file. -make_self_deleting_tempfile bbtemp -cp "$BB_ADMINS" "$bbtemp" -fgrep -v -x "$KEYNAME" <"$bbtemp" >"$BB_ADMINS" +remove_line "$BB_ADMINS" "$KEYNAME" # Make a suggestion: echo From 287b97ccd62fc4de010257b43fad5bf77e971fd0 Mon Sep 17 00:00:00 2001 From: Tyler Akins Date: Mon, 15 Jun 2015 12:29:44 -0500 Subject: [PATCH 6/8] Pulling greps into a common function This will help eliminate problems and make searching consistent --- bin/_blackbox_common.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bin/_blackbox_common.sh b/bin/_blackbox_common.sh index 0cd31ae..a2c90c0 100755 --- a/bin/_blackbox_common.sh +++ b/bin/_blackbox_common.sh @@ -68,8 +68,7 @@ SECRING="${KEYRINGDIR}/secring.gpg" # Return error if not on cryptlist. function is_on_cryptlist() { # Assumes $1 does NOT have the .gpg extension - local rname=$(vcs_relative_path "$1") - grep -F -x -s -q "$rname" "$BB_FILES" + file_contains_line "$(vcs_relative_path "$1")" "$BB_FILES" } # Exit with error if a file exists. @@ -155,10 +154,10 @@ function add_filename_to_cryptlist() { # However no matter what the datestamp is updated. local name=$(vcs_relative_path "$1") - if grep -s -q "$name" "$BB_FILES" ; then - echo ========== File is registered. No need to add to list. + if file_contains_line "$BB_FILES" "$name" ; then + echo "========== File is registered. No need to add to list." else - echo ========== Adding file to list. + echo "========== Adding file to list." touch "$BB_FILES" sort -u -o "$BB_FILES" <(echo "$name") "$BB_FILES" fi @@ -169,7 +168,7 @@ function remove_filename_from_cryptlist() { # If the name is not already on the list, this is a no-op. local name=$(vcs_relative_path "$1") - if ! grep -s -q "$name" "$BB_FILES" ; then + if ! file_contains_line "$BB_FILES" "$name" ; then echo ========== File is not registered. No need to remove from list. else echo ========== Removing file from list. @@ -309,6 +308,11 @@ function remove_line() { rm "$tempfile" } +# Determine if a file contains a given line +function file_contains_line() { + grep -Fxsq "$2" "$1" +} + # # Portability Section: # @@ -515,7 +519,7 @@ function vcs_ignore_generic_file() { file="$(vcs_relative_path "$2")" file="${file/\$\//}" file="$(echo "/$file" | sed 's/\([\*\?]\)/\\\1/g')" - if ! grep -Fsx "$file" "$1" > /dev/null; then + if ! file_contains_line "$1" "$file" ; then echo "$file" >> "$1" vcs_add "$1" fi @@ -556,11 +560,11 @@ function vcs_notice_generic_file() { file="$(vcs_relative_path "$2")" file="${file/\$\//}" file="$(echo "/$file" | sed 's/\([\*\?]\)/\\\1/g')" - if grep -Fsx "$file" "$1" > /dev/null; then + if file_contains_line "$1" "$file" ; then remove_line "$1" "$file" vcs_add "$1" fi - if grep -Fsx "${file:1}" "$1" > /dev/null; then + if file_contains_line "$1" "${file:1}" ; then echo "WARNING: Found a non-absolute ignore match in $1" echo "WARNING: Confirm the pattern is intended to only exclude $file" echo "WARNING: If so, manually update the ignore file" From af7db40263b514785d092c3081aa27b8b72bfc07 Mon Sep 17 00:00:00 2001 From: Tyler Akins Date: Mon, 15 Jun 2015 12:31:50 -0500 Subject: [PATCH 7/8] Using make_self_deleting_tempfile I missed this function earlier --- bin/_blackbox_common.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/_blackbox_common.sh b/bin/_blackbox_common.sh index a2c90c0..c5ecf34 100755 --- a/bin/_blackbox_common.sh +++ b/bin/_blackbox_common.sh @@ -297,7 +297,7 @@ function vcs_relative_path() { function remove_line() { local tempfile - tempfile="${1}.blackbox-temp.${RANDOM}.$$" + make_self_deleting_tempfile tempfile # Ensure source file exists touch "$1" @@ -305,7 +305,6 @@ function remove_line() { # Using cat+rm instead of cp will preserve permissions/ownership cat "$tempfile" > "$1" - rm "$tempfile" } # Determine if a file contains a given line From 3968dab7c8812d00f23c844c2cdc739fc9e3203f Mon Sep 17 00:00:00 2001 From: Tyler Akins Date: Mon, 15 Jun 2015 12:32:44 -0500 Subject: [PATCH 8/8] This line should not have continued to exist The line immediately afterwards was what was intended. --- bin/blackbox_deregister_file | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/blackbox_deregister_file b/bin/blackbox_deregister_file index b5756bd..f4ea76e 100755 --- a/bin/blackbox_deregister_file +++ b/bin/blackbox_deregister_file @@ -27,7 +27,6 @@ fail_if_not_exists "$encrypted_file" "Please specify an existing file." prepare_keychain remove_filename_from_cryptlist "$unencrypted_file" vcs_notice "$unencrypted_file" -git rm "$encrypted_file" vcs_remove "$BB_FILES" vcs_commit "Removing from blackbox: ${unencrypted_file}"