From df1380bc45f0cbf92865687d594a786461c3666e Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Thu, 9 Feb 2017 14:07:07 -0500 Subject: [PATCH 01/19] Remove unneeded shell utils --- shell_utils/create_user.sh | 10 ---------- shell_utils/simple_mount_image.sh | 7 ------- 2 files changed, 17 deletions(-) delete mode 100755 shell_utils/create_user.sh delete mode 100644 shell_utils/simple_mount_image.sh diff --git a/shell_utils/create_user.sh b/shell_utils/create_user.sh deleted file mode 100755 index 80af644..0000000 --- a/shell_utils/create_user.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -useradd -m kitten - -# Useless: the sudoer file comes from the repository -#echo "Cmnd_Alias GROOMER_CMDS = /home/kitten/kitten_mount_src, \ -# /home/kitten/kitten_mount_dst, /home/kitten/kitten_umount" >> /etc/sudoers -#echo "kitten ALL=(ALL) NOPASSWD: GROOMER_CMDS" >> /etc/sudoers - -# /!\ REMOVE SUDO RIGHTS TO USER pi diff --git a/shell_utils/simple_mount_image.sh b/shell_utils/simple_mount_image.sh deleted file mode 100644 index 6c6bf3b..0000000 --- a/shell_utils/simple_mount_image.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -IMAGE='2016-05-12_CIRCLean.img' -OFFSET=$((512 * 131072)) - -mkdir /mnt/rpi -mount -v -o offset=${OFFSET} -t ext4 ${IMAGE} /mnt/rpi From a255251430eaf95e9db748b8c4d7c0c767635d81 Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Thu, 9 Feb 2017 14:41:13 -0500 Subject: [PATCH 02/19] Change python-tk to python3-tk in docs --- doc/image_setup_checklist.md | 2 +- doc/setup_with_proot.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/image_setup_checklist.md b/doc/image_setup_checklist.md index 8c12e59..b9ffc8b 100644 --- a/doc/image_setup_checklist.md +++ b/doc/image_setup_checklist.md @@ -19,7 +19,7 @@ - pmount ntfs-3g - python3 python3-pip - python3-lxml - - libjpeg-dev libtiff-dev libwebp-dev liblcms2-dev tcl-dev tk-dev python-tk + - libjpeg-dev libtiff-dev libwebp-dev liblcms2-dev tcl-dev tk-dev python3-tk * Compile p7zip-rar from source - Change your source.list file - Make a new directory and cd to it diff --git a/doc/setup_with_proot.md b/doc/setup_with_proot.md index 3fea3bc..0c6b1c6 100644 --- a/doc/setup_with_proot.md +++ b/doc/setup_with_proot.md @@ -146,7 +146,7 @@ raspbian-sys-mods related installs may fail - you can ignore them: ``` * Install the linux dependencies (see CONTRIBUTING.md for more details): ``` - apt-get install timidity git p7zip-full python3 python3-pip python3-lxml pmount ntfs-3g libjpeg-dev libtiff-dev libwebp-dev tk-dev python-tk liblcms2-dev tcl-dev + apt-get install timidity git p7zip-full python3 python3-pip python3-lxml pmount ntfs-3g libjpeg-dev libtiff-dev libwebp-dev tk-dev python3-tk liblcms2-dev tcl-dev ``` * Compile p7zip-rar from source. First, uncomment out the second line in /etc/apt/sources.list. Then: ``` From d8215b13f6af90b13af45ec26f65ac6880c20eee Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Wed, 9 Aug 2017 16:02:40 -0400 Subject: [PATCH 03/19] Update changelog --- CHANGELOG.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82a0318..54bbc3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,17 @@ TODO Version 2.2 - 2017-04-18 -New features: + +Version 2.2 - 2017-04-18 +- Updated to newest version of Raspbian Jessie lite (April 10th 2017 release) +- Using the newest version of PyCIRCLean, which includes a new log format and +various bug fixes - Filecheck.py configuration information is now conveniently held in a Config object instead of in globals - New easier to read text-based logger (removed twiggy dependency) - Various filetypes in filecheck.py now have improved descriptions for log -- Improved the interface for adding file descriptions to files + +Version 2.1.1 - 2017-02-28 +- Fix copying PDF documents to the destination key Version 2.1 - 2017-02-02 - Updated to the newest version of Raspbian Jessie lite (January 11th 2017 release) From b5131e295c88ecd57cf609ca5f067ca2325f6048 Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Wed, 9 Aug 2017 16:02:53 -0400 Subject: [PATCH 04/19] Fix typos in setup_with_proot.md --- doc/setup_with_proot.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/setup_with_proot.md b/doc/setup_with_proot.md index 0c6b1c6..ac19707 100644 --- a/doc/setup_with_proot.md +++ b/doc/setup_with_proot.md @@ -120,9 +120,9 @@ to fill the new larger partition using resize2fs: Installing the dependencies =========================== -* Copy circlean_fs/root_partition/systemd/system/rc-local.service into the equivalent location in the image. +* Copy circlean_fs/root_partition/etc/systemd/system/rc-local.service into the equivalent location in the image. ``` - cp circlean_fs/root_partition/systemd/system/rc-local.service /mnt/rpi-root/etc/systemd/system/rc-local.service + sudo cp circlean_fs/root_partition/etc/systemd/system/rc-local.service /mnt/rpi-root/etc/systemd/system/rc-local.service ``` * Use [proot](https://proot-me.github.io/) to enter the equivalent of a chroot inside the mounted image. @@ -152,7 +152,9 @@ raspbian-sys-mods related installs may fail - you can ignore them: ``` cd /home/pi mkdir rar && cd rar/ + apt-get update apt-get build-dep p7zip-rar + apt-get source -b p7zip-rar dpkg -i ${path to p7zip-rar .deb file} ``` * Install the Python dependencies for PyCIRCLean/filecheck.py. PyCIRCLean is 3.3+ @@ -220,5 +222,5 @@ copying process: ``` * Use fsck to verify the root partition: ``` - sudo e2fsck -f /dev/sd2 + sudo e2fsck -f /dev/sd2 ``` From 299743722f9c435564bd6529c539e4c5f963290b Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Wed, 9 Aug 2017 16:05:27 -0400 Subject: [PATCH 05/19] Add shell_utils/start_proot.sh --- shell_utils/start_proot.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 shell_utils/start_proot.sh diff --git a/shell_utils/start_proot.sh b/shell_utils/start_proot.sh new file mode 100755 index 0000000..9ecf4ca --- /dev/null +++ b/shell_utils/start_proot.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# This script runs proot on a mounted image with the proper parameters. +# The root partition should be at /mnt/rpi-root /mnt/rpt-boot +# You should probably run something like basic_mount_image.sh first + +set -e +set -x + +if [ "$(id -u)" != "0" ]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +sudo proot -q qemu-arm -S /mnt/rpi-root -b /mnt/rpi-boot:/boot /bin/bash From b6564d7298395dffc3eefcad92c377a5262bf05f Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Wed, 9 Aug 2017 16:08:18 -0400 Subject: [PATCH 06/19] Update basic_mount_image.sh --- shell_utils/basic_mount_image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shell_utils/basic_mount_image.sh b/shell_utils/basic_mount_image.sh index bcfa3d1..0e37efe 100755 --- a/shell_utils/basic_mount_image.sh +++ b/shell_utils/basic_mount_image.sh @@ -17,11 +17,11 @@ set -x # Double check the path and offsets as noted above! # Path to the image -IMAGE='2017-02-02_CIRCLean.img' +IMAGE='New_Circlean.img' # Start sector of boot (first) partition BOOT_START=8192 # Start sector of root (second) partition -ROOT_START=137216 +ROOT_START=92160 # Locations you'd like the partitions mounted BOOT_PATH='/mnt/rpi-boot' ROOTFS_PATH='/mnt/rpi-root' From 9dd6b2c460d7d8b466c78a868b22dc2ecd99ec4f Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Wed, 9 Aug 2017 17:29:09 -0400 Subject: [PATCH 07/19] Make variables in config.sh readonly --- .../root_partition/opt/groomer/config.sh | 41 ++++++++++--------- .../root_partition/opt/groomer/groomer.sh | 5 +-- .../root_partition/opt/groomer/mount_dest.sh | 2 - .../root_partition/opt/groomer/music.sh | 2 +- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/circlean_fs/root_partition/opt/groomer/config.sh b/circlean_fs/root_partition/opt/groomer/config.sh index e69bf2e..4ef51d9 100755 --- a/circlean_fs/root_partition/opt/groomer/config.sh +++ b/circlean_fs/root_partition/opt/groomer/config.sh @@ -1,29 +1,30 @@ -USERNAME="kitten" -ID=`/usr/bin/id -u` +#!/bin/bash +set -eu + +readonly USERNAME="kitten" +readonly ID=$(/usr/bin/id -u) # Paths used in multiple scripts -SRC="src" -DEV_SRC="/dev/source_key" -SRC_MNT="/media/src" +readonly SRC="src" +readonly DEV_SRC="/dev/source_key" +readonly SRC_MNT="/media/src" -DST="dst" -DEV_DST="/dev/dest_key" -DST_MNT="/media/dst" - -TEMP="${DST_MNT}/temp" -ZIPTEMP="${DST_MNT}/ziptemp" -LOGS="${DST_MNT}/logs" -DEBUG_LOG="/tmp/groomer_debug_log.txt" -MUSIC="/opt/midi/" +readonly DST="dst" +readonly DEV_DST="/dev/dest_key" +readonly DST_MNT="/media/dst" +readonly TEMP="${DST_MNT}/temp" +readonly LOGS_DIR="${DST_MNT}/logs" +readonly DEBUG_LOG="/tmp/groomer_debug_log.txt" +readonly MUSIC_DIR="/opt/midi/" # Commands -SYNC="/bin/sync" -TIMIDITY="/usr/bin/timidity" -MOUNT="/bin/mount" -PMOUNT="/usr/bin/pmount -A -s" -PUMOUNT="/usr/bin/pumount" +readonly SYNC="/bin/sync" +readonly TIMIDITY="/usr/bin/timidity" +readonly MOUNT="/bin/mount" +readonly PMOUNT="/usr/bin/pmount -A -s" +readonly PUMOUNT="/usr/bin/pumount" # Config flags -DEBUG=false +readonly DEBUG=false diff --git a/circlean_fs/root_partition/opt/groomer/groomer.sh b/circlean_fs/root_partition/opt/groomer/groomer.sh index daeb027..7e6b89e 100755 --- a/circlean_fs/root_partition/opt/groomer/groomer.sh +++ b/circlean_fs/root_partition/opt/groomer/groomer.sh @@ -22,7 +22,6 @@ clean(){ # Remove temporary files from destination key rm -rf ${TEMP} - rm -rf ${ZIPTEMP} } trap clean EXIT TERM INT @@ -57,12 +56,12 @@ do echo "GROOMER: ${partition} mounted at /media/${SRC}" # Put the filenames from the current partition in a logfile - find "/media/${SRC}" -fls "${LOGS}/contents_partition_${PARTCOUNT}.txt" + find "/media/${SRC}" -fls "${LOGS_DIR}/contents_partition_${PARTCOUNT}.txt" # Create a directory on ${DST} named PARTION_$PARTCOUNT target_dir="/media/${DST}/FROM_PARTITION_${PARTCOUNT}" mkdir -p "${target_dir}" - LOGFILE="${LOGS}/processing_log.txt" + LOGFILE="${LOGS_DIR}/processing_log.txt" # Run the current partition through filecheck.py echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE} diff --git a/circlean_fs/root_partition/opt/groomer/mount_dest.sh b/circlean_fs/root_partition/opt/groomer/mount_dest.sh index a32b45d..16437b4 100755 --- a/circlean_fs/root_partition/opt/groomer/mount_dest.sh +++ b/circlean_fs/root_partition/opt/groomer/mount_dest.sh @@ -69,10 +69,8 @@ else # Prepare temp dirs and make sure they're empty if they already exist mkdir -p "${TEMP}" - mkdir -p "${ZIPTEMP}" mkdir -p "${LOGS}" rm -rf "${TEMP}/"* - rm -rf "${ZIPTEMP}/"* rm -rf "${LOGS}/"* fi diff --git a/circlean_fs/root_partition/opt/groomer/music.sh b/circlean_fs/root_partition/opt/groomer/music.sh index 0ac07f1..81d8122 100755 --- a/circlean_fs/root_partition/opt/groomer/music.sh +++ b/circlean_fs/root_partition/opt/groomer/music.sh @@ -14,7 +14,7 @@ trap killed EXIT TERM INT # Force output on analog amixer cset numid=3 1 -files=(${MUSIC}*) +files=(${MUSIC_DIR}*) while true; do # -id flags set interface to "dumb" and -qq silences most/all terminal output From 66f2dd25f2636330a9ef459afc1a64b72a8c3803 Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Wed, 9 Aug 2017 18:03:52 -0400 Subject: [PATCH 08/19] Refactor init.sh to use functions --- .../root_partition/opt/groomer/config.sh | 1 + .../root_partition/opt/groomer/init.sh | 60 +++++++++++-------- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/circlean_fs/root_partition/opt/groomer/config.sh b/circlean_fs/root_partition/opt/groomer/config.sh index 4ef51d9..d172c5f 100755 --- a/circlean_fs/root_partition/opt/groomer/config.sh +++ b/circlean_fs/root_partition/opt/groomer/config.sh @@ -28,3 +28,4 @@ readonly PUMOUNT="/usr/bin/pumount" # Config flags readonly DEBUG=false +readonly MUSIC=true diff --git a/circlean_fs/root_partition/opt/groomer/init.sh b/circlean_fs/root_partition/opt/groomer/init.sh index 5c68def..5d3568d 100755 --- a/circlean_fs/root_partition/opt/groomer/init.sh +++ b/circlean_fs/root_partition/opt/groomer/init.sh @@ -1,37 +1,49 @@ #!/bin/bash -# set -e (exit when a line returns non-0 status) and -x (xtrace) flags -set -e -set -x - -# Import constants from config file -source ./config.sh - -if [ ${ID} -ne 0 ]; then - echo "GROOMER: This script has to be run as root." - exit -fi - clean(){ - if [ ${DEBUG} = true ]; then + if [ "${DEBUG}" = true ]; then sleep 20 fi echo "GROOMER: cleaning up after init.sh." - ${SYNC} + "${SYNC}" # Stop the music from playing - kill -9 $(cat /tmp/music.pid) + kill -9 "$(cat /tmp/music.pid)" rm -f /tmp/music.pid } -trap clean EXIT TERM INT +check_is_root() { + if [ "${ID}" -ne 0 ]; then + echo "GROOMER: This script has to be run as root." + exit + fi +} -# Start music -./music.sh & -echo $! > /tmp/music.pid +start_music() { + ./music.sh & + echo $! > /tmp/music.pid +} -# List block storage devices for debugging -if [ ${DEBUG} = true ]; then - lsblk |& tee -a ${DEBUG_LOG} -fi +run_groomer() { + if [ "${DEBUG}" = true ]; then + lsblk |& tee -a "${DEBUG_LOG}" # list block storage devices for debugging + su "${USERNAME}" -c ./mount_dest.sh |& tee -a "${DEBUG_LOG}" + else + su "${USERNAME}" -c ./mount_dest.sh + fi +} -su ${USERNAME} -c ./mount_dest.sh |& tee -a ${DEBUG_LOG} +main() { + set -eu # exit when a line returns non-0 status, treat unset variables as errors + trap clean EXIT TERM INT # run clean when the script ends or is interrupted + check_is_root + source ./config.sh # get config values + if [ "${DEBUG}" = true ]; then + set -x + fi + if [ "${MUSIC}" = true ]; then + start_music + fi + run_groomer +} + +main From fce66420c8479df57c3ce38ef7f985d5683a9674 Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Wed, 9 Aug 2017 18:08:34 -0400 Subject: [PATCH 09/19] Refactor music.sh to use functions --- .../root_partition/opt/groomer/music.sh | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/circlean_fs/root_partition/opt/groomer/music.sh b/circlean_fs/root_partition/opt/groomer/music.sh index 81d8122..20b5235 100755 --- a/circlean_fs/root_partition/opt/groomer/music.sh +++ b/circlean_fs/root_partition/opt/groomer/music.sh @@ -1,22 +1,25 @@ #!/bin/bash -set -e -#set -x - -source ./config.sh - killed(){ echo 'Music stopped.' } -trap killed EXIT TERM INT - -# Force output on analog -amixer cset numid=3 1 - -files=(${MUSIC_DIR}*) - -while true; do - # -id flags set interface to "dumb" and -qq silences most/all terminal output - $TIMIDITY -idqq ${files[RANDOM % ${#files[@]}]} +run_timidity() { + # Force output on analog + amixer cset numid=3 1 + files=(${MUSIC_DIR}*) + while true; do + # -id flags set interface to "dumb" and -qq silences most/all terminal output + "${TIMIDITY}" -idqq "${files[RANDOM % ${#files[@]}]}" done +} + +main() { + set -eu # exit when a line returns non-0 status, treat unset variables as errors + trap killed EXIT TERM INT # run clean when the script ends or is interrupted + check_is_root + source ./config.sh # get config values + run_timidity +} + +main From dd427b8c8e5deb9fbc15f9da3cdae465ccc7e70f Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Wed, 9 Aug 2017 19:04:57 -0400 Subject: [PATCH 10/19] Refactor mount_dest.sh to use functions --- .../root_partition/opt/groomer/init.sh | 4 +- .../root_partition/opt/groomer/mount_dest.sh | 126 +++++++++--------- .../root_partition/opt/groomer/music.sh | 1 - 3 files changed, 64 insertions(+), 67 deletions(-) diff --git a/circlean_fs/root_partition/opt/groomer/init.sh b/circlean_fs/root_partition/opt/groomer/init.sh index 5d3568d..f8ba08a 100755 --- a/circlean_fs/root_partition/opt/groomer/init.sh +++ b/circlean_fs/root_partition/opt/groomer/init.sh @@ -12,7 +12,7 @@ clean(){ } check_is_root() { - if [ "${ID}" -ne 0 ]; then + if [ "${ID}" -ne 0 ]; then # -ne is an integer comparison instead of a string comparison echo "GROOMER: This script has to be run as root." exit fi @@ -35,8 +35,8 @@ run_groomer() { main() { set -eu # exit when a line returns non-0 status, treat unset variables as errors trap clean EXIT TERM INT # run clean when the script ends or is interrupted - check_is_root source ./config.sh # get config values + check_is_root if [ "${DEBUG}" = true ]; then set -x fi diff --git a/circlean_fs/root_partition/opt/groomer/mount_dest.sh b/circlean_fs/root_partition/opt/groomer/mount_dest.sh index 16437b4..5703462 100755 --- a/circlean_fs/root_partition/opt/groomer/mount_dest.sh +++ b/circlean_fs/root_partition/opt/groomer/mount_dest.sh @@ -1,78 +1,76 @@ #!/bin/bash -# set -e (exit when a line returns non-0 status) and -x (xtrace) flags -set -e -set -x - -# Import constants from config file -source ./config.sh - -if ! [ "${ID}" -ge "1000" ]; then - echo "GROOMER: mount_keys.sh cannot run as root." - exit -fi - clean(){ - if [ ${DEBUG} = true ]; then - sleep 20 - # Copy the temporary logfile to the destination key - cp ${DEBUG_LOG} "${DST_MNT}/groomer_debug_log.txt" + if [ "${DEBUG}" = true ]; then + sleep 20 + # Copy the temporary logfile to the destination key + cp "${DEBUG_LOG}" "${DST_MNT}/groomer_debug_log.txt" + fi + echo "GROOMER: Cleaning up in mount_keys.sh." + ${SYNC} # Write anything in memory to disk + # Unmount source and destination + pumount "${SRC}" + pumount "${DST}" + exit + } + +check_not_root() { + if ! [ "${ID}" -ge "1000" ]; then + echo "GROOMER: mount_keys.sh cannot run as root." + exit fi - - echo "GROOMER: Cleaning up in mount_keys.sh." - - # Write anything in memory to disk - ${SYNC} - - # Unmount source and destination - pumount ${SRC} - - # Clean up and unmount destination - pumount ${DST} - - exit } -trap clean EXIT TERM INT +check_source_exists() { + if [ ! -b "${DEV_SRC}" ]; then + echo "GROOMER: Source device (${DEV_SRC}) does not exist." + exit + fi +} -# Check that a device is available on /dev/source_key (symlinked to /dev/sda or sdb) -if [ ! -b ${DEV_SRC} ]; then - echo "GROOMER: Source device (${DEV_SRC}) does not exist." - exit -fi +check_dest_exists() { + if [ ! -b "${DEV_DST}" ]; then + echo "GROOMER: Destination device (${DEV_DST}) does not exist." + exit + fi +} -# Check that a device is available on /dev/dest_key (symlinked to /dev/sda or sdb) -if [ ! -b ${DEV_DST} ]; then - echo "GROOMER: Destination device (${DEV_DST}) does not exist." - exit -fi +unmount_dest_if_mounted() { + if ${MOUNT}|grep "${DST}"; then + ${PUMOUNT} "${DST}" || true + fi +} -# If there is already a device mounted on /media/dst, unmount it -if ${MOUNT}|grep ${DST}; then - ${PUMOUNT} ${DST} || true -fi - -# uid= only works on a vfat FS. What should wedo if we get an ext* FS ? -# What does this ^ comment mean? - -# Mount the first partition of DST (/dev/dest_key1) -# pmount automatically mounts on /media/ (at /media/dst in this case). -${PMOUNT} -w "${DEV_DST}1" ${DST} -if [ ${?} -ne 0 ]; then - echo "GROOMER: Unable to mount ${DEV_DST}1 on ${DST_MNT}" - exit -else - echo "GROOMER: Destination USB device (${DEV_DST}1) mounted at ${DST_MNT}" - - # Remove any existing "FROM_PARTITION_" directories - rm -rf "/media/${DST}/FROM_PARTITION_"* +mount_dest_partition() { + if "${PMOUNT}" -w "${DEV_DST}1" "${DST}"; then # pmount automatically mounts on /media/ (at /media/dst in this case). + echo "GROOMER: Destination USB device (${DEV_DST}1) mounted at ${DST_MNT}" + else + echo "GROOMER: Unable to mount ${DEV_DST}1 on ${DST_MNT}" + exit + fi +} +prepare_dest_partition() { + rm -rf "/media/${DST}/FROM_PARTITION_"* # Remove any existing "FROM_PARTITION_" directories # Prepare temp dirs and make sure they're empty if they already exist mkdir -p "${TEMP}" mkdir -p "${LOGS}" - rm -rf "${TEMP}/"* - rm -rf "${LOGS}/"* -fi + rm -rf "${TEMP:?}/"* + rm -rf "${LOGS:?}/"* +} -# Now that destination is mounted and prepared, run the groomer -./groomer.sh +main() { + set -eu # exit when a line returns non-0 status, treat unset variables as errors + trap clean EXIT TERM INT + set -x + source ./config.sh + check_not_root + check_source_exists + check_dest_exists + unmount_dest_if_mounted + mount_dest_partition + prepare_dest_partition + ./groomer.sh +} + +main \ No newline at end of file diff --git a/circlean_fs/root_partition/opt/groomer/music.sh b/circlean_fs/root_partition/opt/groomer/music.sh index 20b5235..de43f19 100755 --- a/circlean_fs/root_partition/opt/groomer/music.sh +++ b/circlean_fs/root_partition/opt/groomer/music.sh @@ -17,7 +17,6 @@ done main() { set -eu # exit when a line returns non-0 status, treat unset variables as errors trap killed EXIT TERM INT # run clean when the script ends or is interrupted - check_is_root source ./config.sh # get config values run_timidity } From 6845f2b607391c3c9e96dbb0f252392e0d3f416a Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Thu, 10 Aug 2017 11:51:05 -0400 Subject: [PATCH 11/19] Refactor groomer.sh to use functions --- .../root_partition/opt/groomer/groomer.sh | 124 ++++++++++-------- .../root_partition/opt/groomer/init.sh | 2 +- .../root_partition/opt/groomer/mount_dest.sh | 8 +- 3 files changed, 75 insertions(+), 59 deletions(-) diff --git a/circlean_fs/root_partition/opt/groomer/groomer.sh b/circlean_fs/root_partition/opt/groomer/groomer.sh index 7e6b89e..ac24412 100755 --- a/circlean_fs/root_partition/opt/groomer/groomer.sh +++ b/circlean_fs/root_partition/opt/groomer/groomer.sh @@ -1,19 +1,7 @@ #!/bin/bash -# set -e (exit when a line returns non-0 status) and -x (xtrace) flags -set -e -set -x - -# Import constants from config file -source ./config.sh - -if ! [ "${ID}" -ge "1000" ]; then - echo "GROOMER: groomer.sh cannot run as root." - exit -fi - clean(){ - if [ ${DEBUG} = true ]; then + if [ "${DEBUG}" = true ]; then sleep 20 fi @@ -21,58 +9,84 @@ clean(){ ${SYNC} # Remove temporary files from destination key - rm -rf ${TEMP} + rm -rf "${TEMP}" } -trap clean EXIT TERM INT +check_not_root() { + if ! [ "${ID}" -ge "1000" ]; then + echo "GROOMER: groomer.sh cannot run as root." + exit + fi +} -# Find the partition names on the device available at /dev/source_key -DEV_PARTITIONS=`ls "${DEV_SRC}"* | grep "${DEV_SRC}[1-9][0-6]*" || true` -if [ -z "${DEV_PARTITIONS}" ]; then - echo "GROOMER: ${DEV_SRC} does not have any partitions." - exit -fi +check_partitions_not_empty () { + local partitions=$1 + if [ -z "${partitions}" ]; then + echo "GROOMER: ${DEV_SRC} does not have any partitions." + exit + fi +} -PARTCOUNT=1 -for partition in ${DEV_PARTITIONS} -do - echo "GROOMER: Processing partition ${partition}" +unmount_source_partition() { # Unmount anything that is mounted on /media/src - if [ `${MOUNT} | grep -c ${SRC}` -ne 0 ]; then - ${PUMOUNT} ${SRC} + if [ "$(${MOUNT} | grep -c "${SRC}")" -ne 0 ]; then + ${PUMOUNT} "${SRC}" fi +} - # Mount the current partition in write mode - ${PMOUNT} -w ${partition} ${SRC} - # Mark any autorun.inf files as dangerous on the source device - ls ${SRC_MNT} | grep -i autorun.inf | xargs -I {} mv "${SRC_MNT}"/{} "{SRC_MNT}"/DANGEROUS_{}_DANGEROUS || true - # Unmount and remount the current partition in read-only mode - ${PUMOUNT} ${SRC} - ${PMOUNT} -r ${partition} ${SRC} - if [ ${?} -ne 0 ]; then - # Previous command (mounting current partition) failed - echo "GROOMER: Unable to mount ${partition} on /media/${SRC}" - else - echo "GROOMER: ${partition} mounted at /media/${SRC}" +run_groomer() { + local dev_partitions + # Find the partition names on the device + let dev_partitions=$(ls "${DEV_SRC}"* | grep "${DEV_SRC}[1-9][0-6]*" || true) + check_has_partitions dev_partitions + local partcount=1 + local partition + for partition in ${dev_partitions} + do + echo "GROOMER: Processing partition ${partition}" + unmount_source_partition + # Mount the current partition in write mode + ${PMOUNT} -w ${partition} "${SRC}" + # Mark any autorun.inf files as dangerous on the source device to be extra careful + ls "${SRC_MNT}" | grep -i autorun.inf | xargs -I {} mv "${SRC_MNT}"/{} "{SRC_MNT}"/DANGEROUS_{}_DANGEROUS || true + # Unmount and remount the current partition in read-only mode + ${PUMOUNT} "${SRC}" - # Put the filenames from the current partition in a logfile - find "/media/${SRC}" -fls "${LOGS_DIR}/contents_partition_${PARTCOUNT}.txt" + if ${PMOUNT} -r "${partition}" "${SRC}"; then + echo "GROOMER: ${partition} mounted at /media/${SRC}" - # Create a directory on ${DST} named PARTION_$PARTCOUNT - target_dir="/media/${DST}/FROM_PARTITION_${PARTCOUNT}" - mkdir -p "${target_dir}" - LOGFILE="${LOGS_DIR}/processing_log.txt" + # Put the filenames from the current partition in a logfile + find "/media/${SRC}" -fls "${LOGS_DIR}/contents_partition_${partcount}.txt" - # Run the current partition through filecheck.py - echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE} - filecheck.py --source /media/${SRC} --destination ${target_dir} || true - echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE} + # Create a directory on ${DST} named PARTION_$PARTCOUNT + local target_dir="/media/${DST}/FROM_PARTITION_${partcount}" + mkdir -p "${target_dir}" + local logfile="${LOGS_DIR}/processing_log.txt" - # List destination files (recursively) for debugging - ls -lR "${target_dir}" + # Run the current partition through filecheck.py + echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> "${logfile}" + filecheck.py --source /media/"${SRC}" --destination "${target_dir}" || true + echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> "${logfile}" + + # List destination files (recursively) for debugging + ls -lR "${target_dir}" + else + # Previous command (mounting current partition) failed + echo "GROOMER: Unable to mount ${partition} on /media/${SRC}" + fi + let partcount=$((partcount + 1)) + done +} + + +main() { + set -eu # exit when a line returns non-0 status, treat unset variables as errors + trap clean EXIT TERM INT # run clean when the script ends or is interrupted + source ./config.sh # get config values + if [ "${DEBUG}" = true ]; then + set -x fi - let PARTCOUNT=`expr $PARTCOUNT + 1` -done + run_groomer +} -# The cleanup is automatically done in the function clean called when -# the program exits +main diff --git a/circlean_fs/root_partition/opt/groomer/init.sh b/circlean_fs/root_partition/opt/groomer/init.sh index f8ba08a..37ec00a 100755 --- a/circlean_fs/root_partition/opt/groomer/init.sh +++ b/circlean_fs/root_partition/opt/groomer/init.sh @@ -36,10 +36,10 @@ main() { set -eu # exit when a line returns non-0 status, treat unset variables as errors trap clean EXIT TERM INT # run clean when the script ends or is interrupted source ./config.sh # get config values - check_is_root if [ "${DEBUG}" = true ]; then set -x fi + check_is_root if [ "${MUSIC}" = true ]; then start_music fi diff --git a/circlean_fs/root_partition/opt/groomer/mount_dest.sh b/circlean_fs/root_partition/opt/groomer/mount_dest.sh index 5703462..c2e558a 100755 --- a/circlean_fs/root_partition/opt/groomer/mount_dest.sh +++ b/circlean_fs/root_partition/opt/groomer/mount_dest.sh @@ -61,9 +61,11 @@ prepare_dest_partition() { main() { set -eu # exit when a line returns non-0 status, treat unset variables as errors - trap clean EXIT TERM INT - set -x - source ./config.sh + trap clean EXIT TERM INT # run clean when the script ends or is interrupted + source ./config.sh # get config values + if [ "${DEBUG}" = true ]; then + set -x + fi check_not_root check_source_exists check_dest_exists From c6e5caa2bdb0e2371b1065263b46104ab1fa194f Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Thu, 10 Aug 2017 17:28:28 -0400 Subject: [PATCH 12/19] Add IN_PROGRESS canary file to groomer scripts --- circlean_fs/root_partition/opt/groomer/IN_PROGRESS | 1 + circlean_fs/root_partition/opt/groomer/groomer.sh | 11 +++++------ circlean_fs/root_partition/opt/groomer/mount_dest.sh | 10 ++++++++-- 3 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 circlean_fs/root_partition/opt/groomer/IN_PROGRESS diff --git a/circlean_fs/root_partition/opt/groomer/IN_PROGRESS b/circlean_fs/root_partition/opt/groomer/IN_PROGRESS new file mode 100644 index 0000000..ca46e57 --- /dev/null +++ b/circlean_fs/root_partition/opt/groomer/IN_PROGRESS @@ -0,0 +1 @@ +NOTE: This file is copied to the destination key when CIRCLean starts, and deleted once it completes successfully. If you see this file on your destination key it means that the copying process was INTERRUPTED or an ERROR occurred. You should treat files on the source key and destination key with care, and consider repeating the sanitization process. If you think you have found a bug, please report it at https://github.com/CIRCL/Circlean. diff --git a/circlean_fs/root_partition/opt/groomer/groomer.sh b/circlean_fs/root_partition/opt/groomer/groomer.sh index ac24412..3014157 100755 --- a/circlean_fs/root_partition/opt/groomer/groomer.sh +++ b/circlean_fs/root_partition/opt/groomer/groomer.sh @@ -56,20 +56,20 @@ run_groomer() { echo "GROOMER: ${partition} mounted at /media/${SRC}" # Put the filenames from the current partition in a logfile - find "/media/${SRC}" -fls "${LOGS_DIR}/contents_partition_${partcount}.txt" + # find "/media/${SRC}" -fls "${LOGS_DIR}/contents_partition_${partcount}.txt" # Create a directory on ${DST} named PARTION_$PARTCOUNT local target_dir="/media/${DST}/FROM_PARTITION_${partcount}" mkdir -p "${target_dir}" - local logfile="${LOGS_DIR}/processing_log.txt" + # local logfile="${LOGS_DIR}/processing_log.txt" # Run the current partition through filecheck.py - echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> "${logfile}" + # echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> "${logfile}" filecheck.py --source /media/"${SRC}" --destination "${target_dir}" || true - echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> "${logfile}" + # echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> "${logfile}" # List destination files (recursively) for debugging - ls -lR "${target_dir}" + # ls -lR "${target_dir}" else # Previous command (mounting current partition) failed echo "GROOMER: Unable to mount ${partition} on /media/${SRC}" @@ -78,7 +78,6 @@ run_groomer() { done } - main() { set -eu # exit when a line returns non-0 status, treat unset variables as errors trap clean EXIT TERM INT # run clean when the script ends or is interrupted diff --git a/circlean_fs/root_partition/opt/groomer/mount_dest.sh b/circlean_fs/root_partition/opt/groomer/mount_dest.sh index c2e558a..08f8bf6 100755 --- a/circlean_fs/root_partition/opt/groomer/mount_dest.sh +++ b/circlean_fs/root_partition/opt/groomer/mount_dest.sh @@ -7,6 +7,7 @@ clean(){ cp "${DEBUG_LOG}" "${DST_MNT}/groomer_debug_log.txt" fi echo "GROOMER: Cleaning up in mount_keys.sh." + rm -rf "/media/${DST}/IN_PROGRESS"* ${SYNC} # Write anything in memory to disk # Unmount source and destination pumount "${SRC}" @@ -50,7 +51,11 @@ mount_dest_partition() { fi } -prepare_dest_partition() { +copy_in_progress_file() { + cp "/opt/groomer/IN_PROGRESS" "/media/${DST}/IN_PROGRESS" +} + +prepare_dest_key() { rm -rf "/media/${DST}/FROM_PARTITION_"* # Remove any existing "FROM_PARTITION_" directories # Prepare temp dirs and make sure they're empty if they already exist mkdir -p "${TEMP}" @@ -70,7 +75,8 @@ main() { check_source_exists check_dest_exists unmount_dest_if_mounted - mount_dest_partition + mount_dest_key + copy_in_progress_file prepare_dest_partition ./groomer.sh } From a8a57e1ef0fb95703389b89e9822ab75ba1b6e24 Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Thu, 10 Aug 2017 17:44:11 -0400 Subject: [PATCH 13/19] Small bugfixes in groomer scripts --- circlean_fs/root_partition/opt/groomer/groomer.sh | 4 +++- circlean_fs/root_partition/opt/groomer/mount_dest.sh | 2 +- circlean_fs/root_partition/opt/groomer/music.sh | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/circlean_fs/root_partition/opt/groomer/groomer.sh b/circlean_fs/root_partition/opt/groomer/groomer.sh index 3014157..6fb817d 100755 --- a/circlean_fs/root_partition/opt/groomer/groomer.sh +++ b/circlean_fs/root_partition/opt/groomer/groomer.sh @@ -69,7 +69,9 @@ run_groomer() { # echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> "${logfile}" # List destination files (recursively) for debugging - # ls -lR "${target_dir}" + if [ "${DEBUG}" = true ]; then + ls -lR "${target_dir}" + fi else # Previous command (mounting current partition) failed echo "GROOMER: Unable to mount ${partition} on /media/${SRC}" diff --git a/circlean_fs/root_partition/opt/groomer/mount_dest.sh b/circlean_fs/root_partition/opt/groomer/mount_dest.sh index 08f8bf6..9fba305 100755 --- a/circlean_fs/root_partition/opt/groomer/mount_dest.sh +++ b/circlean_fs/root_partition/opt/groomer/mount_dest.sh @@ -13,7 +13,7 @@ clean(){ pumount "${SRC}" pumount "${DST}" exit - } +} check_not_root() { if ! [ "${ID}" -ge "1000" ]; then diff --git a/circlean_fs/root_partition/opt/groomer/music.sh b/circlean_fs/root_partition/opt/groomer/music.sh index de43f19..fc52eec 100755 --- a/circlean_fs/root_partition/opt/groomer/music.sh +++ b/circlean_fs/root_partition/opt/groomer/music.sh @@ -11,7 +11,7 @@ run_timidity() { while true; do # -id flags set interface to "dumb" and -qq silences most/all terminal output "${TIMIDITY}" -idqq "${files[RANDOM % ${#files[@]}]}" -done + done } main() { From 650ad783d76462ca63bf5c2cb1917633bcac2e05 Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Tue, 22 Aug 2017 16:19:31 -0400 Subject: [PATCH 14/19] Mention shellcheck in CONTRIBUTING.md --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7e7b00..30017f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,7 +49,10 @@ IN ALL CASES, PLEASE READ THE COMMENTS IN THE SCRIPTS AT LEAST ONCE. Running the tests ================= -To emulate the Raspberry Pi hardware for testing, we'll be using +* If you've made changes to the shell scripts, start by installing and running +[Shellcheck](https://github.com/koalaman/shellcheck). + +* To emulate the Raspberry Pi hardware for testing, we'll be using [Qemu](http://wiki.qemu.org/Main_Page), an open source machine emulator. The "qemu" package available for Ubuntu/Debian includes all of the required packages (including qemu-system-arm) except for qemu-user-static, which must From ab9c9eb309461e69a47b4fd855667325ea0230e7 Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Wed, 23 Aug 2017 17:05:24 -0400 Subject: [PATCH 15/19] Fix issue with $SRC+$DST and $SRC_MNT+$DST_MNT --- .../root_partition/opt/groomer/config.sh | 2 -- .../root_partition/opt/groomer/groomer.sh | 24 +++++++++---------- .../root_partition/opt/groomer/mount_dest.sh | 20 ++++++++-------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/circlean_fs/root_partition/opt/groomer/config.sh b/circlean_fs/root_partition/opt/groomer/config.sh index d172c5f..dc862d4 100755 --- a/circlean_fs/root_partition/opt/groomer/config.sh +++ b/circlean_fs/root_partition/opt/groomer/config.sh @@ -6,11 +6,9 @@ readonly USERNAME="kitten" readonly ID=$(/usr/bin/id -u) # Paths used in multiple scripts -readonly SRC="src" readonly DEV_SRC="/dev/source_key" readonly SRC_MNT="/media/src" -readonly DST="dst" readonly DEV_DST="/dev/dest_key" readonly DST_MNT="/media/dst" diff --git a/circlean_fs/root_partition/opt/groomer/groomer.sh b/circlean_fs/root_partition/opt/groomer/groomer.sh index 6fb817d..922660d 100755 --- a/circlean_fs/root_partition/opt/groomer/groomer.sh +++ b/circlean_fs/root_partition/opt/groomer/groomer.sh @@ -29,8 +29,8 @@ check_partitions_not_empty () { unmount_source_partition() { # Unmount anything that is mounted on /media/src - if [ "$(${MOUNT} | grep -c "${SRC}")" -ne 0 ]; then - ${PUMOUNT} "${SRC}" + if [ "$(${MOUNT} | grep -c "${SRC_MNT}")" -ne 0 ]; then + ${PUMOUNT} "${SRC_MNT}" fi } @@ -46,27 +46,27 @@ run_groomer() { echo "GROOMER: Processing partition ${partition}" unmount_source_partition # Mount the current partition in write mode - ${PMOUNT} -w ${partition} "${SRC}" + ${PMOUNT} -w ${partition} "${SRC_MNT}" # Mark any autorun.inf files as dangerous on the source device to be extra careful ls "${SRC_MNT}" | grep -i autorun.inf | xargs -I {} mv "${SRC_MNT}"/{} "{SRC_MNT}"/DANGEROUS_{}_DANGEROUS || true # Unmount and remount the current partition in read-only mode - ${PUMOUNT} "${SRC}" + ${PUMOUNT} "${SRC_MNT}" - if ${PMOUNT} -r "${partition}" "${SRC}"; then - echo "GROOMER: ${partition} mounted at /media/${SRC}" + if ${PMOUNT} -r "${partition}" "${SRC_MNT}"; then + echo "GROOMER: ${partition} mounted at ${SRC_MNT}" # Put the filenames from the current partition in a logfile # find "/media/${SRC}" -fls "${LOGS_DIR}/contents_partition_${partcount}.txt" - # Create a directory on ${DST} named PARTION_$PARTCOUNT - local target_dir="/media/${DST}/FROM_PARTITION_${partcount}" + # Create a directory on ${DST_MNT} named PARTION_$PARTCOUNT + local target_dir="${DST_MNT}/FROM_PARTITION_${partcount}" mkdir -p "${target_dir}" # local logfile="${LOGS_DIR}/processing_log.txt" # Run the current partition through filecheck.py - # echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> "${logfile}" - filecheck.py --source /media/"${SRC}" --destination "${target_dir}" || true - # echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> "${logfile}" + # echo "==== Starting processing of ${SRC_MNT} to ${target_dir}. ====" >> "${logfile}" + filecheck.py --source "${SRC_MNT}" --destination "${target_dir}" || true + # echo "==== Done with ${SRC_MNT} to ${target_dir}. ====" >> "${logfile}" # List destination files (recursively) for debugging if [ "${DEBUG}" = true ]; then @@ -74,7 +74,7 @@ run_groomer() { fi else # Previous command (mounting current partition) failed - echo "GROOMER: Unable to mount ${partition} on /media/${SRC}" + echo "GROOMER: Unable to mount ${partition} on ${SRC_MNT}" fi let partcount=$((partcount + 1)) done diff --git a/circlean_fs/root_partition/opt/groomer/mount_dest.sh b/circlean_fs/root_partition/opt/groomer/mount_dest.sh index 9fba305..3daa2f1 100755 --- a/circlean_fs/root_partition/opt/groomer/mount_dest.sh +++ b/circlean_fs/root_partition/opt/groomer/mount_dest.sh @@ -7,11 +7,11 @@ clean(){ cp "${DEBUG_LOG}" "${DST_MNT}/groomer_debug_log.txt" fi echo "GROOMER: Cleaning up in mount_keys.sh." - rm -rf "/media/${DST}/IN_PROGRESS"* + rm -rf "${DST_MNT}/IN_PROGRESS"* ${SYNC} # Write anything in memory to disk # Unmount source and destination - pumount "${SRC}" - pumount "${DST}" + pumount "${SRC_MNT}" + pumount "${DST_MNT}" exit } @@ -37,13 +37,13 @@ check_dest_exists() { } unmount_dest_if_mounted() { - if ${MOUNT}|grep "${DST}"; then - ${PUMOUNT} "${DST}" || true + if ${MOUNT}|grep "${DST_MNT}"; then + ${PUMOUNT} "${DST_MNT}" || true fi } mount_dest_partition() { - if "${PMOUNT}" -w "${DEV_DST}1" "${DST}"; then # pmount automatically mounts on /media/ (at /media/dst in this case). + if "${PMOUNT}" -w "${DEV_DST}1" "${DST_MNT}"; then # pmount automatically mounts on /media/ (at /media/dst in this case). echo "GROOMER: Destination USB device (${DEV_DST}1) mounted at ${DST_MNT}" else echo "GROOMER: Unable to mount ${DEV_DST}1 on ${DST_MNT}" @@ -52,11 +52,11 @@ mount_dest_partition() { } copy_in_progress_file() { - cp "/opt/groomer/IN_PROGRESS" "/media/${DST}/IN_PROGRESS" + cp "/opt/groomer/IN_PROGRESS" "${DST_MNT}/IN_PROGRESS" } -prepare_dest_key() { - rm -rf "/media/${DST}/FROM_PARTITION_"* # Remove any existing "FROM_PARTITION_" directories +prepare_dest_partition() { + rm -rf "${DST_MNT}/FROM_PARTITION_"* # Remove any existing "FROM_PARTITION_" directories # Prepare temp dirs and make sure they're empty if they already exist mkdir -p "${TEMP}" mkdir -p "${LOGS}" @@ -75,7 +75,7 @@ main() { check_source_exists check_dest_exists unmount_dest_if_mounted - mount_dest_key + mount_dest_partition copy_in_progress_file prepare_dest_partition ./groomer.sh From 410f5fce0e37d8e5f145e1626c518c2c3cf1119f Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Sun, 1 Oct 2017 21:35:38 -0400 Subject: [PATCH 16/19] Add .txt extension to IN_PROGRESS file --- .../root_partition/opt/groomer/{IN_PROGRESS => IN_PROGRESS.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename circlean_fs/root_partition/opt/groomer/{IN_PROGRESS => IN_PROGRESS.txt} (100%) diff --git a/circlean_fs/root_partition/opt/groomer/IN_PROGRESS b/circlean_fs/root_partition/opt/groomer/IN_PROGRESS.txt similarity index 100% rename from circlean_fs/root_partition/opt/groomer/IN_PROGRESS rename to circlean_fs/root_partition/opt/groomer/IN_PROGRESS.txt From d9438d03c0ccc5da2ab1bf853fb8ca8b59246fb8 Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Sun, 1 Oct 2017 21:37:10 -0400 Subject: [PATCH 17/19] Rename DEV_SRC and DEV_DST to SRC_DEV, DST_DEV * New order is more consistent with rest of script * This patch also has a few other small bugfixes --- .../root_partition/opt/groomer/config.sh | 4 ++-- .../root_partition/opt/groomer/groomer.sh | 14 ++++------- .../root_partition/opt/groomer/mount_dest.sh | 24 +++++++++---------- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/circlean_fs/root_partition/opt/groomer/config.sh b/circlean_fs/root_partition/opt/groomer/config.sh index dc862d4..64ba840 100755 --- a/circlean_fs/root_partition/opt/groomer/config.sh +++ b/circlean_fs/root_partition/opt/groomer/config.sh @@ -6,10 +6,10 @@ readonly USERNAME="kitten" readonly ID=$(/usr/bin/id -u) # Paths used in multiple scripts -readonly DEV_SRC="/dev/source_key" +readonly SRC_DEV="/dev/source_key" readonly SRC_MNT="/media/src" -readonly DEV_DST="/dev/dest_key" +readonly DST_DEV="/dev/dest_key" readonly DST_MNT="/media/dst" readonly TEMP="${DST_MNT}/temp" diff --git a/circlean_fs/root_partition/opt/groomer/groomer.sh b/circlean_fs/root_partition/opt/groomer/groomer.sh index 922660d..c7919d2 100755 --- a/circlean_fs/root_partition/opt/groomer/groomer.sh +++ b/circlean_fs/root_partition/opt/groomer/groomer.sh @@ -19,10 +19,10 @@ check_not_root() { fi } -check_partitions_not_empty () { +check_has_partitions () { local partitions=$1 if [ -z "${partitions}" ]; then - echo "GROOMER: ${DEV_SRC} does not have any partitions." + echo "GROOMER: ${SRC_DEV} does not have any partitions." exit fi } @@ -37,7 +37,7 @@ unmount_source_partition() { run_groomer() { local dev_partitions # Find the partition names on the device - let dev_partitions=$(ls "${DEV_SRC}"* | grep "${DEV_SRC}[1-9][0-6]*" || true) + dev_partitions=$(ls "${SRC_DEV}"* | grep "${SRC_DEV}[1-9][0-6]*" || true) check_has_partitions dev_partitions local partcount=1 local partition @@ -48,25 +48,19 @@ run_groomer() { # Mount the current partition in write mode ${PMOUNT} -w ${partition} "${SRC_MNT}" # Mark any autorun.inf files as dangerous on the source device to be extra careful - ls "${SRC_MNT}" | grep -i autorun.inf | xargs -I {} mv "${SRC_MNT}"/{} "{SRC_MNT}"/DANGEROUS_{}_DANGEROUS || true + ls "${SRC_MNT}" | grep -i autorun.inf | xargs -I {} mv "${SRC_MNT}"/{} "${SRC_MNT}"/DANGEROUS_{}_DANGEROUS || true # Unmount and remount the current partition in read-only mode ${PUMOUNT} "${SRC_MNT}" if ${PMOUNT} -r "${partition}" "${SRC_MNT}"; then echo "GROOMER: ${partition} mounted at ${SRC_MNT}" - # Put the filenames from the current partition in a logfile - # find "/media/${SRC}" -fls "${LOGS_DIR}/contents_partition_${partcount}.txt" - # Create a directory on ${DST_MNT} named PARTION_$PARTCOUNT local target_dir="${DST_MNT}/FROM_PARTITION_${partcount}" mkdir -p "${target_dir}" - # local logfile="${LOGS_DIR}/processing_log.txt" # Run the current partition through filecheck.py - # echo "==== Starting processing of ${SRC_MNT} to ${target_dir}. ====" >> "${logfile}" filecheck.py --source "${SRC_MNT}" --destination "${target_dir}" || true - # echo "==== Done with ${SRC_MNT} to ${target_dir}. ====" >> "${logfile}" # List destination files (recursively) for debugging if [ "${DEBUG}" = true ]; then diff --git a/circlean_fs/root_partition/opt/groomer/mount_dest.sh b/circlean_fs/root_partition/opt/groomer/mount_dest.sh index 3daa2f1..35ae0ca 100755 --- a/circlean_fs/root_partition/opt/groomer/mount_dest.sh +++ b/circlean_fs/root_partition/opt/groomer/mount_dest.sh @@ -7,7 +7,7 @@ clean(){ cp "${DEBUG_LOG}" "${DST_MNT}/groomer_debug_log.txt" fi echo "GROOMER: Cleaning up in mount_keys.sh." - rm -rf "${DST_MNT}/IN_PROGRESS"* + rm -rf "${DST_MNT}/IN_PROGRESS.txt"* ${SYNC} # Write anything in memory to disk # Unmount source and destination pumount "${SRC_MNT}" @@ -23,15 +23,15 @@ check_not_root() { } check_source_exists() { - if [ ! -b "${DEV_SRC}" ]; then - echo "GROOMER: Source device (${DEV_SRC}) does not exist." + if [ ! -b "${SRC_DEV}" ]; then + echo "GROOMER: Source device (${SRC_DEV}) does not exist." exit fi } check_dest_exists() { - if [ ! -b "${DEV_DST}" ]; then - echo "GROOMER: Destination device (${DEV_DST}) does not exist." + if [ ! -b "${DST_DEV}" ]; then + echo "GROOMER: Destination device (${DST_DEV}) does not exist." exit fi } @@ -43,25 +43,23 @@ unmount_dest_if_mounted() { } mount_dest_partition() { - if "${PMOUNT}" -w "${DEV_DST}1" "${DST_MNT}"; then # pmount automatically mounts on /media/ (at /media/dst in this case). - echo "GROOMER: Destination USB device (${DEV_DST}1) mounted at ${DST_MNT}" + if ${PMOUNT} -w "${DST_DEV}1" "${DST_MNT}"; then # pmount automatically mounts on /media/ (at /media/dst in this case). + echo "GROOMER: Destination USB device (${DST_DEV}1) mounted at ${DST_MNT}" else - echo "GROOMER: Unable to mount ${DEV_DST}1 on ${DST_MNT}" + echo "GROOMER: Unable to mount ${DST_DEV}1 on ${DST_MNT}" exit fi } copy_in_progress_file() { - cp "/opt/groomer/IN_PROGRESS" "${DST_MNT}/IN_PROGRESS" + cp "/opt/groomer/IN_PROGRESS.txt" "${DST_MNT}/IN_PROGRESS.txt" } prepare_dest_partition() { rm -rf "${DST_MNT}/FROM_PARTITION_"* # Remove any existing "FROM_PARTITION_" directories - # Prepare temp dirs and make sure they're empty if they already exist + # Prepare temp dir and make sure it's empty if it already exists: mkdir -p "${TEMP}" - mkdir -p "${LOGS}" rm -rf "${TEMP:?}/"* - rm -rf "${LOGS:?}/"* } main() { @@ -81,4 +79,4 @@ main() { ./groomer.sh } -main \ No newline at end of file +main From 9429ede64cade11a2b1bec56fc5e532178debeec Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Sun, 1 Oct 2017 21:43:57 -0400 Subject: [PATCH 18/19] Update install documentation --- doc/image_setup_checklist.md | 2 +- doc/setup_with_proot.md | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/image_setup_checklist.md b/doc/image_setup_checklist.md index b9ffc8b..0343f89 100644 --- a/doc/image_setup_checklist.md +++ b/doc/image_setup_checklist.md @@ -31,7 +31,7 @@ - exifread - pillow - olefile - - git+https://github.com/decalage2/oletools.git + - oletools - git+https://github.com/grierforensics/officedissector.git - git+https://github.com/CIRCL/PyCIRCLean.git * Add a user named "kitten" diff --git a/doc/setup_with_proot.md b/doc/setup_with_proot.md index ac19707..7c2ad2e 100644 --- a/doc/setup_with_proot.md +++ b/doc/setup_with_proot.md @@ -144,7 +144,9 @@ raspbian-sys-mods related installs may fail - you can ignore them: apt-get dist-upgrade apt-get autoremove ``` -* Install the linux dependencies (see CONTRIBUTING.md for more details): +* Install the linux dependencies (see CONTRIBUTING.md for more details). If you see warnings that +from qemu about "Unsupported syscall: 384", you can ignore them. `getrandom(2)` was implemented in +kernel 3.17 and apt will use /dev/urandom when it fails: ``` apt-get install timidity git p7zip-full python3 python3-pip python3-lxml pmount ntfs-3g libjpeg-dev libtiff-dev libwebp-dev tk-dev python3-tk liblcms2-dev tcl-dev ``` @@ -163,8 +165,7 @@ have to edit your PATH variable or use pip3 to get the correct pip. You also mig verify that these dependencies are current by checking in the PyCIRCLean git repo. ``` pip install -U pip - pip install oletools exifread Pillow - pip install git+https://github.com/decalage2/oletools.git + pip install olefile oletools exifread Pillow pip install git+https://github.com/Rafiot/officedissector.git pip install git+https://github.com/CIRCL/PyCIRCLean.git ``` @@ -197,7 +198,7 @@ manpage for more details. Make sure to include the trailing slashes on the paths exit sudo rsync -vri circlean_fs/boot/ /mnt/rpi-boot/ sudo rsync -vri circlean_fs/root_partition/ /mnt/rpi-root/ - cp -rf midi /mnt/rpi-root/opt/ + sudo cp -rf midi /mnt/rpi-root/opt/ ``` * If have an external hardware led and you're using the led functionality, copy the led files from diode_controller/ as well. From 2fc36b6d23b7ba1498055c9a6f31bdf2bff6f09a Mon Sep 17 00:00:00 2001 From: Dan Puttick Date: Sun, 1 Oct 2017 21:53:59 -0400 Subject: [PATCH 19/19] Update changelog --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54bbc3c..a4117db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,9 @@ Version 2.3 - 2017-09-08 - -TODO - -Version 2.2 - 2017-04-18 - +- Updated to the newest version of Raspbian Stretch lite (2017-08-16 release) +- Using the newest version of PyCIRCLean, including several vulnerability and bug fixes +- Refactored CIRCLean bash scripts according to [Defensive Bash Programming](http://www.kfirlavi.com/blog/2012/11/14/defensive-bash-programming/) +- Added IN_PROGRESS.txt canary file that gets added and then deleted from destination key +- Various smaller bug fixes Version 2.2 - 2017-04-18 - Updated to newest version of Raspbian Jessie lite (April 10th 2017 release)