Merge pull request #58 from dputtick/dev

Changes for v2.3
pull/61/head 2.3
Raphaël Vinot 2017-10-02 17:28:47 +02:00 committed by GitHub
commit 2e93930df5
14 changed files with 251 additions and 216 deletions

View File

@ -1,14 +1,20 @@
Version 2.3 - 2017-09-08
TODO
- Updated to the newest version of Raspbian Stretch lite (2017-08-16 release)
- Using the newest version of PyCIRCLean, including several vulnerability and bug fixes
- Refactored CIRCLean bash scripts according to [Defensive Bash Programming](http://www.kfirlavi.com/blog/2012/11/14/defensive-bash-programming/)
- Added IN_PROGRESS.txt canary file that gets added and then deleted from destination key
- Various smaller bug fixes
Version 2.2 - 2017-04-18
New features:
- Updated to newest version of Raspbian Jessie lite (April 10th 2017 release)
- Using the newest version of PyCIRCLean, which includes a new log format and
various bug fixes
- Filecheck.py configuration information is now conveniently held in a Config object instead of in globals
- New easier to read text-based logger (removed twiggy dependency)
- Various filetypes in filecheck.py now have improved descriptions for log
- Improved the interface for adding file descriptions to files
Version 2.1.1 - 2017-02-28
- Fix copying PDF documents to the destination key
Version 2.1 - 2017-02-02
- Updated to the newest version of Raspbian Jessie lite (January 11th 2017 release)

View File

@ -49,7 +49,10 @@ IN ALL CASES, PLEASE READ THE COMMENTS IN THE SCRIPTS AT LEAST ONCE.
Running the tests
=================
To emulate the Raspberry Pi hardware for testing, we'll be using
* If you've made changes to the shell scripts, start by installing and running
[Shellcheck](https://github.com/koalaman/shellcheck).
* To emulate the Raspberry Pi hardware for testing, we'll be using
[Qemu](http://wiki.qemu.org/Main_Page), an open source machine emulator.
The "qemu" package available for Ubuntu/Debian includes all of the required
packages (including qemu-system-arm) except for qemu-user-static, which must

View File

@ -0,0 +1 @@
NOTE: This file is copied to the destination key when CIRCLean starts, and deleted once it completes successfully. If you see this file on your destination key it means that the copying process was INTERRUPTED or an ERROR occurred. You should treat files on the source key and destination key with care, and consider repeating the sanitization process. If you think you have found a bug, please report it at https://github.com/CIRCL/Circlean.

View File

@ -1,29 +1,29 @@
USERNAME="kitten"
ID=`/usr/bin/id -u`
#!/bin/bash
set -eu
readonly USERNAME="kitten"
readonly ID=$(/usr/bin/id -u)
# Paths used in multiple scripts
SRC="src"
DEV_SRC="/dev/source_key"
SRC_MNT="/media/src"
readonly SRC_DEV="/dev/source_key"
readonly SRC_MNT="/media/src"
DST="dst"
DEV_DST="/dev/dest_key"
DST_MNT="/media/dst"
TEMP="${DST_MNT}/temp"
ZIPTEMP="${DST_MNT}/ziptemp"
LOGS="${DST_MNT}/logs"
DEBUG_LOG="/tmp/groomer_debug_log.txt"
MUSIC="/opt/midi/"
readonly DST_DEV="/dev/dest_key"
readonly DST_MNT="/media/dst"
readonly TEMP="${DST_MNT}/temp"
readonly LOGS_DIR="${DST_MNT}/logs"
readonly DEBUG_LOG="/tmp/groomer_debug_log.txt"
readonly MUSIC_DIR="/opt/midi/"
# Commands
SYNC="/bin/sync"
TIMIDITY="/usr/bin/timidity"
MOUNT="/bin/mount"
PMOUNT="/usr/bin/pmount -A -s"
PUMOUNT="/usr/bin/pumount"
readonly SYNC="/bin/sync"
readonly TIMIDITY="/usr/bin/timidity"
readonly MOUNT="/bin/mount"
readonly PMOUNT="/usr/bin/pmount -A -s"
readonly PUMOUNT="/usr/bin/pumount"
# Config flags
DEBUG=false
readonly DEBUG=false
readonly MUSIC=true

View File

@ -1,19 +1,7 @@
#!/bin/bash
# set -e (exit when a line returns non-0 status) and -x (xtrace) flags
set -e
set -x
# Import constants from config file
source ./config.sh
if ! [ "${ID}" -ge "1000" ]; then
echo "GROOMER: groomer.sh cannot run as root."
exit
fi
clean(){
if [ ${DEBUG} = true ]; then
if [ "${DEBUG}" = true ]; then
sleep 20
fi
@ -21,59 +9,79 @@ clean(){
${SYNC}
# Remove temporary files from destination key
rm -rf ${TEMP}
rm -rf ${ZIPTEMP}
rm -rf "${TEMP}"
}
trap clean EXIT TERM INT
check_not_root() {
if ! [ "${ID}" -ge "1000" ]; then
echo "GROOMER: groomer.sh cannot run as root."
exit
fi
}
# Find the partition names on the device available at /dev/source_key
DEV_PARTITIONS=`ls "${DEV_SRC}"* | grep "${DEV_SRC}[1-9][0-6]*" || true`
if [ -z "${DEV_PARTITIONS}" ]; then
echo "GROOMER: ${DEV_SRC} does not have any partitions."
exit
fi
check_has_partitions () {
local partitions=$1
if [ -z "${partitions}" ]; then
echo "GROOMER: ${SRC_DEV} does not have any partitions."
exit
fi
}
PARTCOUNT=1
for partition in ${DEV_PARTITIONS}
do
echo "GROOMER: Processing partition ${partition}"
unmount_source_partition() {
# Unmount anything that is mounted on /media/src
if [ `${MOUNT} | grep -c ${SRC}` -ne 0 ]; then
${PUMOUNT} ${SRC}
if [ "$(${MOUNT} | grep -c "${SRC_MNT}")" -ne 0 ]; then
${PUMOUNT} "${SRC_MNT}"
fi
}
# Mount the current partition in write mode
${PMOUNT} -w ${partition} ${SRC}
# Mark any autorun.inf files as dangerous on the source device
ls ${SRC_MNT} | grep -i autorun.inf | xargs -I {} mv "${SRC_MNT}"/{} "{SRC_MNT}"/DANGEROUS_{}_DANGEROUS || true
# Unmount and remount the current partition in read-only mode
${PUMOUNT} ${SRC}
${PMOUNT} -r ${partition} ${SRC}
if [ ${?} -ne 0 ]; then
# Previous command (mounting current partition) failed
echo "GROOMER: Unable to mount ${partition} on /media/${SRC}"
else
echo "GROOMER: ${partition} mounted at /media/${SRC}"
run_groomer() {
local dev_partitions
# Find the partition names on the device
dev_partitions=$(ls "${SRC_DEV}"* | grep "${SRC_DEV}[1-9][0-6]*" || true)
check_has_partitions dev_partitions
local partcount=1
local partition
for partition in ${dev_partitions}
do
echo "GROOMER: Processing partition ${partition}"
unmount_source_partition
# Mount the current partition in write mode
${PMOUNT} -w ${partition} "${SRC_MNT}"
# Mark any autorun.inf files as dangerous on the source device to be extra careful
ls "${SRC_MNT}" | grep -i autorun.inf | xargs -I {} mv "${SRC_MNT}"/{} "${SRC_MNT}"/DANGEROUS_{}_DANGEROUS || true
# Unmount and remount the current partition in read-only mode
${PUMOUNT} "${SRC_MNT}"
# Put the filenames from the current partition in a logfile
find "/media/${SRC}" -fls "${LOGS}/contents_partition_${PARTCOUNT}.txt"
if ${PMOUNT} -r "${partition}" "${SRC_MNT}"; then
echo "GROOMER: ${partition} mounted at ${SRC_MNT}"
# Create a directory on ${DST} named PARTION_$PARTCOUNT
target_dir="/media/${DST}/FROM_PARTITION_${PARTCOUNT}"
mkdir -p "${target_dir}"
LOGFILE="${LOGS}/processing_log.txt"
# Create a directory on ${DST_MNT} named PARTION_$PARTCOUNT
local target_dir="${DST_MNT}/FROM_PARTITION_${partcount}"
mkdir -p "${target_dir}"
# Run the current partition through filecheck.py
echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE}
filecheck.py --source /media/${SRC} --destination ${target_dir} || true
echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE}
# Run the current partition through filecheck.py
filecheck.py --source "${SRC_MNT}" --destination "${target_dir}" || true
# List destination files (recursively) for debugging
ls -lR "${target_dir}"
# List destination files (recursively) for debugging
if [ "${DEBUG}" = true ]; then
ls -lR "${target_dir}"
fi
else
# Previous command (mounting current partition) failed
echo "GROOMER: Unable to mount ${partition} on ${SRC_MNT}"
fi
let partcount=$((partcount + 1))
done
}
main() {
set -eu # exit when a line returns non-0 status, treat unset variables as errors
trap clean EXIT TERM INT # run clean when the script ends or is interrupted
source ./config.sh # get config values
if [ "${DEBUG}" = true ]; then
set -x
fi
let PARTCOUNT=`expr $PARTCOUNT + 1`
done
run_groomer
}
# The cleanup is automatically done in the function clean called when
# the program exits
main

View File

@ -1,37 +1,49 @@
#!/bin/bash
# set -e (exit when a line returns non-0 status) and -x (xtrace) flags
set -e
set -x
# Import constants from config file
source ./config.sh
if [ ${ID} -ne 0 ]; then
echo "GROOMER: This script has to be run as root."
exit
fi
clean(){
if [ ${DEBUG} = true ]; then
if [ "${DEBUG}" = true ]; then
sleep 20
fi
echo "GROOMER: cleaning up after init.sh."
${SYNC}
"${SYNC}"
# Stop the music from playing
kill -9 $(cat /tmp/music.pid)
kill -9 "$(cat /tmp/music.pid)"
rm -f /tmp/music.pid
}
trap clean EXIT TERM INT
check_is_root() {
if [ "${ID}" -ne 0 ]; then # -ne is an integer comparison instead of a string comparison
echo "GROOMER: This script has to be run as root."
exit
fi
}
# Start music
./music.sh &
echo $! > /tmp/music.pid
start_music() {
./music.sh &
echo $! > /tmp/music.pid
}
# List block storage devices for debugging
if [ ${DEBUG} = true ]; then
lsblk |& tee -a ${DEBUG_LOG}
fi
run_groomer() {
if [ "${DEBUG}" = true ]; then
lsblk |& tee -a "${DEBUG_LOG}" # list block storage devices for debugging
su "${USERNAME}" -c ./mount_dest.sh |& tee -a "${DEBUG_LOG}"
else
su "${USERNAME}" -c ./mount_dest.sh
fi
}
su ${USERNAME} -c ./mount_dest.sh |& tee -a ${DEBUG_LOG}
main() {
set -eu # exit when a line returns non-0 status, treat unset variables as errors
trap clean EXIT TERM INT # run clean when the script ends or is interrupted
source ./config.sh # get config values
if [ "${DEBUG}" = true ]; then
set -x
fi
check_is_root
if [ "${MUSIC}" = true ]; then
start_music
fi
run_groomer
}
main

View File

@ -1,80 +1,82 @@
#!/bin/bash
# set -e (exit when a line returns non-0 status) and -x (xtrace) flags
set -e
set -x
# Import constants from config file
source ./config.sh
if ! [ "${ID}" -ge "1000" ]; then
echo "GROOMER: mount_keys.sh cannot run as root."
exit
fi
clean(){
if [ ${DEBUG} = true ]; then
sleep 20
# Copy the temporary logfile to the destination key
cp ${DEBUG_LOG} "${DST_MNT}/groomer_debug_log.txt"
fi
echo "GROOMER: Cleaning up in mount_keys.sh."
# Write anything in memory to disk
${SYNC}
# Unmount source and destination
pumount ${SRC}
# Clean up and unmount destination
pumount ${DST}
exit
if [ "${DEBUG}" = true ]; then
sleep 20
# Copy the temporary logfile to the destination key
cp "${DEBUG_LOG}" "${DST_MNT}/groomer_debug_log.txt"
fi
echo "GROOMER: Cleaning up in mount_keys.sh."
rm -rf "${DST_MNT}/IN_PROGRESS.txt"*
${SYNC} # Write anything in memory to disk
# Unmount source and destination
pumount "${SRC_MNT}"
pumount "${DST_MNT}"
exit
}
trap clean EXIT TERM INT
check_not_root() {
if ! [ "${ID}" -ge "1000" ]; then
echo "GROOMER: mount_keys.sh cannot run as root."
exit
fi
}
# Check that a device is available on /dev/source_key (symlinked to /dev/sda or sdb)
if [ ! -b ${DEV_SRC} ]; then
echo "GROOMER: Source device (${DEV_SRC}) does not exist."
exit
fi
check_source_exists() {
if [ ! -b "${SRC_DEV}" ]; then
echo "GROOMER: Source device (${SRC_DEV}) does not exist."
exit
fi
}
# Check that a device is available on /dev/dest_key (symlinked to /dev/sda or sdb)
if [ ! -b ${DEV_DST} ]; then
echo "GROOMER: Destination device (${DEV_DST}) does not exist."
exit
fi
check_dest_exists() {
if [ ! -b "${DST_DEV}" ]; then
echo "GROOMER: Destination device (${DST_DEV}) does not exist."
exit
fi
}
# If there is already a device mounted on /media/dst, unmount it
if ${MOUNT}|grep ${DST}; then
${PUMOUNT} ${DST} || true
fi
unmount_dest_if_mounted() {
if ${MOUNT}|grep "${DST_MNT}"; then
${PUMOUNT} "${DST_MNT}" || true
fi
}
# uid= only works on a vfat FS. What should wedo if we get an ext* FS ?
# What does this ^ comment mean?
mount_dest_partition() {
if ${PMOUNT} -w "${DST_DEV}1" "${DST_MNT}"; then # pmount automatically mounts on /media/ (at /media/dst in this case).
echo "GROOMER: Destination USB device (${DST_DEV}1) mounted at ${DST_MNT}"
else
echo "GROOMER: Unable to mount ${DST_DEV}1 on ${DST_MNT}"
exit
fi
}
# Mount the first partition of DST (/dev/dest_key1)
# pmount automatically mounts on /media/ (at /media/dst in this case).
${PMOUNT} -w "${DEV_DST}1" ${DST}
if [ ${?} -ne 0 ]; then
echo "GROOMER: Unable to mount ${DEV_DST}1 on ${DST_MNT}"
exit
else
echo "GROOMER: Destination USB device (${DEV_DST}1) mounted at ${DST_MNT}"
copy_in_progress_file() {
cp "/opt/groomer/IN_PROGRESS.txt" "${DST_MNT}/IN_PROGRESS.txt"
}
# Remove any existing "FROM_PARTITION_" directories
rm -rf "/media/${DST}/FROM_PARTITION_"*
# Prepare temp dirs and make sure they're empty if they already exist
prepare_dest_partition() {
rm -rf "${DST_MNT}/FROM_PARTITION_"* # Remove any existing "FROM_PARTITION_" directories
# Prepare temp dir and make sure it's empty if it already exists:
mkdir -p "${TEMP}"
mkdir -p "${ZIPTEMP}"
mkdir -p "${LOGS}"
rm -rf "${TEMP}/"*
rm -rf "${ZIPTEMP}/"*
rm -rf "${LOGS}/"*
fi
rm -rf "${TEMP:?}/"*
}
# Now that destination is mounted and prepared, run the groomer
./groomer.sh
main() {
set -eu # exit when a line returns non-0 status, treat unset variables as errors
trap clean EXIT TERM INT # run clean when the script ends or is interrupted
source ./config.sh # get config values
if [ "${DEBUG}" = true ]; then
set -x
fi
check_not_root
check_source_exists
check_dest_exists
unmount_dest_if_mounted
mount_dest_partition
copy_in_progress_file
prepare_dest_partition
./groomer.sh
}
main

View File

@ -1,22 +1,24 @@
#!/bin/bash
set -e
#set -x
source ./config.sh
killed(){
echo 'Music stopped.'
}
trap killed EXIT TERM INT
run_timidity() {
# Force output on analog
amixer cset numid=3 1
files=(${MUSIC_DIR}*)
while true; do
# -id flags set interface to "dumb" and -qq silences most/all terminal output
"${TIMIDITY}" -idqq "${files[RANDOM % ${#files[@]}]}"
done
}
# Force output on analog
amixer cset numid=3 1
main() {
set -eu # exit when a line returns non-0 status, treat unset variables as errors
trap killed EXIT TERM INT # run clean when the script ends or is interrupted
source ./config.sh # get config values
run_timidity
}
files=(${MUSIC}*)
while true; do
# -id flags set interface to "dumb" and -qq silences most/all terminal output
$TIMIDITY -idqq ${files[RANDOM % ${#files[@]}]}
done
main

View File

@ -19,7 +19,7 @@
- pmount ntfs-3g
- python3 python3-pip
- python3-lxml
- libjpeg-dev libtiff-dev libwebp-dev liblcms2-dev tcl-dev tk-dev python-tk
- libjpeg-dev libtiff-dev libwebp-dev liblcms2-dev tcl-dev tk-dev python3-tk
* Compile p7zip-rar from source
- Change your source.list file
- Make a new directory and cd to it
@ -31,7 +31,7 @@
- exifread
- pillow
- olefile
- git+https://github.com/decalage2/oletools.git
- oletools
- git+https://github.com/grierforensics/officedissector.git
- git+https://github.com/CIRCL/PyCIRCLean.git
* Add a user named "kitten"

View File

@ -120,9 +120,9 @@ to fill the new larger partition using resize2fs:
Installing the dependencies
===========================
* Copy circlean_fs/root_partition/systemd/system/rc-local.service into the equivalent location in the image.
* Copy circlean_fs/root_partition/etc/systemd/system/rc-local.service into the equivalent location in the image.
```
cp circlean_fs/root_partition/systemd/system/rc-local.service /mnt/rpi-root/etc/systemd/system/rc-local.service
sudo cp circlean_fs/root_partition/etc/systemd/system/rc-local.service /mnt/rpi-root/etc/systemd/system/rc-local.service
```
* Use [proot](https://proot-me.github.io/) to enter the equivalent of a chroot inside
the mounted image.
@ -144,15 +144,19 @@ raspbian-sys-mods related installs may fail - you can ignore them:
apt-get dist-upgrade
apt-get autoremove
```
* Install the linux dependencies (see CONTRIBUTING.md for more details):
* Install the linux dependencies (see CONTRIBUTING.md for more details). If you see warnings that
from qemu about "Unsupported syscall: 384", you can ignore them. `getrandom(2)` was implemented in
kernel 3.17 and apt will use /dev/urandom when it fails:
```
apt-get install timidity git p7zip-full python3 python3-pip python3-lxml pmount ntfs-3g libjpeg-dev libtiff-dev libwebp-dev tk-dev python-tk liblcms2-dev tcl-dev
apt-get install timidity git p7zip-full python3 python3-pip python3-lxml pmount ntfs-3g libjpeg-dev libtiff-dev libwebp-dev tk-dev python3-tk liblcms2-dev tcl-dev
```
* Compile p7zip-rar from source. First, uncomment out the second line in /etc/apt/sources.list. Then:
```
cd /home/pi
mkdir rar && cd rar/
apt-get update
apt-get build-dep p7zip-rar
apt-get source -b p7zip-rar
dpkg -i ${path to p7zip-rar .deb file}
```
* Install the Python dependencies for PyCIRCLean/filecheck.py. PyCIRCLean is 3.3+
@ -161,8 +165,7 @@ have to edit your PATH variable or use pip3 to get the correct pip. You also mig
verify that these dependencies are current by checking in the PyCIRCLean git repo.
```
pip install -U pip
pip install oletools exifread Pillow
pip install git+https://github.com/decalage2/oletools.git
pip install olefile oletools exifread Pillow
pip install git+https://github.com/Rafiot/officedissector.git
pip install git+https://github.com/CIRCL/PyCIRCLean.git
```
@ -195,7 +198,7 @@ manpage for more details. Make sure to include the trailing slashes on the paths
exit
sudo rsync -vri circlean_fs/boot/ /mnt/rpi-boot/
sudo rsync -vri circlean_fs/root_partition/ /mnt/rpi-root/
cp -rf midi /mnt/rpi-root/opt/
sudo cp -rf midi /mnt/rpi-root/opt/
```
* If have an external hardware led and you're using the led functionality, copy
the led files from diode_controller/ as well.
@ -220,5 +223,5 @@ copying process:
```
* Use fsck to verify the root partition:
```
sudo e2fsck -f /dev/sd<number>2
sudo e2fsck -f /dev/sd<letter>2
```

View File

@ -17,11 +17,11 @@ set -x
# Double check the path and offsets as noted above!
# Path to the image
IMAGE='2017-02-02_CIRCLean.img'
IMAGE='New_Circlean.img'
# Start sector of boot (first) partition
BOOT_START=8192
# Start sector of root (second) partition
ROOT_START=137216
ROOT_START=92160
# Locations you'd like the partitions mounted
BOOT_PATH='/mnt/rpi-boot'
ROOTFS_PATH='/mnt/rpi-root'

View File

@ -1,10 +0,0 @@
#!/bin/bash
useradd -m kitten
# Useless: the sudoer file comes from the repository
#echo "Cmnd_Alias GROOMER_CMDS = /home/kitten/kitten_mount_src, \
# /home/kitten/kitten_mount_dst, /home/kitten/kitten_umount" >> /etc/sudoers
#echo "kitten ALL=(ALL) NOPASSWD: GROOMER_CMDS" >> /etc/sudoers
# /!\ REMOVE SUDO RIGHTS TO USER pi

View File

@ -1,7 +0,0 @@
#!/bin/bash
IMAGE='2016-05-12_CIRCLean.img'
OFFSET=$((512 * 131072))
mkdir /mnt/rpi
mount -v -o offset=${OFFSET} -t ext4 ${IMAGE} /mnt/rpi

15
shell_utils/start_proot.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash
# This script runs proot on a mounted image with the proper parameters.
# The root partition should be at /mnt/rpi-root /mnt/rpt-boot
# You should probably run something like basic_mount_image.sh first
set -e
set -x
if [ "$(id -u)" != "0" ]; then
echo "This script must be run as root" 1>&2
exit 1
fi
sudo proot -q qemu-arm -S /mnt/rpi-root -b /mnt/rpi-boot:/boot /bin/bash