Merge pull request #37 from Dymaxion00/master

Add test files, update documentation, tweak run scripts

README_filecheck.md

@@ -51,7 +51,7 @@ It will be used for the build environment and the final image.
     apt-get update
     apt-get dist-upgrade
     apt-get autoremove
-    apt-get install p7zip-full python-dev libxml2-dev libxslt1-dev pmount
+    apt-get install p7zip-full python-dev libxml2-dev libxslt1-dev pmount python-setuptools libtiff4-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk
 ```
 
 * Install python requirements
@@ -60,6 +60,8 @@ It will be used for the build environment and the final image.
     pip install lxml
     pip install oletools olefile
     pip install officedissector
+    pip install exifread
+    pip install Pillow
     pip install git+https://github.com/Rafiot/python-magic.git@travis
     pip install git+https://github.com/CIRCL/PyCIRCLean.git
 ```

copy_to_final.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# To make debugging easier
+echo "KittenGroomer: in copy_to_final.sh" 1>&2
+
+
 set -e
 set -x
 
@@ -42,3 +46,7 @@ cp -rf midi ${CHROOT_PATH}/opt/
 
 
 # It is also a good idea to run raspi-config once and enable the console login (allow debugging)
+
+# To make debugging easier
+echo "KittenGroomer: done with copy_to_final.sh" 1>&2
+

fs/opt/groomer/groomer.sh

@@ -3,6 +3,9 @@
 set -e
 set -x
 
+# To make debugging easier
+echo "KittenGroomer: in groomer.sh" 1>&2
+
 source ./constraint.sh
 if ! [ "${ID}" -ge "1000" ]; then
     echo "This script cannot run as root."
@@ -101,8 +104,10 @@ do
         mkdir -p "${target_dir}"
         LOGFILE="${LOGS}/processing.txt"
 
+        echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" 1>&2
         echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE}
         generic.py --source /media/${SRC} --destination ${target_dir} || true
+        echo "==== Done with /media/${SRC} to ${target_dir}. ====" 1>&2
         echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE}
 
         ls -lR "${target_dir}"
@@ -110,5 +115,8 @@ do
     let PARTCOUNT=`expr $PARTCOUNT + 1`
 done
 
+# To make debugging easier
+echo "KittenGroomer: done with groomer.sh" 1>&2
+
 # The cleanup is automatically done in the function clean called when
 # the program quits

mount_image.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# To make debugging easier
+echo "KittenGroomer: in mount_image.sh" 1>&2
+
 # Notes:
 # - To chroot in an existing SD card, unset IMAGE. Change the paths to the partitions if needed.
 # - The offsets are thoses of 2013-02-09-wheezy-raspbian.img. It will change on an other image.
@@ -19,6 +22,7 @@ if [ -z "$1" ]; then
     exit
 fi
 COMMAND=${1}
+COMMAND_OPT=${2}
 
 set -e
 set -x
@@ -31,7 +35,7 @@ PARTITION_BOOT='/dev/mmcblk0p1'
 
 # If you use the img
 ##### Debian
-IMAGE='2015-11-06-CIRCLean.img'
+IMAGE='raspbian-wheezy.img'
 OFFSET_ROOTFS=$((122880 * 512))
 OFFSET_BOOT=$((8192 * 512))
 ##### Arch
@@ -93,4 +97,7 @@ cp -pf /etc/resolv.conf ${CHROOT_PATH}/etc
 
 mv ${CHROOT_PATH}/etc/ld.so.preload ${CHROOT_PATH}/etc/ld.so.preload_bkp
 
-${COMMAND} ${CHROOT_PATH}
+# To make debugging easier
+echo "KittenGroomer: Image mounted, executing command from mount_image.sh" 1>&2
+
+${COMMAND} ${CHROOT_PATH} ${COMMAND_OPT}

run_tests.sh

@@ -1,11 +1,22 @@
 #!/bin/bash
 
+TEST_PART_TYPE=${1}
+TEST_SOURCE_TYPE=${2}
+
+if [ -z "$1" ]; then
+    TEST_PART_TYPE="VFAT_NORM"
+fi
+if [ -z "$2" ]; then
+    TEST_SOURCE_TYPE="t_images1"
+fi
+
 set -e
 
 ./mount_image.sh ./copy_to_final.sh
 
 pushd tests/
 
-./run.sh
+./run.sh ${TEST_PART_TYPE} ${TEST_SOURCE_TYPE}
+./check_results.sh ${TEST_SOURCE_TYPE}
 
 popd

tests/check_results.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# To make debugging easier
+echo "KittenGroomer: in tests/check_results.sh" 1>&2
+
+OFFSET_VFAT_NORM=$((8192 * 512))
+IMAGE_DEST="testcase_dest.vfat"
+
+if [ "$(id -u)" != "0" ]; then
+   echo "This script must be run as root" 1>&2
+   exit 1
+fi
+
+if [ -z "$1" ]; then
+    echo "Please tell me which file type to test."
+    echo "t_images1"
+    exit
+fi
+TEST_SOURCE_TYPE=${1}
+
+set -e
+set -x
+
+RESULTS_DIR="results"
+
+# To make debugging easier
+echo "Removing results from previous run." 1>&2
+rm -rf actualResults/*
+
+clean(){
+    umount ${RESULTS_DIR}
+    rm -rf ${RESULTS_DIR}
+}
+
+trap clean EXIT TERM INT
+
+mkdir -p ${RESULTS_DIR}
+
+# Get the run results
+mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_DEST} ${RESULTS_DIR}
+cp -rf ${RESULTS_DIR}/* actualResults
+umount ${RESULTS_DIR}
+
+#TODO: Check if the results were what we expected.
+
+# To make debugging easier
+echo "KittenGroomer: done with tests/check_results.sh" 1>&2

tests/run.exp

@@ -13,6 +13,6 @@ spawn qemu-system-arm -kernel kernel-qemu -cpu arm1176 -m 256 -M versatilepb \
     -mon chardev=mon1,mode=control,pretty=on \
     -vnc 0.0.0.0:1
 
-expect "reboot: System halted"
+expect "System halted."
 
 send "quit\n"

tests/run.sh

@@ -2,6 +2,22 @@
 
 # http://pub.phyks.me/respawn/mypersonaldata/public/2014-05-20-11-08-01/
 
+# To make debugging easier
+echo "KittenGroomer: in tests/run.sh" 1>&2
+
+if [ -z "$1" ]; then
+    echo "Please tell me which partition type to test."
+    echo "VFAT_NORM VFAT_PART NTPS_NORM EXT2 EXT3 EXT4"
+    exit
+fi
+if [ -z "$2" ]; then
+    echo "Please tell me which file type to test."
+    echo "t_images1"
+    exit
+fi
+TEST_PART_TYPE=${1}
+TEST_SOURCE_TYPE=${2}
+
 IMAGE='../raspbian-wheezy.img'
 OFFSET_ROOTFS=$((122880 * 512))
 
@@ -44,50 +60,102 @@ mount -o loop,offset=${OFFSET_ROOTFS} ${IMAGE} ${SETUP_DIR}
 mv ${SETUP_DIR}/etc/ld.so.preload ${SETUP_DIR}/etc/ld.so.preload_bkp
 umount ${SETUP_DIR}
 
+
+
 # Prepare the test source key
-mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_VFAT_NORM} ${SETUP_DIR}
+if [ ${TEST_PART_TYPE} = "VFAT_NORM" ]; then
-cp -rf content_img_vfat_norm/* ${SETUP_DIR}
+    mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_VFAT_NORM} ${SETUP_DIR}
-umount ${SETUP_DIR}
+    rm -rf ${SETUP_DIR}/*
+    cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
+    umount ${SETUP_DIR}
+fi
+
 # Prepare the test source key (with partitions)
-mount -o loop,offset=${OFFSET_VFAT_PART1} ${IMAGE_VFAT_PART} ${SETUP_DIR}
+if [ ${TEST_PART_TYPE} = "VFAT_PART" ]; then
-cp -rf content_img_vfat_part1/* ${SETUP_DIR}
+    mount -o loop,offset=${OFFSET_VFAT_PART1} ${IMAGE_VFAT_PART} ${SETUP_DIR}
-umount ${SETUP_DIR}
+    rm -rf ${SETUP_DIR}/*
-mount -o loop,offset=${OFFSET_VFAT_PART2} ${IMAGE_VFAT_PART} ${SETUP_DIR}
+    cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
-cp -rf content_img_vfat_part2/* ${SETUP_DIR}
+    umount ${SETUP_DIR}
-umount ${SETUP_DIR}
+    mount -o loop,offset=${OFFSET_VFAT_PART2} ${IMAGE_VFAT_PART} ${SETUP_DIR}
+    rm -rf ${SETUP_DIR}/*
+    cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
+    umount ${SETUP_DIR}
+fi
+
 # Prepare the test source key (NTFS)
-mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_NTFS_NORM} ${SETUP_DIR}
+if [ ${TEST_PART_TYPE} = "NTFS_NORM" ]; then
-cp -rf content_img_vfat_norm/* ${SETUP_DIR}
+    mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_NTFS_NORM} ${SETUP_DIR}
-umount ${SETUP_DIR}
+    rm -rf ${SETUP_DIR}/*
+    cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
+    umount ${SETUP_DIR}
+fi
+
 # Prepare the test source key (EXT2)
-mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT2} ${SETUP_DIR}
+if [ ${TEST_PART_TYPE} = "EXT2" ]; then
-cp -rf content_img_vfat_norm/* ${SETUP_DIR}
+    mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT2} ${SETUP_DIR}
-umount ${SETUP_DIR}
+    rm -rf ${SETUP_DIR}/*
+    cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
+    umount ${SETUP_DIR}
+fi
+
 # Prepare the test source key (EXT3)
-mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT3} ${SETUP_DIR}
+if [ ${TEST_PART_TYPE} = "EXT4" ]; then
-cp -rf content_img_vfat_norm/* ${SETUP_DIR}
+    mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT3} ${SETUP_DIR}
-umount ${SETUP_DIR}
+    rm -rf ${SETUP_DIR}/*
+    cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
+    umount ${SETUP_DIR}
+fi
+
 # Prepare the test source key (EXT4)
-mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT4} ${SETUP_DIR}
+if [ ${TEST_PART_TYPE} = "EXT4" ]; then
-cp -rf content_img_vfat_norm/* ${SETUP_DIR}
+    mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT4} ${SETUP_DIR}
+    rm -rf ${SETUP_DIR}/*
+    cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
+    umount ${SETUP_DIR}
+fi
+
+# Prepare the test destination key
+mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_DEST} ${SETUP_DIR}
+rm -rf ${SETUP_DIR}/*
 umount ${SETUP_DIR}
 
+
+# To make debugging easier
+echo "KittenGroomer: about to enter tests/run.exp" 1>&2
+
 chmod a-w ${IMAGE}
-./run.exp ${IMAGE} ${IMAGE_VFAT_NORM} ${IMAGE_DEST}
+
-#sleep 10
+
-#./run.exp ${IMAGE} ${IMAGE_VFAT_PART} ${IMAGE_DEST}
+if [ ${TEST_PART_TYPE} = "VFAT_NORM" ]; then
-#sleep 10
+    ./run.exp ${IMAGE} ${IMAGE_VFAT_NORM} ${IMAGE_DEST}
-#./run.exp ${IMAGE} ${IMAGE_NTFS_NORM} ${IMAGE_DEST}
+    sleep 10
+fi
+
+if [ ${TEST_PART_TYPE} = "VFAT_PART" ]; then
+    ./run.exp ${IMAGE} ${IMAGE_VFAT_PART} ${IMAGE_DEST}
+    sleep 10
+fi
+
+if [ ${TEST_PART_TYPE} = "NTFS_NORM" ]; then
+    ./run.exp ${IMAGE} ${IMAGE_NTFS_NORM} ${IMAGE_DEST}
+    sleep 10
+fi
 
 # EXT* not supported due to permission issues
-#sleep 10
+if [ ${TEST_PART_TYPE} = "EXT2" ]; then
-#./run.exp ${IMAGE} ${IMAGE_EXT2} ${IMAGE_DEST}
+    ./run.exp ${IMAGE} ${IMAGE_EXT2} ${IMAGE_DEST}
-#sleep 10
+    sleep 10
-#./run.exp ${IMAGE} ${IMAGE_EXT3} ${IMAGE_DEST}
+fi
-#sleep 10
+if [ ${TEST_PART_TYPE} = "EXT3" ]; then
-#./run.exp ${IMAGE} ${IMAGE_EXT4} ${IMAGE_DEST}
+    ./run.exp ${IMAGE} ${IMAGE_EXT3} ${IMAGE_DEST}
+    sleep 10
+fi
+if [ ${TEST_PART_TYPE} = "NTFS_EXT4" ]; then
+    ./run.exp ${IMAGE} ${IMAGE_EXT4} ${IMAGE_DEST}
+    sleep 10
+fi
 
-
-#./run.exp ${IMAGE} ${IMAGE_VFAT_PART} ${IMAGE_DEST}
 chmod +w ${IMAGE}
 
+# To make debugging easier
+echo "KittenGroomer: done with tests/run.sh" 1>&2

update_PyCIRCLean.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Change to your repo if doing python dev
+./mount_image.sh chroot "pip install git+https://github.com/Dymaxion00/PyCIRCLean@dev"