diff --git a/README_filecheck.md b/README_filecheck.md index 23b718b..44fcf10 100644 --- a/README_filecheck.md +++ b/README_filecheck.md @@ -51,7 +51,7 @@ It will be used for the build environment and the final image. apt-get update apt-get dist-upgrade apt-get autoremove - apt-get install p7zip-full python-dev libxml2-dev libxslt1-dev pmount + apt-get install p7zip-full python-dev libxml2-dev libxslt1-dev pmount python-setuptools libtiff4-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk ``` * Install python requirements @@ -60,6 +60,8 @@ It will be used for the build environment and the final image. pip install lxml pip install oletools olefile pip install officedissector + pip install exifread + pip install Pillow pip install git+https://github.com/Rafiot/python-magic.git@travis pip install git+https://github.com/CIRCL/PyCIRCLean.git ``` diff --git a/copy_to_final.sh b/copy_to_final.sh index 82f2735..d7ee10b 100755 --- a/copy_to_final.sh +++ b/copy_to_final.sh @@ -1,5 +1,9 @@ #!/bin/bash +# To make debugging easier +echo "KittenGroomer: in copy_to_final.sh" 1>&2 + + set -e set -x @@ -42,3 +46,7 @@ cp -rf midi ${CHROOT_PATH}/opt/ # It is also a good idea to run raspi-config once and enable the console login (allow debugging) + +# To make debugging easier +echo "KittenGroomer: done with copy_to_final.sh" 1>&2 + diff --git a/fs/opt/groomer/groomer.sh b/fs/opt/groomer/groomer.sh index f87ce4d..ec36760 100755 --- a/fs/opt/groomer/groomer.sh +++ b/fs/opt/groomer/groomer.sh @@ -3,6 +3,9 @@ set -e set -x +# To make debugging easier +echo "KittenGroomer: in groomer.sh" 1>&2 + source ./constraint.sh if ! [ "${ID}" -ge "1000" ]; then echo "This script cannot run as root." @@ -101,8 +104,10 @@ do mkdir -p "${target_dir}" LOGFILE="${LOGS}/processing.txt" + echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" 1>&2 echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE} generic.py --source /media/${SRC} --destination ${target_dir} || true + echo "==== Done with /media/${SRC} to ${target_dir}. ====" 1>&2 echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE} ls -lR "${target_dir}" @@ -110,5 +115,8 @@ do let PARTCOUNT=`expr $PARTCOUNT + 1` done +# To make debugging easier +echo "KittenGroomer: done with groomer.sh" 1>&2 + # The cleanup is automatically done in the function clean called when # the program quits diff --git a/mount_image.sh b/mount_image.sh index efbef7e..ada028b 100755 --- a/mount_image.sh +++ b/mount_image.sh @@ -1,5 +1,8 @@ #!/bin/bash +# To make debugging easier +echo "KittenGroomer: in mount_image.sh" 1>&2 + # Notes: # - To chroot in an existing SD card, unset IMAGE. Change the paths to the partitions if needed. # - The offsets are thoses of 2013-02-09-wheezy-raspbian.img. It will change on an other image. @@ -19,6 +22,7 @@ if [ -z "$1" ]; then exit fi COMMAND=${1} +COMMAND_OPT=${2} set -e set -x @@ -31,7 +35,7 @@ PARTITION_BOOT='/dev/mmcblk0p1' # If you use the img ##### Debian -IMAGE='2015-11-06-CIRCLean.img' +IMAGE='raspbian-wheezy.img' OFFSET_ROOTFS=$((122880 * 512)) OFFSET_BOOT=$((8192 * 512)) ##### Arch @@ -93,4 +97,7 @@ cp -pf /etc/resolv.conf ${CHROOT_PATH}/etc mv ${CHROOT_PATH}/etc/ld.so.preload ${CHROOT_PATH}/etc/ld.so.preload_bkp -${COMMAND} ${CHROOT_PATH} +# To make debugging easier +echo "KittenGroomer: Image mounted, executing command from mount_image.sh" 1>&2 + +${COMMAND} ${CHROOT_PATH} ${COMMAND_OPT} diff --git a/run_tests.sh b/run_tests.sh index cff3c18..42e1c8e 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,11 +1,22 @@ #!/bin/bash +TEST_PART_TYPE=${1} +TEST_SOURCE_TYPE=${2} + +if [ -z "$1" ]; then + TEST_PART_TYPE="VFAT_NORM" +fi +if [ -z "$2" ]; then + TEST_SOURCE_TYPE="t_images1" +fi + set -e ./mount_image.sh ./copy_to_final.sh pushd tests/ -./run.sh +./run.sh ${TEST_PART_TYPE} ${TEST_SOURCE_TYPE} +./check_results.sh ${TEST_SOURCE_TYPE} popd diff --git a/tests/content_img_vfat_norm/.keepdir b/tests/actualResults/.keepdir similarity index 100% rename from tests/content_img_vfat_norm/.keepdir rename to tests/actualResults/.keepdir diff --git a/tests/check_results.sh b/tests/check_results.sh new file mode 100755 index 0000000..51cb80f --- /dev/null +++ b/tests/check_results.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# To make debugging easier +echo "KittenGroomer: in tests/check_results.sh" 1>&2 + +OFFSET_VFAT_NORM=$((8192 * 512)) +IMAGE_DEST="testcase_dest.vfat" + +if [ "$(id -u)" != "0" ]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +if [ -z "$1" ]; then + echo "Please tell me which file type to test." + echo "t_images1" + exit +fi +TEST_SOURCE_TYPE=${1} + +set -e +set -x + +RESULTS_DIR="results" + +# To make debugging easier +echo "Removing results from previous run." 1>&2 +rm -rf actualResults/* + +clean(){ + umount ${RESULTS_DIR} + rm -rf ${RESULTS_DIR} +} + +trap clean EXIT TERM INT + +mkdir -p ${RESULTS_DIR} + +# Get the run results +mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_DEST} ${RESULTS_DIR} +cp -rf ${RESULTS_DIR}/* actualResults +umount ${RESULTS_DIR} + +#TODO: Check if the results were what we expected. + +# To make debugging easier +echo "KittenGroomer: done with tests/check_results.sh" 1>&2 diff --git a/tests/content_img_vfat_part2/.keepdir b/tests/content_img_vfat_part2/.keepdir deleted file mode 100644 index e69de29..0000000 diff --git a/tests/content_img_vfat_part1/.keepdir b/tests/expectedResults/.keepdir similarity index 100% rename from tests/content_img_vfat_part1/.keepdir rename to tests/expectedResults/.keepdir diff --git a/tests/run.exp b/tests/run.exp index 3db487f..33c6dbb 100755 --- a/tests/run.exp +++ b/tests/run.exp @@ -13,6 +13,6 @@ spawn qemu-system-arm -kernel kernel-qemu -cpu arm1176 -m 256 -M versatilepb \ -mon chardev=mon1,mode=control,pretty=on \ -vnc 0.0.0.0:1 -expect "reboot: System halted" +expect "System halted." send "quit\n" diff --git a/tests/run.sh b/tests/run.sh index f497d4b..0c5371b 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -2,6 +2,22 @@ # http://pub.phyks.me/respawn/mypersonaldata/public/2014-05-20-11-08-01/ +# To make debugging easier +echo "KittenGroomer: in tests/run.sh" 1>&2 + +if [ -z "$1" ]; then + echo "Please tell me which partition type to test." + echo "VFAT_NORM VFAT_PART NTPS_NORM EXT2 EXT3 EXT4" + exit +fi +if [ -z "$2" ]; then + echo "Please tell me which file type to test." + echo "t_images1" + exit +fi +TEST_PART_TYPE=${1} +TEST_SOURCE_TYPE=${2} + IMAGE='../raspbian-wheezy.img' OFFSET_ROOTFS=$((122880 * 512)) @@ -44,50 +60,102 @@ mount -o loop,offset=${OFFSET_ROOTFS} ${IMAGE} ${SETUP_DIR} mv ${SETUP_DIR}/etc/ld.so.preload ${SETUP_DIR}/etc/ld.so.preload_bkp umount ${SETUP_DIR} + + # Prepare the test source key -mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_VFAT_NORM} ${SETUP_DIR} -cp -rf content_img_vfat_norm/* ${SETUP_DIR} -umount ${SETUP_DIR} +if [ ${TEST_PART_TYPE} = "VFAT_NORM" ]; then + mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_VFAT_NORM} ${SETUP_DIR} + rm -rf ${SETUP_DIR}/* + cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR} + umount ${SETUP_DIR} +fi + # Prepare the test source key (with partitions) -mount -o loop,offset=${OFFSET_VFAT_PART1} ${IMAGE_VFAT_PART} ${SETUP_DIR} -cp -rf content_img_vfat_part1/* ${SETUP_DIR} -umount ${SETUP_DIR} -mount -o loop,offset=${OFFSET_VFAT_PART2} ${IMAGE_VFAT_PART} ${SETUP_DIR} -cp -rf content_img_vfat_part2/* ${SETUP_DIR} -umount ${SETUP_DIR} +if [ ${TEST_PART_TYPE} = "VFAT_PART" ]; then + mount -o loop,offset=${OFFSET_VFAT_PART1} ${IMAGE_VFAT_PART} ${SETUP_DIR} + rm -rf ${SETUP_DIR}/* + cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR} + umount ${SETUP_DIR} + mount -o loop,offset=${OFFSET_VFAT_PART2} ${IMAGE_VFAT_PART} ${SETUP_DIR} + rm -rf ${SETUP_DIR}/* + cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR} + umount ${SETUP_DIR} +fi + # Prepare the test source key (NTFS) -mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_NTFS_NORM} ${SETUP_DIR} -cp -rf content_img_vfat_norm/* ${SETUP_DIR} -umount ${SETUP_DIR} +if [ ${TEST_PART_TYPE} = "NTFS_NORM" ]; then + mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_NTFS_NORM} ${SETUP_DIR} + rm -rf ${SETUP_DIR}/* + cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR} + umount ${SETUP_DIR} +fi + # Prepare the test source key (EXT2) -mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT2} ${SETUP_DIR} -cp -rf content_img_vfat_norm/* ${SETUP_DIR} -umount ${SETUP_DIR} +if [ ${TEST_PART_TYPE} = "EXT2" ]; then + mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT2} ${SETUP_DIR} + rm -rf ${SETUP_DIR}/* + cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR} + umount ${SETUP_DIR} +fi + # Prepare the test source key (EXT3) -mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT3} ${SETUP_DIR} -cp -rf content_img_vfat_norm/* ${SETUP_DIR} -umount ${SETUP_DIR} +if [ ${TEST_PART_TYPE} = "EXT4" ]; then + mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT3} ${SETUP_DIR} + rm -rf ${SETUP_DIR}/* + cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR} + umount ${SETUP_DIR} +fi + # Prepare the test source key (EXT4) -mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT4} ${SETUP_DIR} -cp -rf content_img_vfat_norm/* ${SETUP_DIR} +if [ ${TEST_PART_TYPE} = "EXT4" ]; then + mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT4} ${SETUP_DIR} + rm -rf ${SETUP_DIR}/* + cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR} + umount ${SETUP_DIR} +fi + +# Prepare the test destination key +mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_DEST} ${SETUP_DIR} +rm -rf ${SETUP_DIR}/* umount ${SETUP_DIR} + +# To make debugging easier +echo "KittenGroomer: about to enter tests/run.exp" 1>&2 + chmod a-w ${IMAGE} -./run.exp ${IMAGE} ${IMAGE_VFAT_NORM} ${IMAGE_DEST} -#sleep 10 -#./run.exp ${IMAGE} ${IMAGE_VFAT_PART} ${IMAGE_DEST} -#sleep 10 -#./run.exp ${IMAGE} ${IMAGE_NTFS_NORM} ${IMAGE_DEST} + + +if [ ${TEST_PART_TYPE} = "VFAT_NORM" ]; then + ./run.exp ${IMAGE} ${IMAGE_VFAT_NORM} ${IMAGE_DEST} + sleep 10 +fi + +if [ ${TEST_PART_TYPE} = "VFAT_PART" ]; then + ./run.exp ${IMAGE} ${IMAGE_VFAT_PART} ${IMAGE_DEST} + sleep 10 +fi + +if [ ${TEST_PART_TYPE} = "NTFS_NORM" ]; then + ./run.exp ${IMAGE} ${IMAGE_NTFS_NORM} ${IMAGE_DEST} + sleep 10 +fi # EXT* not supported due to permission issues -#sleep 10 -#./run.exp ${IMAGE} ${IMAGE_EXT2} ${IMAGE_DEST} -#sleep 10 -#./run.exp ${IMAGE} ${IMAGE_EXT3} ${IMAGE_DEST} -#sleep 10 -#./run.exp ${IMAGE} ${IMAGE_EXT4} ${IMAGE_DEST} +if [ ${TEST_PART_TYPE} = "EXT2" ]; then + ./run.exp ${IMAGE} ${IMAGE_EXT2} ${IMAGE_DEST} + sleep 10 +fi +if [ ${TEST_PART_TYPE} = "EXT3" ]; then + ./run.exp ${IMAGE} ${IMAGE_EXT3} ${IMAGE_DEST} + sleep 10 +fi +if [ ${TEST_PART_TYPE} = "NTFS_EXT4" ]; then + ./run.exp ${IMAGE} ${IMAGE_EXT4} ${IMAGE_DEST} + sleep 10 +fi - -#./run.exp ${IMAGE} ${IMAGE_VFAT_PART} ${IMAGE_DEST} chmod +w ${IMAGE} +# To make debugging easier +echo "KittenGroomer: done with tests/run.sh" 1>&2 diff --git a/tests/testFiles/t_images1/image01.jpg b/tests/testFiles/t_images1/image01.jpg new file mode 100644 index 0000000..392069b Binary files /dev/null and b/tests/testFiles/t_images1/image01.jpg differ diff --git a/tests/testFiles/t_images1/image02.png b/tests/testFiles/t_images1/image02.png new file mode 100644 index 0000000..1bb34c1 Binary files /dev/null and b/tests/testFiles/t_images1/image02.png differ diff --git a/tests/testFiles/t_images1/image03.bmp b/tests/testFiles/t_images1/image03.bmp new file mode 100644 index 0000000..a024e7d Binary files /dev/null and b/tests/testFiles/t_images1/image03.bmp differ diff --git a/tests/testFiles/t_images1/image04.gif b/tests/testFiles/t_images1/image04.gif new file mode 100644 index 0000000..fbab751 Binary files /dev/null and b/tests/testFiles/t_images1/image04.gif differ diff --git a/tests/testFiles/t_images1/image05.tif b/tests/testFiles/t_images1/image05.tif new file mode 100644 index 0000000..c374562 Binary files /dev/null and b/tests/testFiles/t_images1/image05.tif differ diff --git a/tests/testFiles/t_images1/image06.tiff b/tests/testFiles/t_images1/image06.tiff new file mode 100644 index 0000000..877cb6f Binary files /dev/null and b/tests/testFiles/t_images1/image06.tiff differ diff --git a/tests/testFiles/t_images1/image07.bmp b/tests/testFiles/t_images1/image07.bmp new file mode 100644 index 0000000..e0b457c Binary files /dev/null and b/tests/testFiles/t_images1/image07.bmp differ diff --git a/tests/testFiles/t_images1/image08.gif b/tests/testFiles/t_images1/image08.gif new file mode 100644 index 0000000..852c9a2 Binary files /dev/null and b/tests/testFiles/t_images1/image08.gif differ diff --git a/tests/testFiles/t_images1/image09.png b/tests/testFiles/t_images1/image09.png new file mode 100644 index 0000000..dbc8edb Binary files /dev/null and b/tests/testFiles/t_images1/image09.png differ diff --git a/tests/testFiles/t_images1/image10.jpeg b/tests/testFiles/t_images1/image10.jpeg new file mode 100644 index 0000000..00bb27d Binary files /dev/null and b/tests/testFiles/t_images1/image10.jpeg differ diff --git a/update_PyCIRCLean.sh b/update_PyCIRCLean.sh new file mode 100755 index 0000000..14f8e0f --- /dev/null +++ b/update_PyCIRCLean.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Change to your repo if doing python dev +./mount_image.sh chroot "pip install git+https://github.com/Dymaxion00/PyCIRCLean@dev"