Compare commits
135 Commits
Author | SHA1 | Date |
---|---|---|
Raphaël Vinot | d484e97332 | |
nosecguy | f7ffb7318a | |
Raphaël Vinot | fe2f7f5a70 | |
Raphaël Vinot | 82b82ec9c4 | |
Raphaël Vinot | 57cb2e2036 | |
Raphaël Vinot | 6f5d09d374 | |
Raphaël Vinot | 4c77d41d69 | |
Raphaël Vinot | 34a2fcdf7d | |
Raphaël Vinot | 6c832c75de | |
Raphaël Vinot | 1d63d3fc14 | |
Steve Clement | 786b736d13 | |
Steve Clement | 0f2f1ba8a6 | |
Raphaël Vinot | eca0eb534c | |
Raphaël Vinot | 7b5aa62edc | |
Raphaël Vinot | 584f72cf12 | |
Raphaël Vinot | 12f7b9ad77 | |
Raphaël Vinot | 4352da90e9 | |
Raphaël Vinot | 87aeb6a919 | |
Dan Puttick | 03b3d68733 | |
Dan Puttick | 46ede0137a | |
Dan Puttick | aaaf22ae78 | |
Dan Puttick | c93941079e | |
Dan Puttick | 624796190b | |
Raphaël Vinot | 2e93930df5 | |
Dan Puttick | 2fc36b6d23 | |
Dan Puttick | 9429ede64c | |
Dan Puttick | d9438d03c0 | |
Dan Puttick | 410f5fce0e | |
Dan Puttick | ab9c9eb309 | |
Dan Puttick | 650ad783d7 | |
Dan Puttick | a8a57e1ef0 | |
Dan Puttick | c6e5caa2bd | |
Dan Puttick | 6845f2b607 | |
Dan Puttick | dd427b8c8e | |
Dan Puttick | fce66420c8 | |
Dan Puttick | 66f2dd25f2 | |
Dan Puttick | 9dd6b2c460 | |
Dan Puttick | b6564d7298 | |
Dan Puttick | 299743722f | |
Dan Puttick | b5131e295c | |
Dan Puttick | d8215b13f6 | |
Dan Puttick | a255251430 | |
Dan Puttick | df1380bc45 | |
Raphaël Vinot | 0e2c3e95e0 | |
Raphaël Vinot | a8d08995cb | |
Raphaël Vinot | 3ee2516a9c | |
Raphaël Vinot | 62753eac6b | |
Raphaël Vinot | 4e5b7b2815 | |
Dan Puttick | df958a1d3b | |
Dan Puttick | b2c0883bf7 | |
Dan Puttick | 5a811438f2 | |
Dan Puttick | 66dee077ce | |
Dan Puttick | 0dd86af8e8 | |
Dan Puttick | 58ae576343 | |
Dan Puttick | a2f2d39582 | |
Dan Puttick | 51de40f2aa | |
Dan Puttick | 67e87e6a55 | |
Dan Puttick | 8549ea35db | |
Dan Puttick | 8f6e8fb87b | |
Dan Puttick | cdf5964392 | |
Dan Puttick | adef197f01 | |
Dan Puttick | 23f9a7b869 | |
Dan Puttick | 47f25c07b5 | |
Dan Puttick | b2cbe7b7e6 | |
Raphaël Vinot | 216a9cf426 | |
Dan Puttick | 1ecf260f67 | |
Dan Puttick | c51531f77c | |
Dan Puttick | 74a22a5ee2 | |
Dan Puttick | deca259529 | |
Dan Puttick | 0d3e3d4ce5 | |
Dan Puttick | 7ffa4552b4 | |
Dan Puttick | 5632c9096a | |
Dan Puttick | b1353fa5bf | |
Dan Puttick | a8a670c08e | |
Dan Puttick | effd4a86a4 | |
Dan Puttick | 05cc4c1155 | |
Dan Puttick | 733b5fd637 | |
Dan Puttick | fca8ac922c | |
Dan Puttick | d5e73354c2 | |
Raphaël Vinot | 52ee749a45 | |
Dan Puttick | 90e4327d5a | |
Dan Puttick | 85a4a43fc2 | |
Dan Puttick | dbf35536be | |
Raphaël Vinot | d9b54ce295 | |
Raphaël Vinot | 3c39266f5a | |
Raphaël Vinot | 1318d1f166 | |
Raphaël Vinot | 1c099abb3e | |
Raphaël Vinot | 3a6e621b6d | |
Alexandre Dulaunoy | f87270ad57 | |
William Robinet | ea793b0968 | |
Raphaël Vinot | 89b54254da | |
William Robinet | 8aac428b7a | |
Alexandre Dulaunoy | 7b973c895d | |
William Robinet | 03f91a75d9 | |
Alexandre Dulaunoy | f54759aaf6 | |
Raphaël Vinot | a0e8295b15 | |
Eleanor Saitta | 6fcbc0a174 | |
Eleanor Saitta | c666b09903 | |
Eleanor Saitta | f2826aa573 | |
Eleanor Saitta | a703d961f3 | |
Eleanor Saitta | 54a6001db1 | |
Eleanor Saitta | 7cd38ee32f | |
Eleanor Saitta | 3d43c32808 | |
Raphaël Vinot | ed23b84dcc | |
Raphaël Vinot | 960f98dd36 | |
Raphaël Vinot | 17602da661 | |
Eleanor Saitta | 8f121f3bf8 | |
Eleanor Saitta | c4879f99b6 | |
Eleanor Saitta | 6b6d7df247 | |
Eleanor Saitta | ce6ee4ad85 | |
Eleanor Saitta | f191621abc | |
Eleanor Saitta | 742071493c | |
Eleanor Saitta | 91e1b13fc8 | |
Eleanor Saitta | 1120f8edf9 | |
Eleanor Saitta | ae1fed5ff2 | |
Eleanor Saitta | 5f984e72bc | |
Raphaël Vinot | 679d688f95 | |
Raphaël Vinot | a92170650a | |
Eleanor Saitta | d2a922b0e0 | |
Steve Clement | a7f986a613 | |
mdurvaux | e41c688869 | |
Raphaël Vinot | 0d45324d4e | |
Raphaël Vinot | 37995d768d | |
Raphaël Vinot | cdfbb9ba0e | |
Raphaël Vinot | def6c26109 | |
Raphaël Vinot | d66be7022a | |
Raphaël Vinot | 90b277f90d | |
Raphaël Vinot | 8abd633782 | |
Raphaël Vinot | c320c28db6 | |
Raphaël Vinot | c6fc4ed21f | |
Raphaël Vinot | 5682a49f5b | |
Raphaël Vinot | 29c4e46ab4 | |
Raphaël Vinot | ff624bf4af | |
Raphaël Vinot | 420a9f8f0d | |
Raphaël Vinot | 622c6ba333 |
|
@ -7,3 +7,7 @@ deb/
|
|||
*img
|
||||
|
||||
ledBtn
|
||||
|
||||
python/build/*
|
||||
pyton/dist/*
|
||||
python/kittengroomer.egg-info/*
|
||||
|
|
26
CHANGELOG
|
@ -1,26 +0,0 @@
|
|||
Version 1.2 - 2015-03-10
|
||||
|
||||
- Rollback the migration to Jessie and use Wheezy again: the only importantant depencendy from Jessie was poppler, which is available in the backports
|
||||
- Use the most recent security patches
|
||||
- Do not wait for user input in case of password protected archive
|
||||
|
||||
Version 1.1.1 - 2014-10-26
|
||||
|
||||
- General upgrade of Debian to avoid the system to fail in case there is no HDMI cable connected.
|
||||
|
||||
Version 1.1 - 2014-10-01
|
||||
|
||||
- NTFS support added for USB key
|
||||
- Updated to Debian Jessie including patches for [bash vulnerabilities CVE-2014-6271 - CVE-2014-7169](/pub/tr-27/)
|
||||
- CIRCLean user are now removed from the sudoer
|
||||
|
||||
Version 1.0 - 2014-05-20
|
||||
|
||||
- Based on Raspbian Jessie
|
||||
- Fully automated tests with Qemu
|
||||
- Mimetype: support of PDF, Office documents, archives, windows executables
|
||||
- Filesystem: USB keys have to be formated in vfat
|
||||
- Support of multiple partitions
|
||||
- Renaming of autorun.inf on the source key
|
||||
- Operating system is read only
|
||||
- Use pdf2htmlEX v0.11
|
|
@ -0,0 +1,77 @@
|
|||
Version 2.4 - 2018-01-29
|
||||
- Bump Raspbian
|
||||
- Bump PyCIRCLean
|
||||
- Wait 15s before shutdown (simplify debug)
|
||||
|
||||
Version 2.3.1 - 2017-10-25
|
||||
- Bugfix for Raspberry Pi 3 that was causing rc.local to fail to boot
|
||||
- Turned off several networking related services: startup is much faster
|
||||
|
||||
Version 2.3 - 2017-09-08
|
||||
- Updated to the newest version of Raspbian Stretch lite (2017-08-16 release)
|
||||
- Using the newest version of PyCIRCLean, including several vulnerability and bug fixes
|
||||
- Refactored CIRCLean bash scripts according to [Defensive Bash Programming](http://www.kfirlavi.com/blog/2012/11/14/defensive-bash-programming/)
|
||||
- Added IN_PROGRESS.txt canary file that gets added and then deleted from destination key
|
||||
- Various smaller bug fixes
|
||||
|
||||
Version 2.2 - 2017-04-18
|
||||
- Updated to newest version of Raspbian Jessie lite (April 10th 2017 release)
|
||||
- Using the newest version of PyCIRCLean, which includes a new log format and
|
||||
various bug fixes
|
||||
- Filecheck.py configuration information is now conveniently held in a Config object instead of in globals
|
||||
- New easier to read text-based logger (removed twiggy dependency)
|
||||
- Various filetypes in filecheck.py now have improved descriptions for log
|
||||
|
||||
Version 2.1.1 - 2017-02-28
|
||||
- Fix copying PDF documents to the destination key
|
||||
|
||||
Version 2.1 - 2017-02-02
|
||||
- Updated to the newest version of Raspbian Jessie lite (January 11th 2017 release)
|
||||
- NTFS files can now be mounted as source or destination keys
|
||||
- Added udev rules that ensure the USB ports map deterministically to source and destination keys
|
||||
- New debug flag and debug logging functionality to make working on Circlean without a monitor easier
|
||||
- Turned off automatic display sleep
|
||||
|
||||
Version 2.0.2 - 2016-05-12
|
||||
- Improve filename encoding
|
||||
|
||||
Version 2.0.1 - 2016-04-26
|
||||
- Re-add [timidity](http://timidity.sourceforge.net/) so the MIDI files are played properly
|
||||
|
||||
Version 2.0 - 2016-04-26
|
||||
- No critical bugs have been identified, this release uses the latest version of Raspbian Jessie lite, with all system updates
|
||||
|
||||
Version 2.0-BETA - 2015-11-06
|
||||
- There a new beta version of CIRCLean which is a significant improvement from the latest version in term of speed and efficiency on low-end hardware like the first version of the Raspberry Pi. The new code base of CIRCLean is now based on [PyCIRCLean](https://github.com/CIRCL/PyCIRCLean)
|
||||
|
||||
Version 1.3 - 2015-05-27
|
||||
- Fix a [critical security bug](https://www.circl.lu/projects/CIRCLean/security/advisory-01) related to [polyglot files](https://github.com/CIRCL/Circlean/issues/9) - thanks to the reporters ([Jann Horn](https://github.com/thejh), [seclab-solutions](http://www.seclab-solutions.com/))
|
||||
- Use [PyCIRCLean](https://github.com/CIRCL/PyCIRCLean) for conversion
|
||||
- Convert PDF files to PDF/A before converting to HTML
|
||||
|
||||
Version 1.2 - 2015-03-10
|
||||
|
||||
- Rollback the migration to Jessie and use Wheezy again: the only important dependency from Jessie was poppler, which is available in the backports
|
||||
- Use the most recent security patches
|
||||
- Do not wait for user input in case of password protected archive
|
||||
|
||||
Version 1.1.1 - 2014-10-26
|
||||
|
||||
- General upgrade of Debian to avoid the system to fail in case there is no HDMI cable connected.
|
||||
|
||||
Version 1.1 - 2014-10-01
|
||||
|
||||
- NTFS support added for USB key
|
||||
- Updated to Debian Jessie including patches for [bash vulnerabilities CVE-2014-6271 - CVE-2014-7169](/pub/tr-27/)
|
||||
- CIRCLean user are now removed from the sudoer
|
||||
|
||||
Version 1.0 - 2014-05-20
|
||||
|
||||
- Based on Raspbian Jessie
|
||||
- Fully automated tests with Qemu
|
||||
- Mimetype: support of PDF, Office documents, archives, windows executables
|
||||
- Filesystem: USB keys have to be formated in vfat
|
||||
- Support of multiple partitions
|
||||
- Renaming of autorun.inf on the source key
|
||||
- Operating system is read only
|
||||
- Use pdf2htmlEX v0.11
|
|
@ -0,0 +1,86 @@
|
|||
Getting started
|
||||
===============
|
||||
|
||||
If you'd like to work on the Python code that processes files for Circlean, you should
|
||||
take a look at [PyCIRCLean](https://github.com/CIRCL/PyCIRCLean), specifically the
|
||||
filecheck.py script. To get started contributing to Circlean, first, fork the project and
|
||||
`git clone` your fork. Then, follow the instructions in [setup_with_proot.md](doc/setup_with_proot.md) to build an image. To make things easier, you can also download a
|
||||
prebuilt image as mentioned in the README, and then mount and make modifications to this
|
||||
image to test your changes.
|
||||
|
||||
The issue tracker
|
||||
=================
|
||||
|
||||
If you find a bug or see a problem with PyCIRCLean, please open an issue in the Github
|
||||
repo. We'll do our best to respond as quickly as possible. Also, feel free to contribute a
|
||||
solution to any of the open issues - we'll do our best to review your pull request in a
|
||||
timely manner. This project is in active development, so any contributions are welcome!
|
||||
|
||||
Dependencies
|
||||
============
|
||||
* Timidity for playing midi files
|
||||
* Git for installing some Python dependencies
|
||||
* 7Zip for unpacking archives
|
||||
* ntfs-3g, exfat-fuse for mounting usb key partitions
|
||||
* Python 3 and pip for installing and running Python dependencies
|
||||
* Python3-lxml for handling ooxml and other Office files in filecheck.py
|
||||
* libjpeg-dev, libtiff-dev, libwebp-dev, liblcms2-dev, tcl-dev, tk-dev, and python-tk for various image formats (dependencies for pillow)
|
||||
* Exifread for file metadata
|
||||
* Pillow for handling images
|
||||
* Olefile, oletools, and officedissector for handling various Office filetypes
|
||||
* PyCIRCLean for main file handling code
|
||||
|
||||
Helper scripts
|
||||
==============
|
||||
|
||||
Use the scripts in shell_utils/ as examples - do not run them blindly as you will most
|
||||
probably have to change some constants/paths accordingly to your configuration.
|
||||
|
||||
IN ALL CASES, PLEASE READ THE COMMENTS IN THE SCRIPTS AT LEAST ONCE.
|
||||
|
||||
* proper_chroot.sh: uses qemu to chroot into a raspbian instance (.img or SD Card)
|
||||
* prepare_rPI.sh: update the system, some configuration
|
||||
* create_user.sh: create the user who will run the scripts, assign the proper sudo rights.
|
||||
* copy_to_final.sh: populate the content of the directory fs/ in the image,
|
||||
contains a sample of dd command to write the image on the SD card.
|
||||
NOTE: TAKE CARE NOT TO USE THE WRONG DESTINATION
|
||||
|
||||
|
||||
Running the tests
|
||||
=================
|
||||
|
||||
* If you've made changes to the shell scripts, start by installing and running
|
||||
[Shellcheck](https://github.com/koalaman/shellcheck).
|
||||
|
||||
* To emulate the Raspberry Pi hardware for testing, we'll be using
|
||||
[Qemu](http://wiki.qemu.org/Main_Page), an open source machine emulator.
|
||||
The "qemu" package available for Ubuntu/Debian includes all of the required
|
||||
packages (including qemu-system-arm) except for qemu-user-static, which must
|
||||
be installed separately.
|
||||
|
||||
```
|
||||
sudo apt-get install qemu qemu-user-static expect
|
||||
```
|
||||
|
||||
* Get the qemu kernel for the image you are using:
|
||||
|
||||
```
|
||||
pushd tests; wget https://github.com/dhruvvyas90/qemu-rpi-kernel/raw/master/kernel-qemu; popd
|
||||
```
|
||||
|
||||
* Put some test data from tests/testFiles into tests/content_img_vfat_norm
|
||||
|
||||
* Comment out the other tests in tests/run.sh or populate those directories as
|
||||
well
|
||||
|
||||
* Make sure to set the filename of the image and the kernel in `tests/run.sh`
|
||||
|
||||
* Run the tests:
|
||||
|
||||
```
|
||||
sudo ./run_tests.sh
|
||||
```
|
||||
|
||||
* If the image run processed images correctly but doesn't exit and unmount the
|
||||
images cleanly, look at tests/run.exp and make sure it's waiting for the
|
||||
string your qemu and kernel actually produce.
|
5
LICENSE
|
@ -1,6 +1,7 @@
|
|||
Copyright (C) 2012, 2013 Maya Bonkowski
|
||||
Copyright (C) 2013 Raphaël Vinot
|
||||
Copyright (C) 2013 CIRCL - Computer Incident Response Center Luxembourg (℅ smile gie)
|
||||
Copyright (C) 2012, 2019 Quinn Norton
|
||||
Copyright (C) 2013, 2019 Raphaël Vinot
|
||||
Copyright (C) 2013, 2019 CIRCL - Computer Incident Response Center Luxembourg (℅ smile gie)
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
|
|
63
NOTES
|
@ -1,63 +0,0 @@
|
|||
Notes
|
||||
=====
|
||||
|
||||
* don't plug in USB devices with a hub because there's no way to tell it which
|
||||
is source and target - its the first drive enumerated (top port) that is the
|
||||
source and the second (bootom port) is the target
|
||||
* don't turn it off without shuting down the system, when grooming is done it
|
||||
shuts down automatically: losing power while it's running can trash the OS
|
||||
on the SD cards because SD cards don't always like dirty shutdowns (ie power loss)
|
||||
* Using a target usb stick that has a status light as long as the device has
|
||||
power is a really useful thing as there the other status lights on the groomer
|
||||
are less than indicative at times: because teh 'OK' led on the rPi toggles on activity
|
||||
it can be off for a long time while processing something and only comes back
|
||||
on when that process finishes - hence why a USB that has some sort of LED activity
|
||||
when jsut plugged in (even if not reading or writing but while the USB port is
|
||||
powered) is helpful in determining when the process is finished - when
|
||||
teh rPI is shutdown, the USB port power is shut off and that LED will also
|
||||
then be off on the USB device
|
||||
* Use a larger target device as all zip files get unpacked and processed onto
|
||||
the target
|
||||
* if you have an hdmi monitor plugged in you can watch what's happening for about
|
||||
30 mintues until the rPI's power saving's kick in and turn off the monitor
|
||||
* if only one usb stick is present at power up, it doesn't groom and looks like
|
||||
a normal rPi
|
||||
* if you want to ssh into the rPi username is 'pi' password 'raspberry' as per defaults
|
||||
|
||||
|
||||
Technical notes
|
||||
===============
|
||||
|
||||
* groomer script is in /opt/groomer/ with the other required files
|
||||
* dependancies are libre-office and OpenJRE
|
||||
* and the ip address is 192.168.1.89
|
||||
* the groomer process is kicked off in /etc/rc.local
|
||||
* the heavy lifting takes place or is dispatched from /opt/groomer/groomer.sh
|
||||
in that script file is what file types get processed (or if not listed there,
|
||||
get ignored)
|
||||
* there are two ways pdf's can get handled -right now they have their text extracted
|
||||
to the target device, the otherway copies it and extracts the text
|
||||
* the pdf text extraction isn't perfect and is the slowest part of it, but should
|
||||
be able to handle unicode stuff and currently doesn't do image extraction from
|
||||
pdf's but could do that too
|
||||
|
||||
|
||||
Discussion
|
||||
==========
|
||||
|
||||
* however image exports of pdf pages only have the images and no text so it's not
|
||||
like saving each page to a jpg which would be a really handy and safe way of
|
||||
converting pdf's
|
||||
* spread sheets and presentations get converted to pdfs to kill off any embedded
|
||||
macros and it's assumed that it's not producing evil pdf's on export but does
|
||||
nothing to sanitize any embedded links within those documents
|
||||
* for spreadsheets, if they are longer than a page, only a page worth from that
|
||||
sheet is exported right from the middle of the sheet (ie the top and bottom of
|
||||
that sheet will get cut off and only the contents in teh middle exported to pdf)
|
||||
dumb but i figure if you want to go back to the source because it's interesting
|
||||
enough on teh groomed side of it, then you can take the extra precautions
|
||||
* the groomed target only copies "safe" files, and does its best to convert any
|
||||
potentiall unsafe files to a safer format
|
||||
* safe files being one that i know of that can't contain malicious embedded macros
|
||||
or other crap like that, and those than can get converted to something that wont
|
||||
contain code after conversion
|
133
README.md
|
@ -1,88 +1,93 @@
|
|||
CIRCLean
|
||||
========
|
||||
|
||||
[![Chatroom](https://badges.gitter.im/CIRCLean/Lobby.svg)](https://gitter.im/CIRCLean/Lobby)
|
||||
|
||||
![CIRCLean logo](https://www.circl.lu/assets/images/logos/circlean.png)
|
||||
![Cleaner in action](http://www.circl.lu/assets/images/CIRCLean/CIRCLean.png)
|
||||
|
||||
How To
|
||||
======
|
||||
How To Install
|
||||
==============
|
||||
|
||||
[Graphical how-to and pre-build image](http://circl.lu/projects/CIRCLean/).
|
||||
[Graphical how-to and pre-built image download](http://circl.lu/projects/CIRCLean/).
|
||||
|
||||
To prepare the SD card on Windows, you can use [Win32DiskImager](http://sourceforge.net/projects/win32diskimager/). On linux/macOS, use dd (see the how-to link for
|
||||
instructions).
|
||||
|
||||
The current prebuilt image is based on the 1-11-17 release of Raspbian Jessie Lite.
|
||||
The smallest SD card that Circlean can fit on is currently 4GB.
|
||||
|
||||
If you'd like to contribute to the project or build the image yourself, see
|
||||
[contributing.md](CONTRIBUTING.md) and the [setup instructions](doc/setup_with_proot.md).
|
||||
This is a work in progress - contributions are welcome.
|
||||
|
||||
FAQ
|
||||
===
|
||||
|
||||
**Question**: I can't login, what is the password?
|
||||
|
||||
**Answer**: For security reasons, it is **not possible** to login on the default image runinng CIRCLean/KittenGroomer (an attacker could exploit that functionality).
|
||||
|
||||
The only thing the default image does is booting, processing the content of the source key, copying over the files to the destination key, and finally shutting down.
|
||||
|
||||
To prepare the SD card on Windows, you can use [Win32DiskImager] (http://sourceforge.net/projects/win32diskimager/).
|
||||
|
||||
And the linux way is in the command line, via dd (see in copy_to_final.sh)
|
||||
|
||||
Why/What
|
||||
========
|
||||
|
||||
This project aims to be used in case you got an USB key you do not know what is
|
||||
contains but still want to have a look.
|
||||
This project aims to be useful when you get/find a USB key that you can't trust,
|
||||
and you want to look at its contents without taking the risk of plugging it into
|
||||
your computer directly. The official project page can be found at [https://www.circl.lu/projects/CIRCLean/]
|
||||
|
||||
Work in progress, contributions welcome:
|
||||
The Raspberry Pi Foundation has a [blog post](https://www.raspberrypi.org/blog/kittengroomercirclean-data-security-for-journalists-and-activists/) with more information
|
||||
about an older version of the project and details of the inspiration behind it.
|
||||
|
||||
The content of the first key will be copyed or/and converted to the second key
|
||||
following theses rules (based on the mime type):
|
||||
- direct copy of plain text files (mime type: text/*)
|
||||
- direct copy of audio files (mime type: audio/*)
|
||||
- direct copy of image files (mime type: image/*)
|
||||
- direct copy of video files (mime type: video/*)
|
||||
- direct copy of example files (mime type: example/*)
|
||||
- direct copy of message files (mime type: message/*)
|
||||
- direct copy of model files (mime type: model/*)
|
||||
- direct copy of multipart files (mime type: multipart/*)
|
||||
- Copying or converting the application files this way (mime type: application/*):
|
||||
- pdf => HTML
|
||||
- msword|vnd.openxmlformats-officedocument.*|vnd.ms-*|vnd.oasis.opendocument* => pdf => html
|
||||
- *xml* => copy as a text file
|
||||
- x-dosexec (executable) => prepend and append DANGEROUS to the filename
|
||||
- x-gzip|x-tar|x-7z-compressed => compressed file
|
||||
- octet-stream => direct copy
|
||||
CIRCLean is currently tested to work with USB keys that have FAT32, NTFS, exFAT or
|
||||
ext2/3/4 filesystems (ext\* filesystems can only be used as source keys, not destination
|
||||
keys).
|
||||
The vast majority of USB keys will be FAT32, NTFS, and exFAT.
|
||||
|
||||
Compressed files (zip|x-rar|x-bzip2|x-lzip|x-lzma|x-lzop|x-xz|x-compress|x-gzip|x-tar|*compressed):
|
||||
- Unpacking of archives
|
||||
- Recursively run the rules on the unpacked files
|
||||
The content of the untrusted key will be copied or/and converted to the second
|
||||
(blank) key following these rules (based on the mime type as determined by libmagic):
|
||||
- Direct copy of:
|
||||
- Plain text files (mime type: text/\*)
|
||||
- Audio files (mime type: audio/\*)
|
||||
- Video files (mime type: video/\*)
|
||||
- Example files (mime type: example/\*)
|
||||
- Multipart files (mime type: multipart/\*)
|
||||
- xml files, after being converted to text files
|
||||
- Octet-stream files
|
||||
- Copied after verification:
|
||||
- Image files after verifying that they are not compression bombs (mime type: image/\*)
|
||||
- PDF files, after marking as dangerous if they contain malicious content
|
||||
- msword|vnd.openxmlformats-officedocument.\*|vnd.ms-\*|vnd.oasis.opendocument\*, after
|
||||
parsing with oletools/olefile and marking as dangerous if the parsing fails.
|
||||
- Copied but marked as dangerous (DANGEROUS\_filename\_DANGEROUS)
|
||||
- Message files (mime type: message/\*)
|
||||
- Model files (mime type: model/\*)
|
||||
- x-dosexec (executable)
|
||||
- Compressed files (zip|x-rar|x-bzip2|x-lzip|x-lzma|x-lzop|x-xz|x-compress|x-gzip|x-tar|\*compressed):
|
||||
- Archives are unpacked, with the unpacking process stopped after 2 levels of archives
|
||||
to prevent archive bombs.
|
||||
- The above rules are applied recursively to the unpacked files.
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
0. Power off the device
|
||||
1. Plug the untrusted key in the top usb slot of the Raspberry Pi
|
||||
2. Plug your own key in the bottom usb slot
|
||||
0. Power off the device and unplug all connections.
|
||||
1. Plug the untrusted key in the top left USB slot of the Raspberry Pi.
|
||||
2. Plug your own key in the bottom USB slot (or use any of the other slots if
|
||||
there are more than 2).
|
||||
|
||||
*Note*: This key should be bigger than the original one because the archives
|
||||
will be copyed
|
||||
*Note*: This key should be bigger than the original one because any archives
|
||||
present on the source key will be expanded and copied.
|
||||
|
||||
3. Optional: connect the HDMI cable to a screen to see what happen
|
||||
4. Connect the power to the micro USB
|
||||
3. Optional: connect the HDMI cable to a screen to monitor the process.
|
||||
4. Connect the power to the micro USB port.
|
||||
|
||||
*Note*: 5V, 700mA regulated power supply
|
||||
*Note*: Use a 5V, 700mA+ regulated power supply
|
||||
|
||||
5. Wait until you do not see any blinking green light on the board, or if you
|
||||
connected the HDMI cable, check the screen
|
||||
it's slow and can take 30-60 minutes depending on how many document
|
||||
conversions take place
|
||||
6. Power off the device and disconnect the drives
|
||||
|
||||
Helper scripts
|
||||
==============
|
||||
|
||||
You should use them as example when you are creating a new image and probably not
|
||||
run them blindly as you will most probably have to change constraints accordingly to
|
||||
your configuration.
|
||||
|
||||
IN ALL CASES, PLEASE READ THE COMMENTS IN THE SCRIPTS AT LEAST ONCE.
|
||||
|
||||
* proper_chroot.sh: uses qemu to chroot into a raspbian instance (.img or SD Card)
|
||||
* prepare_rPI_builder.sh: update the system, add the repositories and install all
|
||||
the dependencies needed to compile poppler and pdf2htmlEX on a rPi
|
||||
* update_builder.sh: compile the latest version of poppler from debian experimental,
|
||||
pull and compile the latest version of pdf2htmlEX from the git repository
|
||||
* prepare_rPI.sh: update the system, install the dependencies of poppler and pdf2htmlEX,
|
||||
install poppler and pdf2htmlEX (the deb packages compiled in the builder)
|
||||
* create_user.sh: create the user who will run the scripts, assign the proper sudo rights.
|
||||
* copy_to_final.sh: populate the content of the directory fs/ in the image,
|
||||
contains a sample of dd command to write the image on the SD card.
|
||||
*TAKE CARE NOT USING THE WRONG DESTINATION*
|
||||
|
||||
|
||||
|
||||
connected the HDMI cable, check the screen. The process is slow and can take
|
||||
30-60 minutes depending on how many document conversions take place.
|
||||
6. Power off the device and disconnect the drives.
|
||||
|
|
|
@ -1,203 +0,0 @@
|
|||
Create a new image from scratch
|
||||
===============================
|
||||
|
||||
* Download the most recent Raspbian version:
|
||||
http://downloads.raspberrypi.org/raspbian_latest
|
||||
|
||||
* Unpack it:
|
||||
|
||||
```
|
||||
unzip 2015-02-16-raspbian-wheezy.zip
|
||||
```
|
||||
|
||||
Prepare the base image
|
||||
======================
|
||||
|
||||
It will be used for the build environment and the final image.
|
||||
|
||||
* [Add empty space to the image](resize_img.md)
|
||||
|
||||
* Edit `mount_image.sh` and change the `IMAGE` variable accordingly
|
||||
|
||||
```
|
||||
IMAGE='2015-02-16-raspbian-wheezy.img'
|
||||
```
|
||||
|
||||
* Chroot in the image
|
||||
|
||||
```
|
||||
sudo ./proper_chroot.sh
|
||||
```
|
||||
|
||||
* Change your user to root (your global variables may be broken)
|
||||
|
||||
```
|
||||
su root
|
||||
```
|
||||
|
||||
* The locales may be broken, fix it (remove `en_GB.UTF-8 UTF-8`, set `en_US.UTF-8 UTF-8`):
|
||||
|
||||
```
|
||||
dpkg-reconfigure locales
|
||||
```
|
||||
|
||||
* In the image, make sure everything is up-to-date, and remove the old packages
|
||||
|
||||
```
|
||||
apt-get update
|
||||
apt-get dist-upgrade
|
||||
apt-get autoremove
|
||||
```
|
||||
|
||||
Setup two images
|
||||
================
|
||||
|
||||
* Create two separate images: one will be used to build the deb packages that are not available in wheezy
|
||||
|
||||
```
|
||||
mv 2015-02-16-raspbian-wheezy.img BUILDENV_2015-02-16-raspbian-wheezy.img
|
||||
cp BUILDENV_2015-02-16-raspbian-wheezy.img FINAL_2015-02-16-raspbian-wheezy.img
|
||||
```
|
||||
|
||||
Build environment specifics
|
||||
===========================
|
||||
|
||||
* Create a symlink to the build image
|
||||
|
||||
```
|
||||
ln -s BUILDENV_2015-02-16-raspbian-wheezy.img 2015-02-16-raspbian-wheezy.img
|
||||
```
|
||||
|
||||
* Chroot in the image
|
||||
|
||||
```
|
||||
sudo ./proper_chroot.sh
|
||||
```
|
||||
|
||||
* Change your user to root (your global variables may be broken)
|
||||
|
||||
```
|
||||
su root
|
||||
```
|
||||
|
||||
* Add Wheezy backports source packages to build a poppler version compatible with pdf2htmlEX
|
||||
|
||||
```
|
||||
echo 'deb-src http://ftp.debian.org/debian/ wheezy-backports main' >> /etc/apt/sources.list
|
||||
gpg --keyserver pgpkeys.mit.edu --recv-key 8B48AD6246925553
|
||||
gpg -a --export 8B48AD6246925553 | sudo apt-key add -
|
||||
apt-get update
|
||||
```
|
||||
|
||||
* Get the required build dependencies and the sources
|
||||
|
||||
```
|
||||
apt-get build-dep poppler
|
||||
apt-get source poppler
|
||||
```
|
||||
|
||||
* Compile the package
|
||||
|
||||
```
|
||||
cd poppler-<VERSION>/
|
||||
dpkg-buildpackage
|
||||
```
|
||||
|
||||
* Install the packages required by pdf2htmlEX
|
||||
|
||||
```
|
||||
apt-get install cmake libfontforge-dev libspiro-dev python-dev default-jre-headless
|
||||
cd ..
|
||||
dpkg -i libpoppler-dev* libpoppler* libpoppler-private-dev*
|
||||
```
|
||||
|
||||
* Download the sources of pdf2htmlEX (we cannot use anything newer than v0.11 because fontforge>=2.0 is not available)
|
||||
|
||||
```
|
||||
wget https://github.com/Rafiot/pdf2htmlEX/archive/KittenGroomer.zip
|
||||
unzip KittenGroomer.zip
|
||||
```
|
||||
|
||||
* Compile the package
|
||||
|
||||
```
|
||||
cd pdf2htmlEX-KittenGroomer/
|
||||
dpkg-buildpackage -uc -b
|
||||
```
|
||||
|
||||
* Get the packages out of the building image (run it outside of the chroot)
|
||||
|
||||
```
|
||||
cp /mnt/arm_rPi/libpoppler46_* /mnt/arm_rPi/pdf2htmlex_* deb/
|
||||
```
|
||||
|
||||
Final image specifics
|
||||
=====================
|
||||
|
||||
* Change the link to the image
|
||||
|
||||
```
|
||||
rm 2015-02-16-raspbian-wheezy.img
|
||||
ln -s FINAL_2015-02-16-raspbian-wheezy.img 2015-02-16-raspbian-wheezy.img
|
||||
```
|
||||
|
||||
* Chroot in the image
|
||||
|
||||
```
|
||||
sudo ./proper_chroot.sh
|
||||
```
|
||||
|
||||
* Change your user to root (your global variables may be broken)
|
||||
|
||||
```
|
||||
su root
|
||||
```
|
||||
|
||||
* Copy the debian packages into the chroot (run it outside of the chroot)
|
||||
|
||||
```
|
||||
cp deb/*.deb /mnt/arm_rPi/
|
||||
```
|
||||
|
||||
* Install repencencies required by the project
|
||||
|
||||
```
|
||||
apt-get install libreoffice p7zip-full libfontforge1 timidity freepats pmount ntfs-3g
|
||||
dpkg -i *.deb
|
||||
```
|
||||
|
||||
* Create the user, make Libreoffice and mtab working on a RO filesystem
|
||||
|
||||
```
|
||||
useradd -m kitten
|
||||
pushd /home/kitten
|
||||
ln -s /tmp/libreoffice
|
||||
popd
|
||||
chown -R kitten:kitten /home/kitten
|
||||
ln -s /proc/mounts /etc/mtab
|
||||
```
|
||||
|
||||
* Copy the script to the image
|
||||
|
||||
```
|
||||
sudo ./copy_to_final.sh
|
||||
```
|
||||
|
||||
Write the image on a SD card
|
||||
============================
|
||||
|
||||
*WARNING*: Make sure you write on the right filesystem
|
||||
|
||||
```
|
||||
sudo dd bs=4M if=FINAL_2015-02-16-raspbian-wheezy.img of=/dev/<FILESYSTEM>
|
||||
```
|
||||
|
||||
Run the tests
|
||||
=============
|
||||
|
||||
Make sure to set the filename of the image in `tests/run.sh`
|
||||
|
||||
```
|
||||
sudo ./run_tests.sh
|
||||
```
|
||||
|
36
TODO
|
@ -1,36 +0,0 @@
|
|||
TODO
|
||||
====
|
||||
|
||||
* the script locations should be changed in the next version so they don't sit
|
||||
next to teh rPi's example development code that ships with teh stock rPi
|
||||
* the system isn't optimised and should be : cleanup and making it as close to
|
||||
stock as possible
|
||||
[Npot sure] Starting process should be more obfuscated
|
||||
* strip exif data and leave it in a .txt file next to the image it came from
|
||||
=> exiftool
|
||||
[Done with remount] set filesystem of OS in RO (physical switch and/or remount OS)
|
||||
[OK] mount source key in RO and noexec <= also nosuid and nodev
|
||||
[OK] mount target key with noexec <= also nosuid and nodev
|
||||
* convert spreadsheets in csv ?
|
||||
[done in HTML] convert documents (pdfs/*office/...) in images ?
|
||||
[Not Needed] Have a look at Ghostscript to work on PDFs (.pdf -> .eps -> .png?)
|
||||
[do everything as user] do not run the conversions as root -> run in chroot
|
||||
* take eth0 down in /etc/netowrk/interfaces or in the groomer script disable the
|
||||
interface before anything happens
|
||||
* hdmi should stay up: solveable by poking the power management timer
|
||||
(better not to disable the PM completely)
|
||||
[Done] get rid of pdfbox. remove need for java
|
||||
[WIP] scripts to generate a SD card automatically (win/mac/linux)
|
||||
* move the scripts away from /opt/
|
||||
* strip back libreoffice to minimum required packages. in particular, if possible,
|
||||
remove libreoffice-java-common package
|
||||
* Write the groomer log on the destination key
|
||||
[Done] use /etc/mime.types and file -b --mime-type <filename> to find out the type of
|
||||
the file
|
||||
* Extract metadata from all the files => https://mat.boum.org/
|
||||
|
||||
HTML Files
|
||||
==========
|
||||
|
||||
- disable JS
|
||||
- cleanup external imports (js/css/images)
|
|
@ -0,0 +1 @@
|
|||
dwc_otg.lpm_enable=0 console=serial0,115200 console=tty1 root=/dev/mmcblk0p2 rootfstype=ext4 elevator=deadline fsck.repair=yes rootwait fastboot noswap ro consoleblank=0
|
|
@ -4,3 +4,4 @@ proc /proc proc defaults 0 0
|
|||
tmpfs /tmp tmpfs rw,size=64M,noexec,nodev,nosuid,mode=1777 0 0
|
||||
tmpfs /media tmpfs rw,size=64M,noexec,nodev,nosuid,mode=1777 0 0
|
||||
# a swapfile is not a swap partition, so no using swapon|off from here on, use dphys-swapfile swap[on|off] for that
|
||||
/dev/dest_key1 /media/kitten/dest_key auto rw,user,noauto,uid=1001,utf8=1 0 2
|
|
@ -19,7 +19,7 @@ cdrom:x:24:pi
|
|||
floppy:x:25:
|
||||
tape:x:26:
|
||||
sudo:x:27:pi
|
||||
audio:x:29:pi
|
||||
audio:x:29:pi,timidity
|
||||
dip:x:30:
|
||||
www-data:x:33:
|
||||
backup:x:34:
|
||||
|
@ -52,3 +52,4 @@ indiecity:x:1001:root
|
|||
spi:x:1002:pi
|
||||
gpio:x:1003:pi
|
||||
kitten:x:1004:
|
||||
timidity:x:110:
|
|
@ -0,0 +1,61 @@
|
|||
#
|
||||
# The PAM configuration file for the Shadow `su' service
|
||||
#
|
||||
|
||||
# This allows root to su without passwords (normal operation)
|
||||
auth sufficient pam_rootok.so
|
||||
|
||||
# Uncomment this to force users to be a member of group root
|
||||
# before they can use `su'. You can also add "group=foo"
|
||||
# to the end of this line if you want to use a group other
|
||||
# than the default "root" (but this may have side effect of
|
||||
# denying "root" user, unless she's a member of "foo" or explicitly
|
||||
# permitted earlier by e.g. "sufficient pam_rootok.so").
|
||||
# (Replaces the `SU_WHEEL_ONLY' option from login.defs)
|
||||
# auth required pam_wheel.so
|
||||
|
||||
# Uncomment this if you want wheel members to be able to
|
||||
# su without a password.
|
||||
# auth sufficient pam_wheel.so trust
|
||||
|
||||
# Uncomment this if you want members of a specific group to not
|
||||
# be allowed to use su at all.
|
||||
# auth required pam_wheel.so deny group=nosu
|
||||
|
||||
# Uncomment and edit /etc/security/time.conf if you need to set
|
||||
# time restrainst on su usage.
|
||||
# (Replaces the `PORTTIME_CHECKS_ENAB' option from login.defs
|
||||
# as well as /etc/porttime)
|
||||
# account requisite pam_time.so
|
||||
|
||||
# This module parses environment configuration file(s)
|
||||
# and also allows you to use an extended config
|
||||
# file /etc/security/pam_env.conf.
|
||||
#
|
||||
# parsing /etc/environment needs "readenv=1"
|
||||
session required pam_env.so readenv=1
|
||||
# locale variables are also kept into /etc/default/locale in etch
|
||||
# reading this file *in addition to /etc/environment* does not hurt
|
||||
session required pam_env.so readenv=1 envfile=/etc/default/locale
|
||||
|
||||
# Defines the MAIL environment variable
|
||||
# However, userdel also needs MAIL_DIR and MAIL_FILE variables
|
||||
# in /etc/login.defs to make sure that removing a user
|
||||
# also removes the user's mail spool file.
|
||||
# See comments in /etc/login.defs
|
||||
#
|
||||
# "nopen" stands to avoid reporting new mail when su'ing to another user
|
||||
session optional pam_mail.so nopen
|
||||
|
||||
# Sets up user limits according to /etc/security/limits.conf
|
||||
# (Replaces the use of /etc/limits in old login)
|
||||
#session required pam_limits.so
|
||||
|
||||
# The standard Unix authentication modules, used with
|
||||
# NIS (man nsswitch) as well as normal /etc/passwd and
|
||||
# /etc/shadow entries.
|
||||
@include common-auth
|
||||
@include common-account
|
||||
@include common-session
|
||||
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
root:x:0:0:root:/root:/bin/bash
|
||||
daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin
|
||||
bin:x:2:2:bin:/bin:/usr/sbin/nologin
|
||||
sys:x:3:3:sys:/dev:/usr/sbin/nologin
|
||||
sync:x:4:65534:sync:/bin:/bin/sync
|
||||
games:x:5:60:games:/usr/games:/usr/sbin/nologin
|
||||
man:x:6:12:man:/var/cache/man:/usr/sbin/nologin
|
||||
lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin
|
||||
mail:x:8:8:mail:/var/mail:/usr/sbin/nologin
|
||||
news:x:9:9:news:/var/spool/news:/usr/sbin/nologin
|
||||
uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin
|
||||
proxy:x:13:13:proxy:/bin:/usr/sbin/nologin
|
||||
www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin
|
||||
backup:x:34:34:backup:/var/backups:/usr/sbin/nologin
|
||||
list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin
|
||||
irc:x:39:39:ircd:/var/run/ircd:/usr/sbin/nologin
|
||||
gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin
|
||||
nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin
|
||||
systemd-timesync:x:100:103:systemd Time Synchronization,,,:/run/systemd:/bin/false
|
||||
systemd-network:x:101:104:systemd Network Management,,,:/run/systemd/netif:/bin/false
|
||||
systemd-resolve:x:102:105:systemd Resolver,,,:/run/systemd/resolve:/bin/false
|
||||
systemd-bus-proxy:x:103:106:systemd Bus Proxy,,,:/run/systemd:/bin/false
|
||||
pi:x:1000:1000:,,,:/home/pi:/bin/false
|
||||
sshd:x:104:65534::/var/run/sshd:/usr/sbin/nologin
|
||||
messagebus:x:105:110::/var/run/dbus:/bin/false
|
||||
avahi:x:106:111:Avahi mDNS daemon,,,:/var/run/avahi-daemon:/bin/false
|
||||
ntp:x:107:112::/home/ntp:/bin/false
|
||||
statd:x:108:65534::/var/lib/nfs:/bin/false
|
||||
lightdm:x:109:114:Light Display Manager:/var/lib/lightdm:/bin/false
|
||||
kitten:x:1001:1004::/home/kitten:/bin/bash
|
|
@ -0,0 +1,14 @@
|
|||
[udisks1]
|
||||
Identity=unix-group:plugdev
|
||||
Action=org.freedesktop.udisks.filesystem-mount;org.freedesktop.udisks.luks-unlock;org.freedesktop.udisks.drive-eject;org.freedesktop.udisks.drive-detach
|
||||
ResultAny=yes
|
||||
|
||||
[udisks2]
|
||||
Identity=unix-group:plugdev
|
||||
Action=org.freedesktop.udisks2.filesystem-mount;org.freedesktop.udisks2.encrypted-unlock;org.freedesktop.udisks2.eject-media;org.freedesktop.udisks2.power-off-drive
|
||||
ResultAny=yes
|
||||
|
||||
[udisks2-other-seat]
|
||||
Identity=unix-group:plugdev
|
||||
Action=org.freedesktop.udisks2.filesystem-mount-other-seat;org.freedesktop.udisks2.filesystem-unmount-others;org.freedesktop.udisks2.encrypted-unlock-other-seat;org.freedesktop.udisks2.eject-media-other-seat;org.freedesktop.udisks2.power-off-drive-other-seat
|
||||
ResultAny=yes
|
|
@ -0,0 +1,25 @@
|
|||
polkit.addRule(function(action, subject) {
|
||||
var YES = polkit.Result.YES;
|
||||
// NOTE: there must be a comma at the end of each line except for the last:
|
||||
var permission = {
|
||||
// required for udisks1:
|
||||
"org.freedesktop.udisks.filesystem-mount": YES,
|
||||
"org.freedesktop.udisks.luks-unlock": YES,
|
||||
"org.freedesktop.udisks.drive-eject": YES,
|
||||
"org.freedesktop.udisks.drive-detach": YES,
|
||||
// required for udisks2:
|
||||
"org.freedesktop.udisks2.filesystem-mount": YES,
|
||||
"org.freedesktop.udisks2.encrypted-unlock": YES,
|
||||
"org.freedesktop.udisks2.eject-media": YES,
|
||||
"org.freedesktop.udisks2.power-off-drive": YES,
|
||||
// required for udisks2 if using udiskie from another seat (e.g. systemd):
|
||||
"org.freedesktop.udisks2.filesystem-mount-other-seat": YES,
|
||||
"org.freedesktop.udisks2.filesystem-unmount-others": YES,
|
||||
"org.freedesktop.udisks2.encrypted-unlock-other-seat": YES,
|
||||
"org.freedesktop.udisks2.eject-media-other-seat": YES,
|
||||
"org.freedesktop.udisks2.power-off-drive-other-seat": YES
|
||||
};
|
||||
if (subject.isInGroup("plugdev")) {
|
||||
return permission[action.id];
|
||||
}
|
||||
});
|
|
@ -0,0 +1,19 @@
|
|||
#!/bin/sh
|
||||
# Part of raspi-config http://github.com/asb/raspi-config
|
||||
#
|
||||
# See LICENSE file for copyright and license details
|
||||
|
||||
# Should be installed to /etc/profile.d/raspi-config.sh to force raspi-config
|
||||
# to run at initial login
|
||||
|
||||
# You may also want to set automatic login in /etc/inittab on tty1 by adding a
|
||||
# line such as:
|
||||
# 1:2345:respawn:/bin/login -f root tty1 </dev/tty1 >/dev/tty1 2>&1 # RPICFG_TO_DISABLE
|
||||
|
||||
if [ $(id -u) -ne 0 ]; then
|
||||
printf "\nNOTICE: the software on this Raspberry Pi has not been fully configured. Please run 'sudo raspi-config'\n\n"
|
||||
else
|
||||
# Disable raspi-config at the first run.
|
||||
# raspi-config
|
||||
exec login -f pi
|
||||
fi
|
|
@ -10,25 +10,24 @@
|
|||
# bits.
|
||||
#
|
||||
# By default this script does nothing.
|
||||
#
|
||||
## The above are the default comments for rc.local. For Circlean, rc.local has
|
||||
## been modified to start the grooming process after booting.
|
||||
|
||||
clean(){
|
||||
echo 'Rc Local done, quit.'
|
||||
echo "GROOMER: Copy over, sleeping 15s before shutdown."
|
||||
sleep 15
|
||||
echo "GROOMER: rc.local done, shutting down."
|
||||
/sbin/shutdown -P -h now
|
||||
}
|
||||
|
||||
# Print the IP address
|
||||
_IP=$(hostname -I) || true
|
||||
if [ "$_IP" ]; then
|
||||
printf "My IP address is %s\n" "$_IP"
|
||||
fi
|
||||
echo "GROOMER: end of boot, running rc.local."
|
||||
|
||||
if [ -e /dev/sda ]; then
|
||||
if [ -e /dev/sdb ]; then
|
||||
# avoid possible misuse
|
||||
/sbin/ifconfig eth0 down
|
||||
trap clean EXIT TERM INT
|
||||
cd /opt/groomer
|
||||
#/usr/sbin/led &
|
||||
/usr/sbin/led &
|
||||
./init.sh
|
||||
fi
|
||||
fi
|
|
@ -0,0 +1,14 @@
|
|||
[Unit]
|
||||
Description=/etc/rc.local Compatibility
|
||||
ConditionPathExists=/etc/rc.local
|
||||
|
||||
[Service]
|
||||
Type=forking
|
||||
ExecStart=/etc/rc.local start
|
||||
TimeoutSec=0
|
||||
StandardOutput=tty
|
||||
RemainAfterExit=yes
|
||||
SysVStartPriority=99
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -0,0 +1,3 @@
|
|||
# The purpose of this rules file is to ensure that the top left usb port and its partitions have a symlink to /dev/source_key[num], and the other ports to /dev/dest_key[num]
|
||||
KERNEL=="sd*", KERNELS=="1-1.2", SUBSYSTEMS=="usb", SYMLINK+="source_key%n" GROUP="kitten" OWNER="kitten" MODE="0666"
|
||||
KERNEL=="sd*", KERNELS=="1-1.[3-5]", SUBSYSTEMS=="usb", SYMLINK+="dest_key%n" GROUP="kitten" OWNER="kitten" MODE="0666"
|
|
@ -0,0 +1 @@
|
|||
SUBSYSTEM=="hidraw", DRIVERS=="usbhid", RUN+="/bin/bash -c 'cd /sys/devices/platform/bcm2708_usb/usb1/1-1 ; for d in $(ls -d 1-1.[2-5]); do if [ $(ls -lR $d | grep -c usbhid) -gt 0 ] ; then echo 0 > $d/authorized ; fi ; done'"
|
|
@ -0,0 +1,5 @@
|
|||
# This udev rule is designed to make the hardware appear more like the actual raspi hardware when emulating in Qemu. You might have to change sdc to sda or sdb depending on where the filesystem you are emulating is available from in /dev/.
|
||||
# See http://pub.phyks.me/respawn/mypersonaldata/public/2014-05-20-11-08-01/ for more info.
|
||||
KERNEL=="sdc", SYMLINK+="mmcblk0"
|
||||
KERNEL=="sdc?", SYMLINK+="mmcblk0p%n",
|
||||
#KERNEL=="sdc2", SYMLINK+="root"
|
|
@ -0,0 +1 @@
|
|||
NOTE: This file is copied to the destination key when CIRCLean starts, and deleted once it completes successfully. If you see this file on your destination key it means that the copying process was INTERRUPTED or an ERROR occurred. You should treat files on the source key and destination key with care, and consider repeating the sanitization process. If you think you have found a bug, please report it at https://github.com/CIRCL/Circlean.
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eu
|
||||
|
||||
readonly USERNAME="kitten"
|
||||
readonly ID=$(/usr/bin/id -u)
|
||||
|
||||
# Paths used in multiple scripts
|
||||
readonly SRC_DEV="/dev/source_key"
|
||||
|
||||
readonly DST_DEV="/dev/dest_key"
|
||||
readonly DST_MNT="/media/kitten/dest_key"
|
||||
|
||||
readonly TEMP="${DST_MNT}/temp"
|
||||
readonly LOGS_DIR="${DST_MNT}/logs"
|
||||
readonly DEBUG_LOG="/tmp/groomer_debug_log.txt"
|
||||
readonly MUSIC_DIR="/opt/midi/"
|
||||
|
||||
# Commands
|
||||
readonly SYNC="/bin/sync"
|
||||
readonly TIMIDITY="/usr/bin/timidity"
|
||||
readonly MOUNT="udisksctl mount"
|
||||
readonly UMOUNT="udisksctl unmount"
|
||||
|
||||
# Config flags
|
||||
readonly DEBUG=false
|
||||
readonly MUSIC=true
|
|
@ -0,0 +1,84 @@
|
|||
#!/bin/bash
|
||||
|
||||
clean(){
|
||||
if [ "${DEBUG}" = true ]; then
|
||||
sleep 20
|
||||
fi
|
||||
|
||||
# Write anything in memory to disk
|
||||
${SYNC}
|
||||
|
||||
# Remove temporary files from destination key
|
||||
rm -rf "${TEMP}"
|
||||
}
|
||||
|
||||
check_not_root() {
|
||||
if ! [ "${ID}" -ge "1000" ]; then
|
||||
echo "GROOMER: groomer.sh cannot run as root."
|
||||
exit
|
||||
fi
|
||||
}
|
||||
|
||||
check_has_partitions () {
|
||||
local partitions=$1
|
||||
if [ -z "${partitions}" ]; then
|
||||
echo "GROOMER: ${SRC_DEV} does not have any partitions."
|
||||
exit
|
||||
fi
|
||||
}
|
||||
|
||||
run_groomer() {
|
||||
local dev_partitions
|
||||
# Find the partition names on the device
|
||||
dev_partitions=$(ls "${SRC_DEV}"* | grep "${SRC_DEV}[1-9][0-6]*" || true)
|
||||
check_has_partitions dev_partitions
|
||||
local partcount=1
|
||||
local partition
|
||||
for partition in ${dev_partitions}
|
||||
do
|
||||
echo "GROOMER: Processing partition ${partition}"
|
||||
# Mount the current partition in write mode
|
||||
SRC_MNT=`${MOUNT} -o rw -b ${partition}| sed -ne 's/Mounted \(.*\) at \(\/media\/kitten\/.*\).$/\2/p'`
|
||||
if [ -z "$SRC_MNT" ]; then
|
||||
echo "Unable to mount source partition (${partition})."
|
||||
continue
|
||||
fi
|
||||
# Mark any autorun.inf files as dangerous on the source device to be extra careful
|
||||
ls "${SRC_MNT}" | grep -i autorun.inf | xargs -I {} mv "${SRC_MNT}"/{} "${SRC_MNT}"/DANGEROUS_{}_DANGEROUS || true
|
||||
# Unmount and remount the current partition in read-only mode
|
||||
${UMOUNT} -b "${partition}"
|
||||
|
||||
if ${MOUNT} -o ro -b "${partition}"; then
|
||||
echo "GROOMER: ${partition} mounted at ${SRC_MNT}"
|
||||
|
||||
# Create a directory on ${DST_MNT} named PARTION_$PARTCOUNT
|
||||
local target_dir="${DST_MNT}/FROM_PARTITION_${partcount}"
|
||||
mkdir -p "${target_dir}"
|
||||
|
||||
# Run the current partition through filecheck.py
|
||||
filecheck.py --source "${SRC_MNT}" --destination "${target_dir}" || true
|
||||
|
||||
# List destination files (recursively) for debugging
|
||||
if [ "${DEBUG}" = true ]; then
|
||||
ls -lR "${target_dir}"
|
||||
fi
|
||||
${UMOUNT} -b "${partition}"
|
||||
else
|
||||
# Previous command (mounting current partition) failed
|
||||
echo "GROOMER: Unable to mount ${partition} on ${SRC_MNT}"
|
||||
fi
|
||||
let partcount=$((partcount + 1))
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
set -eu # exit when a line returns non-0 status, treat unset variables as errors
|
||||
trap clean EXIT TERM INT # run clean when the script ends or is interrupted
|
||||
source ./config.sh # get config values
|
||||
if [ "${DEBUG}" = true ]; then
|
||||
set -x
|
||||
fi
|
||||
run_groomer
|
||||
}
|
||||
|
||||
main
|
|
@ -0,0 +1,49 @@
|
|||
#!/bin/bash
|
||||
|
||||
clean(){
|
||||
if [ "${DEBUG}" = true ]; then
|
||||
sleep 20
|
||||
fi
|
||||
echo "GROOMER: cleaning up after init.sh."
|
||||
"${SYNC}"
|
||||
# Stop the music from playing
|
||||
kill -9 "$(cat /tmp/music.pid)"
|
||||
rm -f /tmp/music.pid
|
||||
}
|
||||
|
||||
check_is_root() {
|
||||
if [ "${ID}" -ne 0 ]; then # -ne is an integer comparison instead of a string comparison
|
||||
echo "GROOMER: This script has to be run as root."
|
||||
exit
|
||||
fi
|
||||
}
|
||||
|
||||
start_music() {
|
||||
./music.sh &
|
||||
echo $! > /tmp/music.pid
|
||||
}
|
||||
|
||||
run_groomer() {
|
||||
if [ "${DEBUG}" = true ]; then
|
||||
lsblk |& tee -a "${DEBUG_LOG}" # list block storage devices for debugging
|
||||
su "${USERNAME}" -c ./mount_dest.sh |& tee -a "${DEBUG_LOG}"
|
||||
else
|
||||
su "${USERNAME}" -c ./mount_dest.sh
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
set -eu # exit when a line returns non-0 status, treat unset variables as errors
|
||||
trap clean EXIT TERM INT # run clean when the script ends or is interrupted
|
||||
source ./config.sh # get config values
|
||||
if [ "${DEBUG}" = true ]; then
|
||||
set -x
|
||||
fi
|
||||
check_is_root
|
||||
if [ "${MUSIC}" = true ]; then
|
||||
start_music
|
||||
fi
|
||||
run_groomer
|
||||
}
|
||||
|
||||
main
|
|
@ -0,0 +1,81 @@
|
|||
#!/bin/bash
|
||||
|
||||
clean(){
|
||||
if [ "${DEBUG}" = true ]; then
|
||||
sleep 20
|
||||
# Copy the temporary logfile to the destination key
|
||||
cp "${DEBUG_LOG}" "${DST_MNT}/groomer_debug_log.txt"
|
||||
fi
|
||||
echo "GROOMER: Cleaning up in mount_keys.sh."
|
||||
rm -rf "${DST_MNT}/IN_PROGRESS.txt"*
|
||||
${SYNC} # Write anything in memory to disk
|
||||
# Unmount destination
|
||||
${UMOUNT} -b "${DST_DEV}1"
|
||||
exit
|
||||
}
|
||||
|
||||
check_not_root() {
|
||||
if ! [ "${ID}" -ge "1000" ]; then
|
||||
echo "GROOMER: mount_keys.sh cannot run as root."
|
||||
exit
|
||||
fi
|
||||
}
|
||||
|
||||
check_source_exists() {
|
||||
if [ ! -b "${SRC_DEV}1" ]; then
|
||||
echo "GROOMER: Source device (${SRC_DEV}1) does not exist."
|
||||
exit
|
||||
fi
|
||||
}
|
||||
|
||||
check_dest_exists() {
|
||||
if [ ! -b "${DST_DEV}1" ]; then
|
||||
echo "GROOMER: Destination device (${DST_DEV}1) does not exist."
|
||||
exit
|
||||
fi
|
||||
}
|
||||
|
||||
unmount_dest_if_mounted() {
|
||||
if /bin/mount|grep "${DST_MNT}"; then
|
||||
${UMOUNT} -b "${DST_DEV}1" || true
|
||||
fi
|
||||
}
|
||||
|
||||
mount_dest_partition() {
|
||||
if ${MOUNT} -o rw -b "${DST_DEV}1"; then # mount automatically mounts on /media/ (at /media/dst in this case).
|
||||
echo "GROOMER: Destination USB device (${DST_DEV}1) mounted at ${DST_MNT}"
|
||||
else
|
||||
echo "GROOMER: Unable to mount ${DST_DEV}1 on ${DST_MNT}"
|
||||
exit
|
||||
fi
|
||||
}
|
||||
|
||||
copy_in_progress_file() {
|
||||
cp "/opt/groomer/IN_PROGRESS.txt" "${DST_MNT}/IN_PROGRESS.txt"
|
||||
}
|
||||
|
||||
prepare_dest_partition() {
|
||||
rm -rf "${DST_MNT}/FROM_PARTITION_"* # Remove any existing "FROM_PARTITION_" directories
|
||||
# Prepare temp dir and make sure it's empty if it already exists:
|
||||
mkdir -p "${TEMP}"
|
||||
rm -rf "${TEMP:?}/"*
|
||||
}
|
||||
|
||||
main() {
|
||||
set -eu # exit when a line returns non-0 status, treat unset variables as errors
|
||||
trap clean EXIT TERM INT # run clean when the script ends or is interrupted
|
||||
source ./config.sh # get config values
|
||||
if [ "${DEBUG}" = true ]; then
|
||||
set -x
|
||||
fi
|
||||
check_not_root
|
||||
check_source_exists
|
||||
check_dest_exists
|
||||
unmount_dest_if_mounted
|
||||
mount_dest_partition
|
||||
copy_in_progress_file
|
||||
prepare_dest_partition
|
||||
./groomer.sh
|
||||
}
|
||||
|
||||
main
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/bash
|
||||
|
||||
killed(){
|
||||
echo 'Music stopped.'
|
||||
}
|
||||
|
||||
run_timidity() {
|
||||
# Force output on analog
|
||||
amixer cset numid=3 1
|
||||
files=(${MUSIC_DIR}*)
|
||||
while true; do
|
||||
# -id flags set interface to "dumb" and -qq silences most/all terminal output
|
||||
"${TIMIDITY}" -idqq "${files[RANDOM % ${#files[@]}]}"
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
set -eu # exit when a line returns non-0 status, treat unset variables as errors
|
||||
trap killed EXIT TERM INT # run clean when the script ends or is interrupted
|
||||
source ./config.sh # get config values
|
||||
run_timidity
|
||||
}
|
||||
|
||||
main
|
|
@ -0,0 +1,930 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
__description__ = 'Tool to test a PDF file'
|
||||
__author__ = 'Didier Stevens'
|
||||
__version__ = '0.2.1'
|
||||
__date__ = '2014/10/18'
|
||||
|
||||
"""
|
||||
|
||||
Tool to test a PDF file
|
||||
|
||||
Source code put in public domain by Didier Stevens, no Copyright
|
||||
https://DidierStevens.com
|
||||
Use at your own risk
|
||||
|
||||
History:
|
||||
2009/03/27: start
|
||||
2009/03/28: scan option
|
||||
2009/03/29: V0.0.2: xml output
|
||||
2009/03/31: V0.0.3: /ObjStm suggested by Dion
|
||||
2009/04/02: V0.0.4: added ErrorMessage
|
||||
2009/04/20: V0.0.5: added Dates
|
||||
2009/04/21: V0.0.6: added entropy
|
||||
2009/04/22: added disarm
|
||||
2009/04/29: finished disarm
|
||||
2009/05/13: V0.0.7: added cPDFEOF
|
||||
2009/07/24: V0.0.8: added /AcroForm and /RichMedia, simplified %PDF header regex, extra date format (without TZ)
|
||||
2009/07/25: added input redirection, option --force
|
||||
2009/10/13: V0.0.9: added detection for CVE-2009-3459; added /RichMedia to disarm
|
||||
2010/01/11: V0.0.10: relaxed %PDF header checking
|
||||
2010/04/28: V0.0.11: added /Launch
|
||||
2010/09/21: V0.0.12: fixed cntCharsAfterLastEOF bug; fix by Russell Holloway
|
||||
2011/12/29: updated for Python 3, added keyword /EmbeddedFile
|
||||
2012/03/03: added PDFiD2JSON; coded by Brandon Dixon
|
||||
2013/02/10: V0.1.0: added http/https support; added support for ZIP file with password 'infected'
|
||||
2013/03/11: V0.1.1: fixes for Python 3
|
||||
2013/03/13: V0.1.2: Added error handling for files; added /XFA
|
||||
2013/11/01: V0.2.0: Added @file & plugins
|
||||
2013/11/02: continue
|
||||
2013/11/04: added options -c, -m, -v
|
||||
2013/11/06: added option -S
|
||||
2013/11/08: continue
|
||||
2013/11/09: added option -o
|
||||
2013/11/15: refactoring
|
||||
2014/09/30: added CSV header
|
||||
2014/10/16: V0.2.1: added output when plugin & file not pdf
|
||||
2014/10/18: some fixes for Python 3
|
||||
|
||||
Todo:
|
||||
- update XML example (entropy, EOF)
|
||||
- code review, cleanup
|
||||
"""
|
||||
|
||||
import optparse
|
||||
import os
|
||||
import re
|
||||
import xml.dom.minidom
|
||||
import traceback
|
||||
import math
|
||||
import operator
|
||||
import os.path
|
||||
import sys
|
||||
import json
|
||||
import zipfile
|
||||
import collections
|
||||
import glob
|
||||
try:
|
||||
import urllib2
|
||||
urllib23 = urllib2
|
||||
except:
|
||||
import urllib.request
|
||||
urllib23 = urllib.request
|
||||
|
||||
#Convert 2 Bytes If Python 3
|
||||
def C2BIP3(string):
|
||||
if sys.version_info[0] > 2:
|
||||
return bytes([ord(x) for x in string])
|
||||
else:
|
||||
return string
|
||||
|
||||
class cBinaryFile:
|
||||
def __init__(self, file):
|
||||
self.file = file
|
||||
if file == '':
|
||||
self.infile = sys.stdin
|
||||
elif file.lower().startswith('http://') or file.lower().startswith('https://'):
|
||||
try:
|
||||
if sys.hexversion >= 0x020601F0:
|
||||
self.infile = urllib23.urlopen(file, timeout=5)
|
||||
else:
|
||||
self.infile = urllib23.urlopen(file)
|
||||
except urllib23.HTTPError:
|
||||
print('Error accessing URL %s' % file)
|
||||
print(sys.exc_info()[1])
|
||||
sys.exit()
|
||||
elif file.lower().endswith('.zip'):
|
||||
try:
|
||||
self.zipfile = zipfile.ZipFile(file, 'r')
|
||||
self.infile = self.zipfile.open(self.zipfile.infolist()[0], 'r', C2BIP3('infected'))
|
||||
except:
|
||||
print('Error opening file %s' % file)
|
||||
print(sys.exc_info()[1])
|
||||
sys.exit()
|
||||
else:
|
||||
try:
|
||||
self.infile = open(file, 'rb')
|
||||
except:
|
||||
print('Error opening file %s' % file)
|
||||
print(sys.exc_info()[1])
|
||||
sys.exit()
|
||||
self.ungetted = []
|
||||
|
||||
def byte(self):
|
||||
if len(self.ungetted) != 0:
|
||||
return self.ungetted.pop()
|
||||
inbyte = self.infile.read(1)
|
||||
if not inbyte or inbyte == '':
|
||||
self.infile.close()
|
||||
return None
|
||||
return ord(inbyte)
|
||||
|
||||
def bytes(self, size):
|
||||
if size <= len(self.ungetted):
|
||||
result = self.ungetted[0:size]
|
||||
del self.ungetted[0:size]
|
||||
return result
|
||||
inbytes = self.infile.read(size - len(self.ungetted))
|
||||
if inbytes == '':
|
||||
self.infile.close()
|
||||
if type(inbytes) == type(''):
|
||||
result = self.ungetted + [ord(b) for b in inbytes]
|
||||
else:
|
||||
result = self.ungetted + [b for b in inbytes]
|
||||
self.ungetted = []
|
||||
return result
|
||||
|
||||
def unget(self, byte):
|
||||
self.ungetted.append(byte)
|
||||
|
||||
def ungets(self, bytes):
|
||||
bytes.reverse()
|
||||
self.ungetted.extend(bytes)
|
||||
|
||||
class cPDFDate:
|
||||
def __init__(self):
|
||||
self.state = 0
|
||||
|
||||
def parse(self, char):
|
||||
if char == 'D':
|
||||
self.state = 1
|
||||
return None
|
||||
elif self.state == 1:
|
||||
if char == ':':
|
||||
self.state = 2
|
||||
self.digits1 = ''
|
||||
else:
|
||||
self.state = 0
|
||||
return None
|
||||
elif self.state == 2:
|
||||
if len(self.digits1) < 14:
|
||||
if char >= '0' and char <= '9':
|
||||
self.digits1 += char
|
||||
return None
|
||||
else:
|
||||
self.state = 0
|
||||
return None
|
||||
elif char == '+' or char == '-' or char == 'Z':
|
||||
self.state = 3
|
||||
self.digits2 = ''
|
||||
self.TZ = char
|
||||
return None
|
||||
elif char == '"':
|
||||
self.state = 0
|
||||
self.date = 'D:' + self.digits1
|
||||
return self.date
|
||||
elif char < '0' or char > '9':
|
||||
self.state = 0
|
||||
self.date = 'D:' + self.digits1
|
||||
return self.date
|
||||
else:
|
||||
self.state = 0
|
||||
return None
|
||||
elif self.state == 3:
|
||||
if len(self.digits2) < 2:
|
||||
if char >= '0' and char <= '9':
|
||||
self.digits2 += char
|
||||
return None
|
||||
else:
|
||||
self.state = 0
|
||||
return None
|
||||
elif len(self.digits2) == 2:
|
||||
if char == "'":
|
||||
self.digits2 += char
|
||||
return None
|
||||
else:
|
||||
self.state = 0
|
||||
return None
|
||||
elif len(self.digits2) < 5:
|
||||
if char >= '0' and char <= '9':
|
||||
self.digits2 += char
|
||||
if len(self.digits2) == 5:
|
||||
self.state = 0
|
||||
self.date = 'D:' + self.digits1 + self.TZ + self.digits2
|
||||
return self.date
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
self.state = 0
|
||||
return None
|
||||
|
||||
def fEntropy(countByte, countTotal):
|
||||
x = float(countByte) / countTotal
|
||||
if x > 0:
|
||||
return - x * math.log(x, 2)
|
||||
else:
|
||||
return 0.0
|
||||
|
||||
class cEntropy:
|
||||
def __init__(self):
|
||||
self.allBucket = [0 for i in range(0, 256)]
|
||||
self.streamBucket = [0 for i in range(0, 256)]
|
||||
|
||||
def add(self, byte, insideStream):
|
||||
self.allBucket[byte] += 1
|
||||
if insideStream:
|
||||
self.streamBucket[byte] += 1
|
||||
|
||||
def removeInsideStream(self, byte):
|
||||
if self.streamBucket[byte] > 0:
|
||||
self.streamBucket[byte] -= 1
|
||||
|
||||
def calc(self):
|
||||
self.nonStreamBucket = map(operator.sub, self.allBucket, self.streamBucket)
|
||||
allCount = sum(self.allBucket)
|
||||
streamCount = sum(self.streamBucket)
|
||||
nonStreamCount = sum(self.nonStreamBucket)
|
||||
return (allCount, sum(map(lambda x: fEntropy(x, allCount), self.allBucket)), streamCount, sum(map(lambda x: fEntropy(x, streamCount), self.streamBucket)), nonStreamCount, sum(map(lambda x: fEntropy(x, nonStreamCount), self.nonStreamBucket)))
|
||||
|
||||
class cPDFEOF:
|
||||
def __init__(self):
|
||||
self.token = ''
|
||||
self.cntEOFs = 0
|
||||
|
||||
def parse(self, char):
|
||||
if self.cntEOFs > 0:
|
||||
self.cntCharsAfterLastEOF += 1
|
||||
if self.token == '' and char == '%':
|
||||
self.token += char
|
||||
return
|
||||
elif self.token == '%' and char == '%':
|
||||
self.token += char
|
||||
return
|
||||
elif self.token == '%%' and char == 'E':
|
||||
self.token += char
|
||||
return
|
||||
elif self.token == '%%E' and char == 'O':
|
||||
self.token += char
|
||||
return
|
||||
elif self.token == '%%EO' and char == 'F':
|
||||
self.token += char
|
||||
return
|
||||
elif self.token == '%%EOF' and (char == '\n' or char == '\r' or char == ' ' or char == '\t'):
|
||||
self.cntEOFs += 1
|
||||
self.cntCharsAfterLastEOF = 0
|
||||
if char == '\n':
|
||||
self.token = ''
|
||||
else:
|
||||
self.token += char
|
||||
return
|
||||
elif self.token == '%%EOF\r':
|
||||
if char == '\n':
|
||||
self.cntCharsAfterLastEOF = 0
|
||||
self.token = ''
|
||||
else:
|
||||
self.token = ''
|
||||
|
||||
def FindPDFHeaderRelaxed(oBinaryFile):
|
||||
bytes = oBinaryFile.bytes(1024)
|
||||
index = ''.join([chr(byte) for byte in bytes]).find('%PDF')
|
||||
if index == -1:
|
||||
oBinaryFile.ungets(bytes)
|
||||
return ([], None)
|
||||
for endHeader in range(index + 4, index + 4 + 10):
|
||||
if bytes[endHeader] == 10 or bytes[endHeader] == 13:
|
||||
break
|
||||
oBinaryFile.ungets(bytes[endHeader:])
|
||||
return (bytes[0:endHeader], ''.join([chr(byte) for byte in bytes[index:endHeader]]))
|
||||
|
||||
def Hexcode2String(char):
|
||||
if type(char) == int:
|
||||
return '#%02x' % char
|
||||
else:
|
||||
return char
|
||||
|
||||
def SwapCase(char):
|
||||
if type(char) == int:
|
||||
return ord(chr(char).swapcase())
|
||||
else:
|
||||
return char.swapcase()
|
||||
|
||||
def HexcodeName2String(hexcodeName):
|
||||
return ''.join(map(Hexcode2String, hexcodeName))
|
||||
|
||||
def SwapName(wordExact):
|
||||
return map(SwapCase, wordExact)
|
||||
|
||||
def UpdateWords(word, wordExact, slash, words, hexcode, allNames, lastName, insideStream, oEntropy, fOut):
|
||||
if word != '':
|
||||
if slash + word in words:
|
||||
words[slash + word][0] += 1
|
||||
if hexcode:
|
||||
words[slash + word][1] += 1
|
||||
elif slash == '/' and allNames:
|
||||
words[slash + word] = [1, 0]
|
||||
if hexcode:
|
||||
words[slash + word][1] += 1
|
||||
if slash == '/':
|
||||
lastName = slash + word
|
||||
if slash == '':
|
||||
if word == 'stream':
|
||||
insideStream = True
|
||||
if word == 'endstream':
|
||||
if insideStream == True and oEntropy != None:
|
||||
for char in 'endstream':
|
||||
oEntropy.removeInsideStream(ord(char))
|
||||
insideStream = False
|
||||
if fOut != None:
|
||||
if slash == '/' and '/' + word in ('/JS', '/JavaScript', '/AA', '/OpenAction', '/JBIG2Decode', '/RichMedia', '/Launch'):
|
||||
wordExactSwapped = HexcodeName2String(SwapName(wordExact))
|
||||
fOut.write(C2BIP3(wordExactSwapped))
|
||||
print('/%s -> /%s' % (HexcodeName2String(wordExact), wordExactSwapped))
|
||||
else:
|
||||
fOut.write(C2BIP3(HexcodeName2String(wordExact)))
|
||||
return ('', [], False, lastName, insideStream)
|
||||
|
||||
class cCVE_2009_3459:
|
||||
def __init__(self):
|
||||
self.count = 0
|
||||
|
||||
def Check(self, lastName, word):
|
||||
if (lastName == '/Colors' and word.isdigit() and int(word) > 2^24): # decided to alert when the number of colors is expressed with more than 3 bytes
|
||||
self.count += 1
|
||||
|
||||
def XMLAddAttribute(xmlDoc, name, value=None):
|
||||
att = xmlDoc.createAttribute(name)
|
||||
xmlDoc.documentElement.setAttributeNode(att)
|
||||
if value != None:
|
||||
att.nodeValue = value
|
||||
|
||||
def PDFiD(file, allNames=False, extraData=False, disarm=False, force=False):
|
||||
"""Example of XML output:
|
||||
<PDFiD ErrorOccured="False" ErrorMessage="" Filename="test.pdf" Header="%PDF-1.1" IsPDF="True" Version="0.0.4" Entropy="4.28">
|
||||
<Keywords>
|
||||
<Keyword Count="7" HexcodeCount="0" Name="obj"/>
|
||||
<Keyword Count="7" HexcodeCount="0" Name="endobj"/>
|
||||
<Keyword Count="1" HexcodeCount="0" Name="stream"/>
|
||||
<Keyword Count="1" HexcodeCount="0" Name="endstream"/>
|
||||
<Keyword Count="1" HexcodeCount="0" Name="xref"/>
|
||||
<Keyword Count="1" HexcodeCount="0" Name="trailer"/>
|
||||
<Keyword Count="1" HexcodeCount="0" Name="startxref"/>
|
||||
<Keyword Count="1" HexcodeCount="0" Name="/Page"/>
|
||||
<Keyword Count="0" HexcodeCount="0" Name="/Encrypt"/>
|
||||
<Keyword Count="1" HexcodeCount="0" Name="/JS"/>
|
||||
<Keyword Count="1" HexcodeCount="0" Name="/JavaScript"/>
|
||||
<Keyword Count="0" HexcodeCount="0" Name="/AA"/>
|
||||
<Keyword Count="1" HexcodeCount="0" Name="/OpenAction"/>
|
||||
<Keyword Count="0" HexcodeCount="0" Name="/JBIG2Decode"/>
|
||||
</Keywords>
|
||||
<Dates>
|
||||
<Date Value="D:20090128132916+01'00" Name="/ModDate"/>
|
||||
</Dates>
|
||||
</PDFiD>
|
||||
"""
|
||||
|
||||
word = ''
|
||||
wordExact = []
|
||||
hexcode = False
|
||||
lastName = ''
|
||||
insideStream = False
|
||||
keywords = ('obj',
|
||||
'endobj',
|
||||
'stream',
|
||||
'endstream',
|
||||
'xref',
|
||||
'trailer',
|
||||
'startxref',
|
||||
'/Page',
|
||||
'/Encrypt',
|
||||
'/ObjStm',
|
||||
'/JS',
|
||||
'/JavaScript',
|
||||
'/AA',
|
||||
'/OpenAction',
|
||||
'/AcroForm',
|
||||
'/JBIG2Decode',
|
||||
'/RichMedia',
|
||||
'/Launch',
|
||||
'/EmbeddedFile',
|
||||
'/XFA',
|
||||
)
|
||||
words = {}
|
||||
dates = []
|
||||
for keyword in keywords:
|
||||
words[keyword] = [0, 0]
|
||||
slash = ''
|
||||
xmlDoc = xml.dom.minidom.getDOMImplementation().createDocument(None, 'PDFiD', None)
|
||||
XMLAddAttribute(xmlDoc, 'Version', __version__)
|
||||
XMLAddAttribute(xmlDoc, 'Filename', file)
|
||||
attErrorOccured = XMLAddAttribute(xmlDoc, 'ErrorOccured', 'False')
|
||||
attErrorMessage = XMLAddAttribute(xmlDoc, 'ErrorMessage', '')
|
||||
|
||||
oPDFDate = None
|
||||
oEntropy = None
|
||||
oPDFEOF = None
|
||||
oCVE_2009_3459 = cCVE_2009_3459()
|
||||
try:
|
||||
attIsPDF = xmlDoc.createAttribute('IsPDF')
|
||||
xmlDoc.documentElement.setAttributeNode(attIsPDF)
|
||||
oBinaryFile = cBinaryFile(file)
|
||||
if extraData:
|
||||
oPDFDate = cPDFDate()
|
||||
oEntropy = cEntropy()
|
||||
oPDFEOF = cPDFEOF()
|
||||
(bytesHeader, pdfHeader) = FindPDFHeaderRelaxed(oBinaryFile)
|
||||
if disarm:
|
||||
(pathfile, extension) = os.path.splitext(file)
|
||||
fOut = open(pathfile + '.disarmed' + extension, 'wb')
|
||||
for byteHeader in bytesHeader:
|
||||
fOut.write(C2BIP3(chr(byteHeader)))
|
||||
else:
|
||||
fOut = None
|
||||
if oEntropy != None:
|
||||
for byteHeader in bytesHeader:
|
||||
oEntropy.add(byteHeader, insideStream)
|
||||
if pdfHeader == None and not force:
|
||||
attIsPDF.nodeValue = 'False'
|
||||
return xmlDoc
|
||||
else:
|
||||
if pdfHeader == None:
|
||||
attIsPDF.nodeValue = 'False'
|
||||
pdfHeader = ''
|
||||
else:
|
||||
attIsPDF.nodeValue = 'True'
|
||||
att = xmlDoc.createAttribute('Header')
|
||||
att.nodeValue = repr(pdfHeader[0:10]).strip("'")
|
||||
xmlDoc.documentElement.setAttributeNode(att)
|
||||
byte = oBinaryFile.byte()
|
||||
while byte != None:
|
||||
char = chr(byte)
|
||||
charUpper = char.upper()
|
||||
if charUpper >= 'A' and charUpper <= 'Z' or charUpper >= '0' and charUpper <= '9':
|
||||
word += char
|
||||
wordExact.append(char)
|
||||
elif slash == '/' and char == '#':
|
||||
d1 = oBinaryFile.byte()
|
||||
if d1 != None:
|
||||
d2 = oBinaryFile.byte()
|
||||
if d2 != None and (chr(d1) >= '0' and chr(d1) <= '9' or chr(d1).upper() >= 'A' and chr(d1).upper() <= 'F') and (chr(d2) >= '0' and chr(d2) <= '9' or chr(d2).upper() >= 'A' and chr(d2).upper() <= 'F'):
|
||||
word += chr(int(chr(d1) + chr(d2), 16))
|
||||
wordExact.append(int(chr(d1) + chr(d2), 16))
|
||||
hexcode = True
|
||||
if oEntropy != None:
|
||||
oEntropy.add(d1, insideStream)
|
||||
oEntropy.add(d2, insideStream)
|
||||
if oPDFEOF != None:
|
||||
oPDFEOF.parse(d1)
|
||||
oPDFEOF.parse(d2)
|
||||
else:
|
||||
oBinaryFile.unget(d2)
|
||||
oBinaryFile.unget(d1)
|
||||
(word, wordExact, hexcode, lastName, insideStream) = UpdateWords(word, wordExact, slash, words, hexcode, allNames, lastName, insideStream, oEntropy, fOut)
|
||||
if disarm:
|
||||
fOut.write(C2BIP3(char))
|
||||
else:
|
||||
oBinaryFile.unget(d1)
|
||||
(word, wordExact, hexcode, lastName, insideStream) = UpdateWords(word, wordExact, slash, words, hexcode, allNames, lastName, insideStream, oEntropy, fOut)
|
||||
if disarm:
|
||||
fOut.write(C2BIP3(char))
|
||||
else:
|
||||
oCVE_2009_3459.Check(lastName, word)
|
||||
|
||||
(word, wordExact, hexcode, lastName, insideStream) = UpdateWords(word, wordExact, slash, words, hexcode, allNames, lastName, insideStream, oEntropy, fOut)
|
||||
if char == '/':
|
||||
slash = '/'
|
||||
else:
|
||||
slash = ''
|
||||
if disarm:
|
||||
fOut.write(C2BIP3(char))
|
||||
|
||||
if oPDFDate != None and oPDFDate.parse(char) != None:
|
||||
dates.append([oPDFDate.date, lastName])
|
||||
|
||||
if oEntropy != None:
|
||||
oEntropy.add(byte, insideStream)
|
||||
|
||||
if oPDFEOF != None:
|
||||
oPDFEOF.parse(char)
|
||||
|
||||
byte = oBinaryFile.byte()
|
||||
(word, wordExact, hexcode, lastName, insideStream) = UpdateWords(word, wordExact, slash, words, hexcode, allNames, lastName, insideStream, oEntropy, fOut)
|
||||
|
||||
# check to see if file ended with %%EOF. If so, we can reset charsAfterLastEOF and add one to EOF count. This is never performed in
|
||||
# the parse function because it never gets called due to hitting the end of file.
|
||||
if byte == None and oPDFEOF != None:
|
||||
if oPDFEOF.token == '%%EOF':
|
||||
oPDFEOF.cntEOFs += 1
|
||||
oPDFEOF.cntCharsAfterLastEOF = 0
|
||||
oPDFEOF.token = ''
|
||||
|
||||
except SystemExit:
|
||||
sys.exit()
|
||||
except:
|
||||
attErrorOccured.nodeValue = 'True'
|
||||
attErrorMessage.nodeValue = traceback.format_exc()
|
||||
|
||||
if disarm:
|
||||
fOut.close()
|
||||
|
||||
attEntropyAll = xmlDoc.createAttribute('TotalEntropy')
|
||||
xmlDoc.documentElement.setAttributeNode(attEntropyAll)
|
||||
attCountAll = xmlDoc.createAttribute('TotalCount')
|
||||
xmlDoc.documentElement.setAttributeNode(attCountAll)
|
||||
attEntropyStream = xmlDoc.createAttribute('StreamEntropy')
|
||||
xmlDoc.documentElement.setAttributeNode(attEntropyStream)
|
||||
attCountStream = xmlDoc.createAttribute('StreamCount')
|
||||
xmlDoc.documentElement.setAttributeNode(attCountStream)
|
||||
attEntropyNonStream = xmlDoc.createAttribute('NonStreamEntropy')
|
||||
xmlDoc.documentElement.setAttributeNode(attEntropyNonStream)
|
||||
attCountNonStream = xmlDoc.createAttribute('NonStreamCount')
|
||||
xmlDoc.documentElement.setAttributeNode(attCountNonStream)
|
||||
if oEntropy != None:
|
||||
(countAll, entropyAll , countStream, entropyStream, countNonStream, entropyNonStream) = oEntropy.calc()
|
||||
attEntropyAll.nodeValue = '%f' % entropyAll
|
||||
attCountAll.nodeValue = '%d' % countAll
|
||||
attEntropyStream.nodeValue = '%f' % entropyStream
|
||||
attCountStream.nodeValue = '%d' % countStream
|
||||
attEntropyNonStream.nodeValue = '%f' % entropyNonStream
|
||||
attCountNonStream.nodeValue = '%d' % countNonStream
|
||||
else:
|
||||
attEntropyAll.nodeValue = ''
|
||||
attCountAll.nodeValue = ''
|
||||
attEntropyStream.nodeValue = ''
|
||||
attCountStream.nodeValue = ''
|
||||
attEntropyNonStream.nodeValue = ''
|
||||
attCountNonStream.nodeValue = ''
|
||||
attCountEOF = xmlDoc.createAttribute('CountEOF')
|
||||
xmlDoc.documentElement.setAttributeNode(attCountEOF)
|
||||
attCountCharsAfterLastEOF = xmlDoc.createAttribute('CountCharsAfterLastEOF')
|
||||
xmlDoc.documentElement.setAttributeNode(attCountCharsAfterLastEOF)
|
||||
if oPDFEOF != None:
|
||||
attCountEOF.nodeValue = '%d' % oPDFEOF.cntEOFs
|
||||
attCountCharsAfterLastEOF.nodeValue = '%d' % oPDFEOF.cntCharsAfterLastEOF
|
||||
else:
|
||||
attCountEOF.nodeValue = ''
|
||||
attCountCharsAfterLastEOF.nodeValue = ''
|
||||
|
||||
eleKeywords = xmlDoc.createElement('Keywords')
|
||||
xmlDoc.documentElement.appendChild(eleKeywords)
|
||||
for keyword in keywords:
|
||||
eleKeyword = xmlDoc.createElement('Keyword')
|
||||
eleKeywords.appendChild(eleKeyword)
|
||||
att = xmlDoc.createAttribute('Name')
|
||||
att.nodeValue = keyword
|
||||
eleKeyword.setAttributeNode(att)
|
||||
att = xmlDoc.createAttribute('Count')
|
||||
att.nodeValue = str(words[keyword][0])
|
||||
eleKeyword.setAttributeNode(att)
|
||||
att = xmlDoc.createAttribute('HexcodeCount')
|
||||
att.nodeValue = str(words[keyword][1])
|
||||
eleKeyword.setAttributeNode(att)
|
||||
eleKeyword = xmlDoc.createElement('Keyword')
|
||||
eleKeywords.appendChild(eleKeyword)
|
||||
att = xmlDoc.createAttribute('Name')
|
||||
att.nodeValue = '/Colors > 2^24'
|
||||
eleKeyword.setAttributeNode(att)
|
||||
att = xmlDoc.createAttribute('Count')
|
||||
att.nodeValue = str(oCVE_2009_3459.count)
|
||||
eleKeyword.setAttributeNode(att)
|
||||
att = xmlDoc.createAttribute('HexcodeCount')
|
||||
att.nodeValue = str(0)
|
||||
eleKeyword.setAttributeNode(att)
|
||||
if allNames:
|
||||
keys = sorted(words.keys())
|
||||
for word in keys:
|
||||
if not word in keywords:
|
||||
eleKeyword = xmlDoc.createElement('Keyword')
|
||||
eleKeywords.appendChild(eleKeyword)
|
||||
att = xmlDoc.createAttribute('Name')
|
||||
att.nodeValue = word
|
||||
eleKeyword.setAttributeNode(att)
|
||||
att = xmlDoc.createAttribute('Count')
|
||||
att.nodeValue = str(words[word][0])
|
||||
eleKeyword.setAttributeNode(att)
|
||||
att = xmlDoc.createAttribute('HexcodeCount')
|
||||
att.nodeValue = str(words[word][1])
|
||||
eleKeyword.setAttributeNode(att)
|
||||
eleDates = xmlDoc.createElement('Dates')
|
||||
xmlDoc.documentElement.appendChild(eleDates)
|
||||
dates.sort(key=lambda x: x[0])
|
||||
for date in dates:
|
||||
eleDate = xmlDoc.createElement('Date')
|
||||
eleDates.appendChild(eleDate)
|
||||
att = xmlDoc.createAttribute('Value')
|
||||
att.nodeValue = date[0]
|
||||
eleDate.setAttributeNode(att)
|
||||
att = xmlDoc.createAttribute('Name')
|
||||
att.nodeValue = date[1]
|
||||
eleDate.setAttributeNode(att)
|
||||
return xmlDoc
|
||||
|
||||
def PDFiD2String(xmlDoc, force):
|
||||
result = 'PDFiD %s %s\n' % (xmlDoc.documentElement.getAttribute('Version'), xmlDoc.documentElement.getAttribute('Filename'))
|
||||
if xmlDoc.documentElement.getAttribute('ErrorOccured') == 'True':
|
||||
return result + '***Error occured***\n%s\n' % xmlDoc.documentElement.getAttribute('ErrorMessage')
|
||||
if not force and xmlDoc.documentElement.getAttribute('IsPDF') == 'False':
|
||||
return result + ' Not a PDF document\n'
|
||||
result += ' PDF Header: %s\n' % xmlDoc.documentElement.getAttribute('Header')
|
||||
for node in xmlDoc.documentElement.getElementsByTagName('Keywords')[0].childNodes:
|
||||
result += ' %-16s %7d' % (node.getAttribute('Name'), int(node.getAttribute('Count')))
|
||||
if int(node.getAttribute('HexcodeCount')) > 0:
|
||||
result += '(%d)' % int(node.getAttribute('HexcodeCount'))
|
||||
result += '\n'
|
||||
if xmlDoc.documentElement.getAttribute('CountEOF') != '':
|
||||
result += ' %-16s %7d\n' % ('%%EOF', int(xmlDoc.documentElement.getAttribute('CountEOF')))
|
||||
if xmlDoc.documentElement.getAttribute('CountCharsAfterLastEOF') != '':
|
||||
result += ' %-16s %7d\n' % ('After last %%EOF', int(xmlDoc.documentElement.getAttribute('CountCharsAfterLastEOF')))
|
||||
for node in xmlDoc.documentElement.getElementsByTagName('Dates')[0].childNodes:
|
||||
result += ' %-23s %s\n' % (node.getAttribute('Value'), node.getAttribute('Name'))
|
||||
if xmlDoc.documentElement.getAttribute('TotalEntropy') != '':
|
||||
result += ' Total entropy: %s (%10s bytes)\n' % (xmlDoc.documentElement.getAttribute('TotalEntropy'), xmlDoc.documentElement.getAttribute('TotalCount'))
|
||||
if xmlDoc.documentElement.getAttribute('StreamEntropy') != '':
|
||||
result += ' Entropy inside streams: %s (%10s bytes)\n' % (xmlDoc.documentElement.getAttribute('StreamEntropy'), xmlDoc.documentElement.getAttribute('StreamCount'))
|
||||
if xmlDoc.documentElement.getAttribute('NonStreamEntropy') != '':
|
||||
result += ' Entropy outside streams: %s (%10s bytes)\n' % (xmlDoc.documentElement.getAttribute('NonStreamEntropy'), xmlDoc.documentElement.getAttribute('NonStreamCount'))
|
||||
return result
|
||||
|
||||
class cCount():
|
||||
def __init__(self, count, hexcode):
|
||||
self.count = count
|
||||
self.hexcode = hexcode
|
||||
|
||||
class cPDFiD():
|
||||
def __init__(self, xmlDoc, force):
|
||||
self.version = xmlDoc.documentElement.getAttribute('Version')
|
||||
self.filename = xmlDoc.documentElement.getAttribute('Filename')
|
||||
self.errorOccured = xmlDoc.documentElement.getAttribute('ErrorOccured') == 'True'
|
||||
self.errorMessage = xmlDoc.documentElement.getAttribute('ErrorMessage')
|
||||
self.isPDF = None
|
||||
if self.errorOccured:
|
||||
return
|
||||
self.isPDF = xmlDoc.documentElement.getAttribute('IsPDF') == 'True'
|
||||
if not force and not self.isPDF:
|
||||
return
|
||||
self.header = xmlDoc.documentElement.getAttribute('Header')
|
||||
self.keywords = {}
|
||||
for node in xmlDoc.documentElement.getElementsByTagName('Keywords')[0].childNodes:
|
||||
self.keywords[node.getAttribute('Name')] = cCount(int(node.getAttribute('Count')), int(node.getAttribute('HexcodeCount')))
|
||||
self.obj = self.keywords['obj']
|
||||
self.endobj = self.keywords['endobj']
|
||||
self.stream = self.keywords['stream']
|
||||
self.endstream = self.keywords['endstream']
|
||||
self.xref = self.keywords['xref']
|
||||
self.trailer = self.keywords['trailer']
|
||||
self.startxref = self.keywords['startxref']
|
||||
self.page = self.keywords['/Page']
|
||||
self.encrypt = self.keywords['/Encrypt']
|
||||
self.objstm = self.keywords['/ObjStm']
|
||||
self.js = self.keywords['/JS']
|
||||
self.javascript = self.keywords['/JavaScript']
|
||||
self.aa = self.keywords['/AA']
|
||||
self.openaction = self.keywords['/OpenAction']
|
||||
self.acroform = self.keywords['/AcroForm']
|
||||
self.jbig2decode = self.keywords['/JBIG2Decode']
|
||||
self.richmedia = self.keywords['/RichMedia']
|
||||
self.launch = self.keywords['/Launch']
|
||||
self.embeddedfile = self.keywords['/EmbeddedFile']
|
||||
self.xfa = self.keywords['/XFA']
|
||||
self.colors_gt_2_24 = self.keywords['/Colors > 2^24']
|
||||
|
||||
def Print(lines, options):
|
||||
print(lines)
|
||||
filename = None
|
||||
if options.scan:
|
||||
filename = 'PDFiD.log'
|
||||
if options.output != '':
|
||||
filename = options.output
|
||||
if filename:
|
||||
logfile = open(filename, 'a')
|
||||
logfile.write(lines + '\n')
|
||||
logfile.close()
|
||||
|
||||
def Quote(value, separator, quote):
|
||||
if isinstance(value, str):
|
||||
if separator in value:
|
||||
return quote + value + quote
|
||||
return value
|
||||
|
||||
def MakeCSVLine(fields, separator=';', quote='"'):
|
||||
formatstring = separator.join([field[0] for field in fields])
|
||||
strings = [Quote(field[1], separator, quote) for field in fields]
|
||||
return formatstring % tuple(strings)
|
||||
|
||||
def ProcessFile(filename, options, plugins):
|
||||
xmlDoc = PDFiD(filename, options.all, options.extra, options.disarm, options.force)
|
||||
if plugins == [] and options.select == '':
|
||||
Print(PDFiD2String(xmlDoc, options.force), options)
|
||||
return
|
||||
|
||||
oPDFiD = cPDFiD(xmlDoc, options.force)
|
||||
if options.select:
|
||||
if options.force or not oPDFiD.errorOccured and oPDFiD.isPDF:
|
||||
pdf = oPDFiD
|
||||
try:
|
||||
selected = eval(options.select)
|
||||
except Exception as e:
|
||||
Print('Error evaluating select expression: %s' % options.select, options)
|
||||
if options.verbose:
|
||||
raise e
|
||||
return
|
||||
if selected:
|
||||
if options.csv:
|
||||
Print(filename, options)
|
||||
else:
|
||||
Print(PDFiD2String(xmlDoc, options.force), options)
|
||||
else:
|
||||
for cPlugin in plugins:
|
||||
if not cPlugin.onlyValidPDF or not oPDFiD.errorOccured and oPDFiD.isPDF:
|
||||
try:
|
||||
oPlugin = cPlugin(oPDFiD)
|
||||
except Exception as e:
|
||||
Print('Error instantiating plugin: %s' % cPlugin.name, options)
|
||||
if options.verbose:
|
||||
raise e
|
||||
return
|
||||
|
||||
try:
|
||||
score = oPlugin.Score()
|
||||
except Exception as e:
|
||||
Print('Error running plugin: %s' % cPlugin.name, options)
|
||||
if options.verbose:
|
||||
raise e
|
||||
return
|
||||
|
||||
if options.csv:
|
||||
if score >= options.minimumscore:
|
||||
Print(MakeCSVLine((('%s', filename), ('%s', cPlugin.name), ('%.02f', score))), options)
|
||||
else:
|
||||
if score >= options.minimumscore:
|
||||
Print(PDFiD2String(xmlDoc, options.force), options)
|
||||
Print('%s score: %.02f' % (cPlugin.name, score), options)
|
||||
else:
|
||||
if options.csv:
|
||||
if oPDFiD.errorOccured:
|
||||
Print(MakeCSVLine((('%s', filename), ('%s', cPlugin.name), ('%s', 'Error occured'))), options)
|
||||
if not oPDFiD.isPDF:
|
||||
Print(MakeCSVLine((('%s', filename), ('%s', cPlugin.name), ('%s', 'Not a PDF document'))), options)
|
||||
else:
|
||||
Print(PDFiD2String(xmlDoc, options.force), options)
|
||||
|
||||
|
||||
def Scan(directory, options, plugins):
|
||||
try:
|
||||
if os.path.isdir(directory):
|
||||
for entry in os.listdir(directory):
|
||||
Scan(os.path.join(directory, entry), options, plugins)
|
||||
else:
|
||||
ProcessFile(directory, options, plugins)
|
||||
except Exception as e:
|
||||
# print directory
|
||||
print(e)
|
||||
# print(sys.exc_info()[2])
|
||||
# print traceback.format_exc()
|
||||
|
||||
#function derived from: http://blog.9bplus.com/pdfidpy-output-to-json
|
||||
def PDFiD2JSON(xmlDoc, force):
|
||||
#Get Top Layer Data
|
||||
errorOccured = xmlDoc.documentElement.getAttribute('ErrorOccured')
|
||||
errorMessage = xmlDoc.documentElement.getAttribute('ErrorMessage')
|
||||
filename = xmlDoc.documentElement.getAttribute('Filename')
|
||||
header = xmlDoc.documentElement.getAttribute('Header')
|
||||
isPdf = xmlDoc.documentElement.getAttribute('IsPDF')
|
||||
version = xmlDoc.documentElement.getAttribute('Version')
|
||||
entropy = xmlDoc.documentElement.getAttribute('Entropy')
|
||||
|
||||
#extra data
|
||||
countEof = xmlDoc.documentElement.getAttribute('CountEOF')
|
||||
countChatAfterLastEof = xmlDoc.documentElement.getAttribute('CountCharsAfterLastEOF')
|
||||
totalEntropy = xmlDoc.documentElement.getAttribute('TotalEntropy')
|
||||
streamEntropy = xmlDoc.documentElement.getAttribute('StreamEntropy')
|
||||
nonStreamEntropy = xmlDoc.documentElement.getAttribute('NonStreamEntropy')
|
||||
|
||||
keywords = []
|
||||
dates = []
|
||||
|
||||
#grab all keywords
|
||||
for node in xmlDoc.documentElement.getElementsByTagName('Keywords')[0].childNodes:
|
||||
name = node.getAttribute('Name')
|
||||
count = int(node.getAttribute('Count'))
|
||||
if int(node.getAttribute('HexcodeCount')) > 0:
|
||||
hexCount = int(node.getAttribute('HexcodeCount'))
|
||||
else:
|
||||
hexCount = 0
|
||||
keyword = { 'count':count, 'hexcodecount':hexCount, 'name':name }
|
||||
keywords.append(keyword)
|
||||
|
||||
#grab all date information
|
||||
for node in xmlDoc.documentElement.getElementsByTagName('Dates')[0].childNodes:
|
||||
name = node.getAttribute('Name')
|
||||
value = node.getAttribute('Value')
|
||||
date = { 'name':name, 'value':value }
|
||||
dates.append(date)
|
||||
|
||||
data = { 'countEof':countEof, 'countChatAfterLastEof':countChatAfterLastEof, 'totalEntropy':totalEntropy, 'streamEntropy':streamEntropy, 'nonStreamEntropy':nonStreamEntropy, 'errorOccured':errorOccured, 'errorMessage':errorMessage, 'filename':filename, 'header':header, 'isPdf':isPdf, 'version':version, 'entropy':entropy, 'keywords': { 'keyword': keywords }, 'dates': { 'date':dates} }
|
||||
complete = [ { 'pdfid' : data} ]
|
||||
result = json.dumps(complete)
|
||||
return result
|
||||
|
||||
def File2Strings(filename):
|
||||
try:
|
||||
f = open(filename, 'r')
|
||||
except:
|
||||
return None
|
||||
try:
|
||||
return list(map(lambda line:line.rstrip('\n'), f.readlines()))
|
||||
except:
|
||||
return None
|
||||
finally:
|
||||
f.close()
|
||||
|
||||
def ProcessAt(argument):
|
||||
if argument.startswith('@'):
|
||||
strings = File2Strings(argument[1:])
|
||||
if strings == None:
|
||||
raise Exception('Error reading %s' % argument)
|
||||
else:
|
||||
return strings
|
||||
else:
|
||||
return [argument]
|
||||
|
||||
def AddPlugin(cClass):
|
||||
global plugins
|
||||
|
||||
plugins.append(cClass)
|
||||
|
||||
def ExpandFilenameArguments(filenames):
|
||||
return list(collections.OrderedDict.fromkeys(sum(map(glob.glob, sum(map(ProcessAt, filenames), [])), [])))
|
||||
|
||||
class cPluginParent():
|
||||
onlyValidPDF = True
|
||||
|
||||
def LoadPlugins(plugins, verbose):
|
||||
if plugins == '':
|
||||
return
|
||||
scriptPath = os.path.dirname(sys.argv[0])
|
||||
for plugin in sum(map(ProcessAt, plugins.split(',')), []):
|
||||
try:
|
||||
if not plugin.lower().endswith('.py'):
|
||||
plugin += '.py'
|
||||
if os.path.dirname(plugin) == '':
|
||||
if not os.path.exists(plugin):
|
||||
scriptPlugin = os.path.join(scriptPath, plugin)
|
||||
if os.path.exists(scriptPlugin):
|
||||
plugin = scriptPlugin
|
||||
exec(open(plugin, 'r').read())
|
||||
except Exception as e:
|
||||
print('Error loading plugin: %s' % plugin)
|
||||
if verbose:
|
||||
raise e
|
||||
|
||||
def PDFiDMain(filenames, options):
|
||||
global plugins
|
||||
plugins = []
|
||||
LoadPlugins(options.plugins, options.verbose)
|
||||
|
||||
if options.csv:
|
||||
if plugins != []:
|
||||
Print(MakeCSVLine((('%s', 'Filename'), ('%s', 'Plugin-name'), ('%s', 'Score'))), options)
|
||||
elif options.select != '':
|
||||
Print('Filename', options)
|
||||
|
||||
for filename in filenames:
|
||||
if options.scan:
|
||||
Scan(filename, options, plugins)
|
||||
else:
|
||||
ProcessFile(filename, options, plugins)
|
||||
|
||||
def Main():
|
||||
moredesc = '''
|
||||
|
||||
Arguments:
|
||||
pdf-file and zip-file can be a single file, several files, and/or @file
|
||||
@file: run PDFiD on each file listed in the text file specified
|
||||
wildcards are supported
|
||||
|
||||
Source code put in the public domain by Didier Stevens, no Copyright
|
||||
Use at your own risk
|
||||
https://DidierStevens.com'''
|
||||
|
||||
oParser = optparse.OptionParser(usage='usage: %prog [options] [pdf-file|zip-file|url|@file] ...\n' + __description__ + moredesc, version='%prog ' + __version__)
|
||||
oParser.add_option('-s', '--scan', action='store_true', default=False, help='scan the given directory')
|
||||
oParser.add_option('-a', '--all', action='store_true', default=False, help='display all the names')
|
||||
oParser.add_option('-e', '--extra', action='store_true', default=False, help='display extra data, like dates')
|
||||
oParser.add_option('-f', '--force', action='store_true', default=False, help='force the scan of the file, even without proper %PDF header')
|
||||
oParser.add_option('-d', '--disarm', action='store_true', default=False, help='disable JavaScript and auto launch')
|
||||
oParser.add_option('-p', '--plugins', type=str, default='', help='plugins to load (separate plugins with a comma , ; @file supported)')
|
||||
oParser.add_option('-c', '--csv', action='store_true', default=False, help='output csv data when using plugins')
|
||||
oParser.add_option('-m', '--minimumscore', type=float, default=0.0, help='minimum score for plugin results output')
|
||||
oParser.add_option('-v', '--verbose', action='store_true', default=False, help='verbose (will also raise catched exceptions)')
|
||||
oParser.add_option('-S', '--select', type=str, default='', help='selection expression')
|
||||
oParser.add_option('-o', '--output', type=str, default='', help='output to log file')
|
||||
(options, args) = oParser.parse_args()
|
||||
|
||||
if len(args) == 0:
|
||||
if options.disarm:
|
||||
print('Option disarm not supported with stdin')
|
||||
options.disarm = False
|
||||
if options.scan:
|
||||
print('Option scan not supported with stdin')
|
||||
options.scan = False
|
||||
filenames = ['']
|
||||
else:
|
||||
try:
|
||||
filenames = ExpandFilenameArguments(args)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return
|
||||
PDFiDMain(filenames, options)
|
||||
|
||||
if __name__ == '__main__':
|
||||
Main()
|
|
@ -0,0 +1,21 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
#2014/10/13
|
||||
|
||||
class cPDFiDEmbeddedFile(cPluginParent):
|
||||
# onlyValidPDF = True
|
||||
name = 'EmbeddedFile plugin'
|
||||
|
||||
def __init__(self, oPDFiD):
|
||||
self.oPDFiD = oPDFiD
|
||||
|
||||
def Score(self):
|
||||
if '/EmbeddedFile' in self.oPDFiD.keywords and self.oPDFiD.keywords['/EmbeddedFile'].count > 0:
|
||||
if self.oPDFiD.keywords['/EmbeddedFile'].hexcode > 0:
|
||||
return 1.0
|
||||
else:
|
||||
return 0.9
|
||||
else:
|
||||
return 0.0
|
||||
|
||||
AddPlugin(cPDFiDEmbeddedFile)
|
|
@ -0,0 +1,3 @@
|
|||
plugin_embeddedfile.py
|
||||
plugin_nameobfuscation.py
|
||||
plugin_triage.py
|
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
#2013/11/04
|
||||
#2013/11/08
|
||||
|
||||
class cPDFiDNameObfuscation(cPluginParent):
|
||||
# onlyValidPDF = True
|
||||
name = 'Name Obfuscation plugin'
|
||||
|
||||
def __init__(self, oPDFiD):
|
||||
self.oPDFiD = oPDFiD
|
||||
|
||||
def Score(self):
|
||||
if sum([oCount.hexcode for oCount in self.oPDFiD.keywords.values()]) > 0:
|
||||
return 1.0
|
||||
else:
|
||||
return 0.0
|
||||
|
||||
AddPlugin(cPDFiDNameObfuscation)
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
#2014/09/30
|
||||
|
||||
class cPDFiDTriage(cPluginParent):
|
||||
# onlyValidPDF = True
|
||||
name = 'Triage plugin'
|
||||
|
||||
def __init__(self, oPDFiD):
|
||||
self.oPDFiD = oPDFiD
|
||||
|
||||
def Score(self):
|
||||
for keyword in ('/ObjStm', '/JS', '/JavaScript', '/AA', '/OpenAction', '/AcroForm', '/JBIG2Decode', '/RichMedia', '/Launch', '/EmbeddedFile', '/XFA', '/Colors > 2^24'):
|
||||
if keyword in self.oPDFiD.keywords and self.oPDFiD.keywords[keyword].count > 0:
|
||||
return 1.0
|
||||
if self.oPDFiD.keywords['obj'].count != self.oPDFiD.keywords['endobj'].count:
|
||||
return 1.0
|
||||
if self.oPDFiD.keywords['stream'].count != self.oPDFiD.keywords['endstream'].count:
|
||||
return 1.0
|
||||
return 0.0
|
||||
|
||||
AddPlugin(cPDFiDTriage)
|
|
@ -1,5 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: in copy_to_final.sh" 1>&2
|
||||
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
|
@ -19,7 +23,7 @@ fi
|
|||
#cp deb/*.deb ${CHROOT_PATH}/
|
||||
|
||||
# prepare fs archive
|
||||
tar -cvpzf backup.tar.gz -C fs/ .
|
||||
tar -cvpzf backup.tar.gz -C fs_filecheck/ .
|
||||
tar -xzf backup.tar.gz -C ${CHROOT_PATH}/
|
||||
chown root:root ${CHROOT_PATH}/etc/sudoers
|
||||
if [ -f deb/led ]; then
|
||||
|
@ -42,3 +46,7 @@ cp -rf midi ${CHROOT_PATH}/opt/
|
|||
|
||||
|
||||
# It is also a good idea to run raspi-config once and enable the console login (allow debugging)
|
||||
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: done with copy_to_final.sh" 1>&2
|
||||
|
||||
|
|
|
@ -1,10 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
useradd -m kitten
|
||||
|
||||
# Useless: the sudoer file comes from the repository
|
||||
#echo "Cmnd_Alias GROOMER_CMDS = /home/kitten/kitten_mount_src, \
|
||||
# /home/kitten/kitten_mount_dst, /home/kitten/kitten_umount" >> /etc/sudoers
|
||||
#echo "kitten ALL=(ALL) NOPASSWD: GROOMER_CMDS" >> /etc/sudoers
|
||||
|
||||
# /!\ REMOVE SUDO RIGHTS TO USER pi
|
|
@ -25,21 +25,24 @@ Mechanical design
|
|||
coordinate (0,0) is located at the PCB corner near the power supply connector.
|
||||
the X axis is along the longest side.
|
||||
|
||||
hole_x1 = 3.50 ; // hole center x offset from PCB edge (uSD side)
|
||||
LedBtn_y = device_y - hole_y - 2.5 * 2.54 + tolerance ; // Y axis center position of LEDs and button(s)
|
||||
R_Led_x = hole_x1 + 29 + 2.54 + tolerance ; // X axis center position of red LED
|
||||
Y_Led_x = R_Led_x - 3 * 2.54 ; // X axis center position of yellow LED
|
||||
G_Led_x = R_Led_x - 6 * 2.54 ; // X axis center position of green LED
|
||||
Btn_x = R_Led_x + 4.5 * 2.54 ; // X axis center position of button
|
||||
|
||||
* hole_x1 = 3.50 ; // hole center x offset from PCB edge (uSD side)
|
||||
* LedBtn_y = device_y - hole_y - 2.5 * 2.54 + tolerance ; // Y axis center position of LEDs and button(s)
|
||||
* R_Led_x = hole_x1 + 29 + 2.54 + tolerance ; // X axis center position of red LED
|
||||
* Y_Led_x = R_Led_x - 3 * 2.54 ; // X axis center position of yellow LED
|
||||
* G_Led_x = R_Led_x - 6 * 2.54 ; // X axis center position of green LED
|
||||
* Btn_x = R_Led_x + 4.5 * 2.54 ; // X axis center position of button
|
||||
|
||||
|
||||
The LEDs and the push-button are positioned on the 2.54 mm (0.1 inch) grid
|
||||
|
||||
Relative to he GPIO connector pin 1
|
||||
along the X axis : -2 * 2.54 mm
|
||||
along the Y axis (in multiples of 2.54 mm)
|
||||
Green LED center : 4.5
|
||||
Yellow LED center : 7.5
|
||||
Red LED center : 10.5
|
||||
Push-button center : 15.0
|
||||
* along the X axis : -2 * 2.54 mm
|
||||
* along the Y axis (in multiples of 2.54 mm)
|
||||
* Green LED center : 4.5
|
||||
* Yellow LED center : 7.5
|
||||
* Red LED center : 10.5
|
||||
* Push-button center : 15.0
|
||||
|
||||
The LEDs have a 3 mm diameter
|
||||
The top of the LEDs are 11 mm above the PCB
|
||||
|
|
|
@ -5,11 +5,12 @@
|
|||
// added spacers on cover (MD)
|
||||
// v 1.5 - 08/02/2015 - fine tuning of connector hole Z position
|
||||
// v 1.7 - 10/02/2015 - fine tuning of holes, cutouts, spacers
|
||||
// v 1.8 - 21/07/2015 - bug correction : inverted riser parameters
|
||||
//
|
||||
|
||||
// design control
|
||||
test_fit = 0 ; // set to one for test fit
|
||||
top_bottom = 1 ; // 0 = bottom only, 1 = top only, 2 = top & bottom
|
||||
top_bottom = 0 ; // 0 = bottom only, 1 = top only, 2 = top & bottom
|
||||
print = 1 ; // set to one for printing configuration
|
||||
|
||||
// parameters
|
||||
|
@ -179,7 +180,7 @@ module box_bottom() {
|
|||
translate([uSD_x, 0, 0])
|
||||
spacers(hole_x1 + tolerance, hole_x2 + tolerance,
|
||||
hole_y + tolerance, device_y - hole_y + tolerance,
|
||||
riser_r, riser_z) ;
|
||||
riser_z, riser_r) ;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,47 @@
|
|||
How to mount a SD card created by NOOBS on Ubuntu 15.04
|
||||
=======================================================
|
||||
|
||||
Related to [this bug](https://github.com/raspberrypi/noobs/issues/262)
|
||||
|
||||
TL;DR
|
||||
=====
|
||||
|
||||
```
|
||||
[122615.777412] usb-storage 4-1.2:1.0: USB Mass Storage device detected
|
||||
[122615.778614] scsi host26: usb-storage 4-1.2:1.0
|
||||
[122616.778643] scsi 26:0:0:0: Direct-Access SD/MMC Card Reader 1.00 PQ: 0 ANSI: 0
|
||||
[122616.779319] sd 26:0:0:0: Attached scsi generic sg1 type 0
|
||||
[122617.072900] sd 26:0:0:0: [sdc] 30392320 512-byte logical blocks: (15.5 GB/14.4 GiB)
|
||||
[122617.075210] sd 26:0:0:0: [sdc] Write Protect is off
|
||||
[122617.075221] sd 26:0:0:0: [sdc] Mode Sense: 03 00 00 00
|
||||
[122617.077407] sd 26:0:0:0: [sdc] No Caching mode page found
|
||||
[122617.077424] sd 26:0:0:0: [sdc] Assuming drive cache: write through
|
||||
[122617.090405] *sdc: [CUMANA/ADFS] sdc1 [ADFS] sdc1*
|
||||
[122617.101930] sd 26:0:0:0: [sdc] Attached SCSI removable disk
|
||||
```
|
||||
|
||||
|
||||
```
|
||||
$ fdisk -l /dev/sdb
|
||||
|
||||
Disk /dev/sdb: 14.5 GiB, 15560867840 bytes, 30392320 sectors
|
||||
Units: sectors of 1 * 512 = *512* bytes
|
||||
Sector size (logical/physical): 512 bytes / 512 bytes
|
||||
I/O size (minimum/optimal): 512 bytes / 512 bytes
|
||||
Disklabel type: dos
|
||||
Disk identifier: 0x000aa00f
|
||||
|
||||
Device Boot Start End Sectors Size Id Type
|
||||
/dev/sdb1 *8192* 1679687 1671496 816.2M e W95 FAT16 (LBA)
|
||||
/dev/sdb2 1687552 30326783 28639232 13.7G 85 Linux extended
|
||||
/dev/sdb3 30326784 30392319 65536 32M 83 Linux
|
||||
/dev/sdb5 1695744 2744319 1048576 512M 83 Linux
|
||||
/dev/sdb6 2752512 2875391 122880 60M c W95 FAT32 (LBA)
|
||||
/dev/sdb7 2883584 30326783 27443200 13.1G 83 Linux
|
||||
|
||||
Partition table entries are not in disk order.
|
||||
```
|
||||
|
||||
```
|
||||
mount -oloop,offset=$((512*8192)) /dev/sdb1 /mnt/sd/
|
||||
```
|
|
@ -0,0 +1,49 @@
|
|||
Usage Notes
|
||||
===========
|
||||
|
||||
* Don't plug in USB devices with a hub because there's no way to tell it which
|
||||
is source and target - its the first drive detected (top port) that is the
|
||||
source and the second (bottom port) is the target
|
||||
* Don't turn it off without shutting down the system, when grooming is done it
|
||||
shuts down automatically: losing power while it's running can trash the OS
|
||||
on the SD cards because SD cards don't always like dirty shutdowns (ie power loss)
|
||||
* Using a target usb stick that has a status light as long as the device has
|
||||
power is a really useful thing as there the other status lights on the groomer
|
||||
are less than indicative at times: because the 'OK' led on the RPi toggles on activity
|
||||
it can be off for a long time while processing something and only comes back
|
||||
on when that process finishes - hence why a USB that has some sort of LED activity
|
||||
when just plugged in (even if not reading or writing but while the USB port is
|
||||
powered) is helpful in determining when the process is finished - when
|
||||
the rPI is shutdown, the USB port power is shut off and that LED will also
|
||||
then be off on the USB device
|
||||
* Use a larger target device as all zip files get unpacked and processed onto
|
||||
the target
|
||||
* If you have an hdmi monitor plugged in you can watch what's happening for about
|
||||
30 minutes until the rPI's power saving kicks in and turns off the monitor
|
||||
* If only one usb stick is present at power up, it doesn't groom and looks like
|
||||
a normal rPi
|
||||
* If you want to ssh into the RPi username is 'pi' password 'raspberry' as per defaults
|
||||
|
||||
|
||||
Technical notes
|
||||
===============
|
||||
|
||||
* Groomer script is in /opt/groomer/ with the other required files and the ip
|
||||
address is 192.168.1.89
|
||||
* The groomer process is kicked off in /etc/rc.local
|
||||
* The heavy lifting is dispatched from /opt/groomer/groomer.sh
|
||||
* All files processing is in filecheck.py
|
||||
|
||||
|
||||
USB Ports
|
||||
=========
|
||||
|
||||
If you connect multiple keys to the RPi, they will be detected in this order:
|
||||
|
||||
First: Top left
|
||||
Second: Top right
|
||||
Third: Bottom left
|
||||
Forth: Bottom right
|
||||
|
||||
* As long as the source key (sda) is connected to the top left port, the
|
||||
destination (sdb) can be connected on any other port.
|
|
@ -0,0 +1,55 @@
|
|||
* Download qemu, qemu-user-static, and proot if not already installed
|
||||
* Download the newest raspbian-lite image from raspberrypi.org
|
||||
* Verify the sha1 hash of the downloaded .zip file
|
||||
* Unzip the image
|
||||
* Expand the image by 2GB using dd
|
||||
* Expand the root partition using fdisk
|
||||
* Mount both partitions in loop mode using /shell_utils/basic_mount_image.sh
|
||||
* Use df to find the larger partition, and resize the filesystem to fill it
|
||||
* Copy circlean_fs/root_partition/etc/systemd/system/rc-local.service into the equivalent location
|
||||
* Use proot to enter a chroot in the image: sudo proot -q qemu-arm -S /mnt/rpi-root -b /mnt/rpi-boot:/boot /bin/bash
|
||||
* Run dpkg-reconfigure locales
|
||||
* apt-get update
|
||||
* apt-get dist-upgrade (might have to run this and autoremove several times)
|
||||
* apt-get autoremove
|
||||
* apt-get install the linux dependencies:
|
||||
- timidity
|
||||
- git
|
||||
- p7zip-full
|
||||
- ntfs-3g exfat-fuse exfat-utils
|
||||
- python3 python3-pip
|
||||
- python3-lxml
|
||||
- libjpeg-dev libtiff-dev libwebp-dev liblcms2-dev tcl-dev tk-dev python3-tk
|
||||
* Compile p7zip-rar from source
|
||||
- Change your source.list file
|
||||
- Make a new directory and cd to it
|
||||
- apt-get build-dep p7zip-rar
|
||||
- dpkg -i <p7zip-rar .deb file path>
|
||||
* Make sure the right pip executable is called by `pip3`, change your path if necessary
|
||||
* Upgrade pip: pip3 install -U pip
|
||||
* pip3 install python dependencies
|
||||
- exifread
|
||||
- pillow
|
||||
- olefile
|
||||
- oletools
|
||||
- git+https://github.com/grierforensics/officedissector.git
|
||||
- git+https://github.com/CIRCL/PyCIRCLean.git
|
||||
* Add a user named "kitten"
|
||||
* Symlink /proc/mounts to /etc/mtab
|
||||
* Turn on rc-local.service `systemctl enable rc-local.service`
|
||||
- If it doesn't work, read these instructions: https://www.linuxbabe.com/linux-server/how-to-enable-etcrc-local-with-systemd
|
||||
* Disable networking.service, bluetooth.service, and dchpcd.service
|
||||
* apt-get autoclean
|
||||
* apt-get autoremove
|
||||
* Exit from proot
|
||||
* Copy all of the project files from circlean_fs/ into the two partitions:
|
||||
- rsync -vnri <source> <destination> will do a dry run of what will be copied, remove the -n to copy. See the rsync manpage for details.
|
||||
- diode_controller/ if you're using the led functionality and have an external led
|
||||
- midi/ files into /opt/midi/
|
||||
- you might want to double check all of the permissions of the new files/directories
|
||||
* Copy the image over to the SD card: sudo dd bs=4M if=<image> of=/dev/sd<letter>
|
||||
- In newer versions of dd, you can add status=progress
|
||||
* Optional: fsck the root partition (sudo e2fsck -f /dev/sd<letter>2).
|
||||
* Test with an rpi
|
||||
- FAT32 filesystem
|
||||
- NTFS filesystem
|
|
@ -0,0 +1,24 @@
|
|||
Modifying an already-built image
|
||||
================================
|
||||
One way to debug the project or test changes quickly is to modify an already built
|
||||
version of the project. Once you've got an image set up on an SD card, you can mount
|
||||
the image and make changes to the files directly or copy changes you've made locally
|
||||
onto the mounted image. The only requirement is a linux distro such as Debian or Ubuntu.
|
||||
If you're using MacOS, you can download and install VirtualBox.
|
||||
|
||||
Mounting an image
|
||||
=================
|
||||
* The steps listed in mount_image.sh are only necessary if you'd like to chroot
|
||||
into and run executables from the image locally.
|
||||
* To mount the image for the purpose of reading/writing to it, the process is much
|
||||
* Plug the SD card into the computer.
|
||||
* If you're on Virtualbox, you'll probably have to unmount the image on the host OS
|
||||
(on MacOS this involves ejecting it or using diskutil unmountDisk) and then mount it
|
||||
on the virtualized OS. You might have to select it under "Devices" first.
|
||||
* Then, in linux, use sudo fdisk -l to find the location of the image.
|
||||
* sudo mount $PATH_TO_IMAGE $PATH_TO_CHOSEN_MOUNT_POINT will mount the image.
|
||||
* The path to the image will need to be the path to the partition with the OS on it,
|
||||
which should be the second partition. So /dev/sdb2, not just dev/sdb.
|
||||
* When you're done, sudo umount $PATH_TO_MOUNT_POINT will unmount it.
|
||||
* If you get a warning about "No caching mode page found," it's safe to skip it
|
||||
by pressing enter.
|
|
@ -0,0 +1,74 @@
|
|||
Various qemu startup commands
|
||||
=============================
|
||||
|
||||
From https://www.raspberrypi.org/forums/viewtopic.php?f=29&t=37386
|
||||
qemu-system-arm -kernel ~/qemu_vms/kernel-qemu-4.4.13-jessie -cpu arm1176 -m 256 -M versatilepb -no-reboot -serial stdio -append "root=/dev/sda2 panic=1" -hda ~/qemu_vms/2016-09-23-raspbian-jessie-lite.img -redir tcp:5022::22
|
||||
|
||||
|
||||
From https://github.com/dhruvvyas90/qemu-rpi-kernel
|
||||
qemu-system-arm -kernel ~/qemu_vms/kernel-qemu-4.4.13-jessie -cpu arm1176 -m 256 -M versatilepb -serial stdio -append "root=/dev/sda2 rootfstype=ext4 rw" -hda ~/qemu_vms/2016-09-23-raspbian-jessie-lite.img
|
||||
|
||||
|
||||
From http://pub.phyks.me/respawn/mypersonaldata/public/2014-05-20-11-08-01/
|
||||
qemu-system-arm -kernel <<<path to kernel>>> -cpu arm1176 -m 256 -M versatilepb -no-reboot -serial stdio -append "root=/dev/sda2 panic=1 rootfstype=ext4 rw init=/bin/bash" -hda <<<path to disk image>>>
|
||||
|
||||
|
||||
Others:
|
||||
qemu-system-arm -kernel ~/qemu_vms/kernel-qemu-3.10.25-wheezy -cpu arm1176 -m 256 -M versatilepb -serial stdio -append "root=/dev/sda2 rootfstype=ext4 rw" -hda ~/qemu_vms/2015-02-16-raspbian-wheezy.img
|
||||
|
||||
qemu-system-arm -kernel qemu-rpi-kernel/kernel-qemu-3.10.25-wheezy -cpu arm1176 -m 256 -M versatilepb -serial stdio -append "root=/dev/sda2 rootfstype=ext4 rw" -hda 2015-02-16-raspbian-wheezy.img
|
||||
|
||||
|
||||
|
||||
Places to get raspbian base images:
|
||||
===================================
|
||||
|
||||
For Raspbian Wheezy image:
|
||||
wget https://downloads.raspberrypi.org/raspbian/images/raspbian-2015-02-17/2015-02-16-raspbian-wheezy.zip
|
||||
|
||||
For Raspbian Jessie Lite image:
|
||||
wget https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2016-09-28/2016-09-23-raspbian-jessie-lite.zip
|
||||
|
||||
|
||||
|
||||
|
||||
Traceback of the qemu failure on digitalocean
|
||||
=============================================
|
||||
|
||||
pulseaudio: pa_context_connect() failed
|
||||
pulseaudio: Reason: Connection refused
|
||||
pulseaudio: Failed to initialize PA contextaudio: Could not init 'pa' audio driver
|
||||
ALSA lib confmisc.c:768:(parse_card) cannot find card '0'
|
||||
ALSA lib conf.c:4259:(_snd_config_evaluate) function snd_func_card_driver returned error: No such file or directory
|
||||
ALSA lib confmisc.c:392:(snd_func_concat) error evaluating strings
|
||||
ALSA lib conf.c:4259:(_snd_config_evaluate) function snd_func_concat returned error: No such file or directory
|
||||
ALSA lib confmisc.c:1251:(snd_func_refer) error evaluating name
|
||||
ALSA lib conf.c:4259:(_snd_config_evaluate) function snd_func_refer returned error: No such file or directory
|
||||
ALSA lib conf.c:4738:(snd_config_expand) Evaluate error: No such file or directory
|
||||
ALSA lib pcm.c:2239:(snd_pcm_open_noupdate) Unknown PCM default
|
||||
alsa: Could not initialize DAC
|
||||
alsa: Failed to open 'default':
|
||||
alsa: Reason: No such file or directory
|
||||
ALSA lib confmisc.c:768:(parse_card) cannot find card '0'
|
||||
ALSA lib conf.c:4259:(_snd_config_evaluate) function snd_func_card_driver returned error: No such file or directory
|
||||
ALSA lib confmisc.c:392:(snd_func_concat) error evaluating strings
|
||||
ALSA lib conf.c:4259:(_snd_config_evaluate) function snd_func_concat returned error: No such file or directory
|
||||
ALSA lib confmisc.c:1251:(snd_func_refer) error evaluating name
|
||||
ALSA lib conf.c:4259:(_snd_config_evaluate) function snd_func_refer returned error: No such file or directory
|
||||
ALSA lib conf.c:4738:(snd_config_expand) Evaluate error: No such file or directory
|
||||
ALSA lib pcm.c:2239:(snd_pcm_open_noupdate) Unknown PCM default
|
||||
alsa: Could not initialize DAC
|
||||
alsa: Failed to open 'default':
|
||||
alsa: Reason: No such file or directory
|
||||
audio: Failed to create voice 'lm4549.out'
|
||||
Could not initialize SDL(No available video device) - exiting
|
||||
|
||||
|
||||
Notes
|
||||
=====
|
||||
- The error message: it is probably not a big deal - can make them not being blocking by modifying https://github.com/CIRCL/Circlean/blob/master/tests/run.exp#L10
|
||||
- https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=760365
|
||||
- Could not initialize SDL(No available video device) - exiting <= this one is blocking
|
||||
- I guess it is the vnc switch - requires x11 installed
|
||||
- If you use a cloud instance, you will need to get qemu to open a port you can connect to with vnc
|
||||
- The good thing of having VNC is that you can see what explodes when you're running the image
|
|
@ -0,0 +1,69 @@
|
|||
Add empty space to the image
|
||||
============================
|
||||
|
||||
* Add 2Gb
|
||||
|
||||
```
|
||||
> dd if=/dev/zero bs=1024k count=2048 >> 2016-03-18-raspbian-jessie-lite.img
|
||||
```
|
||||
|
||||
Expand partition size
|
||||
=====================
|
||||
|
||||
```
|
||||
> fdisk 2016-03-18-raspbian-jessie-lite.img
|
||||
|
||||
Command (m for help): *p*
|
||||
Disk 2016-03-18-raspbian-jessie-lite.img: 3.3 GiB, 3508535296 bytes, 6852608 sectors
|
||||
Units: sectors of 1 * 512 = 512 bytes
|
||||
Sector size (logical/physical): 512 bytes / 512 bytes
|
||||
I/O size (minimum/optimal): 512 bytes / 512 bytes
|
||||
Disklabel type: dos
|
||||
Disk identifier: 0x6f92008e
|
||||
|
||||
Device Boot Start End Sectors Size Id Type
|
||||
2016-03-18-raspbian-jessie-lite.img1 8192 131071 122880 60M c W95 FAT32 (LBA)
|
||||
2016-03-18-raspbian-jessie-lite.img2 131072 2658303 2527232 1.2G 83 Linux
|
||||
|
||||
Command (m for help): *d*
|
||||
Partition number (1,2, default 2): *2*
|
||||
|
||||
Partition 2 has been deleted.
|
||||
|
||||
Command (m for help): *n*
|
||||
Partition type
|
||||
p primary (1 primary, 0 extended, 3 free)
|
||||
e extended (container for logical partitions)
|
||||
Select (default p):
|
||||
|
||||
Using default response p.
|
||||
Partition number (2-4, default 2):
|
||||
First sector (2048-6852607, default 2048): *131072*
|
||||
Last sector, +sectors or +size{K,M,G,T,P} (131072-6852607, default 6852607):
|
||||
|
||||
Created a new partition 2 of type 'Linux' and of size 3.2 GiB.
|
||||
|
||||
Command (m for help): *w*
|
||||
The partition table has been altered.
|
||||
Syncing disks.
|
||||
```
|
||||
|
||||
Resize partition
|
||||
================
|
||||
|
||||
* Chroot in the image
|
||||
|
||||
```
|
||||
sudo ./proper_chroot.sh
|
||||
```
|
||||
|
||||
* Resize the partition (not from the chroot)
|
||||
|
||||
```
|
||||
> df | grep /mnt/arm
|
||||
|
||||
/dev/loop0 3927752 1955672 1794172 53% /mnt/arm_rPi
|
||||
/dev/loop1 57288 18960 38328 34% /mnt/arm_rPi/boot
|
||||
|
||||
> sudo resize2fs /dev/loop0
|
||||
```
|
|
@ -0,0 +1,227 @@
|
|||
Building the image from scratch
|
||||
===============================
|
||||
|
||||
There is always a prebuilt image available for download and installation as
|
||||
described in the [README](../README.md). If you'd like to build the project yourself,
|
||||
there are several steps involved:
|
||||
|
||||
* Downloading a generic Raspbian Lite image
|
||||
* Resizing the image and partition
|
||||
* Downloading and building the dependencies
|
||||
* Modifying the image configuration
|
||||
* Copying the project filesystem into the image
|
||||
|
||||
This procedure will only work on Ubuntu or Debian Linux. If you use MacOS or
|
||||
Windows, the best option is to install Linux in a virtual machine using
|
||||
something like VirtualBox.
|
||||
|
||||
It is recommended that you make a copy of image_setup_checklist.md and √ items off
|
||||
on the list as you go.
|
||||
|
||||
Preparation
|
||||
===========
|
||||
|
||||
* Make sure your development environment is up to date:
|
||||
```
|
||||
apt-get update
|
||||
apt-get dist-upgrade
|
||||
```
|
||||
* Install qemu, qemu-user-static, and proot if not already installed:
|
||||
```
|
||||
apt-get install qemu qemu-user-static qemu-user proot
|
||||
```
|
||||
|
||||
Download the Raspbian image
|
||||
==============================
|
||||
|
||||
* Get the most recent version of Raspbian Jessie Lite from https://downloads.raspberrypi.org/raspbian_lite/images/:
|
||||
|
||||
```
|
||||
wget https://downloads.raspberrypi.org/raspbian_lite_latest
|
||||
```
|
||||
* Verify the hash of the downloaded file and compare it to the hash on the server:
|
||||
```
|
||||
shasum XXXX-XX-XX-raspbian-buster-lite.zip
|
||||
```
|
||||
* Unpack it:
|
||||
```
|
||||
unzip XXXX-XX-XX-raspbian-buster-lite.zip
|
||||
```
|
||||
|
||||
Add space to the image
|
||||
=========================
|
||||
|
||||
* Use dd to add 2GB (2048 blocks of 1024k each). Using /dev/zero as the input
|
||||
file yields an unlimited number of "0x00" bytes.
|
||||
```
|
||||
> dd if=/dev/zero bs=1024k count=2048 >> XXXX-XX-XX-raspbian-jessie-lite.img
|
||||
```
|
||||
|
||||
* Expand the root (second) partition using sfdisk:
|
||||
```
|
||||
> echo ", +" | sfdisk -N 2 XXXX-XX-XX-raspbian-jessie-lite.img
|
||||
Checking that no-one is using this disk right now ... OK
|
||||
|
||||
Disk 2017-11-29-raspbian-stretch-lite.img: 3.7 GiB, 4005560320 bytes, 7823360 sectors
|
||||
Units: sectors of 1 * 512 = 512 bytes
|
||||
Sector size (logical/physical): 512 bytes / 512 bytes
|
||||
I/O size (minimum/optimal): 512 bytes / 512 bytes
|
||||
Disklabel type: dos
|
||||
Disk identifier: 0x37665771
|
||||
|
||||
Old situation:
|
||||
|
||||
Device Boot Start End Sectors Size Id Type
|
||||
2017-11-29-raspbian-stretch-lite.img1 8192 93236 85045 41.5M c W95 FAT32 (LBA)
|
||||
2017-11-29-raspbian-stretch-lite.img2 94208 3629055 3534848 1.7G 83 Linux
|
||||
|
||||
2017-11-29-raspbian-stretch-lite.img2:
|
||||
New situation:
|
||||
Disklabel type: dos
|
||||
Disk identifier: 0x37665771
|
||||
|
||||
Device Boot Start End Sectors Size Id Type
|
||||
2017-11-29-raspbian-stretch-lite.img1 8192 93236 85045 41.5M c W95 FAT32 (LBA)
|
||||
2017-11-29-raspbian-stretch-lite.img2 94208 7823359 7729152 3.7G 83 Linux
|
||||
|
||||
The partition table has been altered.
|
||||
Syncing disks.
|
||||
```
|
||||
|
||||
* Edit `shell_utils/basic_mount_image.sh` to use the correct image path ($IMAGE)
|
||||
* Run the script
|
||||
```
|
||||
sudo shell_utils/basic_mount_image.sh
|
||||
```
|
||||
|
||||
* Resize the filesystem
|
||||
|
||||
Find the loop device of the root filesystem by running `losetup`, and it is the biggest one related to the image you mounted
|
||||
|
||||
```
|
||||
sudo resize2fs /dev/loop<ID of the loop FS mounted as /mnt/rpi-root>
|
||||
```
|
||||
|
||||
|
||||
Installing the dependencies
|
||||
===========================
|
||||
|
||||
* Copy circlean_fs/root_partition/etc/systemd/system/rc-local.service into the equivalent location in the image.
|
||||
```
|
||||
sudo cp circlean_fs/root_partition/etc/systemd/system/rc-local.service /mnt/rpi-root/etc/systemd/system/rc-local.service
|
||||
```
|
||||
* Use [proot](https://proot-me.github.io/) to enter the equivalent of a chroot inside the mounted image.
|
||||
```
|
||||
sudo proot -q qemu-arm -0 -r /mnt/rpi-root -b /mnt/rpi-boot:/boot -b /etc/resolv.conf:/etc/resolv.conf \
|
||||
-b /dev/:/dev/ -b /sys/:/sys/ -b /proc/:/proc/ -b /run/shm:/run/shm /bin/bash
|
||||
```
|
||||
|
||||
**WARNING**: if you have a permission error, make sure the `/tmp` directory is mointed with the `exec` flag.
|
||||
|
||||
* Change your locales (remove "en_GB.UTF-8 UTF-8", add "en_US.UTF-8 UTF-8"). The
|
||||
arrow keys move the cursor, spacebar selects/deselects a locale, tab moves the cursor
|
||||
to a different context, and enter lets you select "ok". This step might take some time,
|
||||
be patient:
|
||||
```
|
||||
sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/g' /etc/locale.gen
|
||||
sed -i -e 's/en_GB.UTF-8 UTF-8/# en_US.UTF-8 UTF-8/g' /etc/locale.gen
|
||||
locale-gen en_US.UTF-8
|
||||
update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
|
||||
```
|
||||
* In the image, make sure everything is up-to-date and remove old packages. You may have to
|
||||
run dist-upgrade and autoremove several times for everything to be installed, and a few
|
||||
raspbian-sys-mods related installs may fail - you can ignore them:
|
||||
```
|
||||
apt-get update
|
||||
apt-get dist-upgrade
|
||||
apt-get autoremove
|
||||
```
|
||||
* Install the linux dependencies (see CONTRIBUTING.md for more details). If you see warnings that
|
||||
from qemu about "Unsupported syscall: 384", you can ignore them. `getrandom(2)` was implemented in
|
||||
kernel 3.17 and apt will use /dev/urandom when it fails:
|
||||
```
|
||||
apt-get install timidity git p7zip-full python3 python3-pip ntfs-3g libjpeg-dev libtiff-dev \
|
||||
libwebp-dev tk-dev python3-tk liblcms2-dev tcl-dev libopenjp2-7 libxml2-dev \
|
||||
libssl-dev libffi-dev libxslt1-dev exfat-fuse exfat-utils udisks2
|
||||
```
|
||||
* Compile p7zip-rar from source. First, uncomment out the second line in /etc/apt/sources.list. Then:
|
||||
```
|
||||
cd /home/pi
|
||||
mkdir rar && cd rar/
|
||||
apt-get update
|
||||
apt-get build-dep p7zip-rar
|
||||
apt-get source -b p7zip-rar
|
||||
dpkg -i ${path to p7zip-rar .deb file}
|
||||
```
|
||||
* Install the Python dependencies for `PyCIRCLean/filecheck.py`. PyCIRCLean is 3.6+
|
||||
compatible, so use `pip -V` to make sure you're using the right version of pip. You might
|
||||
have to edit your PATH variable or use pip3 to get the correct pip. You also might want to
|
||||
verify that these dependencies are current by checking in the PyCIRCLean git repo.
|
||||
```
|
||||
cd /home/pi
|
||||
git clone https://github.com/CIRCL/PyCIRCLean.git
|
||||
cd PyCIRCLean
|
||||
pip3 install -r requirements.txt
|
||||
```
|
||||
* Create a new user named "kitten":
|
||||
```
|
||||
useradd -m kitten
|
||||
chown -R kitten:kitten /home/kitten
|
||||
```
|
||||
* Enable `rc.local`, which ensures that the code in `/etc/rc.local` is run on boot.
|
||||
This is what triggers CIRCLean to run.
|
||||
```
|
||||
systemctl enable rc-local.service
|
||||
```
|
||||
* Turn off several networking related services. This speeds up boot and reduces the attack surface:
|
||||
```
|
||||
systemctl disable networking.service
|
||||
systemctl disable bluetooth.service
|
||||
systemctl disable dhcpcd.service
|
||||
```
|
||||
* Clean up:
|
||||
```
|
||||
apt-get clean
|
||||
apt-get autoremove
|
||||
apt-get autoclean
|
||||
```
|
||||
* Exit proot, and copy the files from your repository into the mounted
|
||||
image. Adding a -n flag will make rsync do a dry run instead of copying. See the rsync
|
||||
manpage for more details. Make sure to include the trailing slashes on the paths:
|
||||
```
|
||||
exit
|
||||
sudo rsync -vri circlean_fs/boot/ /mnt/rpi-boot/
|
||||
sudo rsync -vri circlean_fs/root_partition/ /mnt/rpi-root/
|
||||
sudo cp -rf midi /mnt/rpi-root/opt/
|
||||
```
|
||||
* If have an external hardware led and you're using the led functionality, copy
|
||||
the led files from diode_controller/ as well.
|
||||
|
||||
* Unmount the image
|
||||
```
|
||||
sudo umount /mnt/rpi-boot /mnt/rpi-root
|
||||
```
|
||||
|
||||
Write the image on a SD card
|
||||
============================
|
||||
|
||||
* Plug your SD card into the computer. Then, find where it is mounted using lsblk or df:
|
||||
```
|
||||
lsblk
|
||||
df -h
|
||||
```
|
||||
* If it has been automatically mounted, unmount the SD card (use the path you
|
||||
found in the previous step):
|
||||
```
|
||||
umount $PATH_TO_YOUR_SD
|
||||
```
|
||||
* Write the image to the card. Newer versions of dd include a status option to monitor the
|
||||
copying process:
|
||||
```
|
||||
sudo dd bs=4M if=$PATH_TO_YOUR_IMAGE of=$PATH_TO_YOUR_SD status=progress
|
||||
```
|
||||
* Use fsck to verify the root partition:
|
||||
```
|
||||
sudo fsck.vfat -f /dev/<partition>1
|
||||
sudo e2fsck -f /dev/<partition>2
|
||||
```
|
|
@ -16,14 +16,14 @@ Ideas
|
|||
=====
|
||||
|
||||
Source keys:
|
||||
[DONE] Working documents, one / multiple partitions
|
||||
- Working documents, one / multiple partitions
|
||||
- Non working documents: one / multiple partitions
|
||||
- different FS on different partitions
|
||||
- Non working FS
|
||||
- Malicious documents (very slow, might break the conversions)
|
||||
|
||||
Destinations keys
|
||||
[DONE] empty, big enough
|
||||
- empty, big enough
|
||||
- empty, too small
|
||||
- broken
|
||||
- not empty
|
|
@ -1,5 +0,0 @@
|
|||
# /etc/pmount.allow
|
||||
# pmount will allow users to additionally mount all devices that are
|
||||
# listed here.
|
||||
/dev/sdb1
|
||||
/dev/sda*
|
|
@ -1,2 +0,0 @@
|
|||
KERNEL=="sdc", SYMLINK+="mmcblk0"
|
||||
KERNEL=="sdc?", SYMLINK+="mmcblk0p%n",
|
|
@ -1,23 +0,0 @@
|
|||
DEV_SRC='/dev/sda'
|
||||
DEV_DST='sdb1'
|
||||
|
||||
# User allowed to do the following commands without password
|
||||
USERNAME='kitten'
|
||||
MUSIC="/opt/midi/"
|
||||
|
||||
ID=`/usr/bin/id -u`
|
||||
|
||||
# Paths used in multiple scripts
|
||||
SRC="src"
|
||||
DST="dst"
|
||||
TEMP="/media/${DST}/temp"
|
||||
ZIPTEMP="/media/${DST}/ziptemp"
|
||||
LOGS="/media/${DST}/logs"
|
||||
|
||||
|
||||
# commands
|
||||
SYNC='/bin/sync'
|
||||
TIMIDITY='/usr/bin/timidity'
|
||||
MOUNT='/bin/mount'
|
||||
PMOUNT='/usr/bin/pmount -A -s'
|
||||
PUMOUNT='/usr/bin/pumount'
|
|
@ -1,4 +0,0 @@
|
|||
# Paths to the commands used to convert the files
|
||||
PDF="/usr/bin/pdf2htmlEX"
|
||||
LO="/usr/bin/libreoffice"
|
||||
UNPACKER="/usr/bin/7z"
|
|
@ -1,321 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
import magic
|
||||
import os
|
||||
import mimetypes
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from helpers import FileBase, KittenGroomerBase, main
|
||||
|
||||
LIBREOFFICE = '/usr/bin/unoconv'
|
||||
GS = '/usr/bin/gs'
|
||||
PDF2HTMLEX = '/usr/bin/pdf2htmlEX'
|
||||
SEVENZ = '/usr/bin/7z'
|
||||
|
||||
|
||||
# Prepare application/<subtype>
|
||||
mimes_office = ['msword', 'vnd.openxmlformats-officedocument.', 'vnd.ms-',
|
||||
'vnd.oasis.opendocument']
|
||||
mimes_pdf = ['pdf']
|
||||
mimes_xml = ['xml']
|
||||
mimes_ms = ['x-dosexec']
|
||||
mimes_compressed = ['zip', 'x-rar', 'x-bzip2', 'x-lzip', 'x-lzma', 'x-lzop',
|
||||
'x-xz', 'x-compress', 'x-gzip', 'x-tar', 'compressed']
|
||||
mimes_data = ['octet-stream']
|
||||
|
||||
|
||||
class File(FileBase):
|
||||
|
||||
def __init__(self, src_path, dst_path):
|
||||
''' Init file object, set the mimetype '''
|
||||
super(File, self).__init__(src_path, dst_path)
|
||||
mimetype = magic.from_file(src_path, mime=True)
|
||||
self.main_type, self.sub_type = mimetype.split('/')
|
||||
self.log_details.update({'maintype': self.main_type, 'subtype': self.sub_type})
|
||||
self.expected_mimetype, self.expected_extensions = self.crosscheck_mime()
|
||||
self.is_recursive = False
|
||||
|
||||
def crosscheck_mime(self):
|
||||
'''
|
||||
Set the expected mime and extension variables based on mime type.
|
||||
'''
|
||||
# /usr/share/mime has interesting stuff
|
||||
|
||||
# guess_type uses the extension to get a mime type
|
||||
expected_mimetype, encoding = mimetypes.guess_type(self.src_path, strict=False)
|
||||
if expected_mimetype is not None:
|
||||
expected_extensions = mimetypes.guess_all_extensions(expected_mimetype,
|
||||
strict=False)
|
||||
else:
|
||||
# the extension is unknown...
|
||||
expected_extensions = None
|
||||
|
||||
return expected_mimetype, expected_extensions
|
||||
|
||||
def verify_extension(self):
|
||||
'''Check if the extension is the one we expect'''
|
||||
if self.expected_extensions is None:
|
||||
return None
|
||||
path, actual_extension = os.path.splitext(self.src_path)
|
||||
return actual_extension in self.expected_extensions
|
||||
|
||||
def verify_mime(self):
|
||||
'''Check if the mime is the one we expect'''
|
||||
if self.expected_mimetype is None:
|
||||
return None
|
||||
actual_mimetype = '{}/{}'.format(self.main_type, self.sub_type)
|
||||
return actual_mimetype == self.expected_mimetype
|
||||
|
||||
|
||||
class KittenGroomer(KittenGroomerBase):
|
||||
|
||||
def __init__(self, root_src=None, root_dst=None, max_recursive=5):
|
||||
'''
|
||||
Initialize the basics of the conversion process
|
||||
'''
|
||||
if root_src is None:
|
||||
root_src = os.path.join(os.sep, 'media', 'src')
|
||||
if root_dst is None:
|
||||
root_dst = os.path.join(os.sep, 'media', 'dst')
|
||||
super(KittenGroomer, self).__init__(root_src, root_dst)
|
||||
|
||||
self.recursive = 0
|
||||
self.max_recursive = max_recursive
|
||||
|
||||
subtypes_apps = [
|
||||
(mimes_office, self._office_related),
|
||||
(mimes_pdf, self._pdf),
|
||||
(mimes_xml, self._office_related),
|
||||
(mimes_ms, self._executables),
|
||||
(mimes_compressed, self._archive),
|
||||
(mimes_data, self._binary_app),
|
||||
]
|
||||
self.subtypes_application = self._init_subtypes_application(subtypes_apps)
|
||||
|
||||
self.mime_processing_options = {
|
||||
'text': self.text,
|
||||
'audio': self.audio,
|
||||
'image': self.image,
|
||||
'video': self.video,
|
||||
'application': self.application,
|
||||
'example': self.example,
|
||||
'message': self.message,
|
||||
'model': self.model,
|
||||
'multipart': self.multipart,
|
||||
'inode': self.inode,
|
||||
}
|
||||
|
||||
# ##### Helpers #####
|
||||
def _init_subtypes_application(self, subtypes_application):
|
||||
'''
|
||||
Create the Dict to pick the right function based on the sub mime type
|
||||
'''
|
||||
to_return = {}
|
||||
for list_subtypes, fct in subtypes_application:
|
||||
for st in list_subtypes:
|
||||
to_return[st] = fct
|
||||
return to_return
|
||||
|
||||
def _print_log(self):
|
||||
'''
|
||||
Print the logs related to the current file being processed
|
||||
'''
|
||||
tmp_log = self.log_name.fields(**self.cur_file.log_details)
|
||||
if self.cur_file.log_details.get('dangerous'):
|
||||
tmp_log.warning(self.cur_file.log_string)
|
||||
elif self.cur_file.log_details.get('unknown') or self.cur_file.log_details.get('binary'):
|
||||
tmp_log.info(self.cur_file.log_string)
|
||||
else:
|
||||
tmp_log.debug(self.cur_file.log_string)
|
||||
|
||||
def _run_process(self, command_line):
|
||||
'''Run subprocess, wait until it finishes'''
|
||||
args = shlex.split(command_line)
|
||||
p = subprocess.Popen(args)
|
||||
while True:
|
||||
code = p.poll()
|
||||
if code is not None:
|
||||
break
|
||||
time.sleep(1)
|
||||
return True
|
||||
|
||||
#######################
|
||||
|
||||
# ##### Discarded mime types, reason in the comments ######
|
||||
def inode(self):
|
||||
''' Usually empty file. No reason (?) to copy it on the dest key'''
|
||||
self.cur_file.log_string += 'Inode file'
|
||||
|
||||
def unknown(self):
|
||||
''' This main type is unknown, that should not happen '''
|
||||
self.cur_file.log_string += 'Unknown file'
|
||||
|
||||
# ##### Threated as malicious, no reason to have it on a USB key ######
|
||||
def example(self):
|
||||
'''Way to process example file'''
|
||||
self.cur_file.log_string += 'Example file'
|
||||
self.cur_file.make_dangerous()
|
||||
self._safe_copy()
|
||||
|
||||
def message(self):
|
||||
'''Way to process message file'''
|
||||
self.cur_file.log_string += 'Message file'
|
||||
self.cur_file.make_dangerous()
|
||||
self._safe_copy()
|
||||
|
||||
def model(self):
|
||||
'''Way to process model file'''
|
||||
self.cur_file.log_string += 'Model file'
|
||||
self.cur_file.make_dangerous()
|
||||
self._safe_copy()
|
||||
|
||||
def multipart(self):
|
||||
'''Way to process multipart file'''
|
||||
self.cur_file.log_string += 'Multipart file'
|
||||
self.cur_file.make_dangerous()
|
||||
self._safe_copy()
|
||||
|
||||
#######################
|
||||
|
||||
# ##### Converted ######
|
||||
def text(self):
|
||||
''' LibreOffice should be able to open all the files '''
|
||||
self.cur_file.log_string += 'Text file'
|
||||
self._office_related()
|
||||
|
||||
def application(self):
|
||||
''' Everything can be there, using the subtype to decide '''
|
||||
for subtype, fct in self.subtypes_application.iteritems():
|
||||
if subtype in self.cur_file.sub_type:
|
||||
fct()
|
||||
self.cur_file.log_string += 'Application file'
|
||||
return
|
||||
self.cur_file.log_string += 'Unknown Application file'
|
||||
self._unknown_app()
|
||||
|
||||
def _executables(self):
|
||||
'''Way to process executable file'''
|
||||
self.cur_file.add_log_details('processing_type', 'executable')
|
||||
self.cur_file.make_dangerous()
|
||||
self._safe_copy()
|
||||
|
||||
def _office_related(self):
|
||||
'''Way to process all the files LibreOffice can handle'''
|
||||
self.cur_file.add_log_details('processing_type', 'office')
|
||||
dst_dir, filename = os.path.split(self.cur_file.dst_path)
|
||||
tmpdir = os.path.join(dst_dir, 'temp')
|
||||
name, ext = os.path.splitext(filename)
|
||||
tmppath = os.path.join(tmpdir, name + '.pdf')
|
||||
self._safe_mkdir(tmpdir)
|
||||
lo_command = '{} --format pdf -eSelectPdfVersion=1 --output {} {}'.format(
|
||||
LIBREOFFICE, tmppath, self.cur_file.src_path)
|
||||
self._run_process(lo_command)
|
||||
self._pdfa(tmppath)
|
||||
self._safe_rmtree(tmpdir)
|
||||
|
||||
def _pdfa(self, tmpsrcpath):
|
||||
'''Way to process PDF/A file'''
|
||||
pdf_command = '{} --dest-dir / {} {}'.format(PDF2HTMLEX, tmpsrcpath,
|
||||
self.cur_file.dst_path + '.html')
|
||||
self._run_process(pdf_command)
|
||||
|
||||
def _pdf(self):
|
||||
'''Way to process PDF file'''
|
||||
self.cur_file.add_log_details('processing_type', 'pdf')
|
||||
dst_dir, filename = os.path.split(self.cur_file.dst_path)
|
||||
tmpdir = os.path.join(dst_dir, 'temp')
|
||||
tmppath = os.path.join(tmpdir, filename)
|
||||
self._safe_mkdir(tmpdir)
|
||||
gs_command = '{} -dPDFA -dBATCH -dNOPAUSE -sDEVICE=pdfwrite -sOutputFile={} {}'.format(
|
||||
GS, tmppath, self.cur_file.src_path)
|
||||
self._run_process(gs_command)
|
||||
self._pdfa(tmppath)
|
||||
self._safe_rmtree(tmpdir)
|
||||
|
||||
def _archive(self):
|
||||
'''Way to process Archive'''
|
||||
self.cur_file.add_log_details('processing_type', 'archive')
|
||||
self.cur_file.is_recursive = True
|
||||
self.cur_file.log_string += 'Archive extracted, processing content.'
|
||||
tmpdir = self.cur_file.dst_path + '_temp'
|
||||
self._safe_mkdir(tmpdir)
|
||||
extract_command = '{} -p1 x {} -o{} -bd'.format(SEVENZ, self.cur_file.src_path, tmpdir)
|
||||
self._run_process(extract_command)
|
||||
self.recursive += 1
|
||||
self.processdir(tmpdir, self.cur_file.dst_path)
|
||||
self.recursive -= 1
|
||||
self._safe_rmtree(tmpdir)
|
||||
|
||||
def _unknown_app(self):
|
||||
'''Way to process an unknown file'''
|
||||
self.cur_file.make_unknown()
|
||||
self._safe_copy()
|
||||
|
||||
def _binary_app(self):
|
||||
'''Way to process an unknown binary file'''
|
||||
self.cur_file.make_binary()
|
||||
self._safe_copy()
|
||||
|
||||
#######################
|
||||
|
||||
# ##### Not converted, checking the mime type ######
|
||||
def audio(self):
|
||||
'''Way to process an audio file'''
|
||||
self.cur_file.log_string += 'Audio file'
|
||||
self._media_processing()
|
||||
|
||||
def image(self):
|
||||
'''Way to process an image'''
|
||||
self.cur_file.log_string += 'Image file'
|
||||
self._media_processing()
|
||||
|
||||
def video(self):
|
||||
'''Way to process a video'''
|
||||
self.cur_file.log_string += 'Video file'
|
||||
self._media_processing()
|
||||
|
||||
def _media_processing(self):
|
||||
'''Generic way to process all the media files'''
|
||||
self.cur_log.fields(processing_type='media')
|
||||
if not self.cur_file.verify_mime() or not self.cur_file.verify_extension():
|
||||
# The extension is unknown or doesn't match the mime type => suspicious
|
||||
# TODO: write details in the logfile
|
||||
self.cur_file.make_dangerous()
|
||||
self._safe_copy()
|
||||
|
||||
#######################
|
||||
|
||||
def processdir(self, src_dir=None, dst_dir=None):
|
||||
'''
|
||||
Main function doing the processing
|
||||
'''
|
||||
if src_dir is None:
|
||||
src_dir = self.src_root_dir
|
||||
if dst_dir is None:
|
||||
dst_dir = self.dst_root_dir
|
||||
|
||||
if self.recursive > 0:
|
||||
self._print_log()
|
||||
|
||||
if self.recursive >= self.max_recursive:
|
||||
self.cur_log.warning('ARCHIVE BOMB.')
|
||||
self.cur_log.warning('The content of the archive contains recursively other archives.')
|
||||
self.cur_log.warning('This is a bad sign so the archive is not extracted to the destination key.')
|
||||
self._safe_rmtree(src_dir)
|
||||
if src_dir.endswith('_temp'):
|
||||
archbomb_path = src_dir[:-len('_temp')]
|
||||
self._safe_remove(archbomb_path)
|
||||
|
||||
for srcpath in self._list_all_files(src_dir):
|
||||
self.cur_file = File(srcpath, srcpath.replace(src_dir, dst_dir))
|
||||
|
||||
self.log_name.info('Processing {} ({}/{})', srcpath.replace(src_dir + '/', ''),
|
||||
self.cur_file.main_type, self.cur_file.sub_type)
|
||||
self.mime_processing_options.get(self.cur_file.main_type, self.unknown)()
|
||||
if not self.cur_file.is_recursive:
|
||||
self._print_log()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(KittenGroomer)
|
|
@ -1,221 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
|
||||
source ./constraint.sh
|
||||
source ./constraint_conv.sh
|
||||
|
||||
RECURSIVE_ARCHIVE_MAX=3
|
||||
RECURSIVE_ARCHIVE_CURRENT=0
|
||||
ARCHIVE_BOMB=0
|
||||
LOGFILE="${LOGS}/processing.txt"
|
||||
|
||||
# Something went wrong.
|
||||
error_handler(){
|
||||
echo "FAILED." >> ${LOGFILE}
|
||||
echo -e "\tSomething went wrong during the duplication of the last file." >> ${LOGFILE}
|
||||
echo -e "\tPlease open a bug on https://www.github.com/Rafiot/KittenGroomer" >> ${LOGFILE}
|
||||
continue
|
||||
}
|
||||
|
||||
trap error_handler ERR TERM INT
|
||||
|
||||
office_n_txt(){
|
||||
src_file=${1}
|
||||
dst_file=${2}${1##$CURRENT_SRC}.html
|
||||
temp=${2}/temp
|
||||
${LO} --headless --convert-to pdf --outdir "${temp}" "${src_file}"
|
||||
${PDF} --dest-dir=/ ${temp}/*.pdf ${dst_file}
|
||||
rm -rf "${temp}"
|
||||
}
|
||||
|
||||
copy(){
|
||||
src_file=${1}
|
||||
dst_file=${2}
|
||||
mkdir -p `dirname "${dst_file}"`
|
||||
cp "${src_file}" "${dst_file}"
|
||||
}
|
||||
|
||||
# Plain text
|
||||
text(){
|
||||
echo Text file ${1}
|
||||
office_n_txt ${1} ${2}
|
||||
}
|
||||
|
||||
# Multimedia
|
||||
## WARNING: They are assumed safe.
|
||||
audio(){
|
||||
echo Audio file ${1}
|
||||
copy ${1} ${2}${1##$CURRENT_SRC}
|
||||
}
|
||||
|
||||
image(){
|
||||
echo Image file ${1}
|
||||
copy ${1} ${2}${1##$CURRENT_SRC}
|
||||
}
|
||||
|
||||
video(){
|
||||
echo Video file ${1}
|
||||
copy ${1} ${2}${1##$CURRENT_SRC}
|
||||
}
|
||||
|
||||
# Random - Used
|
||||
|
||||
archive(){
|
||||
echo Archive file ${1}
|
||||
if [ ${ARCHIVE_BOMB} -eq 0 ]; then
|
||||
temp_extract_dir=${2}_temp
|
||||
mkdir -p "${temp_extract_dir}"
|
||||
${UNPACKER} -p1 x "${1}" -o"${temp_extract_dir}" -bd
|
||||
main ${2} ${RECURSIVE_ARCHIVE_CURRENT} ${temp_extract_dir} || true
|
||||
rm -rf "${temp_extract_dir}"
|
||||
fi
|
||||
if [ ${ARCHIVE_BOMB} -eq 1 ]; then
|
||||
rm -rf "${2}"
|
||||
rm -rf "${2}_temp"
|
||||
fi
|
||||
CURRENT_SRC="/media/${SRC}"
|
||||
}
|
||||
|
||||
|
||||
application(){
|
||||
echo App file ${1}
|
||||
src_file=${1}
|
||||
dst_file=${2}${1##$CURRENT_SRC}
|
||||
mime_details=${3}
|
||||
case ${mime_details} in
|
||||
pdf)
|
||||
echo "Got a pdf"
|
||||
${PDF} --dest-dir "${2}" "${src_file}"
|
||||
;;
|
||||
msword|vnd.openxmlformats-officedocument.*|vnd.ms-*|vnd.oasis.opendocument*)
|
||||
# https://blogs.msdn.com/b/vsofficedeveloper/archive/2008/05/08/office-2007-open-xml-mime-types.aspx
|
||||
# http://plan-b-for-openoffice.org/glossary/term/mime-type
|
||||
echo "MS Office or ODF document"
|
||||
office_n_txt ${src_file} ${2}
|
||||
;;
|
||||
*xml*)
|
||||
echo "Got an XML"
|
||||
office_n_txt ${src_file} ${2}
|
||||
;;
|
||||
x-dosexec)
|
||||
echo "Win executable"
|
||||
copy ${src_file} ${2}/DANGEROUS_${1##$CURRENT_SRC/}_DANGEROUS
|
||||
;;
|
||||
zip|x-rar|x-bzip2|x-lzip|x-lzma|x-lzop|x-xz|x-compress|x-gzip|x-tar|*compressed)
|
||||
echo "Compressed file"
|
||||
archive ${src_file} ${dst_file}
|
||||
;;
|
||||
octet-stream)
|
||||
echo "Unknown type."
|
||||
copy ${src_file} ${dst_file}.bin
|
||||
;;
|
||||
*)
|
||||
echo "Unhandled type"
|
||||
copy ${src_file} ${dst_file}
|
||||
;;
|
||||
esac
|
||||
|
||||
}
|
||||
|
||||
# Random - Unused?
|
||||
## WARNING: They are assumed safe.
|
||||
|
||||
example(){
|
||||
echo Example file ${1}
|
||||
copy ${1} ${2}${1##$CURRENT_SRC}
|
||||
}
|
||||
|
||||
message(){
|
||||
echo Message file ${1}
|
||||
copy ${1} ${2}${1##$CURRENT_SRC}
|
||||
}
|
||||
|
||||
model(){
|
||||
echo Model file ${1}
|
||||
copy ${1} ${2}${1##$CURRENT_SRC}
|
||||
}
|
||||
|
||||
multipart(){
|
||||
echo Multipart file ${1}
|
||||
copy ${1} ${2}${1##$CURRENT_SRC}
|
||||
}
|
||||
|
||||
main(){
|
||||
if [ -z ${1} ]; then
|
||||
echo "Please specify the destination directory."
|
||||
exit
|
||||
fi
|
||||
set -e
|
||||
set -x
|
||||
|
||||
if [ -z ${2} ]; then
|
||||
CURRENT_SRC="/media/${SRC}"
|
||||
RECURSIVE_ARCHIVE_CURRENT=0
|
||||
ARCHIVE_BOMB=0
|
||||
else
|
||||
RECURSIVE_ARCHIVE_CURRENT=${2}
|
||||
CURRENT_SRC=${3}
|
||||
if [ ${RECURSIVE_ARCHIVE_CURRENT} -gt ${RECURSIVE_ARCHIVE_MAX} ]; then
|
||||
echo Archive bomb.
|
||||
ARCHIVE_BOMB=1
|
||||
echo "ARCHIVE BOMB." >> ${LOGFILE}
|
||||
echo "The content of the archive contains recursively other archives." >> ${LOGFILE}
|
||||
echo "This is a bad sign so the archive is not extracted to the destination key." >> ${LOGFILE}
|
||||
return
|
||||
else
|
||||
RECURSIVE_ARCHIVE_CURRENT=`expr ${RECURSIVE_ARCHIVE_CURRENT} + 1`
|
||||
fi
|
||||
fi
|
||||
|
||||
FILE_LIST=`find ${CURRENT_SRC} -type f`
|
||||
SAVEIFS=$IFS
|
||||
IFS=$(echo -en "\n\b")
|
||||
for file in ${FILE_LIST}; do
|
||||
# first param is the destination dir
|
||||
dest=${1}
|
||||
|
||||
mime=`file -b --mime-type "${file}"`
|
||||
echo ${mime}
|
||||
main_mime=`echo ${mime} | cut -f1 -d/`
|
||||
details=`echo ${mime} | cut -f2 -d/`
|
||||
echo -n "Processing ${file} (${mime})... " >> ${LOGFILE}
|
||||
case "${main_mime}" in
|
||||
"text")
|
||||
text ${file} ${dest} || error_handler
|
||||
;;
|
||||
"audio")
|
||||
audio ${file} ${dest} || error_handler
|
||||
;;
|
||||
"image")
|
||||
image ${file} ${dest} || error_handler
|
||||
;;
|
||||
"video")
|
||||
video ${file} ${dest} || error_handler
|
||||
;;
|
||||
"application")
|
||||
application ${file} ${dest} ${details} || error_handler
|
||||
;;
|
||||
"example")
|
||||
example ${file} ${dest} || error_handler
|
||||
;;
|
||||
"message")
|
||||
message ${file} ${dest} || error_handler
|
||||
;;
|
||||
"model")
|
||||
model ${file} ${dest} || error_handler
|
||||
;;
|
||||
"multipart")
|
||||
multipart ${file} ${dest} || error_handler
|
||||
;;
|
||||
*)
|
||||
echo "This should never happen... :]"
|
||||
echo $mime $main_mime $details
|
||||
;;
|
||||
esac
|
||||
echo "done." >> ${LOGFILE}
|
||||
done
|
||||
IFS=$SAVEIFS
|
||||
return 0
|
||||
}
|
||||
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
|
||||
from helpers import FileBase, KittenGroomerBase, main
|
||||
|
||||
|
||||
printers = ['.STL', '.obj']
|
||||
cnc = ['.nc', '.tap', '.gcode', '.dxf', '.stl', '.obj', '.iges', '.igs',
|
||||
'.vrml', '.vrl', '.thing', '.step', '.stp', '.x3d']
|
||||
shopbot = ['.ai', '.svg', '.dxf', '.dwg', '.eps']
|
||||
omax = ['.ai', '.svg', '.dxf', '.dwg', '.eps', '.omx', '.obj']
|
||||
epilog_laser = ['.ai', '.svg', '.dxf', '.dwg', '.eps']
|
||||
metabeam = ['.dxf']
|
||||
up = ['.upp', '.up3', '.stl', '.obj']
|
||||
|
||||
|
||||
class FilePier9(FileBase):
|
||||
|
||||
def __init__(self, src_path, dst_path):
|
||||
''' Init file object, set the extension '''
|
||||
super(FilePier9, self).__init__(src_path, dst_path)
|
||||
a, self.extension = os.path.splitext(self.src_path)
|
||||
|
||||
|
||||
class KittenGroomerPier9(KittenGroomerBase):
|
||||
|
||||
def __init__(self, root_src=None, root_dst=None):
|
||||
'''
|
||||
Initialize the basics of the copy
|
||||
'''
|
||||
if root_src is None:
|
||||
root_src = os.path.join(os.sep, 'media', 'src')
|
||||
if root_dst is None:
|
||||
root_dst = os.path.join(os.sep, 'media', 'dst')
|
||||
super(KittenGroomerPier9, self).__init__(root_src, root_dst)
|
||||
|
||||
# The initial version will accept all the file extension for all the machines.
|
||||
self.authorized_extensions = printers + cnc + shopbot + omax + epilog_laser + metabeam + up
|
||||
|
||||
def _print_log(self):
|
||||
'''
|
||||
Print the logs related to the current file being processed
|
||||
'''
|
||||
tmp_log = self.log_name.fields(**self.cur_file.log_details)
|
||||
if not self.cur_file.log_details.get('valid'):
|
||||
tmp_log.warning(self.cur_file.log_string)
|
||||
else:
|
||||
tmp_log.debug(self.cur_file.log_string)
|
||||
|
||||
def processdir(self):
|
||||
'''
|
||||
Main function doing the processing
|
||||
'''
|
||||
for srcpath in self._list_all_files(self.src_root_dir):
|
||||
self.log_name.info('Processing {}', srcpath.replace(self.src_root_dir + '/', ''))
|
||||
self.cur_file = FilePier9(srcpath, srcpath.replace(self.src_root_dir, self.dst_root_dir))
|
||||
if self.cur_file.extension in self.authorized_extensions:
|
||||
self.cur_file.add_log_details('valid', True)
|
||||
self.cur_file.log_string = 'Expected extension: ' + self.cur_file.extension
|
||||
self._safe_copy()
|
||||
else:
|
||||
self.cur_file.log_string = 'Bad extension: ' + self.cur_file.extension
|
||||
self._print_log()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(KittenGroomerPier9)
|
|
@ -1,114 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
source ./constraint.sh
|
||||
if ! [ "${ID}" -ge "1000" ]; then
|
||||
echo "This script cannot run as root."
|
||||
exit
|
||||
fi
|
||||
|
||||
source ./functions.sh
|
||||
|
||||
clean(){
|
||||
echo Cleaning.
|
||||
${SYNC}
|
||||
|
||||
# Cleanup source
|
||||
pumount ${SRC}
|
||||
|
||||
# Cleanup destination
|
||||
rm -rf ${TEMP}
|
||||
rm -rf ${ZIPTEMP}
|
||||
pumount ${DST}
|
||||
|
||||
exit
|
||||
}
|
||||
|
||||
trap clean EXIT TERM INT
|
||||
|
||||
# De we have a source device
|
||||
if [ ! -b ${DEV_SRC} ]; then
|
||||
echo "Source device (${DEV_SRC}) does not exists."
|
||||
exit
|
||||
fi
|
||||
# Find the partition names on the source device
|
||||
DEV_PARTITIONS=`ls "${DEV_SRC}"* | grep "${DEV_SRC}[1-9][0-6]*" || true`
|
||||
if [ -z "${DEV_PARTITIONS}" ]; then
|
||||
echo "${DEV_SRC} does not have any partitions."
|
||||
exit
|
||||
fi
|
||||
|
||||
# Do we have a destination device
|
||||
if [ ! -b "/dev/${DEV_DST}" ]; then
|
||||
echo "Destination device (/dev/${DEV_DST}) does not exists."
|
||||
exit
|
||||
fi
|
||||
|
||||
# mount and prepare destination device
|
||||
if ${MOUNT}|grep ${DST}; then
|
||||
${PUMOUNT} ${DST} || true
|
||||
fi
|
||||
# uid= only works on a vfat FS. What should wedo if we get an ext* FS ?
|
||||
${PMOUNT} -w ${DEV_DST} ${DST}
|
||||
if [ ${?} -ne 0 ]; then
|
||||
echo "Unable to mount /dev/${DEV_DST} on /media/${DST}"
|
||||
exit
|
||||
else
|
||||
echo "Target USB device (/dev/${DEV_DST}) mounted at /media/${DST}"
|
||||
rm -rf "/media/${DST}/FROM_PARTITION_"*
|
||||
|
||||
# prepare temp dirs and make sure it's empty
|
||||
mkdir -p "${TEMP}"
|
||||
mkdir -p "${ZIPTEMP}"
|
||||
mkdir -p "${LOGS}"
|
||||
|
||||
rm -rf "${TEMP}/"*
|
||||
rm -rf "${ZIPTEMP}/"*
|
||||
rm -rf "${LOGS}/"*
|
||||
fi
|
||||
|
||||
# Groom da kitteh!
|
||||
|
||||
# Find the FS types
|
||||
# lsblk -n -o name,fstype,mountpoint,label,uuid -r
|
||||
|
||||
PARTCOUNT=1
|
||||
for partition in ${DEV_PARTITIONS}
|
||||
do
|
||||
# Processing a partition
|
||||
echo "Processing partition: ${partition}"
|
||||
if [ `${MOUNT} | grep -c ${SRC}` -ne 0 ]; then
|
||||
${PUMOUNT} ${SRC}
|
||||
fi
|
||||
|
||||
${PMOUNT} -w ${partition} ${SRC}
|
||||
ls "/media/${SRC}" | grep -i autorun.inf | xargs -I {} mv "/media/${SRC}"/{} "/media/${SRC}"/DANGEROUS_{}_DANGEROUS || true
|
||||
${PUMOUNT} ${SRC}
|
||||
${PMOUNT} -r ${partition} ${SRC}
|
||||
if [ ${?} -ne 0 ]; then
|
||||
echo "Unable to mount ${partition} on /media/${SRC}"
|
||||
else
|
||||
echo "${partition} mounted at /media/${SRC}"
|
||||
|
||||
# Print the filenames on the current partition in a logfile
|
||||
find "/media/${SRC}" -fls "${LOGS}/Content_partition_${PARTCOUNT}.txt"
|
||||
|
||||
# create a directory on ${DST} named PARTION_$PARTCOUNT
|
||||
target_dir="/media/${DST}/FROM_PARTITION_${PARTCOUNT}"
|
||||
echo "copying to: ${target_dir}"
|
||||
mkdir -p "${target_dir}"
|
||||
LOGFILE="${LOGS}/processing.txt"
|
||||
|
||||
echo "==== Starting processing of /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE}
|
||||
main ${target_dir} || true
|
||||
echo "==== Done with /media/${SRC} to ${target_dir}. ====" >> ${LOGFILE}
|
||||
|
||||
ls -lR "${target_dir}"
|
||||
fi
|
||||
let PARTCOUNT=`expr $PARTCOUNT + 1`
|
||||
done
|
||||
|
||||
# The cleanup is automatically done in the function clean called when
|
||||
# the program quits
|
|
@ -1,149 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import shutil
|
||||
from twiggy import quickSetup, log
|
||||
import argparse
|
||||
|
||||
class KittenGroomerError(Exception):
|
||||
def __init__(self, message):
|
||||
'''
|
||||
Base KittenGroomer exception handler.
|
||||
'''
|
||||
super(KittenGroomerError, self).__init__(message)
|
||||
self.message = message
|
||||
|
||||
|
||||
class ImplementationRequired(KittenGroomerError):
|
||||
'''
|
||||
Implementation required error
|
||||
'''
|
||||
pass
|
||||
|
||||
|
||||
class FileBase(object):
|
||||
|
||||
def __init__(self, src_path, dst_path):
|
||||
'''
|
||||
Contains base information for a file on the source USB key,
|
||||
initialised with expected src and dest path
|
||||
'''
|
||||
self.src_path = src_path
|
||||
self.dst_path = dst_path
|
||||
self.log_details = {'filepath': self.src_path}
|
||||
self.log_string = ''
|
||||
|
||||
def add_log_details(self, key, value):
|
||||
'''
|
||||
Add an entry in the log dictionary
|
||||
'''
|
||||
self.log_details[key] = value
|
||||
|
||||
def make_dangerous(self):
|
||||
'''
|
||||
This file should be considered as dangerous and never run.
|
||||
Prepending and appending DANGEROUS to the destination
|
||||
file name avoid double-click of death
|
||||
'''
|
||||
self.log_details['dangerous'] = True
|
||||
path, filename = os.path.split(self.dst_path)
|
||||
self.dst_path = os.path.join(path, 'DANGEROUS_{}_DANGEROUS'.format(filename))
|
||||
|
||||
def make_unknown(self):
|
||||
'''
|
||||
This file has an unknown type and it was not possible to take
|
||||
a decision. Theuser will have to decide what to do.
|
||||
Prepending UNKNOWN
|
||||
'''
|
||||
self.log_details['unknown'] = True
|
||||
path, filename = os.path.split(self.dst_path)
|
||||
self.dst_path = os.path.join(path, 'UNKNOWN_{}'.format(filename))
|
||||
|
||||
def make_binary(self):
|
||||
'''
|
||||
This file is a binary, and should probably not be run.
|
||||
Appending .bin avoir double click of death but the user
|
||||
will have to decide by itself.
|
||||
'''
|
||||
self.log_details['binary'] = True
|
||||
path, filename = os.path.split(self.dst_path)
|
||||
self.dst_path = os.path.join(path, '{}.bin'.format(filename))
|
||||
|
||||
|
||||
class KittenGroomerBase(object):
|
||||
|
||||
def __init__(self, root_src, root_dst):
|
||||
'''
|
||||
Setup the base options of the copy/convert setup
|
||||
'''
|
||||
self.src_root_dir = root_src
|
||||
self.dst_root_dir = root_dst
|
||||
self.log_root_dir = os.path.join(self.dst_root_dir, 'logs')
|
||||
self.log_processing = os.path.join(self.log_root_dir, 'processing.log')
|
||||
|
||||
# quickSetup(file=self.log_processing)
|
||||
quickSetup()
|
||||
self.log_name = log.name('files')
|
||||
|
||||
self.cur_file = None
|
||||
|
||||
# ##### Helpers #####
|
||||
def _safe_rmtree(self, directory):
|
||||
'''Remove a directory tree if it exists'''
|
||||
if os.path.exists(directory):
|
||||
shutil.rmtree(directory)
|
||||
|
||||
def _safe_remove(self, filepath):
|
||||
'''Remove a file if it exists'''
|
||||
if os.path.exists(filepath):
|
||||
os.remove(filepath)
|
||||
|
||||
def _safe_mkdir(self, directory):
|
||||
'''Remove a directory if it exists'''
|
||||
if not os.path.exists(directory):
|
||||
os.makedirs(directory)
|
||||
|
||||
def _safe_copy(self):
|
||||
''' Copy a file and create directory if needed '''
|
||||
try:
|
||||
dst_path, filename = os.path.split(self.cur_file.dst_path)
|
||||
self._safe_mkdir(dst_path)
|
||||
shutil.copy(self.cur_file.src_path, self.cur_file.dst_path)
|
||||
return True
|
||||
except Exception as e:
|
||||
# TODO: Logfile
|
||||
print(e)
|
||||
return False
|
||||
|
||||
def _list_all_files(self, directory):
|
||||
''' Generate an iterator over all the files in a directory tree '''
|
||||
for root, dirs, files in os.walk(directory):
|
||||
for filename in files:
|
||||
filepath = os.path.join(root, filename)
|
||||
yield filepath
|
||||
|
||||
def _print_log(self):
|
||||
'''
|
||||
Print log, should be called after each file.
|
||||
|
||||
You probably want to reimplement it in the subclass
|
||||
'''
|
||||
tmp_log = self.log_name.fields(**self.cur_file.log_details)
|
||||
tmp_log.info('It did a thing.')
|
||||
|
||||
#######################
|
||||
|
||||
def processdir(self, src_dir=None, dst_dir=None):
|
||||
'''
|
||||
Main function doing the work, you have to implement it yourself.
|
||||
'''
|
||||
raise ImplementationRequired('You have to implement the result processdir.')
|
||||
|
||||
|
||||
def main(kg_implementation):
|
||||
parser = argparse.ArgumentParser(prog='KittenGroomer', description='Call the KittenGroomer implementation to do things on files present in the source directory to the destination directory')
|
||||
parser.add_argument('-s', '--source', type=str, help='Source directory')
|
||||
parser.add_argument('-d', '--destination', type=str, help='Destination directory')
|
||||
args = parser.parse_args()
|
||||
kg = kg_implementation(args.source, args.destination)
|
||||
kg.processdir()
|
|
@ -1,31 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
||||
source ./constraint.sh
|
||||
|
||||
if [ ${ID} -ne 0 ]; then
|
||||
echo "This script has to be run as root."
|
||||
exit
|
||||
fi
|
||||
|
||||
clean(){
|
||||
echo Done, cleaning.
|
||||
${SYNC}
|
||||
kill -9 $(cat /tmp/music.pid)
|
||||
rm -f /tmp/music.pid
|
||||
}
|
||||
|
||||
trap clean EXIT TERM INT
|
||||
|
||||
./music.sh &
|
||||
echo $! > /tmp/music.pid
|
||||
|
||||
# Dumb libreoffice wants to write into ~/libreoffice or crash with
|
||||
# com::sun::star::uno::RuntimeException
|
||||
mkdir /tmp/libreoffice
|
||||
chown -R kitten:kitten /tmp/libreoffice
|
||||
|
||||
su ${USERNAME} -c ./groomer.sh
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
||||
source ./constraint.sh
|
||||
|
||||
killed(){
|
||||
echo 'Music stopped.'
|
||||
}
|
||||
|
||||
trap killed EXIT TERM INT
|
||||
|
||||
# Force output on analog
|
||||
amixer cset numid=3 1
|
||||
|
||||
files=(${MUSIC}*)
|
||||
|
||||
while true; do
|
||||
$TIMIDITY ${files[RANDOM % ${#files[@]}]}
|
||||
done
|
|
@ -1,2 +0,0 @@
|
|||
ledBtn: ledBtn.c
|
||||
gcc -o ledBtn ledBtn.c
|
|
@ -1,282 +0,0 @@
|
|||
/***************************************************************************************/
|
||||
/* */
|
||||
/* file : ledBtn.c */
|
||||
/* */
|
||||
/* synopsis : */
|
||||
/* the compiled code should be ran with root privileges to allow for access to */
|
||||
/* the GPIO pins through direct GPIO register manipulation in C-code. */
|
||||
/* After initialization, the code update the LEDs status according to the commands */
|
||||
/* passed on std input (using a pipe). */
|
||||
/* It also monitors the push-button and triggers a reboot sequence */
|
||||
/* when it is depressed. */
|
||||
/* */
|
||||
/* */
|
||||
/* This code is based on examples from */
|
||||
/* http://elinux.org/RPi_Low-level_peripherals#C */
|
||||
/* How to access GPIO registers from C-code on the Raspberry-Pi, Example program */
|
||||
/* Dom and Gert, 15-January-2012, Revised: 15-Feb-2013 */
|
||||
/* */
|
||||
/* and from Raphael Vinot (CIRCL.lu) */
|
||||
/* */
|
||||
/* v 1.00 - 22/02/2015 - initial release (Marc Durvaux) */
|
||||
/* v 1.10 - 27/02/2015 - added 'z' command for debugging, improved handling of */
|
||||
/* concateneted command sequences */
|
||||
/* */
|
||||
/* */
|
||||
/* */
|
||||
/***************************************************************************************/
|
||||
|
||||
// Includes
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
#include <errno.h>
|
||||
|
||||
// Constant for low-level access to GPIO
|
||||
#define BCM2708_PERI_BASE 0x20000000
|
||||
#define GPIO_BASE (BCM2708_PERI_BASE + 0x200000) /* GPIO controller */
|
||||
#define BLOCK_SIZE (4*1024)
|
||||
|
||||
// global variables related to GPIO
|
||||
int mem_fd ;
|
||||
void *gpio_map ;
|
||||
volatile unsigned *gpio ; // I/O access
|
||||
|
||||
// GPIO setup macros. Always use INP_GPIO(x) before using OUT_GPIO(x) or SET_GPIO_ALT(x,y)
|
||||
#define INP_GPIO(g) *(gpio+((g)/10)) &= ~(7<<(((g)%10)*3))
|
||||
#define OUT_GPIO(g) *(gpio+((g)/10)) |= (1<<(((g)%10)*3))
|
||||
#define SET_GPIO_ALT(g,a) *(gpio+(((g)/10))) |= (((a)<=3?(a)+4:(a)==4?3:2)<<(((g)%10)*3))
|
||||
|
||||
#define GPIO_SET *(gpio+7) // sets bits which are 1 ignores bits which are 0
|
||||
#define GPIO_CLR *(gpio+10) // clears bits which are 1 ignores bits which are 0
|
||||
|
||||
#define GET_GPIO(g) (*(gpio+13)&(1<<g)) // 0 if LOW, (1<<g) if HIGH
|
||||
|
||||
#define GPIO_PULL *(gpio+37) // Pull up/pull down
|
||||
#define GPIO_PULLCLK0 *(gpio+38) // Pull up/pull down clock
|
||||
|
||||
// LED and push-button GPIO pin mapping (from schematic)
|
||||
#define GREEN_LED 17
|
||||
#define YELLOW_LED 18
|
||||
#define RED_LED 22
|
||||
#define PUSHBUTTON 23
|
||||
|
||||
// Time tic (in nsec) for loops : 10 ms
|
||||
#define TIME_TIC 10000000L
|
||||
// Blink half-period in tics
|
||||
#define MAX_COUNT 30
|
||||
// Button long pression threshold
|
||||
#define LONG_PUSH 300
|
||||
|
||||
|
||||
// forward declaration of functions
|
||||
void setup_io() ;
|
||||
void do_reboot() ;
|
||||
|
||||
/***************************************************************************************/
|
||||
//
|
||||
// main
|
||||
// input : path and name of the FIFO must be passed as 1st argument
|
||||
//
|
||||
int main(int argc, char **argv) {
|
||||
int fd, nbytes ;
|
||||
int state, count, repeat_count ;
|
||||
int Btn_state, Btn_prev_state, Btn_press_count ;
|
||||
char code ;
|
||||
|
||||
state = 0 ; // initialize state variable
|
||||
count = 0 ; // initialize loop counter
|
||||
repeat_count = 0 ;
|
||||
code = 0 ;
|
||||
|
||||
setup_io() ; // initialize GPIO pointer and GPIO pins
|
||||
Btn_state = GET_GPIO( PUSHBUTTON) ; // get push-button initial state
|
||||
Btn_prev_state = Btn_state ;
|
||||
Btn_press_count = 0 ;
|
||||
|
||||
fd = open(argv[1], O_RDONLY) ;
|
||||
if (fd < 0) {
|
||||
perror("open") ;
|
||||
exit (2) ;
|
||||
}
|
||||
|
||||
while(1) {
|
||||
Btn_state = GET_GPIO( PUSHBUTTON) ;
|
||||
if (Btn_state != 0) { // button released
|
||||
Btn_press_count = 0 ; // reset counter
|
||||
} else { // button pressed
|
||||
Btn_press_count++ ;
|
||||
if (Btn_state != Btn_prev_state) {
|
||||
//printf("Button pressed!\n");
|
||||
if (state >= 4) { // final state, immediate reboot
|
||||
close(fd) ;
|
||||
do_reboot() ;
|
||||
}
|
||||
}
|
||||
if (Btn_press_count == LONG_PUSH) { // trigger forced reboot
|
||||
state = 10 ; // LED animation before reboot
|
||||
repeat_count = 0 ;
|
||||
}
|
||||
}
|
||||
Btn_prev_state = Btn_state ;
|
||||
|
||||
nbytes = read(fd, &code, 1) ;
|
||||
if (nbytes < 0) {
|
||||
perror("read") ;
|
||||
exit (2) ;
|
||||
}
|
||||
|
||||
if (nbytes > 0) {
|
||||
switch (code) { // codes evaluated at every tic
|
||||
case 'z' : // clear without restart (for debugging)
|
||||
GPIO_CLR = 1<<GREEN_LED ;
|
||||
GPIO_CLR = 1<<YELLOW_LED ;
|
||||
GPIO_CLR = 1<<RED_LED ;
|
||||
state = 0 ;
|
||||
break ;
|
||||
case 'r' : // Ready
|
||||
GPIO_SET = 1<<GREEN_LED ;
|
||||
state = 1 ;
|
||||
break ;
|
||||
case 'p' : // Processing
|
||||
GPIO_CLR = 1<<GREEN_LED ;
|
||||
GPIO_SET = 1<<YELLOW_LED ;
|
||||
state = 2 ;
|
||||
break ;
|
||||
case 'e' : // Error (process aborted)
|
||||
GPIO_CLR = 1<<GREEN_LED ;
|
||||
GPIO_CLR = 1<<YELLOW_LED ;
|
||||
GPIO_SET = 1<<RED_LED ;
|
||||
state = 6 ;
|
||||
break ;
|
||||
case 'c' : // task successfully completed
|
||||
GPIO_CLR = 1<<YELLOW_LED ;
|
||||
GPIO_SET = 1<<GREEN_LED ;
|
||||
state = 4 ;
|
||||
count = 0 ;
|
||||
break ;
|
||||
case 'f' : // file processing successfully completed
|
||||
GPIO_SET = 1<<GREEN_LED ;
|
||||
state = 3 ;
|
||||
count = 0 ;
|
||||
break ;
|
||||
} // end switch
|
||||
}
|
||||
|
||||
count++ ;
|
||||
if (count >= MAX_COUNT) {
|
||||
count = 0 ;
|
||||
|
||||
switch (state) { // states evaluated after MAX_COUNT tics
|
||||
case 3 : // green LED flash OFF
|
||||
GPIO_CLR = 1<<GREEN_LED ;
|
||||
state = 2 ;
|
||||
break ;
|
||||
case 4 : // green LED blinks OFF
|
||||
GPIO_CLR = 1<<GREEN_LED ;
|
||||
state = 5 ;
|
||||
break ;
|
||||
case 5 : // green LED blinks ON
|
||||
GPIO_SET = 1<<GREEN_LED ;
|
||||
state = 4 ;
|
||||
break ;
|
||||
case 10 : // start LED animation before reboot
|
||||
GPIO_SET = 1<<GREEN_LED ;
|
||||
GPIO_SET = 1<<YELLOW_LED ;
|
||||
GPIO_SET = 1<<RED_LED ;
|
||||
state = 11 ;
|
||||
break ;
|
||||
case 11 : // LED animation before reboot
|
||||
GPIO_CLR = 1<<GREEN_LED ;
|
||||
GPIO_CLR = 1<<YELLOW_LED ;
|
||||
GPIO_CLR = 1<<RED_LED ;
|
||||
repeat_count++ ;
|
||||
if (repeat_count > 5) {
|
||||
state = 12 ;
|
||||
} else {
|
||||
state = 10 ;
|
||||
}
|
||||
break ;
|
||||
case 12 : // proceed with reboot
|
||||
close(fd) ;
|
||||
do_reboot() ;
|
||||
break ;
|
||||
} // end switch
|
||||
} // end if
|
||||
|
||||
// loop delay
|
||||
nanosleep((struct timespec[]){{0, TIME_TIC}}, NULL) ;
|
||||
}
|
||||
|
||||
return 0 ; // we should never come here!
|
||||
} // main
|
||||
|
||||
/***************************************************************************************/
|
||||
//
|
||||
// Set up a memory region to access GPIO
|
||||
//
|
||||
void setup_io() {
|
||||
/* open /dev/mem */
|
||||
if ((mem_fd = open("/dev/mem", O_RDWR|O_SYNC) ) < 0) {
|
||||
printf("can't open /dev/mem \n");
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
/* mmap GPIO */
|
||||
gpio_map = mmap(
|
||||
NULL, //Any adddress in our space will do
|
||||
BLOCK_SIZE, //Map length
|
||||
PROT_READ|PROT_WRITE,// Enable reading & writting to mapped memory
|
||||
MAP_SHARED, //Shared with other processes
|
||||
mem_fd, //File to map
|
||||
GPIO_BASE //Offset to GPIO peripheral
|
||||
);
|
||||
|
||||
close(mem_fd); //No need to keep mem_fd open after mmap
|
||||
|
||||
if (gpio_map == MAP_FAILED) {
|
||||
printf("mmap error %d\n", (int)gpio_map);//errno also set!
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
// Always use volatile pointer!
|
||||
gpio = (volatile unsigned *)gpio_map ;
|
||||
|
||||
// initializes the LED and push-button pins
|
||||
INP_GPIO( GREEN_LED) ; // must use INP_GPIO before we can use OUT_GPIO
|
||||
OUT_GPIO( GREEN_LED) ;
|
||||
INP_GPIO( YELLOW_LED) ;
|
||||
OUT_GPIO( YELLOW_LED) ;
|
||||
INP_GPIO( RED_LED) ;
|
||||
OUT_GPIO( RED_LED) ;
|
||||
INP_GPIO( PUSHBUTTON) ;
|
||||
|
||||
// initializes LEDs to OFF state
|
||||
GPIO_CLR = 1<<GREEN_LED ;
|
||||
GPIO_CLR = 1<<YELLOW_LED ;
|
||||
GPIO_CLR = 1<<RED_LED ;
|
||||
|
||||
} // setup_io
|
||||
|
||||
/***************************************************************************************/
|
||||
//
|
||||
// Call system reboot
|
||||
//
|
||||
void do_reboot() {
|
||||
static char *execArgv[5] ; /* define arguments for shutdown exec */
|
||||
|
||||
execArgv[0] = "shutdown" ;
|
||||
execArgv[1] = "-r" ;
|
||||
execArgv[2] = "now" ;
|
||||
execArgv[3] = NULL ;
|
||||
|
||||
//printf("going to reboot!\n") ;
|
||||
execv("/sbin/shutdown", execArgv) ;
|
||||
} // do_reboot
|
||||
|
||||
|
||||
|
||||
/*** END OF FILE ***********************************************************************/
|
|
@ -1,27 +0,0 @@
|
|||
#!/bin/sh
|
||||
FIFO="/tmp/ledBtnFIFO"
|
||||
#mkfifo $FIFO
|
||||
|
||||
#sudo ./ledBtn $FIFO
|
||||
|
||||
# send command "READY"
|
||||
echo "r" > $FIFO
|
||||
#sleep 1
|
||||
# send command "PROCESSING"
|
||||
echo "p" > $FIFO
|
||||
sleep 1
|
||||
# send command "FILE processed"
|
||||
echo "f" > $FIFO
|
||||
sleep 3
|
||||
# send command "FILE processed"
|
||||
echo "f" > $FIFO
|
||||
sleep 3
|
||||
# send command "processing successfully COMPLETED"
|
||||
echo "c" > $FIFO
|
||||
sleep 2
|
||||
# send command "ZERO (clear display and return state to 0)"
|
||||
echo "z" > $FIFO
|
||||
sleep 2
|
||||
# send command "ERROR"
|
||||
echo "e" > $FIFO
|
||||
|
|
@ -1,9 +1,12 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Notes:
|
||||
# - To chroot in an existing SD card, unset IMAGE. Change the paths to the partitions if needed.
|
||||
# - The offsets are thoses of 2013-02-09-wheezy-raspbian.img. It will change on an other image.
|
||||
# To get the offsets, use the "file" command.
|
||||
# This script will mount a given image or sd card in loop mode.
|
||||
# Make sure to change the path and offsets for the image you use. You can get
|
||||
# the correct offsets using `file $PATH_TO_IMAGE` or fdisk.
|
||||
# If you want to mount an SD card, unset $IMAGE.
|
||||
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: in mount_image.sh" 1>&2
|
||||
|
||||
if [ "$(id -u)" != "0" ]; then
|
||||
echo "This script must be run as root" 1>&2
|
||||
|
@ -19,31 +22,28 @@ if [ -z "$1" ]; then
|
|||
exit
|
||||
fi
|
||||
COMMAND=${1}
|
||||
COMMAND_OPT=${2}
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# If you use a partition...
|
||||
#PARTITION_ROOTFS='/dev/mmcblk0p2'
|
||||
#PARTITION_BOOT='/dev/mmcblk0p1'
|
||||
PARTITION_ROOTFS='/dev/sdd2'
|
||||
PARTITION_BOOT='/dev/sdd1'
|
||||
PARTITION_ROOTFS='/dev/mmcblk0p2'
|
||||
PARTITION_BOOT='/dev/mmcblk0p1'
|
||||
#PARTITION_ROOTFS='/dev/sdd2'
|
||||
#PARTITION_BOOT='/dev/sdd1'
|
||||
|
||||
# If you use the img
|
||||
# If you use the img...
|
||||
# Double check the path and offsets as noted above!
|
||||
##### Debian
|
||||
IMAGE='2015-02-16-raspbian-wheezy.img'
|
||||
OFFSET_ROOTFS=$((122880 * 512))
|
||||
IMAGE='2017-02-02_CIRCLean.img'
|
||||
OFFSET_BOOT=$((8192 * 512))
|
||||
##### Arch
|
||||
#IMAGE='archlinux-hf-2013-02-11.img'
|
||||
#OFFSET_ROOTFS=$((186368 * 512))
|
||||
#OFFSET_BOOT=$((2048 * 512))
|
||||
############
|
||||
OFFSET_ROOTFS=$((137216 * 512))
|
||||
|
||||
CHROOT_PATH='/mnt/arm_rPi'
|
||||
|
||||
clean(){
|
||||
mv ${CHROOT_PATH}/etc/ld.so.preload_bkp ${CHROOT_PATH}/etc/ld.so.preload
|
||||
mv ${CHROOT_PATH}/etc/ld.so.preload_backup ${CHROOT_PATH}/etc/ld.so.preload
|
||||
rm ${CHROOT_PATH}/etc/resolv.conf
|
||||
rm ${CHROOT_PATH}/usr/bin/qemu*arm*
|
||||
|
||||
|
@ -75,7 +75,7 @@ elif [ -a ${PARTITION_ROOTFS} ]; then
|
|||
mount ${PARTITION_ROOTFS} ${CHROOT_PATH}
|
||||
mount ${PARTITION_BOOT} ${CHROOT_PATH}/boot
|
||||
else
|
||||
print 'You need a SD card or an image'
|
||||
echo 'You need a SD card or an image'
|
||||
exit
|
||||
fi
|
||||
|
||||
|
@ -91,6 +91,9 @@ mount -o bind /tmp ${CHROOT_PATH}/tmp
|
|||
|
||||
cp -pf /etc/resolv.conf ${CHROOT_PATH}/etc
|
||||
|
||||
mv ${CHROOT_PATH}/etc/ld.so.preload ${CHROOT_PATH}/etc/ld.so.preload_bkp
|
||||
mv ${CHROOT_PATH}/etc/ld.so.preload ${CHROOT_PATH}/etc/ld.so.preload_backup
|
||||
|
||||
${COMMAND} ${CHROOT_PATH}
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: Image mounted, executing command from mount_image.sh" 1>&2
|
||||
|
||||
${COMMAND} ${CHROOT_PATH} ${COMMAND_OPT}
|
||||
|
|
|
@ -1,27 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# change locales to en_US.UTF-8
|
||||
dpkg-reconfigure locales
|
||||
|
||||
sed -i "s/wheezy/jessie/" /etc/apt/sources.list
|
||||
apt-get update
|
||||
apt-get dist-upgrade
|
||||
apt-get autoremove
|
||||
apt-get install libreoffice p7zip-full libfontforge1 timidity freepats pmount
|
||||
dpkg -i pdf2htmlex*.deb
|
||||
|
||||
# Make Libreoffice usable on a RO filesystem
|
||||
useradd -m kitten
|
||||
pushd /home/kitten
|
||||
ln -s /tmp/libreoffice
|
||||
popd
|
||||
|
||||
chown -R kitten:kitten /home/kitten
|
||||
|
||||
ln -s /proc/mounts /etc/mtab
|
||||
|
||||
# Disable swap
|
||||
dphys-swapfile uninstall
|
||||
|
||||
# enable rc.local
|
||||
systemctl enable rc-local.service
|
|
@ -1,14 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# change locales to en_US.UTF-8
|
||||
dpkg-reconfigure locales
|
||||
|
||||
# Increase size of image. See resize_img.md
|
||||
|
||||
apt-get update
|
||||
apt-get dist-upgrade
|
||||
apt-get autoremove
|
||||
|
||||
# build dependencies of pdf2htmlEX
|
||||
apt-get install cmake debhelper libpoppler-dev libjpeg-dev libfontforge-dev \
|
||||
libspiro-dev python-dev default-jre-headless libpoppler-private-dev
|
|
@ -1,66 +0,0 @@
|
|||
Add empty space to the image
|
||||
============================
|
||||
|
||||
* Add 2Gb
|
||||
|
||||
```
|
||||
> dd if=/dev/zero bs=1024k count=2048 >> 2015-02-16-raspbian-wheezy.img
|
||||
```
|
||||
|
||||
Expand partition size
|
||||
=====================
|
||||
|
||||
```
|
||||
> fdisk 2015-02-16-raspbian-wheezy.img
|
||||
|
||||
Command (m for help): *p*
|
||||
|
||||
Disk 2013-02-09-wheezy-raspbian.img: 4087 MB, 4087349248 bytes
|
||||
255 heads, 63 sectors/track, 496 cylinders, total 7983104 sectors
|
||||
Units = sectors of 1 * 512 = 512 bytes
|
||||
Sector size (logical/physical): 512 bytes / 512 bytes
|
||||
I/O size (minimum/optimal): 512 bytes / 512 bytes
|
||||
Disk identifier: 0x00014d34
|
||||
|
||||
Device Boot Start End Blocks Id System
|
||||
2015-02-16-raspbian-wheezy.img1 8192 122879 57344 c W95 FAT32 (LBA)
|
||||
2015-02-16-raspbian-wheezy.img2 *122880* 3788799 1832960 83 Linux
|
||||
|
||||
Command (m for help): *d*
|
||||
Partition number (1-4): *2*
|
||||
|
||||
Command (m for help): *n*
|
||||
Partition type:
|
||||
p primary (1 primary, 0 extended, 3 free)
|
||||
e extended
|
||||
Select (default p):
|
||||
Partition number (1-4, default 2):
|
||||
First sector (2048-7983103, default 2048): *122880*
|
||||
Last sector, +sectors or +size{K,M,G} (122880-7983103, default 7983103):
|
||||
Using default value 7983103
|
||||
|
||||
Command (m for help): *w*
|
||||
The partition table has been altered!
|
||||
|
||||
Syncing disks.
|
||||
```
|
||||
|
||||
Resize partition
|
||||
================
|
||||
|
||||
* Chroot in the image
|
||||
|
||||
```
|
||||
sudo ./proper_chroot.sh
|
||||
```
|
||||
|
||||
* Resize the partition (not from the chroot)
|
||||
|
||||
```
|
||||
> df | grep /mnt/arm
|
||||
|
||||
/dev/loop0 3927752 1955672 1794172 53% /mnt/arm_rPi
|
||||
/dev/loop1 57288 18960 38328 34% /mnt/arm_rPi/boot
|
||||
|
||||
> sudo resize2fs /dev/loop0
|
||||
```
|
13
run_tests.sh
|
@ -1,11 +1,22 @@
|
|||
#!/bin/bash
|
||||
|
||||
TEST_PART_TYPE=${1}
|
||||
TEST_SOURCE_TYPE=${2}
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
TEST_PART_TYPE="VFAT_NORM"
|
||||
fi
|
||||
if [ -z "$2" ]; then
|
||||
TEST_SOURCE_TYPE="t_images1"
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
./mount_image.sh ./copy_to_final.sh
|
||||
|
||||
pushd tests/
|
||||
|
||||
./run.sh
|
||||
./run.sh ${TEST_PART_TYPE} ${TEST_SOURCE_TYPE}
|
||||
./check_results.sh ${TEST_SOURCE_TYPE}
|
||||
|
||||
popd
|
||||
|
|
|
@ -0,0 +1,46 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This script will mount a given image in loop mode.
|
||||
# Make sure to change the path and offsets for the image you use. You can get
|
||||
# the correct offsets using `file $PATH_TO_IMAGE` or fdisk.
|
||||
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: in mount_image.sh" 1>&2
|
||||
|
||||
if [ "$(id -u)" != "0" ]; then
|
||||
echo "This script must be run as root" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# Double check the path and offsets as noted above!
|
||||
# Path to the image
|
||||
IMAGE='2019-09-26-raspbian-buster-lite.img'
|
||||
# Start sector of boot (first) partition
|
||||
BOOT_START=`sfdisk -J ${IMAGE} | grep img1 | sed -n 's/.*"start":*\([[:digit:]]*\).*/\1/p'`
|
||||
# Amount of sectors of boot (first) partition
|
||||
BOOT_SIZE=`sfdisk -J ${IMAGE} | grep img1 | sed -n 's/.*"size":*\([[:digit:]]*\).*/\1/p'`
|
||||
# Start sector of root (second) partition
|
||||
ROOT_START=`sfdisk -J ${IMAGE} | grep img2 | sed -n 's/.*"start":*\([[:digit:]]*\).*/\1/p'`
|
||||
# Amount of sectors of root (second) partition
|
||||
ROOT_SIZE=`sfdisk -J ${IMAGE} | grep img2 | sed -n 's/.*"size":*\([[:digit:]]*\).*/\1/p'`
|
||||
|
||||
# Locations you'd like the partitions mounted
|
||||
BOOT_PATH='/mnt/rpi-boot'
|
||||
ROOTFS_PATH='/mnt/rpi-root'
|
||||
|
||||
# Calculate offsets for each partition
|
||||
offset_boot=$((${BOOT_START} * 512))
|
||||
sizelimit_boot=$((${BOOT_SIZE} * 512))
|
||||
offset_rootfs=$((${ROOT_START} * 512))
|
||||
sizelimit_rootfs=$((${ROOT_SIZE} * 512))
|
||||
# TODO: add logic for creating directories if they aren't already there
|
||||
mkdir -p ${BOOT_PATH}
|
||||
mkdir -p ${ROOTFS_PATH}
|
||||
# Mount each partition in loop mode
|
||||
mount -o loop,offset=${offset_boot},sizelimit=${sizelimit_boot} ${IMAGE} ${BOOT_PATH}
|
||||
mount -o loop,offset=${offset_rootfs},sizelimit=${sizelimit_rootfs} ${IMAGE} ${ROOTFS_PATH}
|
||||
|
||||
echo "Image mounted" 1>&2
|
|
@ -0,0 +1,14 @@
|
|||
#!/bin/bash
|
||||
# Filename: cp-metadata
|
||||
|
||||
myecho=echo
|
||||
src_path="$1"
|
||||
dst_path="$2"
|
||||
|
||||
find "$src_path" |
|
||||
while read src_file; do
|
||||
dst_file="$dst_path${src_file#$src_path}"
|
||||
$myecho chmod --reference="$src_file" "$dst_file"
|
||||
$myecho chown --reference="$src_file" "$dst_file"
|
||||
$myecho touch --reference="$src_file" "$dst_file"
|
||||
done
|
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
|
||||
# change locales to en_US.UTF-8
|
||||
dpkg-reconfigure locales
|
||||
|
||||
apt-get update
|
||||
apt-get dist-upgrade
|
||||
apt-get autoremove
|
||||
|
||||
# enable rc.local
|
||||
systemctl enable rc-local.service
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This script runs proot on a mounted image with the proper parameters.
|
||||
# The root partition should be at /mnt/rpi-root /mnt/rpt-boot
|
||||
# You should probably run something like basic_mount_image.sh first
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
if [ "$(id -u)" != "0" ]; then
|
||||
echo "This script must be run as root" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo proot -q qemu-arm -S /mnt/rpi-root -b /mnt/rpi-boot:/boot /bin/bash
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Change to your repo if doing python dev
|
||||
./mount_image.sh chroot "pip install git+https://github.com/Dymaxion00/PyCIRCLean@dev"
|
|
@ -0,0 +1,47 @@
|
|||
#!/bin/bash
|
||||
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: in tests/check_results.sh" 1>&2
|
||||
|
||||
OFFSET_VFAT_NORM=$((8192 * 512))
|
||||
IMAGE_DEST="testcase_dest.vfat"
|
||||
|
||||
if [ "$(id -u)" != "0" ]; then
|
||||
echo "This script must be run as root" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Please tell me which file type to test."
|
||||
echo "t_images1"
|
||||
exit
|
||||
fi
|
||||
TEST_SOURCE_TYPE=${1}
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
RESULTS_DIR="results"
|
||||
|
||||
# To make debugging easier
|
||||
echo "Removing results from previous run." 1>&2
|
||||
rm -rf actualResults/*
|
||||
|
||||
clean(){
|
||||
umount ${RESULTS_DIR}
|
||||
rm -rf ${RESULTS_DIR}
|
||||
}
|
||||
|
||||
trap clean EXIT TERM INT
|
||||
|
||||
mkdir -p ${RESULTS_DIR}
|
||||
|
||||
# Get the run results
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_DEST} ${RESULTS_DIR}
|
||||
cp -rf ${RESULTS_DIR}/* actualResults
|
||||
umount ${RESULTS_DIR}
|
||||
|
||||
#TODO: Check if the results were what we expected.
|
||||
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: done with tests/check_results.sh" 1>&2
|
|
@ -2,17 +2,17 @@
|
|||
|
||||
set timeout -1
|
||||
|
||||
spawn qemu-system-arm -kernel 140801-kernel -cpu arm1176 -m 256 -M versatilepb \
|
||||
spawn qemu-system-arm -kernel kernel-qemu -cpu arm1176 -m 256 -M versatilepb \
|
||||
-append "root=/dev/sdc2 panic=1 rootfstype=ext4 ro console=ttyAMA0 console=ttyS0" \
|
||||
-drive file=[lindex $argv 1],index=0,media=disk \
|
||||
-drive file=[lindex $argv 2],index=1,media=disk \
|
||||
-drive file=[lindex $argv 0],index=2,media=disk \
|
||||
-serial stdio -soundhw all -nographic \
|
||||
-monitor none -soundhw all -nographic -serial pty \
|
||||
-chardev stdio,id=mon0 -mon chardev=mon0,mode=readline \
|
||||
-chardev socket,id=mon1,host=localhost,port=4444,server,nowait \
|
||||
-mon chardev=mon1,mode=control,pretty=on \
|
||||
-vnc 0.0.0.0:1
|
||||
|
||||
expect "reboot: System halted"
|
||||
expect "System halted."
|
||||
|
||||
send "quit\n"
|
||||
|
|
140
tests/run.sh
|
@ -1,8 +1,24 @@
|
|||
#!/bin/bash
|
||||
|
||||
# http://xecdesign.com/qemu-emulating-raspberry-pi-the-easy-way/
|
||||
# http://pub.phyks.me/respawn/mypersonaldata/public/2014-05-20-11-08-01/
|
||||
|
||||
IMAGE='../2015-02-16-raspbian-wheezy.img'
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: in tests/run.sh" 1>&2
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Please tell me which partition type to test."
|
||||
echo "VFAT_NORM VFAT_PART NTPS_NORM EXT2 EXT3 EXT4"
|
||||
exit
|
||||
fi
|
||||
if [ -z "$2" ]; then
|
||||
echo "Please tell me which file type to test."
|
||||
echo "t_images1"
|
||||
exit
|
||||
fi
|
||||
TEST_PART_TYPE=${1}
|
||||
TEST_SOURCE_TYPE=${2}
|
||||
|
||||
IMAGE='../raspbian-wheezy.img'
|
||||
OFFSET_ROOTFS=$((122880 * 512))
|
||||
|
||||
IMAGE_VFAT_NORM="testcase.vfat"
|
||||
|
@ -31,6 +47,7 @@ SETUP_DIR="setup"
|
|||
clean(){
|
||||
mount -o loop,offset=${OFFSET_ROOTFS} ${IMAGE} ${SETUP_DIR}
|
||||
mv ${SETUP_DIR}/etc/ld.so.preload_bkp ${SETUP_DIR}/etc/ld.so.preload
|
||||
sleep 5
|
||||
umount ${SETUP_DIR}
|
||||
rm -rf ${SETUP_DIR}
|
||||
}
|
||||
|
@ -44,50 +61,101 @@ mount -o loop,offset=${OFFSET_ROOTFS} ${IMAGE} ${SETUP_DIR}
|
|||
mv ${SETUP_DIR}/etc/ld.so.preload ${SETUP_DIR}/etc/ld.so.preload_bkp
|
||||
umount ${SETUP_DIR}
|
||||
|
||||
|
||||
|
||||
# Prepare the test source key
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_VFAT_NORM} ${SETUP_DIR}
|
||||
cp -rf content_img_vfat_norm/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
if [ ${TEST_PART_TYPE} = "VFAT_NORM" ]; then
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_VFAT_NORM} ${SETUP_DIR}
|
||||
rm -rf ${SETUP_DIR}/*
|
||||
cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
fi
|
||||
|
||||
# Prepare the test source key (with partitions)
|
||||
mount -o loop,offset=${OFFSET_VFAT_PART1} ${IMAGE_VFAT_PART} ${SETUP_DIR}
|
||||
cp -rf content_img_vfat_part1/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
mount -o loop,offset=${OFFSET_VFAT_PART2} ${IMAGE_VFAT_PART} ${SETUP_DIR}
|
||||
cp -rf content_img_vfat_part2/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
if [ ${TEST_PART_TYPE} = "VFAT_PART" ]; then
|
||||
mount -o loop,offset=${OFFSET_VFAT_PART1} ${IMAGE_VFAT_PART} ${SETUP_DIR}
|
||||
rm -rf ${SETUP_DIR}/*
|
||||
cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
mount -o loop,offset=${OFFSET_VFAT_PART2} ${IMAGE_VFAT_PART} ${SETUP_DIR}
|
||||
rm -rf ${SETUP_DIR}/*
|
||||
cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
fi
|
||||
|
||||
# Prepare the test source key (NTFS)
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_NTFS_NORM} ${SETUP_DIR}
|
||||
cp -rf content_img_vfat_norm/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
if [ ${TEST_PART_TYPE} = "NTFS_NORM" ]; then
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_NTFS_NORM} ${SETUP_DIR}
|
||||
rm -rf ${SETUP_DIR}/*
|
||||
cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
fi
|
||||
|
||||
# Prepare the test source key (EXT2)
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT2} ${SETUP_DIR}
|
||||
cp -rf content_img_vfat_norm/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
if [ ${TEST_PART_TYPE} = "EXT2" ]; then
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT2} ${SETUP_DIR}
|
||||
rm -rf ${SETUP_DIR}/*
|
||||
cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
fi
|
||||
|
||||
# Prepare the test source key (EXT3)
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT3} ${SETUP_DIR}
|
||||
cp -rf content_img_vfat_norm/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
if [ ${TEST_PART_TYPE} = "EXT4" ]; then
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT3} ${SETUP_DIR}
|
||||
rm -rf ${SETUP_DIR}/*
|
||||
cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
fi
|
||||
|
||||
# Prepare the test source key (EXT4)
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT4} ${SETUP_DIR}
|
||||
cp -rf content_img_vfat_norm/* ${SETUP_DIR}
|
||||
if [ ${TEST_PART_TYPE} = "EXT4" ]; then
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_EXT4} ${SETUP_DIR}
|
||||
rm -rf ${SETUP_DIR}/*
|
||||
cp -rf testFiles/${TEST_SOURCE_TYPE}/* ${SETUP_DIR}
|
||||
umount ${SETUP_DIR}
|
||||
fi
|
||||
|
||||
# Prepare the test destination key
|
||||
mount -o loop,offset=${OFFSET_VFAT_NORM} ${IMAGE_DEST} ${SETUP_DIR}
|
||||
rm -rf ${SETUP_DIR}/*
|
||||
umount ${SETUP_DIR}
|
||||
|
||||
chmod -w ${IMAGE}
|
||||
./run.exp ${IMAGE} ${IMAGE_VFAT_NORM} ${IMAGE_DEST}
|
||||
#sleep 10
|
||||
#./run.exp ${IMAGE} ${IMAGE_VFAT_PART} ${IMAGE_DEST}
|
||||
#sleep 10
|
||||
#./run.exp ${IMAGE} ${IMAGE_NTFS_NORM} ${IMAGE_DEST}
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: about to enter tests/run.exp" 1>&2
|
||||
|
||||
chmod a-w ${IMAGE}
|
||||
|
||||
|
||||
if [ ${TEST_PART_TYPE} = "VFAT_NORM" ]; then
|
||||
./run.exp ${IMAGE} ${IMAGE_VFAT_NORM} ${IMAGE_DEST}
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
if [ ${TEST_PART_TYPE} = "VFAT_PART" ]; then
|
||||
./run.exp ${IMAGE} ${IMAGE_VFAT_PART} ${IMAGE_DEST}
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
if [ ${TEST_PART_TYPE} = "NTFS_NORM" ]; then
|
||||
./run.exp ${IMAGE} ${IMAGE_NTFS_NORM} ${IMAGE_DEST}
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
# EXT* not supported due to permission issues
|
||||
#sleep 10
|
||||
#./run.exp ${IMAGE} ${IMAGE_EXT2} ${IMAGE_DEST}
|
||||
#sleep 10
|
||||
#./run.exp ${IMAGE} ${IMAGE_EXT3} ${IMAGE_DEST}
|
||||
#sleep 10
|
||||
#./run.exp ${IMAGE} ${IMAGE_EXT4} ${IMAGE_DEST}
|
||||
if [ ${TEST_PART_TYPE} = "EXT2" ]; then
|
||||
./run.exp ${IMAGE} ${IMAGE_EXT2} ${IMAGE_DEST}
|
||||
sleep 10
|
||||
fi
|
||||
if [ ${TEST_PART_TYPE} = "EXT3" ]; then
|
||||
./run.exp ${IMAGE} ${IMAGE_EXT3} ${IMAGE_DEST}
|
||||
sleep 10
|
||||
fi
|
||||
if [ ${TEST_PART_TYPE} = "NTFS_EXT4" ]; then
|
||||
./run.exp ${IMAGE} ${IMAGE_EXT4} ${IMAGE_DEST}
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
|
||||
#./run.exp ${IMAGE} ${IMAGE_VFAT_PART} ${IMAGE_DEST}
|
||||
chmod +w ${IMAGE}
|
||||
|
||||
# To make debugging easier
|
||||
echo "KittenGroomer: done with tests/run.sh" 1>&2
|
||||
|
|
After Width: | Height: | Size: 101 KiB |
After Width: | Height: | Size: 310 KiB |
After Width: | Height: | Size: 594 KiB |
After Width: | Height: | Size: 114 KiB |
After Width: | Height: | Size: 792 KiB |
After Width: | Height: | Size: 140 KiB |
After Width: | Height: | Size: 598 KiB |
After Width: | Height: | Size: 41 KiB |
|
@ -1,11 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# We cannot use the version 0.12 because it requires fontforge 2.0
|
||||
# The fork use a saner list of dependencies and a patch that allows to build on debian jessie.
|
||||
|
||||
wget https://github.com/Rafiot/pdf2htmlEX/archive/KittenGroomer.zip
|
||||
unzip KittenGroomer.zip
|
||||
|
||||
cd pdf2htmlEX-KittenGroomer/
|
||||
|
||||
dpkg-buildpackage -uc -b
|