MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of github.com:MISP/MISP into galaxy-cluster2.0

pull/6120/head
mokaddem 2020-07-14 16:25:04 +02:00
parent 4b46b06809 c161e25372
commit b3dbecb318
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
47 changed files with 1129 additions and 836 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.travis.yml
View File

@ -147,6 +147,7 @@ before_script:
- popd
script:
- ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
- pushd tests
- ./curl_tests.sh $AUTH
- popd

41
INSTALL/INSTALL.sh
View File

@ -83,27 +83,33 @@ MISPvars () {
# RHEL/CentOS
if [[ -f "/etc/redhat-release" ]]; then
WWW_USER='apache'
SUDO_WWW="sudo -H -u ${WWW_USER} "
# Debian flavoured
elif [[ -f "/etc/debian_version" ]]; then
WWW_USER="www-data"
SUDO_WWW="sudo -H -u ${WWW_USER} "
# OpenBSD
elif [[ "$(uname -s)" == "OpenBSD" ]]; then
WWW_USER="www"
PATH_TO_MISP="/var/www/htdocs/MISP"
SUDO_WWW="doas -u www "
SUDO_CMD="doas "
# NetBSD
elif [[ "$(uname -s)" == "NetBSD" ]]; then
WWW_USER="www"
PATH_TO_MISP="/usr/pkg/share/httpd/htdocs/MISP"
SUDO_WWW="sudo -H -u ${WWW_USER} "
else
# I am feeling lucky
WWW_USER="www-data"
SUDO_WWW="sudo -H -u ${WWW_USER} "
fi
if [ -z "$FQDN" ]; then
if [ -z "${FQDN}" ]; then
FQDN="misp.local"
fi
if [ -z "$MISP_BASEURL" ]; then
if [ -z "${MISP_BASEURL}" ]; then
MISP_BASEURL='""'
fi
@ -118,13 +124,13 @@ MISPvars () {
DBPASSWORD_MISP="$(openssl rand -hex 32)"
# OpenSSL configuration
OPENSSL_CN=$FQDN
OPENSSL_CN=${FQDN}
OPENSSL_C='LU'
OPENSSL_ST='State'
OPENSSL_L='Location'
OPENSSL_O='Organization'
OPENSSL_OU='Organizational Unit'
OPENSSL_EMAILADDRESS="info@$FQDN"
OPENSSL_EMAILADDRESS="info@${FQDN}"
# GPG configuration
GPG_REAL_NAME='Autogenerated Key'
@ -147,7 +153,7 @@ MISPvars () {
max_execution_time=300
memory_limit=2048M
CAKE="$PATH_TO_MISP/app/Console/cake"
CAKE="${PATH_TO_MISP}/app/Console/cake"
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
@ -155,8 +161,7 @@ MISPvars () {
else
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
SUDO_WWW="sudo -H -u ${WWW_USER} "
echo "The following DB Passwords were generated..."
echo "Admin (${DBUSER_ADMIN}) DB Password: ${DBPASSWORD_ADMIN}"
echo "User (${DBUSER_MISP}) DB Password: ${DBPASSWORD_MISP}"
@ -236,6 +241,7 @@ setOpt () {
("-U") echo "upgrade"; UPGRADE=1 ;;
("-N") echo "nuke"; NUKE=1 ;;
("-u") echo "unattended"; UNATTENDED=1 ;;
("-ni") echo "noninteractive"; NONINTERACTIVE=1 ;;
("-f") echo "force"; FORCE=1 ;;
(*) echo "$o is not a valid argument"; exit 1 ;;
esac
@ -1718,9 +1724,9 @@ mispmodules () {
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
$SUDO_CMD git clone https://github.com/MISP/misp-modules.git
$SUDO_CMD git clone git://github.com/stricaud/gtcaca.git
$SUDO_CMD git clone git://github.com/stricaud/faup.git
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/misp-modules.git; done
[[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done
[[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
# Install gtcaca
cd gtcaca
@ -1805,7 +1811,7 @@ mispDashboard () {
sudo mkdir misp-dashboard
sudo chown $WWW_USER:$WWW_USER misp-dashboard
$SUDO_WWW git clone https://github.com/MISP/misp-dashboard.git
false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/MISP/misp-dashboard.git; done
cd misp-dashboard
sudo -H /var/www/misp-dashboard/install_dependencies.sh
sudo sed -i "s/^host\ =\ localhost/host\ =\ 0.0.0.0/g" /var/www/misp-dashboard/config/config.cfg
@ -1889,9 +1895,10 @@ mail2misp () {
debug "Installing Mail2${LBLUE}MISP${NC}"
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
$SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git; done
## TODO: The below fails miserably (obviously) if faup/gtcac dirs exist, let's just make the dangerous assumption (for the sake of the installer, that they exist)
##[[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done
##[[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done
sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
@ -1949,8 +1956,8 @@ viper () {
fi
fi
echo "Cloning Viper"
$SUDO_CMD git clone https://github.com/viper-framework/viper.git
$SUDO_CMD git clone https://github.com/viper-framework/viper-web.git
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/viper-framework/viper.git; done
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/viper-framework/viper-web.git; done
sudo chown -R $MISP_USER:$MISP_USER viper
sudo chown -R $MISP_USER:$MISP_USER viper-web
cd viper
@ -2506,7 +2513,7 @@ mispmodulesRHEL () {
sudo chmod 2777 /usr/local/src
sudo chown root:users /usr/local/src
cd /usr/local/src/
$SUDO_WWW git clone https://github.com/MISP/misp-modules.git
false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/MISP/misp-modules.git; done
cd misp-modules
# pip install
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U -I -r REQUIREMENTS

6
INSTALL/INSTALL.sh.sfv
View File

@ -1,5 +1,5 @@
; Generated by RHash v1.3.9 on 2020-05-22 at 16:25.08
; Generated by RHash v1.3.8 on 2020-07-15 at 04:14.11
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 133066 16:25.07 2020-05-22 INSTALL.sh
INSTALL.sh 79EF825B019669270DBCA0DD922C1E3DE6DA3D89 22A82CD073DA3312DF51089884DE4F3AF88ECD0E359D4C048915178C366327EC 4DA7D94FB036B2CC02120C1AC5AEBA9B57E4200FFC2940CB5BF8D9FE8C8600C72888DD2093590E7E77BB3A9F38D7F656 AA0CFD458A4B5CD84103EB641F59FFBBBB740890CB433108C6E0B8912F795DC521E3C75BD563496088862798D90A5C4D20B862ABA5152A84F62C037E889C3ED3
; 133908 04:14.11 2020-07-15 INSTALL.sh
INSTALL.sh 1048857D0C71A2FB6029A448090BC88E008AA499 2E3E878D35568521B5DEC1E7F6DA3193FE3C51049E4BCC127068659E5375939E ED7092DC612C51D7B81969418B4EEA90CE5E990DDE693A3CE83566DEC11E1CF456452DCF103689F05B7E6CCB63F9BC45 2D4CC5D6E02135B337541CE00CDDB205E10EDE924B89B1EEFE069DD1FCE7CE552970AB65A7342838A3BB41E0DCDAB6460E7F96AB7F63EA554D9A1DB61116AE2A

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
79ef825b019669270dbca0dd922c1e3de6da3d89 INSTALL.sh
1048857d0c71a2fb6029a448090bc88e008aa499 INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
22a82cd073da3312df51089884de4f3af88ecd0e359d4c048915178c366327ec INSTALL.sh
2e3e878d35568521b5dec1e7f6da3193fe3c51049e4bcc127068659e5375939e INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
4da7d94fb036b2cc02120c1ac5aeba9b57e4200ffc2940cb5bf8d9fe8c8600c72888dd2093590e7e77bb3a9f38d7f656 INSTALL.sh
ed7092dc612c51d7b81969418b4eea90ce5e990dde693a3ce83566dec11e1cf456452dcf103689f05b7e6ccb63f9bc45 INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
aa0cfd458a4b5cd84103eb641f59ffbbbb740890cb433108c6e0b8912f795dc521e3c75bd563496088862798d90a5c4d20b862aba5152a84f62c037e889c3ed3 INSTALL.sh
2d4cc5d6e02135b337541ce00cddb205e10ede924b89b1eefe069dd1fce7ce552970ab65a7342838a3bb41e0dcdab6460e7f96ab7f63ea554d9a1db61116ae2a INSTALL.sh

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":128}
{"major":2, "minor":4, "hotfix":129}

2
app/Controller/AppController.php
View File

@ -46,7 +46,7 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');
private $__queryVersion = '106';
private $__queryVersion = '107';
public $pyMispVersion = '2.4.128';
public $phpmin = '7.2';
public $phprec = '7.4';

2
app/Controller/AttributesController.php
View File

@ -8,7 +8,7 @@ App::uses('File', 'Utility');
*/
class AttributesController extends AppController
{
public $components = array('Security', 'RequestHandler', 'Cidr');
public $components = array('Security', 'RequestHandler');
public $paginate = array(
'limit' => 60,

62
app/Controller/Component/CidrComponent.php
View File

@ -1,62 +0,0 @@
<?php
class CidrComponent extends Component
{
public function CIDR($cidr)
{
list($address, $prefix) = explode('/', $cidr, 2);
$address = decbin(ip2long($address));
$address = substr("00000000000000000000000000000000", 0, 32 - strlen($address)) . $address;
$min = '';
$max = '';
for ($i = 0; $i < $prefix; $i++) {
$min .= $address[$i];
}
$max = $min;
$min = str_pad($min, 32, '0', STR_PAD_RIGHT);
$max = str_pad($max, 32, '1', STR_PAD_RIGHT);
$minArray = array();
$maxArray = array();
$searchTermLeft = '';
$searchTermMin = 0;
$searchTermMax = 0;
$results = array();
for ($i = 0; $i < 4; $i++) {
$minArray[] = bindec(substr($min, ($i*8), 8));
$maxArray[] = bindec(substr($max, ($i*8), 8));
if ($minArray[$i] === $maxArray[$i]) {
$searchTermLeft .= $minArray[$i] . '.';
} else {
$searchTermMin = $minArray[$i];
$searchTermMax = $maxArray[$i];
break;
}
}
$length = $i;
for ($i = 0; $i < ($searchTermMax - $searchTermMin + 1); $i++) {
$results[$i] = $searchTermLeft . ($searchTermMin + $i);
if ($length < 3) {
$results[$i] .= '.%';
}
}
return $results;
}
public function checkCIDR($cidr, $ipVersion)
{
if (strpos($cidr, '/') === false || substr_count($cidr, '/') !== 1) {
return false;
}
list($net, $maskbits) = explode('/', $cidr);
if (!is_numeric($maskbits) || $maskbits < 0) {
return false;
}
if ($ipVersion == 4) {
return ($maskbits <= 32) && filter_var($net, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4);
} elseif ($ipVersion == 6) {
return ($maskbits <= 128) && filter_var($net, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6);
} else {
throw new InvalidArgumentException('checkCIDR does only support IPv4 & IPv6');
}
}
}

50
app/Controller/EventsController.php
View File

@ -12,7 +12,6 @@ class EventsController extends AppController
'Email',
'RequestHandler',
'IOCImport',
'Cidr'
);
public $paginate = array(
@ -1410,17 +1409,15 @@ class EventsController extends AppController
}
$this->set('tagConflicts', $tagConflicts);
$startDate = null;
$modificationMap = array();
$modDate = date("Y-m-d", $event['Event']['timestamp']);
$modificationMap = array($modDate => 1);
foreach ($event['Attribute'] as $k => $attribute) {
if ($oldest_timestamp == false || $oldest_timestamp > $attribute['timestamp']) {
$oldest_timestamp = $attribute['timestamp'];
}
if ($startDate === null || $attribute['timestamp'] < $startDate) {
$startDate = $attribute['timestamp'];
}
$modDate = date("Y-m-d", $attribute['timestamp']);
$modificationMap[$modDate] = empty($modificationMap[$modDate])? 1 : $modificationMap[date("Y-m-d", $attribute['timestamp'])] + 1;
$modificationMap[$modDate] = !isset($modificationMap[$modDate]) ? 1 : $modificationMap[$modDate] + 1;
$this->Event->Attribute->removeGalaxyClusterTags($event['Attribute'][$k]);
@ -1436,22 +1433,14 @@ class EventsController extends AppController
$attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event, 'both');
$this->set('attributeTags', array_values($attributeTagsName['tags']));
$this->set('attributeClusters', array_values($attributeTagsName['clusters']));
$startDate = $event['Event']['timestamp'];
$modDate = date("Y-m-d", $event['Event']['timestamp']);
$modificationMap[$modDate] = 1;
foreach ($event['Object'] as $k => $object) {
if ($startDate === null || $object['timestamp'] < $startDate) {
$startDate = $object['timestamp'];
}
$modDate = date("Y-m-d", $object['timestamp']);
$modificationMap[$modDate] = empty($modificationMap[$modDate])? 1 : $modificationMap[date("Y-m-d", $object['timestamp'])] + 1;
$modificationMap[$modDate] = !isset($modificationMap[$modDate])? 1 : $modificationMap[$modDate] + 1;
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $k2 => $attribute) {
if ($startDate === null || $attribute['timestamp'] < $startDate) {
$startDate = $attribute['timestamp'];
}
$modDate = date("Y-m-d", $attribute['timestamp']);
$modificationMap[$modDate] = empty($modificationMap[$modDate])? 1 : $modificationMap[date("Y-m-d", $attribute['timestamp'])] + 1;
$modificationMap[$modDate] = !isset($modificationMap[$modDate])? 1 : $modificationMap[$modDate] + 1;
$this->Event->Attribute->removeGalaxyClusterTags($event['Object'][$k]['Attribute'][$k2]);
@ -1481,11 +1470,11 @@ class EventsController extends AppController
sort($startDate);
$startDate = $startDate[0];
$this->set('startDate', $startDate);
$to = date('Y-m-d', time());
if ((strtotime($to) - 172800) > $startDate) {
$startDate = date('Y-m-d', strtotime($to) - 172800);
$today = strtotime(date('Y-m-d', time()));
if (($today - 172800) > $startDate) {
$startDate = date('Y-m-d', $today - 172800);
}
for ($date = $startDate; strtotime($date) <= strtotime($to); $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
for ($date = $startDate; strtotime($date) <= $today; $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
if (isset($modificationMap[$date])) {
$modificationMapCSV .= $date . ',' . $modificationMap[$date] . '\n';
} else {
@ -1591,9 +1580,6 @@ class EventsController extends AppController
// find the id of the event, change $id to it and proceed to read the event as if the ID was entered.
$id = $this->Toolbox->findIdByUuid($this->Event, $id);
$this->Event->id = $id;
if (!$this->Event->exists()) {
throw new NotFoundException(__('Invalid event'));
}
$conditions = array('eventid' => $id);
if (!$this->_isRest()) {
$conditions['includeAllTags'] = true;
@ -2548,9 +2534,6 @@ class EventsController extends AppController
{
$id = $this->Toolbox->findIdByUuid($this->Event, $id);
$this->Event->id = $id;
if (!$this->Event->exists()) {
throw new NotFoundException(__('Invalid event'));
}
$this->Event->recursive = -1;
$event = $this->Event->read(null, $id);
if (!$this->_isSiteAdmin()) {
@ -2651,9 +2634,6 @@ class EventsController extends AppController
{
$id = $this->Toolbox->findIdByUuid($this->Event, $id);
$this->Event->id = $id;
if (!$this->Event->exists()) {
throw new NotFoundException(__('Invalid event'));
}
// update the event and set the from field to the current instance's organisation from the bootstrap. We also need to save id and info for the logs.
$this->Event->recursive = -1;
$event = $this->Event->read(null, $id);
@ -3699,14 +3679,6 @@ class EventsController extends AppController
$resultArray[$key]['types'] = $temp;
}
// remove all duplicates
foreach ($resultArray as $k => $v) {
for ($i = 0; $i < $k; $i++) {
if (isset($resultArray[$i]) && $v == $resultArray[$i]) {
unset($resultArray[$k]);
}
}
}
if ($this->_isRest()) {
if ($returnMetaAttributes || !empty($this->request->data['Attribute']['returnMetaAttributes'])) {
return $this->RestResponse->viewData($resultArray, $this->response->type());

20
app/Controller/LogsController.php
View File

@ -146,32 +146,18 @@ class LogsController extends AppController
}
// Shows a minimalistic history for the currently selected event
public function event_index($id, $org = null)
public function event_index($id)
{
// check if the user has access to this event...
$mayModify = false;
$mineOrAdmin = false;
$this->loadModel('Event');
if (!is_numeric($id) || $id < 1) {
$id = -1;
}
$event = $this->Event->fetchEvent($this->Auth->user(), array(
'eventid' => $id,
'includeAllTags' => 1,
'sgReferenceOnly' => 1,
'deleted' => 1,
'deleted' => [0, 1],
'deleted_proposals' => 1
));
$conditions = array(
'OR' => array(
array(
'AND' => array(
'Log.model' => 'Event',
'Log.model_id' => $id
)
)
)
);
if (empty($event)) {
throw new NotFoundException('Invalid event.');
}
@ -179,8 +165,6 @@ class LogsController extends AppController
$attribute_ids = array();
$object_ids = array();
$proposal_ids = array();
$event_tag_ids = array();
$attribute_tag_ids = array();
if (!empty($event['Attribute'])) {
foreach ($event['Attribute'] as $aa) {
$attribute_ids[] = $aa['id'];

38
app/Controller/ServersController.php
View File

@ -393,14 +393,7 @@ class ServersController extends AppController
$this->set('externalOrganisations', $externalOrganisations);
$this->set('allOrganisations', $allOrgs);
// list all tags for the rule picker
$this->loadModel('Tag');
$temp = $this->Tag->find('all', array('recursive' => -1));
$allTags = array();
foreach ($temp as $t) {
$allTags[] = array('id' => $t['Tag']['id'], 'name' => $t['Tag']['name']);
}
$this->set('allTags', $allTags);
$this->set('allTags', $this->__getTags());
$this->set('host_org_id', Configure::read('MISP.host_org_id'));
$this->render('edit');
}
@ -586,14 +579,7 @@ class ServersController extends AppController
$this->set('externalOrganisations', $externalOrganisations);
$this->set('allOrganisations', $allOrgs);
// list all tags for the rule picker
$this->loadModel('Tag');
$temp = $this->Tag->find('all', array('recursive' => -1));
$allTags = array();
foreach ($temp as $t) {
$allTags[] = array('id' => $t['Tag']['id'], 'name' => $t['Tag']['name']);
}
$this->set('allTags', $allTags);
$this->set('allTags', $this->__getTags());
$this->set('server', $s);
$this->set('id', $id);
$this->set('host_org_id', Configure::read('MISP.host_org_id'));
@ -2323,4 +2309,24 @@ misp.direct_call(relative_path, body)
$this->set('data', $data);
}
}
/**
* List all tags for the rule picker.
*
* @return array
*/
private function __getTags()
{
$this->loadModel('Tag');
$list = $this->Tag->find('list', array(
'recursive' => -1,
'order' => array('LOWER(TRIM(Tag.name))' => 'ASC'),
'fields' => array('name'),
));
$allTags = array();
foreach ($list as $id => $name) {
$allTags[] = array('id' => $id, 'name' => trim($name));
}
return $allTags;
}
}

24
app/Controller/ShadowAttributesController.php
View File

@ -803,20 +803,18 @@ class ShadowAttributesController extends AppController
public function delete($id)
{
if (strlen($id) == 36) {
$this->ShadowAttribute->Event->recursive = -1;
$temp = $this->ShadowAttribute->Event->Attribute->find('first', array('recursive' => -1, 'conditions' => array('Attribute.uuid' => $id), 'fields' => array('id')));
if ($temp == null) {
throw new NotFoundException(__('Invalid attribute'));
}
$id = $temp['Attribute']['id'];
}
$existingAttribute = $this->ShadowAttribute->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $id)));
if (empty($existingAttribute)) {
throw new NotFoundException(__('Invalid attribute.'));
if (is_numeric($id)) {
$conditions = ['Attribute.id' => $id];
} else if (Validation::uuid($id)) {
$conditions = ['Attribute.uuid' => $id];
} else {
throw new NotFoundException(__('Invalid attribute'));
}
$existingAttribute = $this->ShadowAttribute->Event->Attribute->fetchAttributes(
$this->Auth->user(),
array('conditions' => $conditions, 'flatten' => true)
);
if ($this->request->is('post')) {
if (empty($existingAttribute)) {
return new CakeResponse(array('body'=> json_encode(array('false' => true, 'errors' => 'Invalid Attribute.')), 'status'=>200, 'type' => 'json'));
@ -850,7 +848,7 @@ class ShadowAttributesController extends AppController
}
} else {
if (empty($existingAttribute)) {
throw new NotFoundException(__('Invalid Attribute'));
throw new NotFoundException(__('Invalid attribute'));
}
$existingAttribute = $existingAttribute[0];
$this->set('id', $id);

39
app/Controller/UserSettingsController.php
View File

@ -28,7 +28,6 @@ class UserSettingsController extends AppController
public function beforeFilter()
{
parent::beforeFilter();
$this->Security->unlockedActions = array_merge($this->Security->unlockedActions, array('setHomePage'));
}
public function index()
@ -325,23 +324,27 @@ class UserSettingsController extends AppController
public function setHomePage()
{
if (!$this->request->is('post')) {
throw new MethodNotAllowedException(__('This endpoint only aaccepts POST requests.'));
if ($this->request->is('post')) {
if (isset($this->request->data['UserSetting'])) {
$this->request->data = $this->request->data['UserSetting'];
}
if (!isset($this->request->data['path'])) {
$this->request->data = array('path' => $this->request->data);
}
if (empty($this->request->data['path'])) {
throw new InvalidArgumentException(__('No path POSTed.'));
}
$setting = array(
'UserSetting' => array(
'user_id' => $this->Auth->user('id'),
'setting' => 'homepage',
'value' => json_encode(array('path' => $this->request->data['path']))
)
);
$result = $this->UserSetting->setSetting($this->Auth->user(), $setting);
return $this->RestResponse->saveSuccessResponse('UserSettings', 'setHomePage', false, $this->response->type(), 'Homepage set to ' . $this->request->data['path']);
} else {
$this->layout = false;
}
if (empty($this->request->data['path'])) {
$this->request->data = array('path' => $this->request->data);
}
if (empty($this->request->data['path'])) {
throw new InvalidArgumentException(__('No path POSTed.'));
}
$setting = array(
'UserSetting' => array(
'user_id' => $this->Auth->user('id'),
'setting' => 'homepage',
'value' => json_encode(array('path' => $this->request->data['path']))
)
);
$result = $this->UserSetting->setSetting($this->Auth->user(), $setting);
return $this->RestResponse->saveSuccessResponse('UserSettings', 'setHomePage', false, $this->response->type(), 'Homepage set to ' . $this->request->data['path']);
}
}

2
app/Lib/Export/OpendataExport.php
View File

@ -105,7 +105,7 @@ class OpendataExport
if (!empty($this->__setup['resources'])) {
if (is_array($this->__setup['resources'])) {
foreach ($this->__setup['resources'] as $resource) {
$cmd .= ' ' . $resource;
$cmd .= " '" . $resource . "'";
}
} else {
$cmd .= " '" . $this->__setup['resources'] . "'";

4
app/Lib/Export/StixExport.php
View File

@ -97,7 +97,7 @@ class StixExport
$stix_event = ($this->__return_type == 'stix') ? $file->read() : substr($file->read(), 1, -1);
$file->close();
$file->delete();
unlink($this->__tmp_dir . $filename);
@unlink($this->__tmp_dir . $filename);
$this->__stix_file->append($stix_event . $this->__framing['separator']);
unset($stix_event);
}
@ -131,7 +131,7 @@ class StixExport
{
foreach ($this->__filenames as $f => $filename) {
if ($index >= $f) {
unlink($this->__tmp_dir . $filename);
@unlink($this->__tmp_dir . $filename);
}
}
$this->__stix_file->close();

62
app/Lib/Tools/CIDRTool.php
View File

@ -1,62 +0,0 @@
<?php
class CIDRTool
{
public function CIDR($cidr)
{
list($address, $prefix) = explode('/', $cidr, 2);
$address = decbin(ip2long($address));
$address = substr("00000000000000000000000000000000", 0, 32 - strlen($address)) . $address;
$min = '';
$max = '';
for ($i = 0; $i < $prefix; $i++) {
$min .= $address[$i];
}
$max = $min;
$min = str_pad($min, 32, '0', STR_PAD_RIGHT);
$max = str_pad($max, 32, '1', STR_PAD_RIGHT);
$minArray = array();
$maxArray = array();
$searchTermLeft = '';
$searchTermMin = 0;
$searchTermMax = 0;
$results = array();
for ($i = 0; $i < 4; $i++) {
$minArray[] = bindec(substr($min, ($i*8), 8));
$maxArray[] = bindec(substr($max, ($i*8), 8));
if ($minArray[$i] === $maxArray[$i]) {
$searchTermLeft .= $minArray[$i] . '.';
} else {
$searchTermMin = $minArray[$i];
$searchTermMax = $maxArray[$i];
break;
}
}
$length = $i;
for ($i = 0; $i < ($searchTermMax - $searchTermMin + 1); $i++) {
$results[$i] = $searchTermLeft . ($searchTermMin + $i);
if ($length < 3) {
$results[$i] .= '.%';
}
}
return $results;
}
public function checkCIDR($cidr, $ipVersion)
{
if (strpos($cidr, '/') === false || substr_count($cidr, '/') !== 1) {
return false;
}
list($net, $maskbits) = explode('/', $cidr);
if (!is_numeric($maskbits) || $maskbits < 0) {
return false;
}
if ($ipVersion == 4) {
return ($maskbits <= 32) && filter_var($net, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4);
} elseif ($ipVersion == 6) {
return ($maskbits <= 128) && filter_var($net, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6);
} else {
throw new InvalidArgumentException('checkCIDR does only support IPv4 & IPv6');
}
}
}

291
app/Lib/Tools/ComplexTypeTool.php
View File

@ -4,37 +4,12 @@ class ComplexTypeTool
{
private $__refangRegexTable = array(
array(
'from' => '/^hxxp/i',
'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i',
'to' => 'http',
'types' => array('link', 'url')
),
array(
'from' => '/^meow/i',
'to' => 'http',
'types' => array('link', 'url')
),
array(
'from' => '/^h\[tt\]p/i',
'to' => 'http',
'types' => array('link', 'url')
),
array(
'from' => '/\[\.\]/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'from' => '/\[dot\]/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'from' => '/\(dot\)/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'from' => '/\\\\\./',
'from' => '/(\[\.\]|\[dot\]|\(dot\)|\\\\\.)/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
@ -49,7 +24,7 @@ class ComplexTypeTool
'types' => array('link', 'url')
),
array(
'from' => '/[\@]/',
'from' => '/[\@]|\[at\]/',
'to' => '@',
'types' => array('email-src', 'email-dst')
),
@ -60,24 +35,24 @@ class ComplexTypeTool
)
);
private $__tlds = null;
public function refangValue($value, $type)
{
foreach ($this->__refangRegexTable as $regex) {
if (!isset($regex['types']) || in_array($type, $regex['types'])) {
if (in_array($type, $regex['types'])) {
$value = preg_replace($regex['from'], $regex['to'], $value);
}
}
return $value;
}
private $__tlds = array();
public function setTLDs($tlds = array())
{
if (!empty($tlds)) {
$this->__tlds = $tlds;
$this->__tlds = [];
foreach ($tlds as $tld) {
$this->__tlds[$tld] = true;
}
return true;
}
public function checkComplexRouter($input, $type, $settings = array())
@ -85,17 +60,13 @@ class ComplexTypeTool
switch ($type) {
case 'File':
return $this->checkComplexFile($input);
break;
case 'CnC':
return $this->checkComplexCnC($input);
break;
case 'freetext':
case 'FreeText':
return $this->checkFreeText($input, $settings);
break;
case 'csv':
return $this->checkCSV($input, $settings);
break;
default:
return false;
}
@ -181,6 +152,7 @@ class ComplexTypeTool
{
$delimiter = !empty($settings['delimiter']) ? $settings['delimiter'] : ",";
$rows = str_getcsv($input, "\n");
unset($input);
$data = array();
foreach ($rows as $k => $row) {
if (empty($row[0]) || $row[0] === '#') {
@ -193,7 +165,6 @@ class ComplexTypeTool
}
}
unset($rows);
unset($input);
$values = !empty($settings['value']) ? $settings['value'] : array();
if (!is_array($values)) {
$values = explode(',', $values);
@ -223,7 +194,7 @@ class ComplexTypeTool
public function checkFreeText($input, $settings = array())
{
$charactersToTrim = array('\'', '"', ',', '(', ')', ' ');
$charactersToTrim = '\'",() ' . "\t\n\r\0\x0B"; // custom + default PHP trim
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input);
$quotedText = explode('"', $input);
foreach ($quotedText as $k => $temp) {
@ -236,34 +207,27 @@ class ComplexTypeTool
}
$iocArray = array_merge($iocArray, $this->__returnOddElements($quotedText));
$resultArray = array();
if (!empty($iocArray)) {
foreach ($iocArray as $ioc) {
$ioc = trim($ioc);
$ioc = str_replace("\xc2\xa0", '', $ioc);
foreach ($charactersToTrim as $c) {
$ioc = trim($ioc, $c);
}
$ioc = preg_replace('/\p{C}+/u', '', $ioc);
if (empty($ioc)) {
continue;
}
if (isset($settings['excluderegex']) && !empty($settings['excluderegex'])) {
if (preg_match($settings['excluderegex'], $ioc)) {
continue;
}
}
$typeArray = $this->__resolveType($ioc);
if ($typeArray === false) {
continue;
}
$temp = $typeArray;
if (!isset($temp['value'])) {
$temp['value'] = $ioc;
}
$resultArray[] = $temp;
foreach ($iocArray as $ioc) {
$ioc = str_replace("\xc2\xa0", '', $ioc); // remove non breaking space
$ioc = trim($ioc, $charactersToTrim);
$ioc = preg_replace('/\p{C}+/u', '', $ioc);
if (empty($ioc)) {
continue;
}
if (!empty($settings['excluderegex']) && preg_match($settings['excluderegex'], $ioc)) {
continue;
}
$typeArray = $this->__resolveType($ioc);
if ($typeArray === false) {
continue;
}
// Remove duplicates
if (isset($resultArray[$typeArray['value']])) {
continue;
}
$resultArray[$typeArray['value']] = $typeArray;
}
return $resultArray;
return array_values($resultArray);
}
private $__hexHashTypes = array(
@ -295,13 +259,13 @@ class ComplexTypeTool
return false;
}
private function __checkForBTC($input)
{
if (preg_match("#^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$#i", $input['raw'])) {
return array('types' => array('btc'), 'categories' => array('Financial fraud'), 'to_ids' => true, 'default_type' => 'btc', 'value' => $input['raw']);
private function __checkForBTC($input)
{
if (preg_match("#^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$#i", $input['raw'])) {
return array('types' => array('btc'), 'categories' => array('Financial fraud'), 'to_ids' => true, 'default_type' => 'btc', 'value' => $input['raw']);
}
return false;
}
return false;
}
private function __checkForEmail($input)
{
@ -314,13 +278,14 @@ class ComplexTypeTool
return false;
}
private function __checkForAS($input)
{
if (preg_match('#^as[0-9]+$#i', $input['raw'])) {
$input['raw'] = strtoupper($input['raw']);
return array('types' => array('AS'), 'to_ids' => false, 'default_type' => 'AS', 'value' => $input['raw']);
}
}
private function __checkForAS($input)
{
if (preg_match('#^as[0-9]+$#i', $input['raw'])) {
$input['raw'] = strtoupper($input['raw']);
return array('types' => array('AS'), 'to_ids' => false, 'default_type' => 'AS', 'value' => $input['raw']);
}
return false;
}
private function __checkForHashes($input)
{
@ -329,12 +294,11 @@ class ComplexTypeTool
$compositeParts = explode('|', $input['raw']);
if (count($compositeParts) == 2) {
if ($this->__resolveFilename($compositeParts[0])) {
foreach ($this->__hexHashTypes as $k => $v) {
if (strlen($compositeParts[1]) == $k && preg_match("#[0-9a-f]{" . $k . "}$#i", $compositeParts[1])) {
return array('types' => $v['composite'], 'to_ids' => true, 'default_type' => $v['composite'][0], 'value' => $input['raw']);
}
$hash = $this->__resolveHash($compositeParts[1]);
if ($hash) {
return array('types' => $hash['composite'], 'to_ids' => true, 'default_type' => $hash['composite'][0], 'value' => $input['raw']);
}
if (preg_match('#^[0-9]+:[0-9a-zA-Z\/\+]+:[0-9a-zA-Z\/\+]+$#', $compositeParts[1]) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $compositeParts[1])) {
if ($this->__resolveSsdeep($compositeParts[1])) {
return array('types' => array('filename|ssdeep'), 'to_ids' => true, 'default_type' => 'filename|ssdeep', 'value' => $input['raw']);
}
}
@ -342,17 +306,16 @@ class ComplexTypeTool
}
// check for hashes
foreach ($this->__hexHashTypes as $k => $v) {
if (strlen($input['raw']) == $k && preg_match("#[0-9a-f]{" . $k . "}$#i", $input['raw'])) {
$types = $v['single'];
if (!empty($this->__checkForBTC($input))) {
$types[] = 'btc';
}
return array('types' => $types, 'to_ids' => true, 'default_type' => $v['single'][0], 'value' => $input['raw']);
$hash = $this->__resolveHash($input['raw']);
if ($hash) {
$types = $hash['single'];
if ($this->__checkForBTC($input)) {
$types[] = 'btc';
}
return array('types' => $types, 'to_ids' => true, 'default_type' => $hash['single'][0], 'value' => $input['raw']);
}
// ssdeep has a different pattern
if (preg_match('#^[0-9]+:[0-9a-zA-Z\/\+]+:[0-9a-zA-Z\/\+]+$#', $input['raw']) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $input['raw'])) {
if ($this->__resolveSsdeep($input['raw'])) {
return array('types' => array('ssdeep'), 'to_ids' => true, 'default_type' => 'ssdeep', 'value' => $input['raw']);
}
return false;
@ -362,13 +325,12 @@ class ComplexTypeTool
{
// note down and remove the port if it's a url / domain name / hostname / ip
// input2 from here on is the variable containing the original input with the port removed. It is only used by url / domain name / hostname / ip
$input['comment'] = false;
if (preg_match('/(:[0-9]{2,5})$/', $input['refanged'], $input['port'])) {
$input['comment'] = 'On port ' . substr($input['port'][0], 1);
$input['refanged_no_port'] = str_replace($input['port'][0], '', $input['refanged']);
$input['port'] = substr($input['port'][0], 1);
if (preg_match('/(:[0-9]{2,5})$/', $input['refanged'], $port)) {
$input['comment'] = 'On port ' . substr($port[0], 1);
$input['refanged_no_port'] = str_replace($port[0], '', $input['refanged']);
$input['port'] = substr($port[0], 1);
} else {
unset($input['port']);
$input['comment'] = false;
$input['refanged_no_port'] = $input['refanged'];
}
return $input;
@ -381,13 +343,13 @@ class ComplexTypeTool
$input['refanged'] = preg_replace($regex['from'], $regex['to'], $input['refanged']);
}
$input['refanged'] = rtrim($input['refanged'], ".");
$input['refanged'] = preg_replace_callback(
'/\[.\]/',
function ($matches) {
return trim($matches[0], '[]');
},
$input['refanged']
);
$input['refanged'] = preg_replace_callback(
'/\[.\]/',
function ($matches) {
return trim($matches[0], '[]');
},
$input['refanged']
);
return $input;
}
@ -397,17 +359,16 @@ class ComplexTypeTool
if (preg_match("#^cve-[0-9]{4}-[0-9]{4,9}$#i", $input['raw'])) {
return array('types' => array('vulnerability'), 'categories' => array('External analysis'), 'to_ids' => false, 'default_type' => 'vulnerability', 'value' => $input['raw']);
}
// Phone numbers - for automatic recognition, needs to start with + or include dashes
if (!empty($input['raw'])) {
if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) {
if (!preg_match('#^[0-9]{4}-[0-9]{2}-[0-9]{2}$#i', $input['raw']) && preg_match("#^(\+)?([0-9]{1,3}(\(0\))?)?[0-9\/\-]{5,}[0-9]$#i", $input['raw'])) {
return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'categories' => array('Other'), 'to_ids' => false, 'default_type' => 'phone-number', 'value' => $input['raw']);
}
}
}
// Phone numbers - for automatic recognition, needs to start with + or include dashes
if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) {
if (!preg_match('#^[0-9]{4}-[0-9]{2}-[0-9]{2}$#i', $input['raw']) && preg_match("#^(\+)?([0-9]{1,3}(\(0\))?)?[0-9\/\-]{5,}[0-9]$#i", $input['raw'])) {
return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'categories' => array('Other'), 'to_ids' => false, 'default_type' => 'phone-number', 'value' => $input['raw']);
}
}
return false;
}
private function __checkForIP($input)
private function __checkForIP(array $input)
{
if (filter_var($input['refanged_no_port'], FILTER_VALIDATE_IP)) {
if (isset($input['port'])) {
@ -416,30 +377,48 @@ class ComplexTypeTool
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
}
// IPv6 address that is considered as IP address with port
if (filter_var($input['refanged'], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
return [
'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'],
'to_ids' => true,
'default_type' => 'ip-dst',
'comment' => '',
'value' => $input['refanged'],
];
}
// IPv6 with port in `[1fff:0:a88:85a3::ac1f]:8001` format
if (isset($input['port']) &&
!empty($input['refanged_no_port']) &&
$input['refanged_no_port'][0] === '[' &&
filter_var(substr($input['refanged_no_port'], 1, -1), FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)
) {
$value = substr($input['refanged_no_port'], 1, -1); // remove brackets
return [
'types' => ['ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'],
'to_ids' => true,
'default_type' => 'ip-dst|port',
'comment' => $input['comment'],
'value' => "$value|{$input['port']}",
];
}
// it could still be a CIDR block
if (strpos($input['refanged_no_port'], '/')) {
$temp = explode('/', $input['refanged_no_port']);
if (count($temp) == 2) {
if (filter_var($temp[0], FILTER_VALIDATE_IP) && is_numeric($temp[1])) {
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
if (count($temp) === 2 && filter_var($temp[0], FILTER_VALIDATE_IP) && is_numeric($temp[1])) {
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
}
return false;
}
private function __checkForDomainOrFilename($input)
private function __checkForDomainOrFilename(array $input)
{
if (strpos($input['refanged'], '.') !== false) {
$temp = explode('.', $input['refanged']);
// TODO: use a more flexible matching approach, like the one below (that still doesn't support non-ASCII domains)
//if (filter_var($input, FILTER_VALIDATE_URL)) {
if (strpos($input['refanged_no_port'], '.') !== false) {
$temp = explode('.', $input['refanged_no_port']);
$domainDetection = true;
if (preg_match('/^([-\pL\pN]+\.)+[a-z]+(:[0-9]{2,5})?$/iu', $input['refanged'])) {
if (empty($this->__tlds) || count($this->__tlds) == 1) {
$this->__generateTLDList();
}
$tldExploded = explode(':', $temp[count($temp)-1]);
if (!in_array(strtolower($tldExploded[0]), $this->__tlds)) {
if (preg_match('/^([-\pL\pN]+\.)+[a-z0-9-]+$/iu', $input['refanged_no_port'])) {
if (!$this->isTld(end($temp))) {
$domainDetection = false;
}
} else {
@ -457,12 +436,9 @@ class ComplexTypeTool
if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) {
// Even though some domains are valid, we want to exclude them as they are known security vendors / etc
// TODO, replace that with the appropriate warninglist.
if (preg_match('/^https:\/\/(www.)?virustotal.com\//i', $input['refanged_no_port'])) {
if (preg_match('/^(https:\/\/(www.)?virustotal.com\/|https:\/\/www\.hybrid-analysis\.com\/)/i', $input['refanged_no_port'])) {
return array('types' => array('link'), 'to_ids' => false, 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
if (preg_match('/^https:\/\/www\.hybrid-analysis\.com\//i', $input['refanged_no_port'])) {
return array('types' => array('link'), 'categories' => array('External analysis'), 'to_ids' => false, 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
if (strpos($input['refanged_no_port'], '/')) {
return array('types' => array('url'), 'to_ids' => true, 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
@ -474,11 +450,11 @@ class ComplexTypeTool
}
if (strpos($input['raw'], '\\') !== false) {
$temp = explode('\\', $input['raw']);
if (strpos($temp[count($temp)-1], '.') || preg_match('/^.:/i', $temp[0])) {
if ($this->__resolveFilename($temp[count($temp)-1])) {
if (strpos(end($temp), '.') || preg_match('/^.:/i', $temp[0])) {
if ($this->__resolveFilename(end($temp))) {
return array('types' => array('filename'), 'categories' => array('Payload installation'), 'to_ids' => true, 'default_type' => 'filename', 'value' => $input['raw']);
}
} else {
} else if (!empty($temp[0])) {
return array('types' => array('regkey'), 'to_ids' => false, 'default_type' => 'regkey', 'value' => $input['raw']);
}
}
@ -487,26 +463,61 @@ class ComplexTypeTool
private function __resolveFilename($param)
{
if ((preg_match('/^.:/', $param) || strpos($param, '.') !=0)) {
if ((preg_match('/^.:/', $param) || strpos($param, '.') != 0)) {
$parts = explode('.', $param);
if (!is_numeric($parts[count($parts)-1]) && ctype_alnum($parts[count($parts)-1])) {
if (!is_numeric(end($parts)) && ctype_alnum(end($parts))) {
return true;
}
}
return false;
}
/**
* @param string $value
* @return bool
*/
private function __resolveSsdeep($value)
{
return preg_match('#^[0-9]+:[0-9a-zA-Z\/\+]+:[0-9a-zA-Z\/\+]+$#', $value) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $value);
}
/**
* @param string $value
* @return bool|string[][]
*/
private function __resolveHash($value)
{
$strlen = strlen($value);
if (isset($this->__hexHashTypes[$strlen]) && preg_match("#[0-9a-f]{" . $strlen . "}$#i", $value)) {
return $this->__hexHashTypes[$strlen];
}
return false;
}
/**
* @param string $tld
* @return bool
*/
private function isTld($tld)
{
if ($this->__tlds === null) {
$this->setTLDs($this->__generateTLDList());
}
return isset($this->__tlds[strtolower($tld)]);
}
private function __generateTLDList()
{
$this->__tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion');
$tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion');
$char1 = $char2 = 'a';
for ($i = 0; $i < 26; $i++) {
for ($j = 0; $j < 26; $j++) {
$this->__tlds[] = $char1 . $char2;
$tlds[] = $char1 . $char2;
$char2++;
}
$char1++;
$char2 = 'a';
}
return $tlds;
}
}

94
app/Lib/Tools/TmpFileTool.php Normal file
View File

@ -0,0 +1,94 @@
<?php
class TmpFileTool
{
/**
* @var resource
*/
private $tmpfile;
/**
* @param int $maxInMemory How many bytes should keep in memory before creating file on disk. By default is is 2 MB.
* @throws Exception
*/
public function __construct($maxInMemory = null)
{
if ($maxInMemory === null) {
$maxInMemory = 2 * 1024 * 1024;
}
$this->tmpfile = fopen("php://temp/maxmemory:$maxInMemory", "w+");
if ($this->tmpfile === false) {
throw new Exception('Could not create temporary file.');
}
}
/**
* @param string $content
* @throws Exception
*/
public function write($content)
{
if (fwrite($this->tmpfile, $content) === false) {
if ($this->tmpfile === null) {
throw new Exception('Could not write to finished temporary file.');
}
$tmpFolder = sys_get_temp_dir();
$freeSpace = disk_free_space($tmpFolder);
throw new Exception("Could not write to temporary file in $tmpFolder folder. Maybe not enough space? ($freeSpace bytes left)");
}
}
/**
* @return Generator
* @throws Exception
*/
public function lines()
{
$this->rewind();
while (!feof($this->tmpfile)) {
$result = fgets($this->tmpfile);
if ($result === false) {
throw new Exception('Could not read line from temporary file.');
}
yield $result;
}
fclose($this->tmpfile);
$this->tmpfile = null;
}
/**
* @return string
* @throws Exception
*/
public function finish()
{
$this->rewind();
$final = stream_get_contents($this->tmpfile);
if ($final === false) {
throw new Exception('Could not read from temporary file.');
}
fclose($this->tmpfile);
$this->tmpfile = null;
return $final;
}
/**
* @return string
* @throws Exception
*/
public function __toString()
{
return $this->finish();
}
/**
* Seek to start of file.
*
* @throws Exception
*/
private function rewind()
{
if (fseek($this->tmpfile, 0) === -1) {
throw new Exception('Could not seek to start of temporary file.');
}
}
}

119
app/Model/Attribute.php
View File

@ -6,6 +6,7 @@ App::uses('File', 'Utility');
App::uses('FinancialTool', 'Tools');
App::uses('RandomTool', 'Tools');
App::uses('MalwareTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Attribute extends AppModel
{
@ -822,6 +823,10 @@ class Attribute extends AppModel
),
false
);
if ($this->data['Attribute']['type'] === 'ssdeep') {
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
$this->FuzzyCorrelateSsdeep->purge(null, $this->data['Attribute']['id']);
}
}
}
@ -935,13 +940,15 @@ class Attribute extends AppModel
public function validCategory($fields)
{
$validCategories = array_keys($this->categoryDefinitions);
if (in_array($fields['category'], $validCategories)) {
return true;
}
return false;
return isset($this->categoryDefinitions[$fields['category']]);
}
/**
* Check if the attribute already exists in the same event.
*
* @param array $fields
* @return bool
*/
public function valueIsUnique($fields)
{
if (isset($this->data['Attribute']['deleted']) && $this->data['Attribute']['deleted']) {
@ -951,31 +958,28 @@ class Attribute extends AppModel
if (!empty($this->data['Attribute']['object_relation'])) {
return true;
}
$value = $fields['value'];
if (strpos($value, '|')) {
$value = explode('|', $value);
$value = array(
'Attribute.value1' => $value[0],
'Attribute.value2' => $value[1]
);
} else {
$value = array(
'Attribute.value1' => $value,
);
}
$eventId = $this->data['Attribute']['event_id'];
$type = $this->data['Attribute']['type'];
$category = $this->data['Attribute']['category'];
// check if the attribute already exists in the same event
$eventId = $this->data['Attribute']['event_id'];
$category = $this->data['Attribute']['category'];
$type = $this->data['Attribute']['type'];
$conditions = array(
'Attribute.event_id' => $eventId,
'Attribute.type' => $type,
'Attribute.category' => $category,
'Attribute.deleted' => 0,
'Attribute.object_id' => 0
'Attribute.object_id' => 0,
);
$conditions = array_merge($conditions, $value);
$value = $fields['value'];
if (in_array($type, $this->getCompositeTypes())) {
$value = explode('|', $value);
$conditions['Attribute.value1'] = $value[0];
$conditions['Attribute.value2'] = $value[1];
} else {
$conditions['Attribute.value1'] = $value;
}
if (isset($this->data['Attribute']['id'])) {
$conditions['Attribute.id !='] = $this->data['Attribute']['id'];
}
@ -1625,14 +1629,17 @@ class Attribute extends AppModel
public function getCompositeTypes()
{
// build the list of composite Attribute.type dynamically by checking if type contains a |
// default composite types
$compositeTypes = array('malware-sample'); // TODO hardcoded composite
// dynamically generated list
foreach (array_keys($this->typeDefinitions) as $type) {
$pieces = explode('|', $type);
if (2 == count($pieces)) {
$compositeTypes[] = $type;
static $compositeTypes;
if ($compositeTypes === null) {
// build the list of composite Attribute.type dynamically by checking if type contains a |
// default composite types
$compositeTypes = array('malware-sample'); // TODO hardcoded composite
// dynamically generated list
foreach ($this->typeDefinitions as $type => $foo) {
if (strpos($type, '|') !== false) {
$compositeTypes[] = $type;
}
}
}
return $compositeTypes;
@ -1923,11 +1930,9 @@ class Attribute extends AppModel
'Correlation.attribute_id' => $a['id']))
);
}
if ($a['type'] == 'ssdeep') {
if ($a['type'] === 'ssdeep') {
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
$this->FuzzyCorrelateSsdeep->deleteAll(
array('FuzzyCorrelateSsdeep.attribute_id' => $a['id'])
);
$this->FuzzyCorrelateSsdeep->purge(null, $a['id']);
}
}
@ -2030,18 +2035,13 @@ class Attribute extends AppModel
$ip_version = filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
$cidrList = $this->getSetCIDRList();
foreach ($cidrList as $cidr) {
$cidr_ip = explode('/', $cidr)[0];
if (filter_var($cidr_ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
if ($ip_version === 4) {
if ($this->__ipv4InCidr($ip, $cidr)) {
$ipValues[] = $cidr;
}
if (strpos($cidr, '.') !== false) {
if ($ip_version === 4 && $this->__ipv4InCidr($ip, $cidr)) {
$ipValues[] = $cidr;
}
} else {
if ($ip_version === 6) {
if ($this->__ipv6InCidr($ip, $cidr)) {
$ipValues[] = $cidr;
}
if ($ip_version === 6 && $this->__ipv6InCidr($ip, $cidr)) {
$ipValues[] = $cidr;
}
}
}
@ -2808,6 +2808,10 @@ class Attribute extends AppModel
{
$this->Correlation = ClassRegistry::init('Correlation');
$this->purgeCorrelations($eventId);
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
$this->FuzzyCorrelateSsdeep->purge($eventId, $attributeId);
// get all attributes..
if (!$eventId) {
$eventIds = $this->Event->find('list', array('recursive' => -1, 'fields' => array('Event.id')));
@ -3907,7 +3911,7 @@ class Attribute extends AppModel
{
$redis = $this->setupRedis();
if ($redis) {
if ($redis->sCard('misp:cidr_cache_list') === 0) {
if (!$redis->exists('misp:cidr_cache_list')) {
$cidrList = $this->setCIDRList();
} else {
$cidrList = $redis->smembers('misp:cidr_cache_list');
@ -4588,8 +4592,9 @@ class Attribute extends AppModel
$exportTool->additional_params
);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
@ -4602,22 +4607,15 @@ class Attribute extends AppModel
if (empty($exportTool->mock_query_only)) {
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
}
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size']) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = '';
}
fclose($tmpfile);
return $final;
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
{
$this->Whitelist = ClassRegistry::init('Whitelist');
$continue = true;
while ($continue) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$results = $this->fetchAttributes($user, $params, $continue);
if ($params['includeSightingdb']) {
$this->Sightingdb = ClassRegistry::init('Sightingdb');
@ -4625,7 +4623,6 @@ class Attribute extends AppModel
}
$params['page'] += 1;
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
$i = 0;
$temp = '';
foreach ($results as $attribute) {
@ -4645,7 +4642,7 @@ class Attribute extends AppModel
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
fwrite($tmpfile, $temp);
$tmpfile->write($temp);
}
return true;
}

128
app/Model/Event.php
View File

@ -2,6 +2,7 @@
App::uses('AppModel', 'Model');
App::uses('CakeEmail', 'Network/Email');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Event extends AppModel
{
@ -3169,9 +3170,9 @@ class Event extends AppModel
'conditions' => array(
'User.id' => $event['Event']['user_id'],
'User.disabled' => 0,
'User.org_id' => $event['Event']
'User.org_id' => $event['Event']['orgc_id'],
),
'fields' => array('User.email', 'User.gpgkey', 'User.certif_public'),
'fields' => array('User.email', 'User.gpgkey', 'User.certif_public', 'User.id', 'User.disabled'),
'recursive' => -1
));
if (!empty($temp)) {
@ -3202,10 +3203,10 @@ class Event extends AppModel
$body .= "Someone wants to get in touch with you concerning a MISP event. \n";
$body .= "\n";
$body .= "You can reach them at " . $user['User']['email'] . "\n";
if (!$user['User']['gpgkey']) {
if (!empty($user['User']['gpgkey'])) {
$body .= "Their GnuPG key is added as attachment to this email. \n";
}
if (!$user['User']['certif_public']) {
if (!empty($user['User']['certif_public'])) {
$body .= "Their Public certificate is added as attachment to this email. \n";
}
$body .= "\n";
@ -3652,7 +3653,7 @@ class Event extends AppModel
'conditions' => array('Event.id' => $this->id),
'recursive' => -1
));
if (isset($data['Event']['Attribute']) && !empty($data['Event']['Attribute'])) {
if (!empty($data['Event']['Attribute'])) {
foreach ($data['Event']['Attribute'] as $k => $attribute) {
$block = false;
for ($i = 0; $i < $k; $i++) {
@ -3779,7 +3780,8 @@ class Event extends AppModel
'Server.name',
'Server.id',
'Server.unpublish_event',
'Server.publish_without_email'
'Server.publish_without_email',
'Server.internal',
)
));
} else {
@ -5581,99 +5583,6 @@ class Event extends AppModel
return array('data' => array(), 'csv' => array());
}
public function setSimpleConditions($parameterKey, $parameterValue, $conditions, $restrictScopeToEvents = false)
{
if (is_array($parameterValue)) {
$elements = $parameterValue;
} else {
$elements = explode('&&', $parameterValue);
}
App::uses('CIDRTool', 'Tools');
$cidr = new CIDRTool();
$subcondition = array();
foreach ($elements as $v) {
if ($v === '') {
continue;
}
if (substr($v, 0, 1) === '!') {
// check for an IPv4 address and subnet in CIDR notation (e.g. 127.0.0.1/8)
if ($parameterKey === 'value' && $cidr->checkCIDR(substr($v, 1), 4)) {
$cidrresults = $cidr->CIDR(substr($v, 1));
foreach ($cidrresults as $result) {
$subcondition['AND'][] = array('Attribute.value NOT LIKE' => $result);
}
} else {
if ($parameterKey === 'org') {
$found_orgs = $this->Org->find('all', array(
'recursive' => -1,
'conditions' => array('name' => substr($v, 1)),
));
foreach ($found_orgs as $o) {
$subcondition['AND'][] = array('Event.orgc_id !=' => $o['Org']['id']);
}
} elseif ($parameterKey === 'eventid') {
if ($restrictScopeToEvents) {
$subcondition['AND'][] = array('Event.id !=' => substr($v, 1));
} else {
$subcondition['AND'][] = array('Attribute.event_id !=' => substr($v, 1));
}
} elseif ($parameterKey === 'uuid') {
$subcondition['AND'][] = array('Event.uuid !=' => substr($v, 1));
$subcondition['AND'][] = array('Attribute.uuid !=' => substr($v, 1));
} else {
$lookup = substr($v, 1);
if (strlen($lookup) != strlen(trim($lookup, '%'))) {
$subcondition['AND'][] = array('Attribute.' . $parameterKey . ' NOT LIKE' => $lookup);
} else {
$subcondition['AND'][] = array('NOT' => array('Attribute.' . $parameterKey => $lookup));
}
}
}
} else {
// check for an IPv4 address and subnet in CIDR notation (e.g. 127.0.0.1/8)
if ($parameterKey === 'value' && $cidr->checkCIDR($v, 4)) {
$cidrresults = $cidr->CIDR($v);
foreach ($cidrresults as $result) {
if (!empty($result)) {
$subcondition['OR'][] = array('Attribute.value LIKE' => $result);
}
}
} else {
if ($parameterKey === 'org') {
$found_orgs = $this->Org->find('all', array(
'recursive' => -1,
'conditions' => array('name' => $v),
));
foreach ($found_orgs as $o) {
$subcondition['OR'][] = array('Event.orgc_id' => $o['Org']['id']);
}
} elseif ($parameterKey === 'eventid') {
if ($restrictScopeToEvents) {
$subcondition['OR'][] = array('Event.id' => $v);
} else {
$subcondition['OR'][] = array('Attribute.event_id' => $v);
}
} elseif ($parameterKey === 'uuid') {
$subcondition['OR'][] = array('Attribute.uuid' => $v);
$subcondition['OR'][] = array('Event.uuid' => $v);
} else {
if (!empty($v)) {
if (strlen($v) != strlen(trim($v, '%'))) {
$subcondition['AND'][] = array('Attribute.' . $parameterKey . ' LIKE' => $v);
} else {
$subcondition['AND'][] = array('Attribute.' . $parameterKey => $v);
}
}
}
}
}
}
if (!empty($subcondition)) {
array_push($conditions['AND'], $subcondition);
}
return $conditions;
}
public function cacheSgids($user, $useCache = false)
{
if ($useCache && isset($this->__assetCache['sgids'])) {
@ -5832,7 +5741,7 @@ class Event extends AppModel
$validationIssues = false;
$result = $this->_add($data, true, $user, '', null, false, null, $created_id, $validationIssues);
if ($result) {
if ($original_file) {
if ($original_file && !is_numeric($result)) {
$this->add_original_file($tempFile, $original_file, $created_id, $stix_version);
}
if ($publish && $user['Role']['perm_publish']) {
@ -5861,7 +5770,7 @@ class Event extends AppModel
private function __getTagNamesFromSynonyms($scriptDir)
{
$synonymsToTagNames = $scriptDir . DS . 'synonymsToTagNames.json';
$synonymsToTagNames = $scriptDir . DS . 'tmp' . DS . 'synonymsToTagNames.json';
if (!file_exists($synonymsToTagNames) || (time() - filemtime($synonymsToTagNames)) > 600) {
if (empty($this->GalaxyCluster)) {
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
@ -6796,8 +6705,8 @@ class Event extends AppModel
$filters['published'] = 1;
}
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$i = 0;
if (!empty($filters['withAttachments'])) {
$filters['includeAttachments'] = 1;
@ -6825,7 +6734,7 @@ class Event extends AppModel
if ($i !== 0) {
$temp = $exportTool->separator($exportToolParams) . $temp;
}
fwrite($tmpfile, $temp);
$tmpfile->write($temp);
$i++;
}
}
@ -6833,15 +6742,8 @@ class Event extends AppModel
}
unset($result);
unset($temp);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size'] > 0) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = 0;
}
fclose($tmpfile);
return $final;
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
/*

29
app/Model/Feed.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Feed extends AppModel
{
@ -161,10 +162,10 @@ class Feed extends AppModel
/**
* @param array $feed
* @param HttpSocket $HttpSocket
* @return array
* @return Generator|array
* @throws Exception
*/
public function getCache($feed, $HttpSocket)
public function getCache(array $feed, HttpSocket $HttpSocket)
{
$uri = $feed['Feed']['url'] . '/hashes.csv';
$data = $this->feedGetUri($feed, $uri, $HttpSocket);
@ -173,13 +174,16 @@ class Feed extends AppModel
throw new Exception("File '$uri' with hashes for cache filling is empty.");
}
$data = trim($data);
$data = explode("\n", $data);
$result = array();
foreach ($data as $v) {
$result[] = explode(',', $v);
// CSV file can be pretty big to do operations in memory, so we save content to temp and iterate line by line.
$tmpFile = new TmpFileTool();
$tmpFile->write(trim($data));
unset($data);
foreach ($tmpFile->lines() as $line) {
yield explode(',', rtrim($line));
}
return $result;
return array();
}
/**
@ -320,7 +324,6 @@ class Feed extends AppModel
foreach ($feeds as $k => $v) {
if ($redis->sismember('misp:feed_cache:' . $v['Feed']['id'], md5($value['value']))) {
$data[$key]['feed_correlations'][] = array($v);
} else {
}
}
}
@ -1120,7 +1123,7 @@ class Feed extends AppModel
}
}
private function __cacheFreetextFeed($feed, $redis, $HttpSocket, $jobId = false)
private function __cacheFreetextFeed(array $feed, $redis, HttpSocket $HttpSocket, $jobId = false)
{
$feedId = $feed['Feed']['id'];
@ -1128,19 +1131,23 @@ class Feed extends AppModel
$values = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], 'all');
} catch (Exception $e) {
$this->logException("Could not get freetext feed $feedId", $e);
$this->jobProgress($jobId, 'Could not fetch freetext feed. See log for more details.');
$this->jobProgress($jobId, 'Could not fetch freetext feed. See error log for more details.');
return false;
}
$pipe = $redis->multi(Redis::PIPELINE);
foreach ($values as $k => $value) {
$md5Value = md5($value['value']);
$redis->sAdd('misp:feed_cache:' . $feedId, $md5Value);
$redis->sAdd('misp:feed_cache:combined', $md5Value);
if ($k % 1000 == 0) {
$this->jobProgress($jobId, "Feed $feedId: $k/" . count($values) . " values cached.");
$pipe->exec();
$pipe = $redis->multi(Redis::PIPELINE);
}
}
$redis->set('misp:feed_cache_timestamp:' . $feedId, time());
$pipe->exec();
return true;
}

23
app/Model/FuzzyCorrelateSsdeep.php
View File

@ -89,4 +89,27 @@ class FuzzyCorrelateSsdeep extends AppModel
$this->saveAll($to_save);
return $result;
}
/**
* @param int|null $eventId
* @param int|null $attributeId
* @return bool True on success, false on failure
*/
public function purge($eventId = null, $attributeId = null)
{
if (!$eventId && !$attributeId) {
$this->query('TRUNCATE TABLE fuzzy_correlate_ssdeep;');
} elseif (!$attributeId) {
$this->Attribute = ClassRegistry::init('Attribute');
$attributeId = $this->Attribute->find('list', array(
'conditions' => array(
'Attribute.event_id' => $eventId,
'Attribute.type' => 'ssdeep',
),
'fields' => 'Attribute.id',
));
}
return $this->deleteAll(array('FuzzyCorrelateSsdeep.attribute_id' => $attributeId));
}
}

21
app/Model/MispObject.php
View File

@ -1,6 +1,6 @@
<?php
App::uses('AppModel', 'Model');
App::uses('TmpFileTool', 'Tools');
class MispObject extends AppModel
{
@ -1426,8 +1426,8 @@ class MispObject extends AppModel
$exportTool->additional_params
);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
@ -1438,18 +1438,11 @@ class MispObject extends AppModel
$params['page'] = 1;
}
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size']) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = '';
}
fclose($tmpfile);
return $final;
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
{
$continue = true;
while ($continue) {
@ -1485,7 +1478,7 @@ class MispObject extends AppModel
if (!$loop) {
$continue = false;
}
fwrite($tmpfile, $temp);
$tmpfile->write($temp);
}
return true;
}

2
app/Model/Server.php
View File

@ -5662,7 +5662,7 @@ class Server extends AppModel
public function extensionDiagnostics()
{
$results = array();
$extensions = array('redis', 'gd', 'ssdeep');
$extensions = array('redis', 'gd', 'ssdeep', 'zip');
foreach ($extensions as $extension) {
$results['web']['extensions'][$extension] = extension_loaded($extension);
}

19
app/Model/Sighting.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Sighting extends AppModel
{
@ -634,11 +635,11 @@ class Sighting extends AppModel
$allowedContext = array('event', 'attribute');
// validate context
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
throw new MethodNotAllowedException(_('Invalid context.'));
throw new MethodNotAllowedException(__('Invalid context.'));
}
// ensure that an id is provided if context is set
if (!empty($filters['context']) && !isset($filters['id'])) {
throw new MethodNotAllowedException(_('An id must be provided if the context is set.'));
throw new MethodNotAllowedException(__('An id must be provided if the context is set.'));
}
if (!isset($this->validFormats[$returnFormat][1])) {
@ -749,8 +750,8 @@ class Sighting extends AppModel
'filters' => $filters
);
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$temp = '';
$i = 0;
@ -763,13 +764,9 @@ class Sighting extends AppModel
}
$i++;
}
fwrite($tmpfile, $temp);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
return $final;
$tmpfile->write($temp);
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
/**

428
app/Test/ComplexTypeToolTest.php Normal file
View File

@ -0,0 +1,428 @@
<?php
require_once __DIR__ . '/../Lib/Tools/ComplexTypeTool.php';
use PHPUnit\Framework\TestCase;
class ComplexTypeToolTest extends TestCase
{
public function testCheckFreeTextHeader(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText(<<<EOT
# LAST 1000 # UTC UPDATE 2020-07-13 08:15:00
127.0.0.1,(127.0.0.2), <127.0.0.3>; "127.0.0.4" '127.0.0.5'
EOT
);
$this->assertCount(5, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
$this->assertEquals('127.0.0.2', $results[1]['value']);
$this->assertEquals('ip-dst', $results[1]['default_type']);
$this->assertEquals('127.0.0.3', $results[2]['value']);
$this->assertEquals('ip-dst', $results[2]['default_type']);
$this->assertEquals('127.0.0.4', $results[3]['value']);
$this->assertEquals('ip-dst', $results[3]['default_type']);
$this->assertEquals('127.0.0.5', $results[4]['value']);
$this->assertEquals('ip-dst', $results[4]['default_type']);
}
public function testCheckFreeTextIpv4(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('127.0.0.1');
$this->assertCount(1, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
}
public function testCheckFreeTextIpv4WithPort(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('127.0.0.1:8080');
$this->assertCount(1, $results);
$this->assertEquals('127.0.0.1|8080', $results[0]['value']);
$this->assertEquals('ip-dst|port', $results[0]['default_type']);
$this->assertEquals('On port 8080', $results[0]['comment']);
}
public function testCheckFreeTextIpv4Cidr(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('127.0.0.1/32');
$this->assertCount(1, $results);
$this->assertEquals('127.0.0.1/32', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
}
// Issue https://github.com/MISP/MISP/issues/6009
public function testCheckFreeTextIpv6(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('2a00:1450:4005:80a::2003');
$this->assertCount(1, $results);
$this->assertEquals('2a00:1450:4005:80a::2003', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
}
// Issue https://github.com/MISP/MISP/issues/3383
public function testCheckFreeTextIpv6Invalid(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('fe80:0000:f2cd:7d80:3f37:52c6');
$this->assertCount(0, $results);
}
public function testCheckFreeTextIpv6Cidr(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('2a00:1450:4005:80a::2003/128');
$this->assertCount(1, $results);
$this->assertEquals('2a00:1450:4005:80a::2003/128', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
}
public function testCheckFreeTextIpv6WithPort(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('[1fff:0:a88:85a3::ac1f]:8001');
$this->assertCount(1, $results);
$this->assertEquals('1fff:0:a88:85a3::ac1f|8001', $results[0]['value']);
$this->assertEquals('ip-dst|port', $results[0]['default_type']);
$this->assertEquals('On port 8001', $results[0]['comment']);
}
public function testCheckFreeTextDomain(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('example.com');
$this->assertCount(1, $results);
$this->assertEquals('example.com', $results[0]['value']);
$this->assertEquals('domain', $results[0]['default_type']);
}
public function testCheckFreeTextDomainThirdLevel(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('example.example.com');
$this->assertCount(1, $results);
$this->assertEquals('example.example.com', $results[0]['value']);
$this->assertEquals('hostname', $results[0]['default_type']);
}
public function testCheckFreeTextDomainDot(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('example.com.');
$this->assertCount(1, $results);
$this->assertEquals('example.com', $results[0]['value']);
$this->assertEquals('domain', $results[0]['default_type']);
}
public function testCheckFreeTextDomainNotExistsTld(): void
{
$complexTypeTool = new ComplexTypeTool();
$complexTypeTool->setTLDs(['com']);
$results = $complexTypeTool->checkFreeText('example.example');
$this->assertCount(1, $results);
$this->assertEquals('example.example', $results[0]['value']);
$this->assertEquals('filename', $results[0]['default_type']);
}
public function testCheckFreeTextFilenameMultipleExt(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('example.txt.zip');
$this->assertCount(1, $results);
$this->assertEquals('example.txt.zip', $results[0]['value']);
$this->assertEquals('filename', $results[0]['default_type']);
}
public function testCheckFreeTextFilenameWithPathUnix(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('/var/log/example.txt.zip');
$this->assertCount(1, $results);
$this->assertEquals('/var/log/example.txt.zip', $results[0]['value']);
$this->assertEquals('filename', $results[0]['default_type']);
}
public function testCheckFreeTextFilenameWithPathWindows(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('C:\example.txt.zip');
$this->assertCount(1, $results);
$this->assertEquals('C:\example.txt.zip', $results[0]['value']);
$this->assertEquals('filename', $results[0]['default_type']);
}
public function testCheckFreeTextRegkey(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion');
$this->assertCount(1, $results);
$this->assertEquals('HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion', $results[0]['value']);
$this->assertEquals('regkey', $results[0]['default_type']);
}
public function testCheckFreeTextDomainWithPort(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('example.com:80');
$this->assertCount(1, $results);
$this->assertEquals('example.com', $results[0]['value']);
$this->assertEquals('domain', $results[0]['default_type']);
$this->assertEquals('On port 80', $results[0]['comment']);
}
public function testCheckFreeTextDomainUppercase(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('EXAMPLE.COM');
$this->assertCount(1, $results);
$this->assertEquals('EXAMPLE.COM', $results[0]['value']);
$this->assertEquals('domain', $results[0]['default_type']);
}
// Issue https://github.com/MISP/MISP/issues/657
public function testCheckFreeTextPunycode(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('xn--ghq549cb2anjl2suxo.com');
$this->assertCount(1, $results);
$this->assertEquals('xn--ghq549cb2anjl2suxo.com', $results[0]['value']);
$this->assertEquals('domain', $results[0]['default_type']);
}
// Issue https://github.com/MISP/MISP/issues/657
public function testCheckFreeTextPunycodeTld(): void
{
$complexTypeTool = new ComplexTypeTool();
$complexTypeTool->setTLDs(['xn--fiqs8s']);
$results = $complexTypeTool->checkFreeText('xn--lbrs59br5a.xn--fiqs8s');
$this->assertCount(1, $results);
$this->assertEquals('xn--lbrs59br5a.xn--fiqs8s', $results[0]['value']);
$this->assertEquals('domain', $results[0]['default_type']);
}
// Issue https://github.com/MISP/MISP/issues/3580
public function testCheckFreeTextDate(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('2018-08-21');
$this->assertCount(0, $results);
}
public function testCheckFreeTextEmail(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('test@example.com');
$this->assertCount(1, $results);
$this->assertEquals('test@example.com', $results[0]['value']);
$this->assertEquals('email-src', $results[0]['default_type']);
}
public function testCheckFreeTextEmailBracket(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('test[@]example.com');
$this->assertCount(1, $results);
$this->assertEquals('test@example.com', $results[0]['value']);
$this->assertEquals('email-src', $results[0]['default_type']);
}
// Issue https://github.com/MISP/MISP/issues/4805
public function testCheckFreeTextEmailBracketAt(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('test[at]example.com');
$this->assertCount(1, $results);
$this->assertEquals('test@example.com', $results[0]['value']);
$this->assertEquals('email-src', $results[0]['default_type']);
}
public function testCheckFreeTextUrlHttp(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('http://example.com');
$this->assertCount(1, $results);
$this->assertEquals('http://example.com', $results[0]['value']);
$this->assertEquals('url', $results[0]['default_type']);
}
public function testCheckFreeTextUrlHttps(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('https://example.com');
$this->assertCount(1, $results);
$this->assertEquals('https://example.com', $results[0]['value']);
$this->assertEquals('url', $results[0]['default_type']);
}
public function testCheckFreeTextUrlWithoutProtocol(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('github.com/MISP/MISP');
$this->assertCount(1, $results);
$this->assertEquals('github.com/MISP/MISP', $results[0]['value']);
$this->assertEquals('url', $results[0]['default_type']);
}
public function testCheckFreeTextUrlVirusTotal(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('https://www.virustotal.com/example https://virustotal.com/example');
$this->assertCount(2, $results);
$this->assertEquals('https://www.virustotal.com/example', $results[0]['value']);
$this->assertEquals('link', $results[0]['default_type']);
$this->assertFalse($results[0]['to_ids']);
$this->assertEquals('https://virustotal.com/example', $results[1]['value']);
$this->assertEquals('link', $results[1]['default_type']);
$this->assertFalse($results[1]['to_ids']);
}
public function testCheckFreeTextUrlHybridAnalysis(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('https://www.hybrid-analysis.com/example');
$this->assertCount(1, $results);
$this->assertEquals('https://www.hybrid-analysis.com/example', $results[0]['value']);
$this->assertEquals('link', $results[0]['default_type']);
$this->assertFalse($results[0]['to_ids']);
}
// Issue https://github.com/MISP/MISP/issues/4908
public function testCheckFreeTextUrlReplace(): void
{
$complexTypeTool = new ComplexTypeTool();
foreach (['hxxp://example.com', 'hxtp://example.com', 'htxp://example.com'] as $test) {
$results = $complexTypeTool->checkFreeText($test);
$this->assertCount(1, $results);
$this->assertEquals('http://example.com', $results[0]['value']);
$this->assertEquals('url', $results[0]['default_type']);
}
}
// Issue https://github.com/MISP/MISP/issues/4908
public function testCheckFreeTextUrlReplaceHttps(): void
{
$complexTypeTool = new ComplexTypeTool();
foreach (['hxxps://example.com', 'hxtps://example.com', 'htxps://example.com'] as $test) {
$results = $complexTypeTool->checkFreeText($test);
$this->assertCount(1, $results);
$this->assertEquals('https://example.com', $results[0]['value']);
$this->assertEquals('url', $results[0]['default_type']);
}
}
public function testCheckFreeTextBtc(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa');
$this->assertCount(1, $results);
$this->assertEquals('1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa', $results[0]['value']);
$this->assertEquals('btc', $results[0]['default_type']);
}
public function testCheckFreeTextSsdeep(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('24:VGXGP7L5e/Ixt3af/WKPPaYpzg4m3XWMCsXNCRs0:kYDxcfPZpelCs9Cm0');
$this->assertCount(1, $results);
$this->assertEquals('24:VGXGP7L5e/Ixt3af/WKPPaYpzg4m3XWMCsXNCRs0:kYDxcfPZpelCs9Cm0', $results[0]['value']);
$this->assertEquals('ssdeep', $results[0]['default_type']);
}
public function testCheckFreeTextCve(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('CVE-2019-16202');
$this->assertCount(1, $results);
$this->assertEquals('CVE-2019-16202', $results[0]['value']);
$this->assertEquals('vulnerability', $results[0]['default_type']);
}
public function testCheckFreeTextAs(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('as0 AS0');
$this->assertCount(1, $results);
$this->assertEquals('AS0', $results[0]['value']);
$this->assertEquals('AS', $results[0]['default_type']);
}
public function testCheckFreeTextMd5(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('9e107d9d372bb6826bd81d3542a419d6');
$this->assertCount(1, $results);
$this->assertEquals('9e107d9d372bb6826bd81d3542a419d6', $results[0]['value']);
$this->assertEquals('md5', $results[0]['default_type']);
}
public function testCheckFreeTextSha1(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('da39a3ee5e6b4b0d3255bfef95601890afd80709');
$this->assertCount(1, $results);
$this->assertEquals('da39a3ee5e6b4b0d3255bfef95601890afd80709', $results[0]['value']);
$this->assertEquals('sha1', $results[0]['default_type']);
}
public function testCheckFreeTextFilenameWithMd5(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('ahoj.txt|9e107d9d372bb6826bd81d3542a419d6');
$this->assertCount(1, $results);
$this->assertEquals('ahoj.txt|9e107d9d372bb6826bd81d3542a419d6', $results[0]['value']);
$this->assertEquals('filename|md5', $results[0]['default_type']);
}
public function testCheckFreeTextRandomString(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('cK753n3MVw');
$this->assertCount(0, $results);
}
public function testCheckFreeTextEmpty(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('');
$this->assertCount(0, $results);
}
public function testCheckFreeTextEmptyValues(): void
{
$complexTypeTool = new ComplexTypeTool();
foreach (['|', '&', '$', '0', ':80', '1.2', '[]:80', '\.', '.', ':', 'a:b', 'a:b:c'] as $char) {
$results = $complexTypeTool->checkFreeText($char);
$this->assertCount(0, $results);
}
}
public function testCheckFreeTextRemoveDuplicates(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText('1.2.3.4 1.2.3.4');
$this->assertCount(1, $results);
}
public function testRefangValueUrl(): void
{
$complexTypeTool = new ComplexTypeTool();
foreach (['meow://example.com', 'h[tt]p://example.com'] as $test) {
$this->assertEquals('http://example.com', $complexTypeTool->refangValue($test, 'url'));
$this->assertEquals('http://example.com', $complexTypeTool->refangValue($test, 'link'));
}
}
public function testRefangValueDot(): void
{
$complexTypeTool = new ComplexTypeTool();
foreach (['127.0.0.1', '127[.]0.0.1', '127[.]0[.]0[.]1', '127[dot]0[dot]0[dot]1', '127(dot)0(dot)0(dot)1', '127\.0.0.1'] as $test) {
$this->assertEquals('127.0.0.1', $complexTypeTool->refangValue($test, 'ip-src'));
}
}
}

31
app/View/Attributes/add.ctp
View File

@ -91,30 +91,17 @@
)
));
if (!$ajax) {
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => $this->action === 'add' ? 'add' : 'editEvent'));
$event = ['Event' => ['id' => $event_id, 'published' => $published ]];
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event', 'menuItem' => 'addAttribute', 'event' => $event));
}
?>
<script type="text/javascript">
var notice_list_triggers = <?php echo $notice_list_triggers; ?>;
var composite_types = <?php echo json_encode($compositeTypes); ?>;
var category_type_mapping = new Array();
<?php
foreach ($categoryDefinitions as $category => $def) {
echo "category_type_mapping['" . addslashes($category) . "'] = {";
$first = true;
foreach ($def['types'] as $type) {
if ($first) {
$first = false;
} else {
echo ', ';
}
echo "'" . addslashes($type) . "' : '" . addslashes($type) . "'";
}
echo "}; \n";
}
?>
var category_type_mapping = <?php echo json_encode(array_map(function($value) {
return array_combine($value['types'], $value['types']);
}, $categoryDefinitions)); ?>;
$('#AttributeDistribution').change(function() {
checkSharingGroup('Attribute');
@ -133,7 +120,7 @@
checkNoticeList('attribute');
});
$(document).ready(function() {
$(function() {
<?php
if ($action == 'edit'):
?>
@ -143,7 +130,7 @@
?>
checkSharingGroup('Attribute');
var $form = $('#AttributeType').closest('form').submit(function( event ) {
$('#AttributeType').closest('form').submit(function( event ) {
if ($('#AttributeType').val() === 'datetime') {
// add timezone of the browser if not set
var allowLocalTZ = true;
@ -153,7 +140,7 @@
if (dateValue.creationData().format !== "YYYY-MM-DDTHH:mm:ssZ" && dateValue.creationData().format !== "YYYY-MM-DDTHH:mm:ss.SSSSZ") {
// Missing timezone data
var confirm_message = '<?php echo __('Timezone missing, auto-detected as: ') ?>' + dateValue.format('Z')
confirm_message += '<?php echo '\r\n' . __('The following value will be submited instead: '); ?>' + dateValue.toISOString(allowLocalTZ)
confirm_message += '<?php echo '\r\n' . __('The following value will be submitted instead: '); ?>' + dateValue.toISOString(allowLocalTZ)
if (confirm(confirm_message)) {
$valueInput.val(dateValue.toISOString(allowLocalTZ));
} else {
@ -161,7 +148,7 @@
}
}
} else {
textStatus = '<?php echo __('Value is not a valid datetime. Excpected format YYYY-MM-DDTHH:mm:ssZ') ?>'
textStatus = '<?php echo __('Value is not a valid datetime. Expected format YYYY-MM-DDTHH:mm:ssZ') ?>'
showMessage('fail', textStatus);
return false;
}

147
app/View/Attributes/edit.ctp
View File

@ -1,147 +0,0 @@
<div class="attributes <?php if (!isset($ajax) || !$ajax) echo 'form';?>">
<?php echo $this->Form->create('Attribute');?>
<fieldset>
<legend><?php echo __('Edit Attribute'); ?></legend>
<?php
echo $this->Form->hidden('event_id');
echo $this->Form->input('category', array(
'empty' => __('(choose one)'),
'label' => __('Category ') . $this->element('formInfo', array('type' => 'category'))
));
$typeInputData = array(
'empty' => __('(first choose category)'),
'label' => __('Type ') . $this->element('formInfo', array('type' => 'type')),
);
if ($objectAttribute) {
$typeInputData[] = __('disabled');
}
echo $this->Form->input('type', $typeInputData);
?>
<div class="input clear"></div>
<?php
echo $this->Form->input('distribution', array(
'options' => array($distributionLevels),
'label' => __('Distribution ') . $this->element('formInfo', array('type' => 'distribution'))
));
?>
<div id="SGContainer" style="display:none;">
<?php
if (!empty($sharingGroups)) {
echo $this->Form->input('sharing_group_id', array(
'options' => array($sharingGroups),
'label' => __('Sharing Group'),
));
}
?>
</div>
<?php
echo $this->Form->input('value', array(
'type' => 'textarea',
'error' => array('escape' => false),
'div' => 'input clear',
'class' => 'input-xxlarge'
));
echo $this->Form->input('comment', array(
'type' => 'text',
'label' => __('Contextual Comment'),
'error' => array('escape' => false),
'div' => 'input clear',
'class' => 'input-xxlarge'
));
?>
<div class="input clear"></div>
<?php
echo $this->Form->input('to_ids', array(
'label' => __('for Intrusion Detection System'),
));
if (!$objectAttribute) {
echo $this->Form->input('batch_import', array(
'type' => 'checkbox',
));
}
echo '<div class="input clear"></div>';
echo $this->Form->input('disable_correlation', array(
'type' => 'checkbox'
));
?>
</fieldset>
<p style="color:red;font-weight:bold;display:none;<?php if (isset($ajax) && $ajax) echo "text-align:center;";?> " id="warning-message"><?php echo __('Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.'); ?></p>
<?php
echo $this->Form->button('Submit', array('class' => 'btn btn-primary'));
echo $this->Form->end();
?>
</div>
<?php
if (!$ajax) {
$event['Event']['id'] = $this->request->data['Attribute']['event_id'];
$event['Event']['published'] = $published;
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event', 'menuItem' => 'addAttribute', 'event' => $event));
}
?>
<script type="text/javascript">
var fieldsArray = new Array('AttributeCategory', 'AttributeType', 'AttributeValue', 'AttributeDistribution', 'AttributeComment', 'AttributeToIds', 'AttributeBatchImport', 'AttributeSharingGroupId');
<?php
$formInfoTypes = array('distribution' => 'Distribution', 'category' => 'Category', 'type' => 'Type');
echo 'var formInfoFields = ' . json_encode($formInfoTypes) . PHP_EOL;
foreach ($formInfoTypes as $formInfoType => $humanisedName) {
echo 'var ' . $formInfoType . 'FormInfoValues = {' . PHP_EOL;
foreach ($info[$formInfoType] as $key => $formInfoData) {
echo '"' . $key . '": "<span class=\"blue bold\">' . h($formInfoData['key']) . '</span>: ' . h($formInfoData['desc']) . '<br />",' . PHP_EOL;
}
echo '}' . PHP_EOL;
}
?>
//
//Generate Category / Type filtering array
//
var category_type_mapping = new Array();
<?php
foreach ($categoryDefinitions as $category => $def) {
echo "category_type_mapping['" . addslashes($category) . "'] = {";
$first = true;
foreach ($def['types'] as $type) {
if ($first) $first = false;
else echo ', ';
echo "'" . addslashes($type) . "' : '" . addslashes($type) . "'";
}
echo "}; \n";
}
?>
$(document).ready(function() {
initPopoverContent('Attribute');
$('#AttributeDistribution').change(function() {
if ($('#AttributeDistribution').val() == 4) $('#SGContainer').show();
else $('#SGContainer').hide();
});
<?php
if (!$objectAttribute):
?>
$("#AttributeCategory").on('change', function(e) {
formCategoryChanged('Attribute');
if ($(this).val() === 'Attribution' || $(this).val() === 'Targeting data') {
$("#warning-message").show();
} else {
$("#warning-message").hide();
}
if ($(this).val() === 'Internal reference') {
$("#AttributeDistribution").val('0');
$('#SGContainer').hide();
}
});
var start = $("#AttributeType").val();
formCategoryChanged('Attribute');
$("#AttributeType").val(start);
<?php
endif;
?>
$("#AttributeCategory, #AttributeType, #AttributeDistribution").change(function() {
var start = $("#AttributeType").val();
initPopoverContent('Attribute');
$("#AttributeType").val(start);
});
});
</script>
<?php echo $this->Js->writeBuffer(); // Write cached scripts

5
app/View/Elements/global_menu.ctp
View File

@ -417,8 +417,9 @@
'type' => 'root',
'url' => '#',
'html' => sprintf(
'<span class="fas fa-star %s" id="setHomePage" title="Set the current page as your home page in MISP"></span>',
(!empty($homepage['path']) && $homepage['path'] === $this->here) ? 'orange' : ''
'<span class="fas fa-star %s" id="setHomePage" title="Set the current page as your home page in MISP" data-current-page="%s"></span>',
(!empty($homepage['path']) && $homepage['path'] === $this->here) ? 'orange' : '',
$this->here
)
),
array(

1
app/View/Layouts/default.ctp
View File

@ -88,6 +88,7 @@
<div id = "ajax_fail_container" class="ajax_container">
<div id="ajax_fail" class="ajax_result ajax_fail"></div>
</div>
<div id = "ajax_hidden_container" class="hidden"></div>
<div class="loading">
<div class="spinner"></div>
<div class="loadingText"><?php echo __('Loading');?></div>

132
app/View/ShadowAttributes/edit.ctp Normal file
View File

@ -0,0 +1,132 @@
<div class="shadow_attributes <?php if (!isset($ajax) || !$ajax) echo 'form';?>">
<?php echo $this->Form->create('ShadowAttribute');?>
<fieldset>
<legend><?php echo __('Add Proposal'); ?></legend>
<?php
echo $this->Form->input('id');
echo $this->Form->input('category', array(
'empty' => __('(choose one)'),
'div' => 'input',
'label' => __('Category ') . $this->element('formInfo', array('type' => 'category')),
));
$typeInputData = array(
'empty' => __('(first choose category)'),
'label' => __('Type ') . $this->element('formInfo', array('type' => 'type')),
);
if ($objectAttribute) {
$typeInputData[] = 'disabled';
}
if (!$attachment) {
echo $this->Form->input('type', $typeInputData);
}
?>
<div class="input clear"></div>
<?php
echo $this->Form->input('value', array(
'type' => 'textarea',
'error' => array('escape' => false),
'class' => 'input-xxlarge clear'
));
echo $this->Form->input('comment', array(
'type' => 'text',
'label' => __('Contextual Comment'),
'error' => array('escape' => false),
'div' => 'input clear',
'class' => 'input-xxlarge'
));
?>
<div class="input clear"></div>
<?php
echo $this->Form->input('to_ids', array(
'label' => __('IDS Signature?'),
));
echo $this->Form->input('first_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
echo $this->Form->input('last_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
?>
<div id="bothSeenSliderContainer"></div>
</fieldset>
<p style="color:red;font-weight:bold;display:none;<?php if (isset($ajax) && $ajax) echo "text-align:center;"?>" id="warning-message"><?php echo __('Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.');?></p>
<?php if (isset($ajax) && $ajax): ?>
<div class="overlay_spacing">
<table>
<tr>
<td style="vertical-align:top">
<span role="button" tabindex="0" aria-label="<?php echo __('Propose');?>" title="<?php echo __('Propose');?>" id="submitButton" class="btn btn-primary" onClick="submitPopoverForm('<?php echo $event_id;?>', 'propose')"><?php echo __('Propose');?></span>
</td>
<td style="width:540px;">
<p style="color:red;font-weight:bold;display:none;<?php if (isset($ajax) && $ajax) echo "text-align:center;"?>" id="warning-message"><?php echo __('Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.');?></p>
</td>
<td style="vertical-align:top;">
<span class="btn btn-inverse" id="cancel_attribute_add"><?php echo __('Cancel');?></span>
</td>
</tr>
</table>
</div>
<?php
else:
echo $this->Form->button('Propose', array('class' => 'btn btn-primary'));
endif;
echo $this->Form->end();
?>
</div>
<?php
$event['Event']['id'] = $this->request->data['ShadowAttribute']['event_id'];
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event', 'menuItem' => 'proposeAttribute', 'event' => $event));
echo $this->element('form_seen_input');
?>
<script type="text/javascript">
<?php
$formInfoTypes = array('category' => 'Category', 'type' => 'Type');
echo 'var formInfoFields = ' . json_encode($formInfoTypes) . PHP_EOL;
foreach ($formInfoTypes as $formInfoType => $humanisedName) {
echo 'var ' . $formInfoType . 'FormInfoValues = {' . PHP_EOL;
foreach ($info[$formInfoType] as $key => $formInfoData) {
echo '"' . $key . '": "<span class=\"blue bold\">' . h($formInfoData['key']) . '</span>: ' . h($formInfoData['desc']) . '<br />",' . PHP_EOL;
}
echo '}' . PHP_EOL;
}
?>
//
//Generate Category / Type filtering array
//
var category_type_mapping = new Array();
<?php
foreach ($categoryDefinitions as $category => $def) {
echo "category_type_mapping['" . addslashes($category) . "'] = {";
$first = true;
foreach ($def['types'] as $type) {
if ($first) $first = false;
else echo ', ';
echo "'" . addslashes($type) . "' : '" . addslashes($type) . "'";
}
echo "}; \n";
}
?>
$(document).ready(function() {
initPopoverContent('ShadowAttribute');
$("#ShadowAttributeCategory").on('change', function(e) {
formCategoryChanged('ShadowAttribute');
if ($(this).val() === 'Attribution' || $(this).val() === 'Targeting data') {
$("#warning-message").show();
} else {
$("#warning-message").hide();
}
});
$("#ShadowAttributeCategory, #ShadowAttributeType").change(function() {
initPopoverContent('ShadowAttribute');
});
});
</script>
<?php echo $this->Js->writeBuffer(); // Write cached scripts

3
app/composer.json
View File

@ -11,6 +11,9 @@
"pear/crypt_gpg": "1.6.3",
"monolog/monolog": "1.24.0"
},
"require-dev": {
"phpunit/phpunit": "^8"
},
"suggest": {
"elasticsearch/elasticsearch": "For logging to elasticsearch",
"aws/aws-sdk-php": "To upload samples to S3"

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 8c3c224e6ad1d3fa848f0accaeab8161343fca4a
Subproject commit 439993200d815283c627e6134636857b51beb2c2

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit b7c2562a4f2b79b3764a8e3fcd38d24bb5abfa33
Subproject commit b768b2d735b7702d60b5efd4ee76158f433f69d0

2
app/files/scripts/misp2stix.py
View File

@ -7,6 +7,7 @@ import datetime
import re
import ntpath
import socket
import traceback
from misp2stix_mapping import *
from collections import defaultdict
from copy import deepcopy
@ -156,6 +157,7 @@ class StixBuilder(object):
print(json.dumps({'success': 1}))
except Exception as e:
print(json.dumps({'error': e.__str__()}))
traceback.print_tb()
def generate_package(self, event):
self.objects_to_parse = defaultdict(dict)

2
app/files/scripts/misp2stix_mapping.py
View File

@ -1,7 +1,7 @@
# mappings
status_mapping = {'0' : 'New', '1' : 'Open', '2' : 'Closed'}
threat_level_mapping = {'1' : 'High', '2' : 'Medium', '3' : 'Low', '4' : 'Undefined'}
TLP_order = {'RED' : 4, 'AMBER' : 3, 'GREEN' : 2, 'WHITE' : 1}
TLP_order = {'RED' : 4, 'AMBER' : 3, 'AMBER NATO ALLIANCE': 3, 'GREEN' : 2, 'WHITE' : 1}
confidence_mapping = {False : 'None', True : 'High'}
not_implemented_attributes = ('yara', 'snort', 'pattern-in-traffic', 'pattern-in-memory')

2
app/files/scripts/selftest.php
View File

@ -1,5 +1,5 @@
<?php
$extensions = array('redis', 'gd', 'ssdeep');
$extensions = array('redis', 'gd', 'ssdeep', 'zip');
$results = array();
$results['phpversion'] = phpversion();
foreach ($extensions as $extension) {

38
app/files/scripts/stix2/stix2misp.py
View File

@ -142,7 +142,7 @@ class StixParser():
else:
print(f'Unknown content disposition in the following email body: {body_multipart}', file=sys.stderr)
continue
if isinstance(reference, stix2.Artifact):
if isinstance(reference, stix2.v20.observables.Artifact):
attribute.update({
'value': body_multipart['content_disposition'].split('=')[-1].strip("'"),
'data': reference.payload_bin,
@ -216,9 +216,9 @@ class StixParser():
self.fill_misp_object(section_object, section, 'pe_section_mapping')
if hasattr(section, 'hashes'):
self.fill_misp_object(section_object, section.hashes, 'pe_section_mapping')
self.misp_event.add_object(section_object)
pe_object.add_reference(section_object.uuid, 'includes')
self.misp_event.add_object(**section_object)
self.misp_event.add_object(**pe_object)
self.misp_event.add_object(pe_object)
return pe_object.uuid
def parse_relationships(self):
@ -338,11 +338,11 @@ class StixParser():
@staticmethod
def _process_test_filter(value, main_type):
_is_main_process = any(feature in value for feature in ('parent_ref', 'child_refs'))
return isinstance(value, getattr(stix2, main_type)) and _is_main_process
return isinstance(value, getattr(stix2.v20.observables, main_type)) and _is_main_process
@staticmethod
def _standard_test_filter(value, main_type):
return isinstance(value, getattr(stix2, main_type))
return isinstance(value, getattr(stix2.v20.observables, main_type))
def update_marking_refs(self, attribute_uuid, marking_refs):
try:
@ -515,7 +515,7 @@ class StixFromMISPParser(StixParser):
misp_object.add_reference(target_uuid, 'includes')
for attribute in attributes:
misp_object.add_attribute(**attribute)
self.misp_event.add_object(**misp_object)
self.misp_event.add_object(misp_object)
def parse_observable_attribute(self, observable):
attribute = self.create_attribute_dict(observable)
@ -541,7 +541,7 @@ class StixFromMISPParser(StixParser):
misp_object.add_reference(target_uuid, 'includes')
for attribute in attributes:
misp_object.add_attribute(**attribute)
self.misp_event.add_object(**misp_object)
self.misp_event.add_object(misp_object)
def parse_vulnerability(self, vulnerability):
attributes = self.fill_observable_attributes(vulnerability, 'vulnerability_mapping')
@ -617,7 +617,7 @@ class StixFromMISPParser(StixParser):
attributes = []
mapping = 'asn_mapping'
for observable_object in observable.values():
if isinstance(observable_object, stix2.AutonomousSystem):
if isinstance(observable_object, stix2.v20.observables.AutonomousSystem):
attributes.extend(self.fill_observable_attributes(observable_object, mapping))
else:
attributes.append(self._parse_observable_reference(observable_object, mapping))
@ -855,7 +855,7 @@ class StixFromMISPParser(StixParser):
def parse_url_observable(observable):
attributes = []
for object in observable.values():
feature = 'dst_port' if isinstance(object, stix2.NetworkTraffic) else 'value'
feature = 'dst_port' if isinstance(object, stix2.v20.observables.NetworkTraffic) else 'value'
attribute = deepcopy(stix2misp_mapping.url_mapping[object._type])
attribute.update({'value': getattr(object, feature), 'to_ids': False})
attributes.append(attribute)
@ -1066,9 +1066,9 @@ class StixFromMISPParser(StixParser):
attribute = deepcopy(stix2misp_mapping.pe_section_mapping[feature])
attribute['value'] = value
pe_section.add_attribute(**attribute)
self.misp_event.add_object(pe_section)
pe.add_reference(pe_section.uuid, 'includes')
self.misp_event.add_object(**pe_section)
self.misp_event.add_object(**pe)
self.misp_event.add_object(pe)
return attributes, pe.uuid
def parse_process_pattern(self, pattern):
@ -1299,7 +1299,7 @@ class ExternalStixParser(StixParser):
@staticmethod
def _fetch_reference_type(references, object_type):
for key, reference in references.items():
if isinstance(reference, getattr(stix2, object_type)):
if isinstance(reference, getattr(stix2.v20.observables, object_type)):
return key
return None
@ -1342,7 +1342,7 @@ class ExternalStixParser(StixParser):
file.add_reference(pe_uuid, 'includes')
for attribute in attributes:
file.add_attribute(**attribute)
self.misp_event.add_object(**file)
self.misp_event.add_object(file)
@staticmethod
def _is_reference(network_traffic, reference):
@ -1422,7 +1422,7 @@ class ExternalStixParser(StixParser):
if references:
filename = file_object.name if hasattr(file_object, 'name') else 'unknown_filename'
for key, reference in references.items():
if isinstance(reference, stix2.Artifact):
if isinstance(reference, stix2.v20.observables.Artifact):
_is_content_ref = 'content_ref' in file_object and file_object.content_ref == key
attribute_type, value = self._handle_attachment_type(reference, _is_content_ref, filename)
attribute = {
@ -1434,7 +1434,7 @@ class ExternalStixParser(StixParser):
if hasattr(reference, 'payload_bin'):
attribute['data'] = reference.payload_bin
attributes.append(attribute)
elif isinstance(reference, stix2.Directory):
elif isinstance(reference, stix2.v20.observables.Directory):
attribute = {
'type': 'text',
'object_relation': 'path',
@ -1757,15 +1757,15 @@ class ExternalStixParser(StixParser):
pe_object = MISPObject('pe', misp_objects_path_custom=_misp_objects_path)
sections = self._get_sections(pe_extension)
self.fill_misp_object_from_dict(pe_object, pe_extension, 'pe_mapping')
file_object.add_reference(pe_object.uuid, 'includes')
if sections:
for section in sections:
section_object = MISPObject('pe-section')
self.fill_misp_object_from_dict(section_object, section, 'pe_section_mapping')
self.misp_event.add_object(section_object)
pe_object.add_reference(section_object.uuid, 'includes')
self.misp_event.add_object(**section_object)
self.misp_event.add_object(**pe_object)
self.misp_event.add_object(**file_object)
self.misp_event.add_object(pe_object)
file_object.add_reference(pe_object.uuid, 'includes')
self.misp_event.add_object(file_object)
def parse_ip_address_pattern(self, indicator, separator):
self.add_attributes_from_indicator(indicator, 'ip-dst', separator)

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 8cd3e148cbf21d7fd2c15f62f72e1b1d0acf3daf
Subproject commit 1acdef286a3ac6f188f5028c6d1c0dd788dbadc4

58
app/webroot/js/misp.js
View File

@ -23,16 +23,22 @@ function stringToRGB(str){
return "#" + "00000".substring(0, 6 - c.length) + c;
}
function deleteObject(type, action, id, event) {
var destination = 'attributes';
var alternateDestinations = ['shadow_attributes', 'template_elements', 'taxonomies', 'galaxy_clusters', 'objects', 'object_references'];
if (alternateDestinations.indexOf(type) > -1) destination = type;
else destination = type;
url = "/" + destination + "/" + action + "/" + id;
function xhrFailCallback(xhr) {
if (xhr.status === 403) {
showMessage('fail', 'Not allowed.');
} else if (xhr.status === 404) {
showMessage('fail', 'Resource not found.');
} else {
showMessage('fail', 'Something went wrong - the queried function returned an exception. Contact your administrator for further details.');
}
}
function deleteObject(type, action, id) {
var url = "/" + type + "/" + action + "/" + id;
$.get(url, function(data) {
openPopup("#confirmation_box");
$("#confirmation_box").html(data);
});
}).fail(xhrFailCallback)
}
function quickDeleteSighting(id, rawId, context) {
@ -40,7 +46,7 @@ function quickDeleteSighting(id, rawId, context) {
$.get(url, function(data) {
$("#confirmation_box").html(data);
openPopup("#confirmation_box");
});
}).fail(xhrFailCallback)
}
function fetchAddSightingForm(type, attribute_id, page, onvalue) {
@ -1799,11 +1805,7 @@ function getPopup(id, context, target, admin, popupType) {
error:function(xhr) {
$(".loading").hide();
$("#gray_out").fadeOut();
if (xhr.status === 403) {
showMessage('fail', 'Not allowed.');
} else {
showMessage('fail', 'Something went wrong - the queried function returned an exception. Contact your administrator for further details (the exception has been logged).');
}
xhrFailCallback(xhr);
},
url: url
});
@ -1925,13 +1927,7 @@ function simplePopup(url) {
error:function(xhr) {
$(".loading").hide();
$("#gray_out").fadeOut();
if (xhr.status == 403) {
showMessage('fail', 'Not allowed.');
} else if (xhr.status == 404) {
showMessage('fail', 'Resource not found.');
} else {
showMessage('fail', 'Something went wrong - the queried function returned an exception. Contact your administrator for further details (the exception has been logged).');
}
xhrFailCallback(xhr);
},
url: url,
});
@ -5056,15 +5052,23 @@ function resetDashboardGrid(grid) {
function setHomePage() {
$.ajax({
type: 'POST',
type: 'GET',
url: baseurl + '/userSettings/setHomePage',
data: {
path: window.location.pathname
},
success:function (data, textStatus) {
showMessage('success', 'Homepage set.');
$('#setHomePage').addClass('orange');
},
$('#ajax_hidden_container').html(data);
var currentPage = $('#setHomePage').data('current-page');
$('#UserSettingPath').val(currentPage);
$.ajax({
type: 'POST',
url: baseurl + '/userSettings/setHomePage',
data: $('#UserSettingSetHomePageForm').serialize(),
success:function (data, textStatus) {
showMessage('success', 'Homepage set.');
$('#setHomePage').addClass('orange');
},
});
}
});
}

2
cti-python-stix2

@ -1 +1 @@
Subproject commit 808dd94f676243e2f1932878d511de6d74557918
Subproject commit 0d6db44c80afd81976f54f58c8cb02e4d33acc16

5
docs/generic/mail_to_misp-debian.md
View File

@ -9,8 +9,9 @@ mail2misp () {
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git; done
[[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done
[[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done
## TODO: The below fails miserably (obviously) if faup/gtcac dirs exist, let's just make the dangerous assumption (for the sake of the installer, that they exist)
##[[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done
##[[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done
sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build

1
docs/generic/supportFunctions.md
View File

@ -74,6 +74,7 @@ setOpt () {
("-U") echo "upgrade"; UPGRADE=1 ;;
("-N") echo "nuke"; NUKE=1 ;;
("-u") echo "unattended"; UNATTENDED=1 ;;
("-ni") echo "noninteractive"; NONINTERACTIVE=1 ;;
("-f") echo "force"; FORCE=1 ;;
(*) echo "$o is not a valid argument"; exit 1 ;;
esac

13
tools/misp_retention.py Normal file → Executable file
View File

@ -42,10 +42,19 @@ class misphelper(object):
print("Removing IDS flag in event '{}' on attr '{}'".format(mevent.id, attr["value"]))
changed = True
attr["to_ids"] = False
self.misp.update_attribute(attr)
for obj in mevent.objects:
for attr in obj.Attribute:
if (attr["type"] == "ip-dst" or attr["type"] == "ip-src") and attr["to_ids"]:
print("Removing IDS flag in event '{}' on attr '{}'".format(mevent.id, attr["value"]))
changed = True
attr["to_ids"] = False
self.misp.update_attribute(attr)
self.misp.tag(mevent, self.expiredTag, True)
if changed:
res = self.misp.update_event(mevent.id, mevent)
self.misp.update_event(mevent.id, mevent)
self.misp.publish(mevent)
def findEventsAfterRetention(self, events, retention):
for event in events:
@ -70,7 +79,7 @@ class misphelper(object):
for tag in res['entries']:
m = re.match(r"^retention:([0-9]+)([d,w,m,y])$", tag["tag"])
if m:
tagSearch = self.misp.build_complex_query(and_parameters = tag["tag"], not_parameters = self.expiredTag)
tagSearch = self.misp.build_complex_query(and_parameters = [tag["tag"]], not_parameters = [self.expiredTag])
events = self.misp.search(published=True, tags=tagSearch)
self.findEventsAfterRetention(events, (m.group(1), m.group(2)))