2013-04-25 14:04:08 +02:00
< ? php
App :: uses ( 'AppModel' , 'Model' );
App :: uses ( 'Folder' , 'Utility' );
App :: uses ( 'File' , 'Utility' );
2020-05-05 15:23:26 +02:00
App :: uses ( 'AttachmentTool' , 'Tools' );
2013-04-25 14:04:08 +02:00
2020-07-24 15:46:59 +02:00
/**
* @ property Event $Event
*/
2018-07-19 11:48:22 +02:00
class ShadowAttribute extends AppModel
{
public $combinedKeys = array ( 'event_id' , 'category' , 'type' );
2019-02-10 13:08:12 +01:00
public $name = 'ShadowAttribute' ; // TODO general
2018-07-19 11:48:22 +02:00
2019-09-30 08:23:36 +02:00
public $recursive = - 1 ;
2018-07-19 11:48:22 +02:00
public $actsAs = array (
2019-02-10 13:08:12 +01:00
'SysLogLogable.SysLogLogable' => array ( // TODO Audit, logable
2018-07-19 11:48:22 +02:00
'userModel' => 'User' ,
'userKey' => 'user_id' ,
'change' => 'full' ),
'Trim' ,
'Containable' ,
'Regexp' => array ( 'fields' => array ( 'value' , 'value2' )),
);
public $belongsTo = array (
'Event' => array (
'className' => 'Event' ,
'foreignKey' => 'event_id' ,
'conditions' => '' ,
'fields' => '' ,
'order' => '' ,
'counterCache' => true
),
'Org' => array (
'className' => 'Organisation' ,
'foreignKey' => 'org_id'
),
'EventOrg' => array (
'className' => 'Organisation' ,
'foreignKey' => 'event_org_id'
),
2019-07-11 22:57:58 +02:00
'Attribute' => array (
'className' => 'Attribute' ,
'foreignKey' => 'old_id'
)
2018-07-19 11:48:22 +02:00
);
public $displayField = 'value' ;
public $virtualFields = array (
'value' => " CASE WHEN ShadowAttribute.value2 = '' THEN ShadowAttribute.value1 ELSE CONCAT(ShadowAttribute.value1, '|', ShadowAttribute.value2) END " ,
); // TODO hardcoded
// explanations of certain fields to be used in various views
public $fieldDescriptions = array (
'signature' => array ( 'desc' => 'Is this attribute eligible to automatically create an IDS signature (network IDS or host IDS) out of it ?' ),
//'private' => array('desc' => 'Prevents upload of this single Attribute to other CyDefSIG servers', 'formdesc' => 'Prevents upload of <em>this single Attribute</em> to other CyDefSIG servers.<br/>Used only when the Event is NOT set as Private')
);
// if these then a category my have upload to be zipped
public $zippedDefinitions = array (
'malware-sample'
);
// if these then a category my have upload
public $uploadDefinitions = array (
'attachment'
);
// definitions of categories
public $categoryDefinitions ;
public $order = array ( " ShadowAttribute.event_id " => " DESC " , " ShadowAttribute.type " => " ASC " );
public $validate = array (
'event_id' => array (
'numeric' => array (
'rule' => array ( 'numeric' )
)
),
'org_id' => array (
'numeric' => array (
'rule' => array ( 'numeric' )
)
),
'event_org_id' => array (
'numeric' => array (
'rule' => array ( 'numeric' )
)
),
'type' => array (
// currently when adding a new attribute type we need to change it in both places
'rule' => array ( 'validateTypeValue' ),
'message' => 'Options depend on the selected category.' ,
//'allowEmpty' => false,
'required' => true ,
//'last' => false, // Stop validation after this rule
//'on' => 'create', // Limit validation to 'create' or 'update' operations
),
// this could be initialized from categoryDefinitions but dunno how at the moment
'category' => array (
'validCategory' => array (
'rule' => array ( 'validCategory' ),
'message' => 'Options : Payload delivery, Antivirus detection, Payload installation, Files dropped ...'
),
),
'value' => array (
'stringNotEmpty' => array (
'rule' => array ( 'stringNotEmpty' ),
),
'userdefined' => array (
'rule' => array ( 'validateAttributeValue' ),
'message' => 'Value not in the right type/format. Please double check the value or select type "other".' ,
),
),
'to_ids' => array (
'boolean' => array (
'rule' => array ( 'boolean' ),
'required' => false ,
),
),
'uuid' => array (
'uuid' => array (
'rule' => array ( 'custom' , '/^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/' ),
'message' => 'Please provide a valid UUID'
),
),
'proposal_to_delete' => array (
'boolean' => array (
'rule' => array ( 'boolean' ),
),
),
2019-06-24 10:28:55 +02:00
'first_seen' => array (
'rule' => array ( 'datetimeOrNull' ),
'required' => false ,
'message' => array ( 'Invalid ISO 8601 format' )
),
'last_seen' => array (
'rule' => array ( 'datetimeOrNull' ),
'required' => false ,
'message' => array ( 'Invalid ISO 8601 format' )
)
2018-07-19 11:48:22 +02:00
);
public function __construct ( $id = false , $table = null , $ds = null )
{
parent :: __construct ( $id , $table , $ds );
$this -> virtualFields = Set :: merge ( $this -> virtualFields , array (
//'distribution' => 'IF (Attribute.private=true, "Your organization only", IF (Attribute.cluster=true, "This Community-only", "All communities"))',
//'distribution' => 'IF (ShadowAttribute.private=true AND ShadowAttribute.cluster=false, "Your organization only", IF (ShadowAttribute.private=true AND ShadowAttribute.cluster=true, "This server-only", IF (ShadowAttribute.private=false AND ShadowAttribute.cluster=true, "This Community-only", IF (ShadowAttribute.communitie=true, "Connected communities" , "All communities"))))',
));
$this -> fieldDescriptions = Set :: merge ( $this -> fieldDescriptions , array (
//'distribution' => array('desc' => 'This fields indicates the intended distribution of the attribute (same as when adding an event, see Add Event)'),
));
$this -> categoryDefinitions = $this -> Event -> Attribute -> categoryDefinitions ;
$this -> typeDefinitions = $this -> Event -> Attribute -> typeDefinitions ;
}
// The Associations below have been created with all possible keys, those that are not needed can be removed
public function beforeSave ( $options = array ())
{
// explode value of composite type in value1 and value2
// or copy value to value1 if not composite type
if ( ! empty ( $this -> data [ 'ShadowAttribute' ][ 'type' ])) {
$compositeTypes = $this -> getCompositeTypes ();
// explode composite types in value1 and value2
$pieces = explode ( '|' , $this -> data [ 'ShadowAttribute' ][ 'value' ]);
if ( in_array ( $this -> data [ 'ShadowAttribute' ][ 'type' ], $compositeTypes )) {
if ( 2 != count ( $pieces )) {
throw new InternalErrorException ( 'Composite type, but value not explodable' );
}
$this -> data [ 'ShadowAttribute' ][ 'value1' ] = $pieces [ 0 ];
$this -> data [ 'ShadowAttribute' ][ 'value2' ] = $pieces [ 1 ];
} else {
$total = implode ( '|' , $pieces );
$this -> data [ 'ShadowAttribute' ][ 'value1' ] = $total ;
$this -> data [ 'ShadowAttribute' ][ 'value2' ] = '' ;
}
}
if ( ! isset ( $this -> data [ 'ShadowAttribute' ][ 'deleted' ])) {
$this -> data [ 'ShadowAttribute' ][ 'deleted' ] = 0 ;
}
if ( $this -> data [ 'ShadowAttribute' ][ 'deleted' ]) {
$this -> __beforeDeleteCorrelation ( $this -> data [ 'ShadowAttribute' ]);
}
2019-06-24 10:28:55 +02:00
// convert into utc and micro sec
2019-12-16 10:47:07 +01:00
$this -> data = $this -> Attribute -> ISODatetimeToUTC ( $this -> data , $this -> alias );
2018-07-19 11:48:22 +02:00
return true ;
}
private function __beforeDeleteCorrelation ( $sa )
{
if ( isset ( $sa [ 'ShadowAttribute' ])) {
$sa = $sa [ 'ShadowAttribute' ];
}
$this -> ShadowAttributeCorrelation = ClassRegistry :: init ( 'ShadowAttributeCorrelation' );
$this -> ShadowAttributeCorrelation -> deleteAll ( array ( 'ShadowAttributeCorrelation.1_shadow_attribute_id' => $sa [ 'id' ]));
}
private function __afterSaveCorrelation ( $sa )
{
if ( isset ( $sa [ 'ShadowAttribute' ])) {
$sa = $sa [ 'ShadowAttribute' ];
}
if ( in_array ( $sa [ 'type' ], $this -> Event -> Attribute -> nonCorrelatingTypes )) {
return ;
}
$this -> ShadowAttributeCorrelation = ClassRegistry :: init ( 'ShadowAttributeCorrelation' );
$shadow_attribute_correlations = array ();
$fields = array ( 'value1' , 'value2' );
$correlatingValues = array ( $sa [ 'value1' ]);
if ( ! empty ( $sa [ 'value2' ])) {
$correlatingValues [] = $sa [ 'value2' ];
}
foreach ( $correlatingValues as $k => $cV ) {
$correlatingAttributes [ $k ] = $this -> Event -> Attribute -> find ( 'all' , array (
'conditions' => array (
'AND' => array (
'OR' => array (
'Attribute.value1' => $cV ,
'Attribute.value2' => $cV
),
'Attribute.type !=' => $this -> Event -> Attribute -> nonCorrelatingTypes ,
'Attribute.deleted' => 0 ,
'Attribute.event_id !=' => $sa [ 'event_id' ]
),
),
'recursive => -1' ,
'fields' => array ( 'Attribute.event_id' , 'Attribute.id' , 'Attribute.distribution' , 'Attribute.sharing_group_id' ),
'contain' => array ( 'Event' => array ( 'fields' => array ( 'Event.id' , 'Event.date' , 'Event.info' , 'Event.org_id' , 'Event.distribution' , 'Event.sharing_group_id' ))),
'order' => array (),
));
foreach ( $correlatingAttributes as $key => $cA ) {
foreach ( $cA as $corr ) {
$shadow_attribute_correlations [] = array (
'value' => $correlatingValues [ $key ],
'1_event_id' => $sa [ 'event_id' ],
'1_shadow_attribute_id' => $sa [ 'id' ],
'event_id' => $corr [ 'Attribute' ][ 'event_id' ],
'attribute_id' => $corr [ 'Attribute' ][ 'id' ],
'org_id' => $corr [ 'Event' ][ 'org_id' ],
'distribution' => $corr [ 'Event' ][ 'distribution' ],
'a_distribution' => $corr [ 'Attribute' ][ 'distribution' ],
'sharing_group_id' => $corr [ 'Event' ][ 'sharing_group_id' ],
'a_sharing_group_id' => $corr [ 'Attribute' ][ 'sharing_group_id' ],
'date' => $corr [ 'Event' ][ 'date' ],
'info' => $corr [ 'Event' ][ 'info' ],
);
}
}
}
if ( ! empty ( $shadow_attribute_correlations )) {
$this -> ShadowAttributeCorrelation -> saveMany ( $shadow_attribute_correlations );
}
}
public function afterSave ( $created , $options = array ())
{
$result = true ;
// if the 'data' field is set on the $this->data then save the data to the correct file
if ( isset ( $this -> data [ 'ShadowAttribute' ][ 'deleted' ]) && $this -> data [ 'ShadowAttribute' ][ 'deleted' ]) {
$sa = $this -> find ( 'first' , array ( 'conditions' => array ( 'ShadowAttribute.id' => $this -> data [ 'ShadowAttribute' ][ 'id' ]), 'recursive' => - 1 , 'fields' => array ( 'ShadowAttribute.id' , 'ShadowAttribute.event_id' , 'ShadowAttribute.type' )));
if ( $this -> typeIsAttachment ( $sa [ 'ShadowAttribute' ][ 'type' ])) {
2020-08-13 15:58:42 +02:00
$this -> loadAttachmentTool () -> deleteShadow ( $sa [ 'ShadowAttribute' ][ 'event_id' ], $sa [ 'ShadowAttribute' ][ 'id' ]);
2018-07-19 11:48:22 +02:00
}
} else {
if ( isset ( $this -> data [ 'ShadowAttribute' ][ 'type' ]) && $this -> typeIsAttachment ( $this -> data [ 'ShadowAttribute' ][ 'type' ]) && ! empty ( $this -> data [ 'ShadowAttribute' ][ 'data' ])) {
$result = $result && $this -> saveBase64EncodedAttachment ( $this -> data [ 'ShadowAttribute' ]);
}
}
if (( isset ( $this -> data [ 'ShadowAttribute' ][ 'deleted' ]) && $this -> data [ 'ShadowAttribute' ][ 'deleted' ]) || ( isset ( $this -> data [ 'ShadowAttribute' ][ 'proposal_to_delete' ]) && $this -> data [ 'ShadowAttribute' ][ 'proposal_to_delete' ])) {
// this is a deletion
// Could be a proposal to delete or flagging a proposal that it was discarded / accepted - either way, we don't want to correlate here for now
} else {
$this -> __afterSaveCorrelation ( $this -> data [ 'ShadowAttribute' ]);
}
2019-03-05 12:24:56 +01:00
if ( empty ( $this -> data [ 'ShadowAttribute' ][ 'deleted' ])) {
$action = $created ? 'add' : 'edit' ;
$this -> publishKafkaNotification ( 'shadow_attribute' , $this -> data , $action );
}
2018-07-19 11:48:22 +02:00
return $result ;
}
public function beforeDelete ( $cascade = true )
{
// delete attachments from the disk
$this -> read (); // first read the attribute from the db
if ( $this -> typeIsAttachment ( $this -> data [ 'ShadowAttribute' ][ 'type' ])) {
2020-08-13 15:58:42 +02:00
$this -> loadAttachmentTool () -> deleteShadow ( $this -> data [ 'ShadowAttribute' ][ 'event_id' ], $this -> data [ 'ShadowAttribute' ][ 'id' ]);
2018-07-19 11:48:22 +02:00
}
}
public function beforeValidate ( $options = array ())
{
parent :: beforeValidate ();
// remove leading and trailing blanks
//$this->trimStringFields(); // TODO
if ( isset ( $this -> data [ 'ShadowAttribute' ][ 'value' ])) {
$this -> data [ 'ShadowAttribute' ][ 'value' ] = trim ( $this -> data [ 'ShadowAttribute' ][ 'value' ]);
}
if ( ! isset ( $this -> data [ 'ShadowAttribute' ][ 'comment' ])) {
$this -> data [ 'ShadowAttribute' ][ 'comment' ] = '' ;
}
if ( ! isset ( $this -> data [ 'ShadowAttribute' ][ 'type' ])) {
return false ;
}
if ( empty ( $this -> data [ 'ShadowAttribute' ][ 'timestamp' ])) {
$date = new DateTime ();
$this -> data [ 'ShadowAttribute' ][ 'timestamp' ] = $date -> getTimestamp ();
}
if ( ! isset ( $this -> data [ 'ShadowAttribute' ][ 'proposal_to_delete' ])) {
$this -> data [ 'ShadowAttribute' ][ 'proposal_to_delete' ] = 0 ;
}
// make some last changes to the inserted value
$this -> data [ 'ShadowAttribute' ][ 'value' ] = $this -> Event -> Attribute -> modifyBeforeValidation ( $this -> data [ 'ShadowAttribute' ][ 'type' ], $this -> data [ 'ShadowAttribute' ][ 'value' ]);
// generate UUID if it doesn't exist
if ( empty ( $this -> data [ 'ShadowAttribute' ][ 'uuid' ])) {
$this -> data [ 'ShadowAttribute' ][ 'uuid' ] = CakeText :: uuid ();
}
2019-07-29 11:17:18 +02:00
if ( ! empty ( $this -> data [ 'ShadowAttribute' ][ 'type' ]) && empty ( $this -> data [ 'ShadowAttribute' ][ 'category' ])) {
$this -> data [ 'ShadowAttribute' ][ 'category' ] = $this -> Event -> Attribute -> typeDefinitions [ $this -> data [ 'ShadowAttribute' ][ 'type' ]][ 'default_category' ];
}
2018-07-19 11:48:22 +02:00
// always return true, otherwise the object cannot be saved
return true ;
}
2019-06-24 10:28:55 +02:00
public function afterFind ( $results , $primary = false )
{
foreach ( $results as $k => $v ) {
2019-12-16 10:47:07 +01:00
$results [ $k ] = $this -> Attribute -> UTCToISODatetime ( $results [ $k ], $this -> alias );
2019-06-24 10:28:55 +02:00
}
return $results ;
}
2018-07-19 11:48:22 +02:00
public function validateTypeValue ( $fields )
{
$category = $this -> data [ 'ShadowAttribute' ][ 'category' ];
if ( isset ( $this -> categoryDefinitions [ $category ][ 'types' ])) {
return in_array ( $fields [ 'type' ], $this -> categoryDefinitions [ $category ][ 'types' ]);
}
return false ;
}
public function validCategory ( $fields )
{
return $this -> Event -> Attribute -> validCategory ( $fields );
}
public function validateAttributeValue ( $fields )
{
$value = $fields [ 'value' ];
return $this -> Event -> Attribute -> runValidation ( $value , $this -> data [ 'ShadowAttribute' ][ 'type' ]);
}
public function getCompositeTypes ()
{
// build the list of composite Attribute.type dynamically by checking if type contains a |
// default composite types
2019-02-10 13:08:12 +01:00
$compositeTypes = array ( 'malware-sample' ); // TODO hardcoded composite
2018-07-19 11:48:22 +02:00
// dynamically generated list
foreach ( array_keys ( $this -> typeDefinitions ) as $type ) {
$pieces = explode ( '|' , $type );
if ( 2 == count ( $pieces )) {
$compositeTypes [] = $type ;
}
}
return $compositeTypes ;
}
public function typeIsMalware ( $type )
{
2020-05-05 15:23:26 +02:00
return in_array ( $type , $this -> zippedDefinitions );
2018-07-19 11:48:22 +02:00
}
public function typeIsAttachment ( $type )
{
2020-05-05 15:23:26 +02:00
return in_array ( $type , $this -> zippedDefinitions ) || in_array ( $type , $this -> uploadDefinitions );
2018-07-19 11:48:22 +02:00
}
public function base64EncodeAttachment ( $attribute )
{
2020-08-13 15:58:42 +02:00
$content = $this -> loadAttachmentTool () -> getShadowContent ( $attribute [ 'event_id' ], $attribute [ 'id' ]);
2018-07-19 11:48:22 +02:00
return base64_encode ( $content );
}
public function saveBase64EncodedAttachment ( $attribute )
{
2020-05-05 15:23:26 +02:00
$data = base64_decode ( $attribute [ 'data' ]);
2020-08-13 15:58:42 +02:00
return $this -> loadAttachmentTool () -> saveShadow ( $attribute [ 'event_id' ], $attribute [ 'id' ], $data );
}
/**
* @ param array $shadowAttribute
* @ param string $path_suffix
* @ return File
* @ throws Exception
*/
public function getAttachmentFile ( array $shadowAttribute , $path_suffix = '' )
{
return $this -> loadAttachmentTool () -> getShadowFile ( $shadowAttribute [ 'event_id' ], $shadowAttribute [ 'id' ], $path_suffix );
2018-07-19 11:48:22 +02:00
}
public function checkComposites ()
{
$compositeTypes = $this -> getCompositeTypes ();
$fails = array ();
$attributes = $this -> find ( 'all' , array ( 'recursive' => 0 ));
foreach ( $attributes as $attribute ) {
if (( in_array ( $attribute [ 'ShadowAttribute' ][ 'type' ], $compositeTypes )) && ( ! strlen ( $attribute [ 'ShadowAttribute' ][ 'value1' ]) || ! strlen ( $attribute [ 'ShadowAttribute' ][ 'value2' ]))) {
$fails [] = $attribute [ 'ShadowAttribute' ][ 'event_id' ] . ':' . $attribute [ 'ShadowAttribute' ][ 'id' ];
}
}
return $fails ;
}
2019-06-24 10:28:55 +02:00
// check whether the variable is null or datetime
public function datetimeOrNull ( $fields )
{
$k = array_keys ( $fields )[ 0 ];
$seen = $fields [ $k ];
try {
new DateTime ( $seen );
$returnValue = true ;
} catch ( Exception $e ) {
$returnValue = false ;
}
return $returnValue || is_null ( $seen );
}
2018-07-19 11:48:22 +02:00
public function setDeleted ( $id )
{
$this -> Behaviors -> detach ( 'SysLogLogable.SysLogLogable' );
$sa = $this -> find ( 'first' , array ( 'conditions' => array ( 'ShadowAttribute.id' => $id ), 'recusive' => - 1 ));
if ( empty ( $sa )) {
return false ;
}
$date = new DateTime ();
$sa [ 'ShadowAttribute' ][ 'deleted' ] = 1 ;
$sa [ 'ShadowAttribute' ][ 'timestamp' ] = $date -> getTimestamp ();
$this -> save ( $sa );
return true ;
}
public function findOldProposal ( $sa )
{
$oldsa = $this -> find ( 'first' , array (
'conditions' => array (
2019-07-14 08:24:10 +02:00
'ShadowAttribute.event_uuid' => $sa [ 'event_uuid' ],
'ShadowAttribute.uuid' => $sa [ 'uuid' ],
'ShadowAttribute.value' => $sa [ 'value' ],
'ShadowAttribute.type' => $sa [ 'type' ],
'ShadowAttribute.category' => $sa [ 'category' ],
'ShadowAttribute.to_ids' => $sa [ 'to_ids' ],
'ShadowAttribute.comment' => $sa [ 'comment' ]
2018-07-19 11:48:22 +02:00
),
));
if ( empty ( $oldsa )) {
return false ;
} else {
return $oldsa [ 'ShadowAttribute' ];
}
}
public function getEventContributors ( $id )
{
2019-09-30 08:23:36 +02:00
$orgs = $this -> find ( 'all' , array ( 'fields' => array (
'DISTINCT(ShadowAttribute.org_id)' ),
'conditions' => array ( 'event_id' => $id ),
'recursive' => - 1 ,
'order' => false
));
2018-07-19 11:48:22 +02:00
$org_ids = array ();
2019-02-28 08:30:17 +01:00
$this -> Organisation = ClassRegistry :: init ( 'Organisation' );
2018-07-19 11:48:22 +02:00
foreach ( $orgs as $org ) {
2019-02-28 08:30:17 +01:00
$org_ids [] = $this -> Organisation -> find ( 'first' , array ( 'recursive' => - 1 , 'fields' => array ( 'id' , 'name' ), 'conditions' => array ( 'Organisation.id' => $org [ 'ShadowAttribute' ][ 'org_id' ])));
2018-07-19 11:48:22 +02:00
}
return $org_ids ;
}
/**
* Sends an email to members of the organization that owns the event
* @ param int $id The event id
* @ return boolean False if no email at all was sent , true if at least an email was sent
*/
public function sendProposalAlertEmail ( $id )
{
$this -> Event -> recursive = - 1 ;
$event = $this -> Event -> read ( null , $id );
// If the event has an e-mail lock, return
if ( $event [ 'Event' ][ 'proposal_email_lock' ] == 1 ) {
return false ;
} else {
$this -> setProposalLock ( $id );
}
$this -> User = ClassRegistry :: init ( 'User' );
$this -> User -> recursive = - 1 ;
$orgMembers = $this -> User -> find ( 'all' , array (
'conditions' => array (
'org_id' => $event [ 'Event' ][ 'orgc_id' ],
'contactalert' => 1 ,
'disabled' => 0
),
'fields' => array ( 'email' , 'gpgkey' , 'certif_public' , 'contactalert' , 'id' )
));
$body = " Hello, \n \n " ;
$body .= " A user of another organisation has proposed a change to an event created by you or your organisation. \n \n " ;
$body .= 'To view the event in question, follow this link: ' . Configure :: read ( 'MISP.baseurl' ) . '/events/view/' . $id . " \n " ;
$subject = " [ " . Configure :: read ( 'MISP.org' ) . " MISP] Proposal to event # " . $id ;
$result = false ;
foreach ( $orgMembers as $user ) {
$result = $this -> User -> sendEmail ( $user , $body , $body , $subject ) or $result ;
}
return $result ;
}
public function setProposalLock ( $id , $lock = true )
{
$this -> Event -> recursive = - 1 ;
$event = $this -> Event -> read ( null , $id );
if ( $lock ) {
$event [ 'Event' ][ 'proposal_email_lock' ] = 1 ;
} else {
$event [ 'Event' ][ 'proposal_email_lock' ] = 0 ;
}
$fieldList = array ( 'proposal_email_lock' , 'id' , 'info' );
$event [ 'Event' ][ 'skip_zmq' ] = 1 ;
2019-03-05 12:24:56 +01:00
$event [ 'Event' ][ 'skip_kafka' ] = 1 ;
2018-07-19 11:48:22 +02:00
$this -> Event -> save ( $event , array ( 'fieldList' => $fieldList ));
}
public function generateCorrelation ( $jobId = false )
{
$this -> ShadowAttributeCorrelation = ClassRegistry :: init ( 'ShadowAttributeCorrelation' );
$this -> ShadowAttributeCorrelation -> deleteAll ( array ( 'id !=' => 0 ), false );
// get all proposals..
$proposals = $this -> find ( 'all' , array ( 'recursive' => - 1 , 'conditions' => array ( 'ShadowAttribute.deleted' => 0 , 'ShadowAttribute.proposal_to_delete' => 0 )));
$proposalCount = count ( $proposals );
if ( $jobId && Configure :: read ( 'MISP.background_jobs' )) {
$this -> Job = ClassRegistry :: init ( 'Job' );
$this -> Job -> id = $jobId ;
}
if ( $proposalCount > 0 ) {
foreach ( $proposals as $k => $proposal ) {
$this -> __afterSaveCorrelation ( $proposal [ 'ShadowAttribute' ]);
if ( $jobId && Configure :: read ( 'MISP.background_jobs' ) && $k > 0 && $proposalCount % $k == 10 ) {
$this -> Job -> saveField ( 'progress' , ( $k / $proposalCount * 100 ));
}
}
}
if ( $jobId && Configure :: read ( 'MISP.background_jobs' )) {
$this -> Job -> saveField ( 'progress' , 100 );
$this -> Job -> saveField ( 'status' , 4 );
$this -> Job -> saveField ( 'message' , 'Job done.' );
}
return $proposalCount ;
}
2019-07-12 08:56:06 +02:00
private function __preCaptureMassage ( $proposal )
{
if ( empty ( $proposal [ 'event_uuid' ]) || empty ( $proposal [ 'Org' ])) {
return false ;
}
2019-07-12 16:03:08 +02:00
if ( isset ( $proposal [ 'id' ])) {
2019-07-12 08:56:06 +02:00
unset ( $proposal [ 'id' ]);
}
$event = $this -> Event -> find ( 'first' , array (
'recursive' => - 1 ,
'conditions' => array ( 'Event.uuid' => $proposal [ 'event_uuid' ]),
'fields' => array ( 'Event.id' , 'Event.uuid' , 'Event.org_id' )
));
if ( empty ( $event )) {
return false ;
}
$proposal [ 'event_id' ] = $event [ 'Event' ][ 'id' ];
$proposal [ 'event_org_id' ] = $event [ 'Event' ][ 'org_id' ];
return $proposal ;
}
2019-07-11 22:57:58 +02:00
public function capture ( $proposal , $user )
{
2019-07-12 08:56:06 +02:00
$proposal = $this -> __preCaptureMassage ( $proposal );
if ( $proposal === false ) {
return false ;
}
2019-07-11 22:57:58 +02:00
$oldsa = $this -> findOldProposal ( $proposal );
if ( ! $oldsa || $oldsa [ 'timestamp' ] < $proposal [ 'timestamp' ]) {
if ( $oldsa ) {
2019-07-12 16:03:08 +02:00
$this -> delete ( $oldsa [ 'id' ]);
2019-07-11 22:57:58 +02:00
}
if ( isset ( $proposal [ 'old_id' ])) {
2019-07-14 08:24:10 +02:00
$oldAttribute = $this -> Attribute -> find ( 'first' , array ( 'recursive' => - 1 , 'conditions' => array ( 'Attribute.uuid' => $proposal [ 'uuid' ])));
2019-07-11 22:57:58 +02:00
if ( $oldAttribute ) {
$proposal [ 'old_id' ] = $oldAttribute [ 'Attribute' ][ 'id' ];
} else {
$proposal [ 'old_id' ] = 0 ;
}
2019-07-12 16:03:08 +02:00
} else {
$proposal [ 'old_id' ] = 0 ;
2019-07-11 22:57:58 +02:00
}
2019-07-12 16:03:08 +02:00
$proposal [ 'org_id' ] = $this -> Event -> Orgc -> captureOrg ( $proposal [ 'Org' ], $user );
2019-07-11 22:57:58 +02:00
unset ( $proposal [ 'Org' ]);
$this -> create ();
2019-07-12 08:56:06 +02:00
if ( $this -> save ( $proposal )) {
if ( ! isset ( $proposal [ 'deleted' ]) || ! $proposal [ 'deleted' ]) {
2019-07-11 22:57:58 +02:00
$this -> sendProposalAlertEmail ( $proposal [ 'event_id' ]);
}
2019-07-12 16:03:08 +02:00
return true ;
2019-07-11 22:57:58 +02:00
}
}
2019-07-12 16:03:08 +02:00
return false ;
2019-07-11 22:57:58 +02:00
}
2019-07-12 08:56:06 +02:00
public function pullProposals ( $user , $server , $HttpSocket = null )
{
2019-07-12 16:03:08 +02:00
$version = explode ( '.' , $server [ 'Server' ][ 'version' ]);
2019-07-12 08:56:06 +02:00
if (
( $version [ 0 ] == 2 && $version [ 1 ] == 4 && $version [ 2 ] < 111 )
) {
return 0 ;
}
$url = $server [ 'Server' ][ 'url' ];
$HttpSocket = $this -> setupHttpSocket ( $server , $HttpSocket );
$request = $this -> setupSyncRequest ( $server );
$i = 1 ;
$fetchedCount = 0 ;
$chunk_size = 1000 ;
2019-07-12 16:03:08 +02:00
$timestamp = strtotime ( " -90 day " );
2019-07-12 08:56:06 +02:00
while ( true ) {
$uri = sprintf (
2019-07-12 16:03:08 +02:00
'%s/shadow_attributes/index/all:1/timestamp:%s/limit:%s/page:%s/deleted[]:0/deleted[]:1.json' ,
2019-07-12 08:56:06 +02:00
$url ,
$timestamp ,
$chunk_size ,
$i
);
$i += 1 ;
$response = $HttpSocket -> get ( $uri , false , $request );
2019-07-12 16:03:08 +02:00
if ( $response -> code == 200 ) {
2019-07-12 08:56:06 +02:00
$data = json_decode ( $response -> body , true );
if ( empty ( $data )) {
return $fetchedCount ;
}
$returnSize = count ( $data );
foreach ( $data as $k => $proposal ) {
2019-07-12 16:03:08 +02:00
$result = $this -> capture ( $proposal [ 'ShadowAttribute' ], $user );
2019-07-12 08:56:06 +02:00
if ( $result ) {
$fetchedCount += 1 ;
}
}
if ( $returnSize < $chunk_size ) {
return $fetchedCount ;
}
} else {
return $fetchedCount ;
}
}
}
2019-07-11 22:57:58 +02:00
public function buildConditions ( $user )
{
$conditions = array ();
if ( ! $user [ 'Role' ][ 'perm_site_admin' ]) {
$sgids = $this -> Event -> cacheSgids ( $user , true );
$attributeDistribution = array (
'Attribute.distribution' => array ( 1 , 2 , 3 , 5 )
);
$objectDistribution = array (
'(SELECT distribution FROM objects WHERE objects.id = Attribute.object_id)' => array ( 1 , 2 , 3 , 5 )
);
if ( ! empty ( $sgids ) && ( ! isset ( $sgids [ 0 ]) || $sgids [ 0 ] != - 1 )) {
$objectDistribution [ '(SELECT sharing_group_id FROM objects WHERE objects.id = Attribute.object_id)' ] = $sgids ;
$attributeDistribution [ 'Attribute.sharing_group_id' ] = $sgids ;
}
2020-07-24 15:46:59 +02:00
$unpublishedPrivate = Configure :: read ( 'MISP.unpublishedprivate' );
2019-07-11 22:57:58 +02:00
$conditions = array (
'AND' => array (
'OR' => array (
'Event.org_id' => $user [ 'org_id' ],
2020-07-24 15:46:59 +02:00
[ 'AND' => [
'Event.distribution' => array ( 1 , 2 , 3 , 5 ),
$unpublishedPrivate ? [ 'Event.published' => 1 ] : [],
]],
[ 'AND' => [
'Event.distribution' => 4 ,
'Event.sharing_group_id' => $sgids ,
$unpublishedPrivate ? [ 'Event.published' => 1 ] : [],
]],
2019-07-11 22:57:58 +02:00
),
array (
'OR' => array (
'ShadowAttribute.old_id' => '0' ,
'AND' => array (
array (
'OR' => array (
'Attribute.object_id' => '0' ,
array (
'OR' => $objectDistribution
)
)
),
array (
'OR' => $attributeDistribution
)
)
)
)
)
);
}
return $conditions ;
}
2018-07-19 11:48:22 +02:00
public function upgradeToProposalCorrelation ()
{
$this -> Log = ClassRegistry :: init ( 'Log' );
if ( ! Configure :: read ( 'MISP.background_jobs' )) {
$this -> Log -> create ();
$this -> Log -> save ( array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => 'Starting proposal correlation generation' ,
'change' => 'The generation of Proposal correlations as part of the 2.4.20 datamodel upgrade has started'
));
$count = $this -> generateCorrelation ();
$this -> Log -> create ();
if ( is_numeric ( $count )) {
$this -> Log -> save ( array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => 'Proposal correlation generation complete' ,
'change' => 'The generation of Proposal correlations as part of the 2.4.20 datamodel upgrade is completed. ' . $count . ' proposals used.'
));
} else {
$this -> Log -> save ( array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => 'Proposal correlation generation failed' ,
'change' => 'The generation of Proposal correlations as part of the 2.4.20 has failed. You can rerun it from the administrative tools.'
));
}
} else {
$job = ClassRegistry :: init ( 'Job' );
$job -> create ();
$data = array (
'worker' => 'default' ,
'job_type' => 'generate proposal correlation' ,
'job_input' => 'All attributes' ,
'retries' => 0 ,
'status' => 1 ,
'org' => 'SYSTEM' ,
'message' => 'Correlating Proposals.' ,
);
$job -> save ( $data );
$jobId = $job -> id ;
$process_id = CakeResque :: enqueue (
'default' ,
'AdminShell' ,
array ( 'jobGenerateShadowAttributeCorrelation' , $jobId ),
true
);
$job -> saveField ( 'process_id' , $process_id );
$this -> Log -> create ();
$this -> Log -> save ( array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => 'Proposal correlation generation job queued' ,
'change' => 'The job for the generation of Proposal correlations as part of the 2.4.20 datamodel upgrade has been queued'
));
}
}
2016-06-06 10:09:55 +02:00
}