User guide and UI changes

- first set of changes to the user guide, still missing updated images

- some UI changes to make the looks a bit more appealing
pull/226/head
iglocska 2014-03-06 09:20:05 +01:00
parent 239324dfbb
commit eb69a6e78f
10 changed files with 388 additions and 184 deletions

View File

@ -1,6 +1,7 @@
<div class="footer <?php echo $debugMode;?>">
<div class="navbar navbar-inverse" style="padding-left:20px;">
<div class="navbar-inner row">
<div class="navbar navbar-inverse">
<div class="glass"></div>
<div class="navbar-inner" style="border-radius: 10px;">
<div class="pull-left footerText" style="float:left;position:absolute;padding-top:12px;z-index:2;">
<?php
$gpgpath = ROOT.DS.APP_DIR.DS.WEBROOT_DIR.DS.'gpg.asc';

View File

@ -1,12 +1,18 @@
<div class="navbar-wrapper header <?php echo $debugMode;?>">
<div class="navbar-wrapper header <?php echo $debugMode;?>" style="height:42px;">
<div class="glass"></div>
<div class="navbar navbar-inverse">
<div class="navbar-inner">
<div class="navbar-inner" style="border-radius: 10px;">
<!-- .btn-navbar is used as the toggle for collapsed navbar content -->
<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</a>
<?php if ($me != false ):?>
<div class="nav-collapse collapse">
<ul class="nav">
<li class="active"><a href="/">home
<li><a href="/" style="color:white">Home
</a></li>
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#">
Event Actions
@ -140,24 +146,47 @@
</div>
<div class="nav-collapse collapse pull-right">
<ul class="nav">
<li>
<a href ="/events/proposalEventIndex" <?php if ($proposalCount > 0) echo 'style="font-weight:bold;"'; ?>>
<?php
$proposalPluralOrZero = 's';
if ($proposalCount == 1) $proposalPluralOrZero = '';
$proposalEventPluralOrZero = 's';
if ($proposalEventCount == 1) $proposalEventPluralOrZero = '';
echo $proposalCount . ' proposal' . $proposalPluralOrZero . ' in ' . $proposalEventCount . ' event' . $proposalEventPluralOrZero;
?>
</a>
</li>
<li>
<a href="/" id="fullLogo" style="font-weight:bold;">
<span class="logoBlue">M</span><span class="logoGray">alware</span>
<span class="logoBlue">I</span><span class="logoGray">nformation </span>
<span class="logoBlue">S</span><span class="logoGray">haring</span>
<span class="logoBlue">P</span><span class="logoGray">latform</span>
</a>
<a href="/" id="smallLogo" style="display:none;font-weight:bold;">
<span class="logoBlue">MISP</span>
</a>
</li>
<li><a href="/users/logout">Log out</a></li>
</ul>
</div>
<div class="nav-collapse collapse pull-right" style="margin-top:10px">
<div class="nav" style="font-weight:bold">
<?php if ($proposalCount > 0): ?>
<span class="proposal_span"><a href="/events/proposalEventIndex" class="proposal_link"><?php echo $proposalCount . ' proposals in ' . $proposalEventCount; ?> events</a></span>
<?php else: ?>
<span><a href="/events/proposalEventIndex" class="proposal_link"><?php echo $proposalCount . ' proposals in ' . $proposalEventCount; ?> events</a></span>
<?php endif;?>
<span class="logoBlue">M</span><span class="logoGray">alware</span>
<span class="logoBlue">I</span><span class="logoGray">nformation </span>
<span class="logoBlue">S</span><span class="logoGray">haring</span>
<span class="logoBlue">P</span><span class="logoGray">latform</span>
</div>
</div>
<?php endif;?>
</div>
</div>
</div>
</div>
<script>
window.onload = resizeLogo;
window.onresize = resizeLogo;
function resizeLogo() {
if ($(window).width() < 1400) {
document.getElementById('fullLogo').style.display='none';
document.getElementById('smallLogo').style.display='block';
}
if ($(window).width() > 1399) {
document.getElementById('fullLogo').style.display='block';
document.getElementById('smallLogo').style.display='none';
}
}
</script>

View File

@ -8,7 +8,6 @@
</title>
<?php
echo $this->Html->meta('icon');
// echo $this->Html->css('cake.generic');
echo $this->Html->css('roboto');
echo $this->Html->css('bootstrap'); // see http://twitter.github.io/bootstrap/base-css.html
@ -33,6 +32,7 @@
<body>
<div id="container">
<?php echo $this->element('global_menu');
//throw new Exception(serialize($this->Session->read('Message.flash.message')));
if ($debugMode == 'debugOff') {
?>
<div class="container-fluid debugOff" style="padding-top:50px;width:98%;">
@ -50,7 +50,11 @@
</div>
<div
<?php
if (Configure::read('debug') == 0) echo 'class="topGap"';
//echo 'class="topGap"';
$message = $this->Session->check('Message');
if (Configure::read('debug') == 0 && $this->Session->check('Message')) {
echo 'class="topGap"';
}
?>
>
<?php echo $this->fetch('content'); ?>
@ -70,10 +74,16 @@
<div class="spinner"></div>
<div class="loadingText">Loading</div>
</div>
<?php
if ($debugMode == 'debugOff'):
?>
<script type="text/javascript">
$(window).scroll(function(e) {
$('.actions').css('left',-$(window).scrollLeft());
});
</script>
<?php
endif;
?>
</body>
</html>

View File

@ -13,6 +13,8 @@
<li><a href="#user">User Management</a></li>
<li><a href="#roles">Role Management</a></li>
<li><a href="#logs">Logging</a></li>
<li><a href="#admin_tools">Administrative Tools</a></li>
<li><a href="#background">Background Processing</a></li>
</ul>
</li>
<li><?php echo $this->Html->link('Categories and Types', array('controller' => 'pages', 'action' => 'display', 'doc', 'categories_and_types')); ?></li>
@ -109,9 +111,12 @@ Privileges are assigned to users by assigning them to rule groups, which use one
<b>Manage Organization Events:</b> allows users to create events or modify and delete events created by a member of their organisation. <br />
<b>Manage &amp; Publish Organisation Events:</b> This last setting, gives users the right to do all of the above and also to publish the events of their organisation.<br />
<b>Perm sync:</b> This setting allows the users of the role to be used as a synchronisation user. The authentication key of this user can be handed out to the administrator of a remote MISP instance to allow the synchronisation features to work.<br />
<b>Perm admin:</b> Gives the user administrator privileges, this setting is used for the organisation admins. <br />
<b>Perm audit:</b> Grants access to the logs. With the exception of site admins, only logs generated by the user's own org are visible. <br />
<b>Perm auth:</b> This setting enables the authentication key of the role's users to be used for rest requests. <br />
<b>Perm admin:</b> Gives the user limited administrator privileges, this setting is used for the organisation admins. <br />
<b>Perm site admin:</b> Gives the user full administrator privileges, this setting is used for the site admins. <br />
<b>Perm audit:</b> Grants access to the logs. With the exception of site admins, only logs generated by the user's own org are visible. <br />
<b>Perm regexp access:</b> Allows the users with this permission enabled to edit the regular expression table. Be careful when giving out this permission, incorrect regular expressions can be very harmful (infinite loops, loss of data, etc.). <br />
<b>Perm tagger:</b> Allows the user with this permission to create custom tags and assign them to events. <br />
<h3>Creating roles:</h3>
When creating a new role, you will have to enter a name for the role to be created and set up the permissions (as described above) using the radio toggle and the four check-boxes.<br />
<h3>Listing roles:</h3>
@ -120,11 +125,12 @@ By clicking on the List Roles button, you can view a list of all the currently r
<hr />
<h2><a id="logs"></a>Using the logs of MISP</h2>
Users with audit permissions are able to browse or search the logs that MISP automatically appends each time certain actions are taken (actions that modify data or if a user logs in and out).<br />
Generally, the following actions are logged:<br />
Generally, the following actions are logged:<br /><br />
<ul>
<li><b>User:</b> Creation, deletion, modification, Login / Logout<br /></li>
<li><b>Event:</b>Creation, deletion, modification, publishing<br /></li>
<li><b>Attribute:</b> Creation, deletion, modification<br /></li>
<li><b>ShadowAttribute:</b> Creation, deletion, Accept, Discard<br /></li>
<li><b>Roles:</b> Creation, deletion, modification<br /></li>
<li><b>Blacklist:</b> Creation, deletion, modification<br /></li>
<li><b>Whitelist:</b> Creation, deletion, modification<br /></li>
@ -133,7 +139,7 @@ Generally, the following actions are logged:<br />
<br />
<h3>Browsing the logs:</h3>
Listing all the log entries will show the following columns generated by the users of your organisation (or all organisations in the case of site admins):<br />
<img src="/img/doc/list_logs.png" alt = "List logs" title = "Here you can view a list of all logged actions."/><br />
<img src="/img/doc/list_logs.png" alt = "List logs" title = "Here you can view a list of all logged actions."/><br /><br />
<ul>
<li><b>Id:</b> The automatically assigned ID number of the entry.<br /></li>
<li><b>Email:</b> The e-mail address of the user whose actions triggered the entry.<br /></li>
@ -148,7 +154,7 @@ Listing all the log entries will show the following columns generated by the use
</ul>
<img src="/img/doc/search_log.png" alt = "Search log" style="float:right;" title = "You can search the logs using this form, narrow down your search by filling out several fields."/>
<h3>Searching the Logs:</h3>
Another way to browse the logs is to search it by filtering the results according to the following fields (the search is a sub-string search, the sub-string has to be an exact match for the entry in the field that is being searched for):<br />
Another way to browse the logs is to search it by filtering the results according to the following fields (the search is a sub-string search, the sub-string has to be an exact match for the entry in the field that is being searched for):<br /><br />
<ul>
<li><b>Email:</b> By searching by Email, it is possible to view the log entries of a single user.<br /></li>
<li><b>Org:</b> Searching for an organisation allows you to see all actions taken by any member of the organisation.<br /></li>
@ -156,5 +162,46 @@ Another way to browse the logs is to search it by filtering the results accordin
<li><b>Title:</b> There are several ways in which to use this field, since the title fields contain several bits of information and the search searches for any substrings contained within the field, it is possible to just search for the ID number of a logged event, the username / server's name / event's name / attribute's name of the event target.<br /></li>
<li><b>Change:</b> With the help of this field, you can search for various specific changes or changes to certain variables (such as published will find all the log entries where an event has gotten published, ip-src will find all attributes where a source IP address has been entered / edited, etc).<br /></li>
</ul>
<hr />
<h2><a id="admin_tools"></a>Administrative Tools</h2>
MISP has a couple of administrative tools that help administrators keep their instance up to date and healthy. The list of these small tools can change rapidly with each new version, but they should be self-explanatory. Make sure to check this section after upgrading to a new version, just in case there is a new upgrade script in there - though if this is the case it will be mentioned in the upgrade instructions.<br /><br />
<hr />
<h2><a id="background"></a>Background Processing</h2>
If enabled, MISP can delegate a lot of the time intensive tasks to the background workers. These will then be executed in order, allowing the users of the instance to keep using the system without a hiccup and without having to wait for the process to finish. It also allows for certain tasks to be scheduled and automated.
<h3>Command Line Tools for the Background Workers</h3>
The background workers are powered by <a href="https://github.com/kamisama/Cake-Resque">CakeResque</a>, so all of the CakeResque commands work.
To start all of the workers needed by MISP go to your <code>/var/www/MISP/app/Console/worker</code> (assuming a standard installation path) and execute start.sh.
To interact with the workers, here is a list of useful commands. Go to your <code>/var/www/MISP/app/Console</code> (assuming a standard installation path) and execute one of the following commands as a parameter to <code>./cake CakeResque.CakeResque</code> (for example: <code>./cake CakeResque.CakeResque tail</code>):<br /><br />
<ul>
<li><b>tail</b>: tail the various log files that CakeResque creates, just choose the one from the list that you are interested in.</li>
<li><b>cleanup</b>: terminate the job that a worker is working on immediately. You will be presented with a choice of workers to choose from when executing this command.</li>
<li><b>clear</b>: Clear the queue of a worker immediately.</li>
<li><b>stats</b>: shows some statistics about your workers including the count of successful and failed jobs.</li>
</ul>
The other commands should not be needed, instead of starting / stopping or restarting workers use the supplied start.sh (it stops all workers and starts them all up again). For further instructions on how to use the console commands for the workers, visit the <a href="http://cakeresque.kamisama.me/commands#cleanup">CakeResque list of commands</a>.<br />
<h3>Monitoring the Background Processes</h3>
The "Jobs" menu item within the Administration menu allows site admins to get an overview of all of the currently and in the past scheduled jobs. Admins can see the status of each job, and what the queued job is trying to do. If a job fails, it will try to set an error message here too. The following columns are shown in the jobs table:<br /><br />
<ul>
<li><b>Id</b>: The job's ID (this is the ID of the job's metadata stored in the default datastore, not to be confused with the process ID stored in the redis database and used by the workers)</li>
<li><b>Process</b>: The process's ID.</li>
<li><b>Worker</b>: The name of the worker queue. There are 3+1 workers running if background jobs are enabled: default, cache, email, and a special Scheduler (this should never show up in the jobs table).</li>
<li><b>Job Type</b>: The name of the queued job.</li>
<li><b>Input</b>: Shows a basic input handled by the job - such as "Event:50" for a publish email alert job for event 50.</li>
<li><b>Message</b>: This will show what the job is currently doing or alternatively an error message describing why a job failed. </li>
<li><b>Org</b>: The string identifier of the organisation that has scheduled the job. </li>
<li><b>Status</b>: The status reported by the worker.</li>
<li><b>Retries</b>: Currently unused, it is planned to introduced automatic delayed retries for the background processing to add resilience.</li>
<li><b>Progress</b>: A progress bar showing how the job is coming along.</li>
</ul>
<h3>Scheduling Jobs and Recurring Jobs</h3>
Apart from off-loading long-lasting jobs to the background workers, there is a second major benefit of enabling the background workers: Site-administrators can schedule recurring tasks for the jobs that generally take the longest to execute. At the moment this includes pushing / pulling other instances and generating a full export cache for every organisation and export type. MISP comes with these 3 tasks pre-defined, but further tasks are planned. The following fields make up the scheduled tasks table: <br /><br />
<ul>
<li><b>Id</b>: The ID of the task.</li>
<li><b>Type</b>: The type of the task.</li>
<li><b>Frequency (h)</b>: This number sets how often the job should be executed in hours. Setting this to 168 and picking the next execution on Sunday at 01:00 would execute the task every Sunday at 1 AM. Setting this value to 0 will make the task only run once on the scheduled date / time without rescheduling it afterwards.</li>
<li><b>Scheduled Time</b>: The time (in 24h format) when the task should be executed the next time it runs (and all consecutive times if a multiple of 24 is chosen for frequency).</li>
<li><b>Next Run</b>: The date on which the task should be executed.</li>
<li><b>Description</b>: A brief description of the task.</li>
<li><b>Message</b>: This field shows when the job was queued by the scheduler for execution. </li><br />
</ul>
</div>

View File

@ -12,22 +12,32 @@
<div class="index">
<h2><a id="general"></a>General Concepts</h2>
<h3>Admins and Site Admins</h3>
There are two types of admins in MISP: Admins (also refered to as org admins) and Site Admins. Whilst the former can only do some limited administration of users of his/her own organisation, site admins have access to all of the features and data of the system. They are in charge of making sure that the system runs correctly and the maintenance of MISP.
<h3>Background Jobs</h3>
A lot of the heavier tasks are a burden to users, in that their actions can cause long delays (and in some cases timeouts) while the application logic is executing. To alleviate this, long processes have been (if enabled) moved to background jobs, meaning that their execution happens asynchronously in the background, allowing the user to freely interact with the platform whilst the request is being processed.
<h3>MISP Instance</h3>
A MISP instance is an installation of the MISP software and the connected database. All the data visible to the users is stored locally in the database and data that is shareable (based on the distribution settings) can be synchronised with other instances via the Sync actions. The instance that you are reading this manual on will be refered to as "this instance" or "your instance". The instances that your instance synchronises with will be refered to as "remote instances".
<h3>Organisation administrators and Site administrators</h3>
We have two types of administrators, site and organisation admins. The former has access to every administrator feature for all the data located on the system including global features such as the creation and modification of user roles and instance links, whilst organisation admins can administer users, events and logs of their own respective organisations.<br />
<h3>Synchronisation</h3>
What we call synchronisation is an exchange of data between two (or more) MISP instances through our pull and push mechanisms.
<h3>Pivot path</h3>
The (branching) path taken by a user from event to event while following correlation links. This is represented by the branching graph in the event view.
<h3>Pivoting</h3>
The act of navigating from event to event through correlation links.
<h3>Proposals</h3>
Each event can only be directly edited by users of the original creator organisation (and site admins). However, if another organisation would like to amend an event with extra information on an event, or if they'd like to correct a mistake in an attribute, they can create a Proposal. These proposals could then be accepted by the original creator organisation. These proposals can be pulled to another server, allowing users on connected instances to propose changes which then could be accepted by the original creators on another instance (and subsequently pushed back).
<h3>Publishing</h3>
When an event is first created by a user, it is visible to everyone on the instance based on the access rights ("Your organisation only" events will not be visible to users of other organisations), but they will not be synchronised and they won't be exportable. For this, a user with publishing permission of the organisation that created the event has to publish the event. The system will then inform all the users of the instance that are subscribing to e-mail notifications and who have access to view the published event via an e-mail.
<h3>Pull</h3>
Pulling is the process of using the configured sync user on a remote instance to REST GET all of the accessible data (based on the distribution rights) to your instance and store it.
<h3>Push</h3>
Pushing is the process of using a configured instance link to send an event or all accessible events (limited by the distribution rights) through the REST interface to a remote instance.
<h3>Publishing</h3>
When an event is first created by a user, it is visible to everyone on the instance based on the access rights ("Your organisation only" events will not be visible to users of other organisations), but they will not be synchronised and they won't be exportable. For this, a user with publishing permission of the organisation that created the event has to publish the event. The system will then inform all the users of the instance that are subscribing to e-mail notifications and who have access to view the published event via an e-mail.
<h3>MISP Instance</h3>
A MISP instance is an installation of the MISP software and the connected database. All the data visible to the users is stored locally in the database and data that is shareable (based on the distribution settings) can be synchronised with other instances via the Sync actions. The instance that you are reading this manual on will be refered to as "this instance" or "your instance". The instances that your instance synchronises with will be refered to as "remote instances".
<h3>Sync user</h3>
<h3>Scheduled Tasks</h3>
Certain common tasks can be scheduled for a later execution or for regular recurring executions. These tasks currently include caching all of the export formats, pulling from all eligible instances and pushing to all eligible instances.
<h3>Sync User</h3>
A user of a role that grants sync permissions, these users (and their authentication keys) are used to serve as the points of connection between instances. Events pushed to an instance are pushed to a sync user, who then creates the events on the remote instance. Events pulled are added by the sync user that is used to connect the remote instance to your instance. As an administrator, keep in mind that a sync user needs auth key and publish permissions, has to have undergone the mandatory password change and has to have accepted the Terms of Use in order for the sync to work. Please make sure that all of these steps are taken before attempting to push or pull.
<h3>Admins and Site Admins</h3>
There are two types of admins in MISP: Admins (also refered to as org admins) and Site Admins. Whilst the former can only do some limited administration of users of his/her own organisation, site admins have access to all of the features and data of the system. They are in charge of making sure that the system runs correctly and the maintenance of MISP.
<h3>Background jobs</h3>
A lot of the heavier tasks are a burden to users, in that their actions can cause long delays (and in some cases timeouts) while the application logic is executing. To alleviate this, long processes have been (if enabled) moved to background jobs, meaning that their execution happens asynchronously in the background, allowing the user to freely interact with the platform whilst the request is being processed.
<h3>Synchronisation</h3>
What we call synchronisation is an exchange of data between two (or more) MISP instances through our pull and push mechanisms.
<h3>Tagging</h3>
Users with tagging rights can assigned various dynamically created tags to events, allowing an arbitrary link between events to be created. It is possible to filter events based on these tags and they can also be used to filter events for the automation.
</div>

View File

@ -23,6 +23,8 @@
<li><b>Sync Actions:</b> With administrator access rights, shows a list of the connected instances and allows the initiation of a push and a pull (more about the synchronisation mechanisms later).</li>
<li><b>Administration:</b> Administrators can add, edit or remove user accounts and user roles. Roles define the access rights to certain features such as publishing of events, usage of the REST interface or synchronisation of any user belonging to the given role. Site administrators can also access a contact form, through which it is possible to reset the passwords of users, or to just get in touch with them via encrypted e-mails.</li>
<li><b>Audit:</b> If you have audit permissions, you can view the logs for your organisation (or for site admins for the entire system) here or even search the logs if you are interested in something specific.</li>
<li><b>Discussions:</b> Link to the discussion threads.</li>
<li><b>Proposal Notifications: </b> This shows how many proposals your organisation has received and across how many events they are spread out. Clicking this will take you to the list of proposals.</li>
<li><b>Log out:</b> Logs you out of the system.</li>
</ul>
@ -35,16 +37,16 @@
<li><b>Search Attributes:</b> You can set search terms for a filtered attribute index view here.</li>
<li><b>View Proposals:</b> Shows a list of all proposals that you are eligible to see.</li>
<li><b>Events with proposals: </b> Shows all of the events created by your organsiation that has pending proposals.</li>
<li><b>List Tags:</b>List all the tags that have been created by users with tag creation rights on this instance</li>
<li><b>Add Tag:</b>Create a new tag</li>
<li><b>List Tags:</b>List all the tags that have been created by users with tag creation rights on this instance.</li>
<li><b>Add Tag:</b>Create a new tag.</li>
<li><b>Export:</b> Export the data accessible to you in various formats.</li>
<li><b>Automation:</b> If you have authentication key access, you can view how to use your key to use the REST interface for automation here.</li>
</ul>
<h5>Input filters</h5>
<ul>
<li><b>Import Whitelist:</b> You can view the whitelist rules, which contain the values that are blocked from being used for exports and automation on this instance. Site administrators have access to editing this list.</li>
<li><b>Import Regexp:</b> You can view the Regular Expression rules, which modify the data that can be entered into the system. This can and should be used to help filter out personal information from automatic imports (such as removing the username from windows file paths), having unified representation for certain common values for easier correlation or simply standardising certain input. It is also possible to block certain values from being inserted. As a site administrator or a user with regex permission, you can also edit these rules.</li>
<li><b>Signature Whitelist:</b> You can view the whitelist rules, which contain the values that are blocked from being used for exports and automation on this instance. Site administrators have access to editing this list.</li>
</ul>
<h5>Global Actions</h5>
@ -55,7 +57,7 @@
<li><b>Role Permissions:</b> You can view the role permissions here. </li>
<li><b>User Guide:</b> A link to this user guide.</li>
<li><b>Terms &amp; Conditions:</b> View the terms &amp; conditions again.</li>
<li><b>Statistics: </b> View a series of statistics about the users and the data on this instane</li>
<li><b>Statistics: </b> View a series of statistics about the users and the data on this instance.</li>
<li><b>Log out:</b> Logs the current user out.</li>
</ul>

View File

@ -52,10 +52,19 @@ MISP also provides its users with some information about itself and its users th
<li><b>Member statistics:</b> By using the Members List menu button on the left, you can get a quick overview over how many users each organisation has registered on your server, and a histogram, depicting the distribution of attribute types created by each organisation.<br /><br /></li>
<li><b>User Guide:</b> The user guide is also accessible via the Global Actions menu. You can find out more about how to use the system by reading this.<br /><br /></li>
<li><b>Terms &amp; Conditions:</b> It is possible to review the terms &amp; conditions that were shown during the first run of the system by clicking on the terms &amp; conditions link in the Global Actions menu.<br /><br /></li>
<li><b>Statistics:</b> View statistics about the users and the data contained within this instance.</li>
<li><b>Statistics:</b> View statistics about the users and the data contained within this instance.
<ul>
<li>
<b>General Statistics:</b> View a set of statistics such as the number of Events and Attributes currently in existance on the platform. The number in the bracket shows the number of new items added during this week.
</li>
<li>
<b>Activity Heatmap:</b> This graph shows a heatmap of all activity related to creating event related data on a day by day basis. By default, the graph shows the sum of the contributions of all organisations, but using the buttons representing each organisation in existance on the platform you can switch to the activity heatmap of a single organisation. If you'd like to see the activity further back in the past, just use the arrow buttons to navigate the heatmap.
</li>
</ul>
</li>
</ul>
<a id="filters"></a><h3>Inspecting the input filters:</h3>
All the events and attributes that get entered into MISP will be run through a series of input filters. These are defined by the site administrators, but every user can take a look at the currently active lists.<br><br>
All the events and attributes that get entered into MISP will be run through a series of input filters. These are defined by the site administrators or users with special privileges to edit the filters, but every user can take a look at the currently active lists.<br><br>
<ul>
<li><b>Import Regexp:</b> All Attribute value and Event info fields will be parsed for a set of regular expressions and replaced based on the replacement values contained in this section. This has many uses, such as unifying similar data for better correlation, removing personal data from file-paths or simply for clarity. It is also possible to blacklist data by not defining a replacement for a regular expression. <br /><br /></li>
<li><b>Signature Whitelist:</b> This list (can) contain a set of addresses that are allowed to be entered as attribute values but will be blocked from being exported to NIDS-es.<br /><br /> </li>

View File

@ -28,7 +28,7 @@ and attachments and finally publishing it.<br /><br />
During this first step, you will be create a basic event without any actual attributes, but storing general information such as a description, time and risk level of the incident. To start creating the event, click on the New Event button on the left and fill out the form you are presented with. The following fields need to be filled out:<br /><br />
<p><img src="/img/doc/add_event.png" alt = "" style="float:right;" title = "Fill this form out to create a skeleton event, before proceeding to populate it with attributes and attachments."/></p>
<ul>
<li><b>Date:</b> The date when the incident has happened.<br /><br /></li>
<li><b>Date:</b> The date when the incident has happened. Just click this field and a date-picker will pop up where you can select the desired date.<br /><br /></li>
<li><b>Distribution:</b>
<a id="distribution"></a>This setting controls, who will be able to see this event once it becomes published and eventually when it becomes pulled.
Apart from being able to set which users on this server are allowed to see the event, this also controls whether
@ -60,7 +60,7 @@ and attachments and finally publishing it.<br /><br />
<br /><br /></li>
</ul>
</li>
<li><b>Risk:</b> This field indicates the risk level of the event. Incidents can be categorised into three different threat categories (low, medium, high). This field can alternatively be left as undefined. The 3 options are:</li>
<li><b>Threat Level:</b> This field indicates the risk level of the event. Incidents can be categorised into three different threat categories (low, medium, high). This field can alternatively be left as undefined. The 3 options are:</li>
<li style="list-style: none;"><ul>
<li><i>Low:</i> General mass malware.</li>
<li><i>Medium:</i> Advanced Persistent Threats (APT)</li>
@ -87,9 +87,10 @@ Keep in mind that the system searches for regular expressions in the value field
<li><b>Distribution:</b> This drop-down list allows you to control who will be able to see this attribute.
The distribution is inherited by attributes: the most restrictive setting wins.
For more info <a href="#distribution">click here</a>.<br /><br /></li>
<li><b>Contextual Comment:</b> Add a comment to the attribute. This will not be used for correlation<br /><br /></li>
<li><b>IDS Signature:</b> This option allows the attribute to be used as an IDS signature when exporting the NIDS data, unless it is being overruled by the white-list. For more information about the whitelist, head over to the <?php echo $this->Html->link(__('administration', true), array('controller' => 'pages', 'action' => 'display', 'doc', 'administration', '#' => 'whitelist')); ?> section.<br /><br /></li>
<li><b>Contextual Comment:</b> Add a comment to the attribute. This will not be used for correlation.<br /><br /></li>
<li><b>Value:</b> The actual value of the attribute, enter data about the value based on what is valid for the chosen attribute type. For example, for an attribute of type ip-src (source IP address), 11.11.11.11 would be a valid value. For more information on types and values, <?php echo $this->Html->link(__('click here', true), array('controller' => 'pages', 'action' => 'display', 'doc', 'categories_and_types')); ?>.<br /><br /></li>
<li><b>Contextual Comment:</b> You can add some comments to the attribute that will not be used for correlation but instead serves as purely an informational field.<br /><br /></li>
<li><b>For Intrusion Detection System:</b> This option allows the attribute to be used as an IDS signature when exporting the NIDS data, unless it is being overruled by the white-list. For more information about the whitelist, head over to the <?php echo $this->Html->link(__('administration', true), array('controller' => 'pages', 'action' => 'display', 'doc', 'administration', '#' => 'whitelist')); ?> section.<br /><br /></li>
<li><b>Batch import:</b> If there are several attributes of the same type to enter (such as a list of IP addresses, it is possible to enter them all into the same value-field, separated by a line break between each line. This will allow the system to create separate lines for the each attribute. <br /><br /></li>
</ul>
<hr />
@ -103,11 +104,12 @@ You can also upload attachments, such as the malware itself, report files from e
<p><img src="/img/doc/add_attachment.png" alt = "Add attachment" title = "Point the uploader to the file you want to upload. Make sure to mark it as malware if the uploaded file is harmful, that way it will be neutralised."/></p><br />
<ul>
<li><b>Category:</b> The category is the same as with the attributes, it answers the question of what the uploaded file is meant to describe.<br /><br /></li>
<li><b>Upload field:</b> By hitting browse, you can browse your file system and point the uploader to the file that you want to attach to the attribute. This will then be uploaded when the upload button is pushed.<br /><br /></li>
<li><b>Malware:</b> This check-box marks the file as malware and as such it will be zipped and passworded, to protect the users of the system from accidentally downloading and executing the file. Make sure to tick this if you suspect that the filed is infected, before uploading it.<br /><br /></li>
<li><b>Distribution:</b> This drop-down list allows you to control who will be able to see this attachment.
The distribution is inherited by attributes: the most restrictive setting wins.
For more info <a href="#distribution">click here</a>.<br /><br /></li>
<li><b>Upload field:</b> By hitting browse, you can browse your file system and point the uploader to the file that you want to attach to the attribute. This will then be uploaded when the upload button is pushed.<br /><br /></li>
<li><b>Malware:</b> This check-box marks the file as malware and as such it will be zipped and passworded, to protect the users of the system from accidentally downloading and executing the file. Make sure to tick this if you suspect that the filed is infected, before uploading it.<br /><br /></li>
<li><b>Contextual Comment:</b> You can add some comments to the attribute that will not be used for correlation but instead serves as purely an informational field.<br /><br /></li>
</ul>
<hr />
<h3>Populate from IOC</h3>
@ -115,10 +117,15 @@ It is also possible to attempt to import the data contained in a .ioc file, The
<p><img src="/img/doc/ioc1.png" alt = "OpenIOC1" title = "The import tool will list the successful and failed entries after the process is done."/></p><br />
<p><img src="/img/doc/ioc2.png" alt = "OpenIOC2" title = "You'll also be able to see a graph of the imported .ioc file and how successful the import was."/></p><br />
<hr />
<h3>Populate from ThreatConnect</h3>
You can also import the data from a ThreatConnect export csv file. The following columns are used by the import tool (and are thus mandatory fields to select during the export): <br /><ul><li>Type</li><li>Value</li><li>Confidence</li><li>Description</li><li>Source</li></ul>The result will be a list of attributes that get added to the currently selected event, each of which will be marked with a comment that indicates that its origin being from a ThreatConnect import.
<hr />
<h3>Publish an event:</h3>
<p><img src="/img/doc/publish.png" alt = "Publish" style="float:right;" title = "Only use publish (no email) for minor changes such as the correction of typos."/></p><br />
Once all the attributes and attachments that you want to include with the event are uploaded / set, it is time to finalise its creation by publishing the event (click on publish event in the event view). This will alert the eligible users of it (based on the private-controls of the event and its attributes/attachments and whether they have auto-alert turned on), push the event to instances that your instance connects to and propagate it further based on the distribution rules. It also readies the network related attributes for NIDS signature creation (through the NIDS signature export feature, for more information, go to the export section.).<br /><br />
There is an alternate way of publishing an event without alerting any other users, by using the "publish (no email)" button. This should only be used for minor edits (such as correcting a typo). <br />
<br />
If your instance has background jobs enabled then the event might not get published immediately.
<hr />
<a id="browsing_events"></a><h2>Browsing past events:</h2>
The MISP interface allows the user to have an overview over or to search for events and attributes of events that are already stored in the system in various ways.<br /><br />
@ -126,15 +133,15 @@ The MISP interface allows the user to have an overview over or to search for eve
On the left menu bar, the option "List events" will generate a list of the last 60 events. While the attributes themselves aren't shown in this view, the following pieces of information can be seen:<br /><br />
<img src="/img/doc/list_events2.png" alt = "List events" title = "This is the list of events in the system. Use the buttons to the right to alter or view any of the events."/><br /><br />
<ul>
<li><b>Valid.:</b> Validation, an event that has been published counts as validated, marked by a checkmark. Unpublished events are marked by a cross.<br /><br /></li>
<li><b>Published:</b> Already published events are marked by a checkmark. Unpublished events are marked by a cross.<br /><br /></li>
<li><b>Org:</b> The organisation that created the event.<br /><br /></li>
<li><b>Owner Org:</b> The organisation that owns the event on this instance. This field is only visible to administrators. <br /><br /></li>
<li><b>ID:</b> The event's ID number, assigned by the system when the event was first entered (or in the case of an event that was synchronized, when it was first copied over - more on synchronisation in chapter xy)<br /><br /></li>
<li><b>Tags:</b> Tags that are assigned to this event.<br /><br /></li>
<li><b>#:</b> The number of attributes that the event has.<br /><br /></li>
<li><b>Email:</b> The e-mail address of the event's reporter.<br /><br /></li>
<li><b>#Attr.:</b> The number of attributes that the event has.<br /><br /></li>
<li><b>Email:</b> The e-mail address of the event's reporter. This is not visible to regular users. Organisation administrators can see the e-mail addresses of their own organisation's users.<br /><br /></li>
<li><b>Date:</b> The date of the attack.<br /><br /></li>
<li><b>Risk:</b> The risk level of the attack, the following levels are possible:<br /><br /></li>
<li><b>Threat Level:</b> The risk level of the attack, the following levels are possible:<br /><br /></li>
<li style="list-style: none;"><ul>
<li><b>Low:</b> General Malware</li>
<li><b>Medium:</b> Advanced Persistent Threats (APTs)</li>
@ -159,31 +166,98 @@ On the left menu bar, the option "List events" will generate a list of the last
<h3>Filters</h3>It is also possible to filter the events shown by clicking on the small magnifying glass icons next to the field names and entering a filter term.<br /><br />
<h3>Event view</h3>
<img src="/img/doc/event_detail.png" alt = "Event" title = "This view includes the basic information about an event, a link to related events, all attributes and attachments with tools to modify or delete them and extra functions for publishing the event or getting in touch with the event's reporter."/><br /><br /></li>
<li><b>List of related events:</b> Events can be related by having one or more attributes that are exact matches. For example, if two events both contain a source IP attribute of 11.11.11.11 then they are related. The list of events that are related the currently shown one, are listed under "Related Events", as links (titled the related event's date and ID number) to the events themselves.<br /><br /></li>
<li><b>Attributes:</b> A list of all attributes attached to the event, including its category, type, value, whether the attribute in itself is related to another event, whether the flag signalling that the attribute can be turned into an IDS signature is on, and a field showing the current privacy setting of the attribute.Attributes can also be modified or deleted via the 3 buttons at the end of each line.<br /><br />
Using the modify button will bring up the attribute creation view, with all data filled out with the attribute's currently stored data.<br /><br /></li>
<li><b>Event History:</b> View the logs of the event that show how the event has changed over time, including the contribution from other organisations in the form of proposals.<br /><br /></li>
<li><b>Contributors:</b> Shows a list of the organisations that have contributed to the event via proposals. If you click any of the logos listed here, you'll get redirected to a filtered event history view, including only the changes made by the organisation.</li>
<li><b><u>General Event Information</u></b><br /><br /><ul>
<li><b>ID:</b> The ID of the event.</li><br />
<li><b>Uuid:</b> In order to avoid collisions between events and attributes (during for example a sync) a Uuid is assigned that uniquely identifies each of them.</li><br />
<li><b>Org</b> The organisation that has originally created the event. The logo (if it exists on the server, alternatively a string) representing the organisation is also shown int he right upper corner.</li><br />
<li><b>Contributors:</b> Shows a list of the organisations that have contributed to the event via proposals. If you click any of the logos listed here, you'll get redirected to a filtered event history view, including only the changes made by the organisation.</li><br />
<li><b>Tags:</b> A list of tags associated with the event. Clicking a tag will show a list of events with the same tag attached. The little cross next to each tag allows you to remove the tag from the event, whilst the '+' button allows you to assign a tag. For the latter two options to be visible, you have to have tagging permission.</li><br />
<li><b>Date:</b> The date of detection, set by the user that creates the event, not to be confused with the creation date of the event.</li><br />
<li><b>Threat Level:</b> The assigned threat level of the event.</li><br />
<li><b>Analysis:</b> The status of the analysis.</li><br />
<li><b>Distribution:</b> This shows the distribution rules applied to this event, controlling whether only the creating organisation can see (Your organisation only) it or everyone on the instance (This community only). The two remaining settings allow the event to be propagated to organisations on remote connected instances.</li><br />
<li><b>Info:</b> A short description of the event itself. Make sure not to put information in here that could be used for correlation purposes and be better suited as an Attribute. </li><br />
<li><b>Published:</b> Whether the event has been published or not. Publishing allows the attributes of the event to be used for all eligible exports and it notifies users that have subscribed to the event alerts. Also, a publish initiates a push to all eligible instances. </li><br />
</ul></li>
<li><b><u>List of Related Events</u></b><br /><br />The list of relations is shown on the right hand side of the general event information. Events can be related by having one or more attributes that are exact matches. For example, if two events both contain a source IP attribute of 11.11.11.11 then they are related. The list of events that are related the currently shown one, are listed under "Related Events", as links (titled the related event's date and ID number) to the events themselves.</li><br />
<li><b><u>Data Element Toggles</u></b><br /><br />
<p>You can control some of the data that is shown on this page using three toggles. The elements that can be disabled are the pivot threads, the attributes (and proposals) and the Discussions. You can collapse these elements and then expand them again using the same button.</p>
</li>
<li><b><u>Pivot Threads</u></b><br /><br />While moving from event to event through the relation links (a process that we refer to as pivoting), you create a path that shows which events you have traversed. This path is reset by leaving the event view and navigating elsewhere in the application or by deleting the root pivot element. <br /><br />
Each event visited is represented by a bubble in the pivot thread graph, connected by lines that show how the user has arrived at the next connected event. It is possible to jump back to an earlier relation and pivot to another event through that, creating branches in the graph.<br /><br />The currently selected event is coloured blue in the graph. If you would like to delete an element from the graph (including all of elements that branch off of it) just click on the small x within a pivot bubble. For a deletion to be possible the following conditions have to be met:<br /><br />
<ul><li>The pivot element to be deleted cannot be on the path that leads to the currently selected event</li><br />
<li>The pivot element residing in the graph's root can always be deleted - this will simply reset the current pivot thread</li></ul><br />
</li>
<li><b><u>Attributes and Proposals</u></b><br /><br /><ul> A list of all attributes and proposals attached to the event. The fields for each of them only differ in the available actions and the fact that for proposals to attributes all fields are blank that would stay unchanged if the proposal was accepted (for example, proposing a change to an attribute to turn the IDS flag on will have all fields apart from the IDS flag blank in the proposal. Here is a list of what each of the fields represents: <br /><br />
<ul>
<li><b>Date</b>: The date of the last modification to the attribute. Proposals don't have a date of last edit.</li><br />
<li><b>Category</b>: The category of the attribute or proposal. For a list of possible categories visit the section on <a href="/pages/display/doc/categories_and_types">categories and types</a>.</li><br />
<li><b>Type</b>: The type of the attribute or proposal. For a list of possible categories visit the section on <a href="/pages/display/doc/categories_and_types">categories and types</a>.</li><br />
<li><b>Value</b>: The value or value-pair of the attribute. This is the main payload of the attribute, which is described by the category and type columns. For certain types of attributes that are made up of value-pairs the two parts will be split by a pipe (|), such as for filename|md5. The value field(s) are used by the correlation engine to find relations between events. In value-pair attributes both values are correlated individually. </li><br />
<li><b>Comment</b>: Attributes can have a contextual comment to further describe the attribute. These comments are not used for correlation and are purely informative. </li><br />
<li><b>Related Events</b>: A list of the event IDs that also contain an attribute with the same value. </li><br />
<li><b>IDS</b>: Flags an attribute as an indicator of compromise, allowing it to be included in all of the eligible exports. </li><br />
<li><b>Distribution</b>: Defines the distribution of the attribute individually. An attribute can have a different distribution level than the event. In any case, the lowest distribution level of the two is used. </li><br />
<li><b>Actions</b>: The user can interact with the events through these buttons, which will be further described in the next portion of the guide as they differ for attributes and proposals. </li><br />
</ul> <br />
Depending on the colour coding of the row, you can have an attribute, a proposal to the event or a proposal to an attribute:<br /><br />
<ul>
<li><b>Attributes</b>: Each uncoloured line represents an Attribute.</li><br />
<li><b>Proposals to an Event</b>: Each gray line at the end of the list represents a Proposal to an event. These are proposals for a new attribute, mostly unrelated to any of the currently existing attributes. If the creator of the event accepts one of these a new attribute will be created.</li><br />
<li><b>Proposals to an Attribute</b>: Each attribute can have several edit proposals. These will be placed right below the attribute that the proposal affects and - as with the event proposals - is coloured grey. The original attribute's row is coloured blue if a proposal exists for it.</li><br />
</ul>
Using the modify button will bring up the attribute creation view, with all data filled out with the attribute's currently stored data.<br /><br /></li>
</ul></li>
<li><b><u>Event Discussion Thread</u></b></li><br />
Each event has its own assigned discussion where users (that are eligible to see the event) can participate in an open discussion. The users are anonymised in the messages, all that other users will see is their user ID number and their organisation. To post a message on the Event Discussion, either use the reply button on a previous post or use the quickresponse field at the bottom of the page.
Each post is made up of the following:<br /><br />
<ul>
<li><b>Date: </b>The date when the post was created.</li><br />
<li><b>Post navigation: </b>This should the post's ID as well as a link to jump to the top of the discussion thread on the page itself.</li><br />
<li><b>Organisation logo: </b>If such an image exists for the organisation that has posted the message, then the logo is shown.</li><br />
<li><b>Message: </b>The body of the post itself. This can also include automatically generated links to other events and threads as well as show quoted test in embedded bubbles. Editing an event will also append a post with a message indicating that it was edited together with the timestamp of the edit.</li><br />
<li><b>User: </b>The e-mail address of the poster if he/she is from the organisation as the current user. Alternatively a generated sting is shown that includes the user ID of the user, so that his/her e-mail address could remain hidden whilst still being identifiable.</li><br />
<li><b>Action buttons: </b>Edit, Delete and Reply. The first two of the three options are only available to the poster of the message or a site admin. Quoting a post will automatically include the original message in [quote] tags. </li><br />
</li>
</ul>
Here is a list of the various tools you can use while using this feature:<br /><br />
<ul>
<li><b>Pagination:</b> There are 5 posts visible on each event page, if there have been more messages posted, use the previous and next button to navigate through the thread. This will not reload the rest of the page.</li><br />
<li><b>Discussion Tags:</b> Users can quote something by encapsulating it in [quote][/quote] tags, they can create a link to another event with the [event][/event] tags or to another discussion thread with [thread][/thread].</li><br />
<li><b>Quick Post:</b> Adding a post will take the user to a separate add Post page, something that can be a bit of an inconvenience. To avoid this, there is a quick post button, where users can add messages on the fly without having to reload the page. On top of the quick post field, 3 buttons allow users to generate quote, event and thread tags quickly. </li>
</ul>
</ul>
<hr />
<h3>Event History:</h3>
View the logs of the event that show how the event has changed over time, including the contribution from other organisations in the form of proposals. There are two ways to get to this view, either by clicking on View Event History on the side menu of an event view, or by clicking on a contribing organisation's logo on the event view. The latter will show a restricted form of the logs, showing only Proposals created by the selected organisation. The fields shown in this view are as described as follows:<br /><br />
<ul>
<li><b>Org</b>: The logo (or in the lack thereof a string representation) of the organisation.</li><br />
<li><b>Action</b>: Each entry in the log happens during an action, such as the creation, modification or deletion of data and some special actions (such as accepting a proposal). This field shows which action caused the entry to be created. </li><br />
<li><b>Model</b>: As described above, a log entry is generated on certain actions. This field shows which type of data was affected that caused the log entry to be created (such as a change to the event, the creation of an attribute, the discarding of a proposal, etc).</li><br />
<li><b>Title</b>: This is a short description of the change itself and it is not nearly as detailed as the information administrators get in the audit logs. However, for attributes and proposals the category / type and value of the created or edited attribute is shown. </li><br />
<li><b>Created</b>: The date and time of the log entry's creation.</li>
</ul>
<hr />
<h3>Listing all attributes:</h3>
Apart from having a list of all the events, it is also possible to get a list of all the stored attributes in the system by clicking on the list attributes button. The produced list of attributes will include the followings fields:<br /><br />
<img src="/img/doc/list_attributes2.png" alt = "" title = "Use the buttons to the right to view the event that this attribute belongs to or to modify/delete the attribute."/><br /><br />
<ul>
<li><b>Event:</b> This is the ID number of the event that the attribute is tied to. If an event belongs to your organisation, then this field will be coloured red.<br /><br /></li>
<li><b>Org:</b> The organisation that has created the event.</li>
<li><b>Org:</b> The organisation that has created the event.</li><br />
<li><b>Category:</b> The category of the attribute, showing what the attribute describes (for example the malware's payload). For more information on categories, go to section xy<br /><br /></li>
<li><b>Type:</b> The type of the value contained in the attribute (for example a source IP address). For more information on types, go to section xy<br /><br /></li>
<li><b>Value:</b> The actual value of the attribute, describing an aspect, defined by the category and type fields of the malware (for example 11.11.11.11).<br /><br /></li>
<li><b>Signature:</b> Shows whether the attribute has been flagged for NIDS signature generation or not.<br /><br /></li>
<li><b>Comment:</b> An optional contextual comment attached to the attribute.<br /><br /></li>
<li><b>IDS:</b> Shows whether the attribute has been flagged for NIDS signature generation or not.<br /><br /></li>
<li><b>Actions:</b> A set of buttons that allow you to view the event that the attribute is tied to, to edit the attribute (using the same view as what is used to set up attributes, but filled out with the attribute's current data) and a delete button. <br /><br /></li>
</ul>
<hr />
<h3>Searching for attributes:</h3>
Apart from being able to list all events, it is also possible to search for data contained in the value field of an attribute, by clicking on the "Search Attributes" button.<br /><br />
<img src="/img/doc/search_attribute.png" alt = "Search attribute" title = "You can search for attributes by searching for a phrase contained in its value. Narrow your search down by selecting a type and/or a category which the event has to belong to."/><br /><br />
This will bring up a form that lets you enter one or several search strings (separate search strings with line breaks) that will be compared to the values of all attributes, along with options to narrow down the search based on category and type. The entered search string has to be an exact match with (the sub-string of) a value. A second text field makes it possible to enter event IDs for events that should be excluded from the search (again, each line represents an event ID to be excluded).<br /><br />
The list generated by the search will look exactly the same as listing all attributes, except that only the attributes that matched the search criteria will be listed (to find out more about the list attributes view, <?php echo $this->Html->link(__('click here', true), array('controller' => 'pages', 'action' => 'display', 'doc', 'categories_and_types')); ?>.). The search parameters will be shown above the produced list and the search terms will be highlighted.<br />
This will bring up a form that lets you enter one or several search strings (separate search strings with line breaks) that will be compared to the values of all attributes, along with options to narrow down the search based on category and type. The entered search string has to be an exact match with (the sub-string of) a value. A second text field makes it possible to enter event IDs for events that should be excluded from the search (again, each line represents an event ID to be excluded). The third text field allows the user to restrict the results to attributes from certain organisations or to attributes not created by certain other organisations, using the above described syntax.<br /><br />
The list generated by the search will look exactly the same as listing all attributes, except that only the attributes that matched the search criteria will be listed (to find out more about the list attributes view, <?php echo $this->Html->link(__('click here', true), array('controller' => 'pages', 'action' => 'display', 'doc', 'categories_and_types')); ?>.). The search parameters will be shown above the produced list and the search terms will be highlighted.<br /><br />
The last option is a checkbox that restricts all of the results to attributes that are marked as IDS signatures.<br />
<br /><img src="/img/doc/search_attribute_result.png" alt = "" title = "You can view the event that an attribute belongs to with the view button, or you can edit/delete the attribute via the buttons on the right."/><br />
<hr />
<a id="update_events"></a><h2>Updating and modifying events and attributes:</h2>
@ -199,26 +273,29 @@ By default, the message will be sent to every member of the organisation that po
<hr />
<a id="automation"></a><h2>Automation:</h2>
It is possible to quickly and conveniently export the data contained within the system using the automation features located in the main menu on the left (available to users with authentication key access only). There are various sets of data that can be exported, by using the authentication key provided by the system (also shown on the export page). If for whatever reason you would need to invalidate your current key and get a new one instead (for example due to the old one becoming compromised) just hit the reset link next to the authentication key in the export view or in your "my profile" view.<br /><br />
The following types of export are possible:<br /><br />
<h3>XML export:</h3>
Exports all attributes and the event data of every single event in the database (that you are eligible to see) in the XML format. The usage is:<br /><br /><i>&lt;server&gt;/events/xml/&lt;authentication_key&gt;</i><br /><br />
In order to export the data about a single event and its attributes, use the following syntax:<br /><br />
<i>&lt;server&gt;/events/xml/&lt;authentication_key&gt;/&lt;EventID&gt;</i><br /><br />
<h3>NIDS export:</h3>
This allows the user to export all network related attributes (that you are eligible to see) under the Snort format. The attributes have to belong to a published event and they have to have IDS signature generation enabled. The types that will be used when creating the export are: email-dst, ip-src, ip-dst, snort, url, domain. The usage is as follows:<br /><br /><i>&lt;server&gt;/events/nids/&lt;authentication_key&gt;</i><br /><br />
<h3>Hash database export:</h3>
There are two hash formats (sha1 and md5) in which all filenames stored in the system can be exported. Events need to be published and the IDS Signature field needs to be turned on for this export. The usage is as follows:<br /><br />
For MD5: <i>&lt;server&gt;events/hids_md5/&lt;authentication_key&gt;</i><br /><br />
For SHA1: <i>&lt;server&gt;events/hids_sha1/&lt;authentication_key&gt;</i><br /><br />
<h3>Text export:</h3>
It is also possible to export a list of all eligible attributes that match a specific type into a plain text file. The format to do this is:<br /><br />
<i>&lt;server&gt;/events/text/&lt;authentication_key&gt;/&lt;type&gt;</i><br /><br />
Type could be any valid type (as according to the list of <?php echo $this->Html->link(__('categories and types', true), array('controller' => 'pages', 'action' => 'display', 'doc', 'categories_and_types')); ?>), for example md5, ip-src or comment.<br />
To find out about the various export formats and the usage within the automation functions, please read the page on <a href="/events/automation">automation</a>.
<hr />
<a id="export"></a><h2>Exporting data:</h2>
For users that do not have authentication key access, an alternate export feature is available that relies on your interactive login to the site. To access these, just use the automation menu button to the left and you'll be presented with a list of export options. The results of the export will automatically be offered as a file download.<br /><br/>
<br /><img src="/img/doc/export.png" alt = "" title = "Use the export features here to quickly download data in various formats"/><br /><br />
Apart from that, it's also possible to export all events involved in a search attribute result table, by using the "Download results as XML" button on the left menu bar. <br /><br />
For users that do not have authentication key access, an alternate export feature is available that relies on your interactive login to the site. To access these, just use the export menu button to the left and you'll be presented with a list of export options.<br /><br/>
Depending on your server's configuration, you will be presented with one of two possible pages, depending on whether you have background processing enabled or not. (The setting on this instance is currently set to: <code><?php echo (Configure::read('MISP.background_jobs') == true ? 'On' : 'Off'); ?></code>)<br /><br />
<br /><h4>Export page with background jobs <code>disabled</code></h4>
The page will list a set of export formats that you can immediately download as a file. Just click on the desired export format and MISP will start collecting all the data that you will receive in a file. Keep in mind that this can be a lengthy process. To avoid having to wait, consult with your instance's site administrator about enabling the background processing.<br />
<br /><img src="/img/doc/export.png" alt = "" title = "Use the export features here to quickly download data in various formats"/><br />
<br /><h4>Export page with background jobs <code>enabled</code></h4>
If the background jobs are enabled, you'll be redirected to a different version of the export page. Here you will see a table with all of the major export formats and the current status of the cached export files. Keep in mind that these are generated on an organisation by organisation basis, so even though others have generated newer export caches your organisation may have an outdated cache. You can simply issue a generate command (by clicking the "Generate" button) on the desired export type and the background workers will start fetching and assembling your cache. A progress bar will show the progress of the export process.<br />
Once done, you can click "Download" to download the freshly generated cache file. If the cache is already up to date from before, then you don't have to regenerate the cache, just click on the "download" button. <br /> <br />
You may have noticed that the TEXT export only has a generate button - this is because TEXT exports are made up of a lot of types of exports, all of which get generated together. To download any of these files, just click on any of the attribute types at the bottom of the table.</br /><br />
A quick description of each of the fields in the table:<br /><br />
<ul>
<li><b>Type</b>: The type of the export (such as XML, Suricata, MD5, etc.).</li><br />
<li><b>Last Update</b>: The generation date of the current cache for the given export type.</li><br />
<li><b>Description</b>: A description of the export format. </li><br />
<li><b>Outdated</b>: This compares the cache generation date to the last timestamp when an event was updated and lets you know whether the cache is outdated or not. </li><br />
<li><b>Progress</b>: Shows the progress of the last initiated generation process.</li><br />
<li><b>Actions</b>: Download or Generate the given cache with these buttons. </li><br />
</ul>
<br /><h4>Exporting search results and individual events</h4>
Apart from the options offered by the export pages, it's also possible to export all events involved in a search attribute result table, by using the "Download results as XML" button on the left menu bar. <br /><br />
<br /><img src="/img/doc/export_search.png" alt = "" title = "Download a .xml from all the events that are shown through an attribute in the search results."/><br /><br />
Each event's view has its own export feature, both as an XML export and as a .ioc file. To reach these features, just navigate to an event and use the appropriate buttons on the right side.<br /><br />
<br /><img src="/img/doc/export_event.png" alt = "" title = "Download a .xml or a .ioc of the event."/><br /><br />
@ -234,6 +311,8 @@ In order to share data with a remote server via pushes and pulls, you need to re
<li><b>Authkey:</b> The authentication key that you have received from the hosting organisation of the remote instance.<br /><br /></li>
<li><b>Push:</b> This check-box controls whether your server is allowed to push to the remote instance.<br /><br /></li>
<li><b>Pull:</b> This check-box controls whether your server can request to pull all data from the remote instance.<br /><br /></li>
<li><b>Self Signed:</b> Ticking this checkbox will allow syncing with instances using self-signed certificates.<br /><br /></li>
<li><b>Certificate File:</b> If the instance that you want to connect to has their entire own certificate chain, you can use this to import a .pem file with it and override CakePHP's standard root CA file.<br /><br /></li>
</ul>
<b>If you are an administrator</b>, trying to allow another instance to connect to your own, it is vital that two rules are followed when setting up a synchronisation account: <br /><br />
<ul>
@ -256,7 +335,7 @@ The platform is also <a href="http://en.wikipedia.org/wiki/Representational_stat
Use any HTTP compliant library to perform requests. However to make clear you are doing a REST request you need to either specify the Accept type to application/xml, or append .xml to the url<br /><br />
The following table shows the relation of the request type and the resulting action:<br /><br />
<table style="width:250px;" summary="">
<table style="width:350px;" summary="">
<colgroup>
<col width="18%">
<col width="34%">
@ -312,32 +391,37 @@ Authorization: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX</pre>
&lt;response&gt;
&lt;Event&gt;
&lt;id&gt;57&lt;/id&gt;
&lt;date&gt;2012-11-19&lt;/date&gt;
&lt;risk&gt;Undefined&lt;/risk&gt;
&lt;info&gt;Test&lt;/info&gt;
&lt;user_id&gt;1&lt;/user_id&gt;
&lt;published&gt;0&lt;/published&gt;
&lt;org&gt;NCIRC&lt;/org&gt;
&lt;date&gt;2014-03-04&lt;/date&gt;
&lt;threat_level_id&gt;1&lt;/threat_level_id&gt;
&lt;info&gt;Code monkey doing code monkey stuff&lt;/info&gt;
&lt;published&gt;1&lt;/published&gt;
&lt;uuid&gt;50aa54aa-f7a0-4d74-910d-10f0ff32448e&lt;/uuid&gt;
&lt;revision&gt;0&lt;/revision&gt;
&lt;private&gt;0&lt;/private&gt;
&lt;attribute_count&gt;0&lt;/attribute_count&gt;
&lt;communitie&gt;0&lt;/communitie&gt;
&lt;distribution&gt;This Community-only&lt;/distribution&gt;
&lt;attribute_count&gt;1&lt;/attribute_count&gt;
&lt;analysis&gt;1&lt;/analysis&gt;
&lt;timestamp&gt;1393327600&lt;/timestamp&gt;
&lt;distribution&gt;1&lt;/distribution&gt;
&lt;proposal_email_lock&gt;0&lt;/proposal_email_lock&gt;
&lt;orgc&gt;Iglocska&lt;/orgc&gt;
&lt;locked&gt;0&lt;/locked&gt;
&lt;publish_timestamp&gt;1393327600&lt;/publish_timestamp&gt;
&lt;Attribute&gt;
&lt;id&gt;9577&lt;/id&gt;
&lt;event_id&gt;123&lt;/event_id&gt;
&lt;category&gt;Artifacts dropped&lt;/category&gt;
&lt;type&gt;other&lt;/type&gt;
&lt;category&gt;Artifacts dropped&lt;/category&gt;
&lt;to_ids&gt;1&lt;/to_ids&gt;
&lt;uuid&gt;50aa54bd-adec-4544-b494-10f0ff32448e&lt;/uuid&gt;
&lt;revision&gt;1&lt;/revision&gt;
&lt;private&gt;0&lt;/private&gt;
&lt;cluster&gt;0&lt;/cluster&gt;
&lt;communitie&gt;0&lt;/communitie&gt;
&lt;value&gt;0&lt;/value&gt;
&lt;distribution&gt;0&lt;/distribution&gt;
&lt;event_id&gt;57&lt;/event_id&gt;
&lt;distribution&gt;1&lt;/distribution&gt;
&lt;timestamp&gt;1393327600&lt;/timestamp&gt;
&lt;comment&gt;This is an Attribute&lt;/comment&gt;
&lt;value&gt;Some_attribute&lt;/value&gt;
&lt;ShadowAttribute /&gt;
&lt;/Attribute&gt;
&lt;ShadowAttribute /&gt;
&lt;RelatedEvent /&gt;
&lt;/Event&gt;
&lt;xml_version&gt;2.2.0&lt;/xml_version&gt;
&lt;/response&gt;</pre>
<h4>Example - Add new Event</h4>
@ -348,101 +432,100 @@ Accept: application/xml
Authorization: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX</pre>
<p>And the request body:</p>
<pre>&lt;Event&gt;
&lt;id&gt;14&lt;/id&gt;
&lt;org&gt;ORG&lt;/org&gt;
&lt;date&gt;2012-11-26&lt;/date&gt;
&lt;risk&gt;Undefined&lt;/risk&gt;
&lt;info&gt;Event information&lt;/info&gt;
&lt;user_id&gt;1&lt;/user_id&gt;
&lt;alerted&gt;0&lt;/alerted&gt;
&lt;uuid&gt;4f8c2c4e-00dc-42c9-83ad-76e9ff32448e&lt;/uuid&gt;
&lt;private&gt;0&lt;/private&gt;
&lt;published&gt;0&lt;/published&gt;
&lt;id&gt;72&lt;/id&gt;
&lt;org&gt;NCIRC&lt;/org&gt;
&lt;date&gt;2014-03-04&lt;/date&gt;
&lt;threat_level_id&gt;1&lt;/threat_level_id&gt;
&lt;info&gt;Something concise&lt;/info&gt;
&lt;published&gt;1&lt;/published&gt;
&lt;uuid&gt;50aa54aa-f7a0-4d74-920d-10f0ff32448e&lt;/uuid&gt;
&lt;attribute_count&gt;1&lt;/attribute_count&gt;
&lt;analysis&gt;1&lt;/analysis&gt;
&lt;timestamp&gt;1393328991&lt;/timestamp&gt;
&lt;distribution&gt;1&lt;/distribution&gt;
&lt;proposal_email_lock&gt;0&lt;/proposal_email_lock&gt;
&lt;orgc&gt;Iglocska&lt;/orgc&gt;
&lt;locked&gt;0&lt;/locked&gt;
&lt;publish_timestamp&gt;1393329599&lt;/publish_timestamp&gt;
&lt;Attribute&gt;
&lt;id&gt;116&lt;/id&gt;
&lt;event_id&gt;14&lt;/event_id&gt;
&lt;type&gt;ip-dst&lt;/type&gt;
&lt;category&gt;Network activity&lt;/category&gt;
&lt;id&gt;9577&lt;/id&gt;
&lt;type&gt;other&lt;/type&gt;
&lt;category&gt;Artifacts dropped&lt;/category&gt;
&lt;to_ids&gt;1&lt;/to_ids&gt;
&lt;uuid&gt;4f8c2cc3-0410-4bf0-8559-5b9dff32448e&lt;/uuid&gt;
&lt;revision&gt;1&lt;/revision&gt;
&lt;private&gt;0&lt;/private&gt;
&lt;value&gt;1.1.1.111&lt;/value&gt;
&lt;category_order&gt;g&lt;/category_order&gt;
&lt;/Attribute&gt;
&lt;Attribute&gt;
&lt;id&gt;117&lt;/id&gt;
&lt;event_id&gt;14&lt;/event_id&gt;
&lt;type&gt;malware-sample&lt;/type&gt;
&lt;category&gt;Payload delivery&lt;/category&gt;
&lt;to_ids&gt;0&lt;/to_ids&gt;
&lt;uuid&gt;4f8c2d08-7e6c-4648-8730-50a7ff32448e&lt;/uuid&gt;
&lt;revision&gt;1&lt;/revision&gt;
&lt;private&gt;0&lt;/private&gt;
&lt;value&gt;.doc|3f6f1aaab6171925c81de9b34a8fcf8e&lt;/value&gt;
&lt;category_order&gt;c&lt;/category_order&gt;
&lt;data /&gt;
&lt;uuid&gt;50aa54bd-adec-4544-b412-10f0ff32448e&lt;/uuid&gt;
&lt;event_id&gt;57&lt;/event_id&gt;
&lt;distribution&gt;1&lt;/distribution&gt;
&lt;timestamp&gt;1393328991&lt;/timestamp&gt;
&lt;comment&gt;This is an Attribute&lt;/comment&gt;
&lt;value&gt;Some_attribute&lt;/value&gt;
&lt;ShadowAttribute /&gt;
&lt;/Attribute&gt;
&lt;ShadowAttribute /&gt;
&lt;RelatedEvent /&gt;
&lt;/Event&gt;</pre>
<p>The response you're going to get is the following data:</p>
<pre>
HTTP/1.1 100 Continue
HTTP/1.1 200 Continue
Date: Mon, 26 Nov 2012 14:17:11 GMT
Server: Apache/2.2.13 (Win32) PHP/5.2.10
X-Powered-By: PHP/5.2.10
Set-Cookie: CAKEPHP=deleted; expires=Sun, 27-Nov-2012 14:17:11 GMT; path=/
Set-Cookie: CAKEPHP=a4ok3lr5p9n5drqj27025i4le3; expires Mon, 26-Nov-2012 18:17:11 GMT; path=/; HttpOnly
Content-Length: 1466
Date: Tue, 04-Mar-2014 15:00:00
Server: Apache/2.2.22 (Ubuntu) PHP/5.4.9-4ubuntu2.3
X-Powered-By: PHP/5.4.9-4ubuntu2.3
Set-Cookie: CAKEPHP=deleted; expires=Wed, 05-Mar-2014 15:00:00 GMT; path=/
Set-Cookie: CAKEPHP=a4ok3lr5p9n5drqj27025i4le3; expires Tue, 04-Mar-2014 15:00:00 GMT; path=/; HttpOnly
Content-Length: 1 kB
Content-Type: application/xml
&lt;?xml version="1.0" encoding="UTF-8"&gt;
&lt;response&gt;
&lt;Event&gt;
&lt;id&gt;14&lt;/id&gt;
&lt;org&gt;ORG&lt;/org&gt;
&lt;date&gt;2012-11-26&lt;/date&gt;
&lt;risk&gt;Undefined&lt;/risk&gt;
&lt;info&gt;Event information&lt;/info&gt;
&lt;user_id&gt;1&lt;/user_id&gt;
&lt;published&gt;0&lt;/published&gt;
&lt;uuid&gt;4f8c2c4e-00dc-42c9-83ad-76e9ff32448e&lt;/uuid&gt;
&lt;revision&gt;0&lt;/revision&gt;
&lt;private&gt;0&lt;/private&gt;
&lt;attribute_count&gt;0&lt;/attribute_count&gt;
&lt;communitie&gt;0&lt;/communitie&gt;
&lt;distribution&gt;All communities&lt;/distribution&gt;
&lt;id&gt;76&lt;/id&gt;
&lt;org&gt;NCIRC&lt;/org&gt;
&lt;date&gt;2014-03-04&lt;/date&gt;
&lt;threat_level_id&gt;1&lt;/threat_level_id&gt;
&lt;info&gt;Something concise&lt;/info&gt;
&lt;published&gt;1&lt;/published&gt;
&lt;uuid&gt;50aa54aa-f7a0-4d74-920d-10f0ff32448e&lt;/uuid&gt;
&lt;attribute_count&gt;1&lt;/attribute_count&gt;
&lt;analysis&gt;1&lt;/analysis&gt;
&lt;timestamp&gt;1393328991&lt;/timestamp&gt;
&lt;distribution&gt;1&lt;/distribution&gt;
&lt;proposal_email_lock&gt;0&lt;/proposal_email_lock&gt;
&lt;orgc&gt;Iglocska&lt;/orgc&gt;
&lt;locked&gt;0&lt;/locked&gt;
&lt;publish_timestamp&gt;1393947960&lt;/publish_timestamp&gt;
&lt;Attribute&gt;
&lt;id&gt;116&lt;/id&gt;
&lt;event_id&gt;14&lt;/event_id&gt;
&lt;category&gt;Network activity&lt;/category&gt;
&lt;type&gt;ip-dst&lt;/type&gt;
&lt;id&gt;10462&lt;/id&gt;
&lt;type&gt;other&lt;/type&gt;
&lt;category&gt;Artifacts dropped&lt;/category&gt;
&lt;to_ids&gt;1&lt;/to_ids&gt;
&lt;uuid&gt;4f8c2cc3-0410-4bf0-8559-5b9dff32448e&lt;/uuid&gt;
&lt;revision&gt;1&lt;/revision&gt;
&lt;private&gt;0&lt;/private&gt;
&lt;cluster&gt;0&lt;/cluster&gt;
&lt;communitie&gt;0&lt;/communitie&gt;
&lt;value&gt;1.1.1.111&lt;/value&gt;
&lt;distribution&gt;All communities&lt;/distribution&gt;
&lt;category_order&gt;g&lt;/category_order&gt;
&lt;/Attribute&gt;
&lt;Attribute&gt;
&lt;id&gt;117&lt;/id&gt;
&lt;event_id&gt;14&lt;/event_id&gt;
&lt;category&gt;Payload delivery&lt;/category&gt;
&lt;type&gt;malware-sample&lt;/type&gt;
&lt;to_ids&gt;0&lt;/to_ids&gt;
&lt;uuid&gt;4f8c2d08-7e6c-4648-8730-50a7ff32448e&lt;/uuid&gt;
&lt;revision&gt;1&lt;/revision&gt;
&lt;private&gt;0&lt;/private&gt;
&lt;cluster&gt;0&lt;/cluster&gt;
&lt;communitie&gt;0&lt;/communitie
&lt;value&gt;.doc|3f6f1aaab6171925c81de9b34a8fcf8e&lt;/value&gt;
&lt;distribution&gt;All communities&lt;/distribution&gt;
&lt;category_order&gt;c&lt;/category_order&gt;
&lt;uuid&gt;50aa54bd-adec-4544-b412-10f0ff32448e&lt;/uuid&gt;
&lt;event_id&gt;76&lt;/event_id&gt;
&lt;distribution&gt;1&lt;/distribution&gt;
&lt;timestamp&gt;1393328991&lt;/timestamp&gt;
&lt;comment/&gt;
&lt;value&gt;Some_attribute&lt;/value&gt;
&lt;ShadowAttribute/&gt;
&lt;/Attribute&gt;
&lt;ShadowAttribute/&gt;
&lt;RelatedEvent&gt;
&lt;id&gt;75&lt;/id&gt;
&lt;org&gt;NCIRC&lt;/org&gt;
&lt;date&gt;2012-11-19&lt;/date&gt;
&lt;info&gt;Code monkey doing code monkey stuff&lt;/info&gt;
&lt;uuid&gt;50aa54aa-f7a0-4d74-910d-10f0ff32448e&lt;/uuid&gt;
&lt;published&gt;1&lt;/published&gt;
&lt;analysis&gt;1&lt;/analysis&gt;
&lt;attribute_count&gt;1&lt;/attribute_count&gt;
&lt;orgc&gt;Iglocska&lt;/orgc&gt;
&lt;timestamp&gt;1393327600&lt;/timestamp&gt;
&lt;distribution&gt;1&lt;/distribution&gt;
&lt;proposal_email_lock&gt;0&lt;/proposal_email_lock&gt;
&lt;locked&gt;0&lt;/locked&gt;
&lt;threat_level_id&gt;1&lt;/threat_level_id&gt;
&lt;publish_timestamp&gt;1393947655&lt;/publish_timestamp&gt;
&lt;/RelatedEvent&gt;
&lt;/Event&gt;
&lt;xml_version&gt;2.2.0&lt;/xml_version&gt;
&lt;/response&gt;
</pre>
<p>The respone from requesting an invalid page</p>
@ -450,7 +533,7 @@ Content-Type: application/xml
&lt;?xml version = "1.0" encoding = "UTF-8"?&gt;
&lt;response&gt;
&lt;name&gt;Not Found&lt;/name&gt;
&lt;url&gt;/Waldo/&lt;/url&gt;
&lt;url&gt;/The_meaning_of_life&lt;/url&gt;
&lt;/response&gt;
</pre>
</div>

View File

@ -195,7 +195,7 @@ div.message{
float: top;
color: #FFFFFF;
background-color: #C11B17;
border-radius: 7px;
border-radius: 10px;
box-shadow: 5px 5px 8px #B2B2B2;
}
@ -407,12 +407,25 @@ dd {
.logoBlue {
color: #0088cc;
text-shadow: #0088cc 0 0 5px;
}
.logoGray {
color: #999999;
}
.glass {
width: 100%;
height: 100%;
position: absolute;
padding: 0;
-webkit-border-radius: 10px;
-moz-border-radius: 10px;
border-radius: 10px;
background: rgba(0,0,0,0.25);
box-shadow: 0 2px 6px rgba(0,0,0,0.5), inset 0 1px rgba(255,255,255,0.3), inset 0 10px rgba(255,255,255,0.1), inset 0 10px 20px rgba(255,255,255,0.3), inset 0 -15px 30px rgba(0,0,0,0.3);
}
.footerText {
color: #999999;
}

View File

@ -1,5 +1,5 @@
curl -i -H "Accept: application/xml" -H "content-type: text/xml" -H "Authorization: FoKoqC6hr5Ke3Sb81iDBVuvem9HSCRAhflGFXMXp" \
--data "@input/event.xml" -X POST http://192.168.14.10:3600/events
curl -i -H "Accept: application/xml" -H "content-type: text/xml" -H "Authorization: vlf4o42bYSVVWLm28jLB85my4HBZWXTri8vGdySb" \
--data "@input/event.xml" -X POST http://localhost/events
#curl -i -H "Accept: application/json" -H "content-type: text/json" -H "Authorization: vlf4o42bYSVVWLm28jLB85my4HBZWXTri8vGdySb" \
#--data "@input/event.json" -X POST http://localhost/events