MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

Added function capability to be applied on value + better input parsing 

and output generation
megaRefact
Sami Mokaddem 2018-08-27 06:35:10 +00:00
parent 76c7b5fb10
commit 61d01176b7
2 changed files with 266 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
static/js/proxyMapper.js
View File

@ -12,14 +12,18 @@
var ProxyMapper = function(mapping, data, options) { var ProxyMapper = function(mapping, data, options) {
this.mapping = mapping; this.mapping = mapping;
this.data = data; this.data = data;
console.log(options);
this._default_options = { this._default_options = {
fillValue: 0, fillValue: 0,
functions: { functions: {
dates: function (value) {return value;}, dates: function (value, datum) {return value;},
labels: function (value) {return value;}, labels: function (value, datum) {return value;},
values: function (value) {return value;} values: function (value, datum) {return value;}
} },
prefillData: {
dates: [],
labels: []
},
datum: false // the tree data to walk in parallel
}; };
this.options = $.extend({}, this._default_options, options); this.options = $.extend({}, this._default_options, options);
this.result = {}; this.result = {};
@ -36,19 +40,33 @@
perform_mapping: function(data) { perform_mapping: function(data) {
if (this.mapping.dates.length > 0) { if (this.mapping.dates.length > 0) {
for (var x of this.options.prefillData.dates) { this.result['dates'].push(x); }
this.c_dates(this.data, this.mapping.dates); // probe and fetch all dates this.c_dates(this.data, this.mapping.dates); // probe and fetch all dates
} }
let fillArray = [];
for (var i=0; i<this.result.dates.length; i++) {
if ((this.options.fillValue !== undefined && this.options.fillValue !== '')) {
fillArray.push(this.options.fillValue);
}
}
for (var x of this.options.prefillData.labels) {
this.result[x] = fillArray.slice(0);
this.i1_prefill = x;
}
if (this.mapping.labels.length > 0) { if (this.mapping.labels.length > 0) {
this.c_labels(this.data, this.mapping.labels); // probe and fetch all labels this.c_labels(this.data, this.mapping.labels); // probe and fetch all labels
} }
if (this.mapping.labels.length > 0 && this.mapping.values.length > 0) {
//if (this.mapping.labels.length > 0 && this.mapping.values.length > 0) {
if (Object.keys(this.result).length > 1 && this.mapping.values.length > 0) {
this.c_values(this.data, this.mapping.values); // fetch values and overwrite default values this.c_values(this.data, this.mapping.values); // fetch values and overwrite default values
for (var k in this.result) { for (var k in this.result) {
this.result[k] = this.result[k].filter(function(n){ return n != undefined }); this.result[k] = this.result[k].filter(function(n){ return n != undefined });
} }
} }
}, },
c_dates: function(intermediate, instructions) { c_dates: function(intermediate, instructions) {
var that = this; var that = this;
var matchingFun = function (intermediate, instructions, additionalData) { var matchingFun = function (intermediate, instructions, additionalData) {
@ -56,27 +74,32 @@
let val = intermediate[index]; let val = intermediate[index];
if (that.mappingI2[val] === undefined) { if (that.mappingI2[val] === undefined) {
that.mappingI2[val] = that.result['dates'].length; that.mappingI2[val] = that.result['dates'].length;
let nval = that.options.functions.dates(val); let nval = that.options.functions.dates(val, additionalData.datum);
that.result['dates'].push(nval); that.result['dates'].push(nval);
} }
}; };
this.iter(intermediate, instructions, matchingFun, {}); this.iter(intermediate, instructions, matchingFun, { datum: this.options.datum });
}, },
c_labels: function(intermediate, instructions, valuesLength) { c_labels: function(intermediate, instructions, valuesLength) {
var that = this; var that = this;
var matchingFun = function (intermediate, instructions, additionalData) { var matchingFun = function (intermediate, instructions, additionalData) {
let reg = /\{(\w+)\}/;
let res = reg.exec(instructions);
if (res !== null) {
instructions = res[1];
}
let index = instructions; let index = instructions;
if (index == 'l') { // labels are the key themself if (index == 'l') { // labels are the keys themself
for (let label in intermediate) { for (let label in intermediate) {
let val = []; let val = [];
for (var i=0; i<additionalData.valueLength; i++) { for (var i=0; i<additionalData.valueLength; i++) {
if ((that.options.fillValue !== undefined && that.options.fillValue != '')) { if ((that.options.fillValue !== undefined && that.options.fillValue !== '')) {
val.push(that.options.fillValue); val.push(that.options.fillValue);
} }
} }
let nval = that.options.functions.dates(val); let nlabel = that.options.functions.labels(label, additionalData.datum);
that.result[label] = nval; that.result[nlabel] = val;
} }
} else { } else {
let label = intermediate[index]; let label = intermediate[index];
@ -86,26 +109,37 @@
val.push(that.options.fillValue); val.push(that.options.fillValue);
} }
} }
let nlabel = that.options.functions.labels(label); let nlabel = that.options.functions.labels(label, additionalData.datum);
that.result[nlabel] = val; that.result[nlabel] = val;
} }
}; };
this.iter(intermediate, instructions, matchingFun, {valueLength: this.result.dates.length}); this.iter(intermediate, instructions, matchingFun, {valueLength: this.result.dates.length, datum: this.options.datum});
}, },
c_values: function(intermediate, instructions) { c_values: function(intermediate, instructions) {
var that = this; var that = this;
var matchingFun = function (intermediate, instructions, additionalData) { var matchingFun = function (intermediate, instructions, additionalData) {
let index = instructions; let val;
let val = intermediate[index]; if (!instructions) { // value is self (intermediate)
val = intermediate;
} else {
let reg = /\{(\w+)\}/;
let res = reg.exec(instructions);
if (res !== null) {
instructions = res[1];
}
let index = instructions;
val = intermediate[index];
}
let i1 = additionalData.i1; let i1 = additionalData.i1;
i1 = i1 !== undefined ? i1 : that.i1_prefill;
let i2 = additionalData.i2; let i2 = additionalData.i2;
let i2_adjusted = that.mappingI2[i2]; let i2_adjusted = that.mappingI2[i2];
let ni1 = that.options.functions.labels(i1); let ni1 = that.options.functions.labels(i1, additionalData.datum);
let nval = that.options.functions.values(val); let nval = that.options.functions.values(val, additionalData.datum);
that.result[ni1][i2_adjusted] = nval; that.result[ni1][i2_adjusted] = nval;
}; };
this.iter(intermediate, instructions, matchingFun, {mapping: this.mapping}); this.iter(intermediate, instructions, matchingFun, {mapping: this.mapping, datum: this.options.datum});
}, },
// deterministic function, always follow the indexes // deterministic function, always follow the indexes
@ -119,16 +153,22 @@
}, },
iter: function(intermediate, instructions, matchingFun, additionalData) { iter: function(intermediate, instructions, matchingFun, additionalData) {
if (instructions === undefined || instructions.length == 0) { if (instructions === undefined) {
return; return;
} }
if (instructions.length == 0 || instructions[0] === '') {
return matchingFun(intermediate, false, additionalData);
}
var flag_register_i = false; var flag_register_i = false;
var i_type; var i_type;
if (instructions.length == 1) { if (instructions.length == 1) {
matchingFun(intermediate, instructions[0], additionalData); return matchingFun(intermediate, instructions[0], additionalData);
} else { } else {
switch (instructions[0]) { let tmp = new String(instructions[0]).split(',');
let record_inst = tmp[0]
let ind_inst = tmp.length == 2 ? tmp[1] : tmp[0];
switch (record_inst) {
case 'i1': case 'i1':
if (additionalData.mapping) { if (additionalData.mapping) {
flag_register_i = true; flag_register_i = true;
@ -141,13 +181,32 @@
i_type = 'i2'; i_type = 'i2';
} }
break; break;
case 'l':
break;
case '': case '':
break; break;
default: default:
break; break;
} }
let inst = ind_inst;
let reg = /\{(\w+)\}/;
let res = reg.exec(inst);
if (res !== null) { // check if index requested
let i = res[1];
if (flag_register_i) {
let sub_instructions = additionalData.mapping.index[i_type]
let curI;
if (sub_instructions.length > 0) {
curI = this.fetch_value(intermediate[i], sub_instructions);
} else {
curI = i;
}
additionalData[i_type] = curI;
}
additionalData.datum = this.update_datum(additionalData.datum, i);
return this.iter(intermediate[i], instructions.slice(1), matchingFun, additionalData);
}
// fallback to standard loop
} }
if (!(Array.isArray(intermediate) || this.isObject(intermediate))) { if (!(Array.isArray(intermediate) || this.isObject(intermediate))) {
@ -155,7 +214,8 @@
} }
if (Array.isArray(intermediate)) { if (Array.isArray(intermediate)) {
for (var node of intermediate) { for (var k=0; k<intermediate.length; k++) {
var node = intermediate[k];
if (flag_register_i) { if (flag_register_i) {
let sub_instructions = additionalData.mapping.index[i_type] let sub_instructions = additionalData.mapping.index[i_type]
let curI; let curI;
@ -166,6 +226,8 @@
} }
additionalData[i_type] = curI; additionalData[i_type] = curI;
} }
// update datum object
additionalData.datum = this.update_datum(additionalData.datum, k);
this.iter(node, instructions.slice(1), matchingFun, additionalData); this.iter(node, instructions.slice(1), matchingFun, additionalData);
} }
} else if (this.isObject(intermediate)) { } else if (this.isObject(intermediate)) {
@ -181,6 +243,7 @@
} }
additionalData[i_type] = curI; additionalData[i_type] = curI;
} }
additionalData.datum = this.update_datum(additionalData.datum, k, true);
this.iter(node, instructions.slice(1), matchingFun, additionalData); this.iter(node, instructions.slice(1), matchingFun, additionalData);
} }
} }
@ -188,6 +251,27 @@
isObject: function(v) { isObject: function(v) {
return v !== null && typeof v === 'object'; return v !== null && typeof v === 'object';
},
update_datum: function(d, k, should_look_into_linkname) {
if (!d) { // no datum, ignoring update
return;
} else if (d.children == undefined) {
return d;
}
var next;
if (should_look_into_linkname) {
for (var n in d.children) {
var c = d.children[n];
if (c.linkname == k) {
next = c;
break;
}
}
} else {
next = d.children[k];
}
return next;
} }
}; };

191
static/js/treeFromJson.js
View File

@ -65,9 +65,11 @@
if (this.options.toBeMapped.length > 0 ) { if (this.options.toBeMapped.length > 0 ) {
this.instructions = {}; this.instructions = {};
this.prefillData = {};
var that = this; var that = this;
this.options.toBeMapped.forEach(function(item, index) { this.options.toBeMapped.forEach(function(item, index) {
that.instructions[item] = []; that.instructions[item] = [];
that.prefillData[item] = [];
that.itemColors.set(item, that.options.itemColors[index]); that.itemColors.set(item, that.options.itemColors[index]);
}); });
@ -142,7 +144,7 @@
var nodeEnterNotArray = nodeEnter.filter(function(d) { var nodeEnterNotArray = nodeEnter.filter(function(d) {
var not_add = d.additionalNode === undefined || !d.additionalNode; var not_add = d.additionalNode === undefined || !d.additionalNode;
var not_arr = d.children === undefined || d.children[0].linkname === undefined; var not_arr = d.children === undefined || d.children[0].linkname === undefined || d.children[0].linkname !== '';
return not_add && not_arr; return not_add && not_arr;
}); });
nodeEnterNotArray nodeEnterNotArray
@ -152,8 +154,8 @@
var nodeEnterArray = nodeEnter.filter(function(d) { var nodeEnterArray = nodeEnter.filter(function(d) {
var not_add = d.additionalNode === undefined || !d.additionalNode; var not_add = d.additionalNode === undefined || !d.additionalNode;
var is_arr = d.children !== undefined && d.children[0].linkname !== undefined; var not_arr = d.children === undefined || d.children[0].linkname === undefined || d.children[0].linkname !== '';
return not_add && is_arr; return not_add && !not_arr;
}); });
nodeEnterArray nodeEnterArray
.append("rect") .append("rect")
@ -319,8 +321,8 @@
var children = par.children; var children = par.children;
for (var i=0; i<children.length; i++) { for (var i=0; i<children.length; i++) {
if (children[i].id == c_id) { if (children[i].id == c_id) {
return i; var isObj = child.linkname !== undefined && child.linkname !== '';
break; return isObj ? child.linkname : i;
} }
} }
}, },
@ -356,7 +358,18 @@
return false; return false;
} }
var c1 = d.depth == o_depth; var c1 = d.depth == o_depth;
var c2 = d.parent.id - c_index -1 == d.id; //var c2 = d.parent.id - c_index -1 == d.id;
var c21 = d.parent.id - c_index -1 == d.id
// consider linkname if label has been picked manually
let il_last = that.instructions.labels.length-1;
var labelIsManual = that.instructions.labels[il_last] != 'l';
var c22 = true;
if (labelIsManual) {
c22 = d.linkname === c_index;
} else {
}
var c2 = c21 || c22;
var notClicked = d.id != c_id; var notClicked = d.id != c_id;
return c1 && c2; return c1 && c2;
}); });
@ -364,17 +377,73 @@
// check if children are leaf // check if children are leaf
var child = clicked.data()[0].children[0]; var child = clicked.data()[0].children[0];
if (that.isObject(child) || Array.isArray(child)) { // children are not leaves if (that.isObject(child) || Array.isArray(child)) { // children are not leaves
// First child is not a node, should highlight the label instead // First child is not a node, should highlight all labels instead
// --> simulate label click
let source = clicked.data()[0]; var itemColor = this.itemColors.get(this.currentPicking);
let target = child; var resRect = this.svg.selectAll(".rectText")
if (target.linkname !== undefined && target.linkname !== '') { .filter(function(d) {
var resL = this.svg.selectAll("path.link").filter(function(d) { if (d.depth == 0) {
return d.source.id == source.id && d.target.id == target.id; return false;
}
var c1 = d.source.depth == o_depth;
return c1;
});
var resText = this.svg.selectAll(".linkText")
.filter(function(d) {
if (d.depth == 0) {
return false;
}
var c1 = d.source.depth == o_depth;
return c1;
});
resRect.data().forEach(function(elem) {
if (elem.picked !== undefined && elem.picked != '') {
console.log('Possible collision with '+elem.picked);
}
elem.picked = that.currentPicking;
});
resRect.style('fill', itemColor)
resText.style('fill', that.should_invert_text_color(itemColor) ? 'white' : 'black');
resCircle = that.svg.selectAll(".node circle")
.filter(function(d) {
return d.parent.depth == clicked.data()[0].depth;
//return d.parent !== null && d.parent.id == clicked.data()[0].id;
});
var nodesData = [];
if(resCircle !== undefined) {
resCircle.data().forEach(function(elem) {
if (elem.picked !== undefined && elem.picked != '') {
// alert || repick conflicting ????
console.log('Possible collision with '+elem.picked);
//alert('Possible collision with '+elem.picked);
}
elem.picked = that.currentPicking;
nodesData.push(elem);
}); });
that.clickLabel(resL.data()[0]);
} }
// find all paths
var paths = [];
nodesData.forEach(function(d, i) {
paths[i] = that.find_full_path(d, []);
});
var instructions = this.compute_mapping_instructions(paths);
this.add_instruction(instructions);
return; return;
//let source = clicked.data()[0];
//let target = child;
//if (target.linkname !== undefined && target.linkname !== '') {
// var resL = this.svg.selectAll("path.link").filter(function(d) {
// return d.source.id == source.id && d.target.id == target.id;
// });
// that.clickLabel(resL.data()[0]);
//}
//return;
} else { // children are leaves } else { // children are leaves
resCircle = that.svg.selectAll(".node circle") resCircle = that.svg.selectAll(".node circle")
.filter(function(d) { .filter(function(d) {
@ -434,6 +503,7 @@
var that = this; var that = this;
var o_depth = d.source.depth; var o_depth = d.source.depth;
var dest_depth = d.target.depth; var dest_depth = d.target.depth;
var c_label = d.target.linkname;
var c_id = d.source.id; var c_id = d.source.id;
var c_index; // no index as the index is the label itself var c_index; // no index as the index is the label itself
var itemColor = this.itemColors.get(this.currentPicking); var itemColor = this.itemColors.get(this.currentPicking);
@ -447,7 +517,8 @@
return false; return false;
} }
var c1 = d.source.depth == o_depth; var c1 = d.source.depth == o_depth;
return c1; var c2 = d.target.linkname === c_label;
return c1 && c2;
}); });
var resText = this.svg.selectAll(".linkText") var resText = this.svg.selectAll(".linkText")
.filter(function(d) { .filter(function(d) {
@ -455,7 +526,8 @@
return false; return false;
} }
var c1 = d.source.depth == o_depth; var c1 = d.source.depth == o_depth;
return c1; var c2 = d.target.linkname === c_label;
return c1 && c2;
}); });
@ -473,15 +545,28 @@
// find all paths // find all paths
var paths = []; var paths = [];
var nodes = that.svg.selectAll(".node circle").filter( var nodesCircle = that.svg.selectAll(".node circle").filter(
function(d) { return d.depth == dest_depth;} function(d) {
return d.depth == dest_depth && d.linkname == c_label;
//return d.depth == dest_depth;
}
); );
nodesCircle.data().forEach(function(d, i) {
nodes.data().forEach(function(d, i) {
paths[i] = that.find_full_path(d, []); paths[i] = that.find_full_path(d, []);
}); });
var instructions = this.compute_mapping_instructions(paths); var nodesRect = that.svg.selectAll(".node rect").filter(
this.add_instruction(instructions); function(d) {
return d.depth == dest_depth && d.linkname == c_label;
//return d.depth == dest_depth;
}
);
nodesRect.data().forEach(function(d, i) {
paths[i] = that.find_full_path(d, []);
});
//var instructions = this.compute_mapping_instructions(paths);
//this.add_instruction(instructions);
this.add_prefill_data([c_label]);
}, },
@ -510,7 +595,7 @@
elem.picked = ''; elem.picked = '';
}); });
this.add_instruction(''); this.add_instruction([]);
}, },
@ -524,7 +609,7 @@
if (prevVal === null) { if (prevVal === null) {
prevVal = arr[i]; prevVal = arr[i];
} else { } else {
if (prevVal != arr[i]) { // value different, nood to loop over them if (prevVal != arr[i]) { // value different, need to loop over them
instruction = 'l' instruction = 'l'
break; break;
} }
@ -617,7 +702,7 @@
set_current_mapping_item: function(name) { set_current_mapping_item: function(name) {
if (name === undefined) { if (name === undefined) {
for (var entry of this.options.toBeMapped) { for (var entry of this.options.toBeMapped) {
if (this.instructions[entry].length == 0) { if (this.instructions[entry].length == 0 && this.prefillData[entry].length == 0) {
name = entry; name = entry;
break; break;
} }
@ -645,6 +730,13 @@
this.update_result_tree(); this.update_result_tree();
}, },
add_prefill_data: function(data) {
this.prefillData[this.currentPicking] = data;
this.currentPickingCell.text(data.toString());
this.set_current_mapping_item();
this.update_result_tree();
},
// destroy and redraw // destroy and redraw
update_result_tree: function() { update_result_tree: function() {
var options = { var options = {
@ -661,13 +753,15 @@
$('.mappingTable textarea').each(function() { $('.mappingTable textarea').each(function() {
var dom = $(this); var dom = $(this);
var f_body = dom.val(); var f_body = dom.val();
functions[dom[0].id] = new Function('value', 'd', f_body); functions[dom[0].id] = new Function('value', 'datum', f_body);
}); });
// perform mapping // perform mapping
var pm_options = { var pm_options = {
fillValue: this.fillValueDomInput.val(), fillValue: this.fillValueDomInput.val(),
functions: functions functions: functions,
datum: this.root,
prefillData: this.prefillData
}; };
var adjustedInstructions = this.adjust_instruction(); var adjustedInstructions = this.adjust_instruction();
var result = new $.proxyMapper(adjustedInstructions, this.data, pm_options); var result = new $.proxyMapper(adjustedInstructions, this.data, pm_options);
@ -687,13 +781,21 @@
// label & value // label & value
if (l.length != 0 && v.length != 0) { if (l.length != 0 && v.length != 0) {
var smaller_array = v.length < l.length ? v : l; var smaller_array = v.length < l.length ? v : l;
var has_matched = false;
for (var i=0; i<smaller_array.length; i++) { for (var i=0; i<smaller_array.length; i++) {
if (v[i] != l[i]) { if (v[i] != l[i]) {
matchingIndex = i-1; matchingIndex = i-1;
has_matched = true;
break; break;
} }
} }
adjustedInstructions.values[matchingIndex] = 'i1'; // in case no match, last one should be registered
matchingIndex = has_matched ? matchingIndex : smaller_array.length-1;
//adjustedInstructions.values[matchingIndex] = 'i1,l';
//adjustedInstructions.values[matchingIndex] = 'i1,'+adjustedInstructions.values[matchingIndex];
let inst = adjustedInstructions.values[matchingIndex];
inst = inst == 'l' ? 'l' : '{'+inst+'}';
adjustedInstructions.values[matchingIndex] = 'i1,'+inst;
adjustedInstructions.index['i1'] = adjustedInstructions.labels.slice(matchingIndex+1); adjustedInstructions.index['i1'] = adjustedInstructions.labels.slice(matchingIndex+1);
} }
@ -707,8 +809,18 @@
break; break;
} }
} }
adjustedInstructions.values[matchingIndex] = 'i2'; //adjustedInstructions.values[matchingIndex] = 'i2,l';
adjustedInstructions.values[matchingIndex] = 'i2,'+adjustedInstructions.values[matchingIndex];
adjustedInstructions.index['i2'] = adjustedInstructions.dates.slice(matchingIndex+1); adjustedInstructions.index['i2'] = adjustedInstructions.dates.slice(matchingIndex+1);
// add '' at the end for value only
var end_i = adjustedInstructions.values.length-1;
var last_i = adjustedInstructions.values[end_i];
last_i = last_i.split(',');
last_i = last_i.length == 2 ? last_i[1] : last_i[0];
if (last_i == 'l') {
adjustedInstructions.values[end_i+1] = '';
}
} }
return adjustedInstructions; return adjustedInstructions;
@ -724,14 +836,21 @@
var funXOuput = $('#funXOuput-'+c_id); var funXOuput = $('#funXOuput-'+c_id);
// check if valid function // check if valid function
try { try {
var f = new Function('value', 'd', f_body); var f = new Function('value', 'datum', f_body);
var nodes = that.svg.selectAll(".node circle").filter( var nodes = that.svg.selectAll(".node, .rectText").filter(
function(d) { return d.picked === c_id;} function(d) { return d.picked === c_id;}
); );
var x = nodes.data()[0].name; // fetch first name occurence
var d = nodes.data()[0];
var x;
if (d.source !== undefined && d.target !== undefined) { // is a link label
x = d.target.linkname;
} else {
x = d.name;
}
funXInput.text('"'+that.adjust_text_length(x)+'"'); funXInput.text('"'+that.adjust_text_length(x)+'"');
funXInput[0].innerHTML = '"'+that.adjust_text_length(x)+'"'; funXInput[0].innerHTML = '"'+that.adjust_text_length(x)+'"';
funXOuput[0].innerHTML = that.adjust_text_length('"'+f(x)+'"'); funXOuput[0].innerHTML = that.adjust_text_length('"'+f(x, d)+'"');
} catch(err) { // Error } catch(err) { // Error
if (err.name == 'SyntaxError') { if (err.name == 'SyntaxError') {
flag_continue = false; flag_continue = false;
@ -784,7 +903,8 @@
if (root.length > maxWidth) { if (root.length > maxWidth) {
var addNode = {}; var addNode = {};
var remaining = root.length - maxWidth; var remaining = root.length - maxWidth;
addNode.name = ''+remaining+'...'; //addNode.name = ''+remaining+'...';
addNode.name = '['+remaining+' more]';
addNode.parent = null; addNode.parent = null;
addNode.additionalNode = true; addNode.additionalNode = true;
child['children'].push(addNode); child['children'].push(addNode);
@ -805,7 +925,8 @@
if (Object.keys(root).length > maxWidth) { if (Object.keys(root).length > maxWidth) {
var addNode = {}; var addNode = {};
var remaining = root.length - maxWidth; var remaining = root.length - maxWidth;
addNode.name = ''+remaining+' ...'; //addNode.name = ''+remaining+' ...';
addNode.name = '['+remaining+' more]';
addNode.parent = null; addNode.parent = null;
addNode.additionalNode = true; addNode.additionalNode = true;
child.children.push(addNode); child.children.push(addNode);