mirror of https://github.com/CIRCL/lookyloo
new: Add screenshot thumbnail on tree, move links to the end of the node
Raphaël Vinot
parent
ba7c0f4f2e
commit
3c6eca3567
|
|
@ -24,6 +24,7 @@ from defang import refang # type: ignore
|
|||
import dns.resolver
|
||||
import dns.rdatatype
|
||||
from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode
|
||||
from PIL import Image
|
||||
from pymisp import MISPEvent
|
||||
from pymisp.tools import URLObject, FileObject
|
||||
from redis import Redis
|
||||
|
|
@ -650,6 +651,18 @@ class Lookyloo():
|
|||
def get_screenshot(self, capture_uuid: str, all_images: bool=False) -> BytesIO:
|
||||
return self._get_raw(capture_uuid, 'png', all_images)
|
||||
|
||||
def get_screenshot_thumbnail(self, capture_uuid: str, all_images: bool=False, for_datauri=False) -> Union[str, BytesIO]:
|
||||
size = 64, 64
|
||||
screenshot = Image.open(self._get_raw(capture_uuid, 'png', False))
|
||||
c_screenshot = screenshot.crop((0, 0, screenshot.width, screenshot.width))
|
||||
c_screenshot.thumbnail(size)
|
||||
to_return = BytesIO()
|
||||
c_screenshot.save(to_return, 'png')
|
||||
if for_datauri:
|
||||
return base64.b64encode(to_return.getvalue()).decode()
|
||||
else:
|
||||
return to_return
|
||||
|
||||
def get_capture(self, capture_uuid: str) -> BytesIO:
|
||||
return self._get_raw(capture_uuid)
|
||||
|
||||
|
|
|
|||
|
|
@ -596,6 +596,14 @@ category = "dev"
|
|||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[[package]]
|
||||
name = "pillow"
|
||||
version = "8.0.1"
|
||||
description = "Python Imaging Library (Fork)"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[[package]]
|
||||
name = "prompt-toolkit"
|
||||
version = "3.0.8"
|
||||
|
|
@ -1127,7 +1135,7 @@ misp = ["python-magic", "pydeep"]
|
|||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.7"
|
||||
content-hash = "bd69eeeb9e624116448b67542215105ea09782d51240ec7ba4769adf77cdc521"
|
||||
content-hash = "f35e1b5ea6f84ceadfe782d669bec094e9d486e30f4541b3c46390de46b7caa6"
|
||||
|
||||
[metadata.files]
|
||||
aiohttp = [
|
||||
|
|
@ -1586,6 +1594,36 @@ pickleshare = [
|
|||
{file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"},
|
||||
{file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"},
|
||||
]
|
||||
pillow = [
|
||||
{file = "Pillow-8.0.1-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3"},
|
||||
{file = "Pillow-8.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302"},
|
||||
{file = "Pillow-8.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c"},
|
||||
{file = "Pillow-8.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11"},
|
||||
{file = "Pillow-8.0.1-cp36-cp36m-win32.whl", hash = "sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e"},
|
||||
{file = "Pillow-8.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3"},
|
||||
{file = "Pillow-8.0.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09"},
|
||||
{file = "Pillow-8.0.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae"},
|
||||
{file = "Pillow-8.0.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a"},
|
||||
{file = "Pillow-8.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8"},
|
||||
{file = "Pillow-8.0.1-cp37-cp37m-win32.whl", hash = "sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0"},
|
||||
{file = "Pillow-8.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039"},
|
||||
{file = "Pillow-8.0.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11"},
|
||||
{file = "Pillow-8.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"},
|
||||
{file = "Pillow-8.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792"},
|
||||
{file = "Pillow-8.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015"},
|
||||
{file = "Pillow-8.0.1-cp38-cp38-win32.whl", hash = "sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271"},
|
||||
{file = "Pillow-8.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7"},
|
||||
{file = "Pillow-8.0.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5"},
|
||||
{file = "Pillow-8.0.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce"},
|
||||
{file = "Pillow-8.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3"},
|
||||
{file = "Pillow-8.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544"},
|
||||
{file = "Pillow-8.0.1-cp39-cp39-win32.whl", hash = "sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140"},
|
||||
{file = "Pillow-8.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021"},
|
||||
{file = "Pillow-8.0.1-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6"},
|
||||
{file = "Pillow-8.0.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb"},
|
||||
{file = "Pillow-8.0.1-pp37-pypy37_pp73-win32.whl", hash = "sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8"},
|
||||
{file = "Pillow-8.0.1.tar.gz", hash = "sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e"},
|
||||
]
|
||||
prompt-toolkit = [
|
||||
{file = "prompt_toolkit-3.0.8-py3-none-any.whl", hash = "sha256:7debb9a521e0b1ee7d2fe96ee4bd60ef03c6492784de0547337ca4433e46aa63"},
|
||||
{file = "prompt_toolkit-3.0.8.tar.gz", hash = "sha256:25c95d2ac813909f813c93fde734b6e44406d1477a9faef7c915ff37d39c0a8c"},
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ pymisp = {version = "^2.4.135", extras = ["url"]}
|
|||
python-magic = {version = "^0.4.18", optional = true}
|
||||
# pydeep requires libfuzzy-dev, and is only used in the MISP export module
|
||||
pydeep = {version = "^0.4", optional = true}
|
||||
Pillow = "^8.0.1"
|
||||
|
||||
[tool.poetry.extras]
|
||||
misp = ['python-magic', 'pydeep']
|
||||
|
|
|
|||
|
|
@ -355,8 +355,10 @@ def tree(tree_uuid: str, urlnode_uuid: Optional[str]=None):
|
|||
|
||||
try:
|
||||
tree_json, start_time, user_agent, root_url, meta = lookyloo.load_tree(tree_uuid)
|
||||
b64_thumbnail = lookyloo.get_screenshot_thumbnail(tree_uuid, for_datauri=True)
|
||||
return render_template('tree.html', tree_json=tree_json, start_time=start_time,
|
||||
user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid,
|
||||
screenshot_thumbnail=b64_thumbnail,
|
||||
meta=meta, enable_mail_notification=enable_mail_notification,
|
||||
enable_context_by_users=enable_context_by_users,
|
||||
enable_categorization=enable_categorization,
|
||||
|
|
|
|||
|
|
@ -380,35 +380,6 @@ function update(root, computed_node_width=0) {
|
|||
.attr("id", d => `node_${d.data.uuid}`)
|
||||
.attr("transform", `translate(${root.y0}, ${root.x0})`);
|
||||
|
||||
node_group
|
||||
// Add Circle for the nodes
|
||||
.append('circle')
|
||||
.attr('class', 'node')
|
||||
.attr('r', 1e-6)
|
||||
.style("fill", d => d._children ? "lightsteelblue" : "#fff")
|
||||
.on('mouseover', (event, d) => {
|
||||
if (d.children || d._children) {
|
||||
d3.select('#tooltip')
|
||||
.style('opacity', 1)
|
||||
.style('left', `${event.pageX + 10}px`)
|
||||
.style('top', `${event.pageY + 10}px`)
|
||||
.text(d.children ? 'Collapse the childrens of this node.' : 'Expand the childrens of this node.');
|
||||
};
|
||||
}
|
||||
)
|
||||
.on('mouseout', (event, d) => {
|
||||
if (d.children || d._children) {
|
||||
d3.select('#tooltip').style('opacity', 0)
|
||||
};
|
||||
}
|
||||
)
|
||||
.on('click', (event, d) => {
|
||||
if (d.children || d._children) {
|
||||
toggle_children_collapse(event, d)
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
let node_data = node_group
|
||||
.append('svg')
|
||||
.attr('class', 'node_data')
|
||||
|
|
@ -418,7 +389,7 @@ function update(root, computed_node_width=0) {
|
|||
node_data.append('rect')
|
||||
.attr("rx", 6)
|
||||
.attr("ry", 6)
|
||||
.attr('x', 12)
|
||||
.attr('x', 0)
|
||||
.attr('y', 0)
|
||||
.attr('width', 0)
|
||||
.style("opacity", "0.5")
|
||||
|
|
@ -426,15 +397,14 @@ function update(root, computed_node_width=0) {
|
|||
.attr('stroke-opacity', "0.8")
|
||||
.attr("stroke-width", "2")
|
||||
.attr("stroke-linecap", "round")
|
||||
.attr("fill", "white");
|
||||
.attr("fill", "white")
|
||||
|
||||
// Set Hostname text
|
||||
node_data
|
||||
.append(d => text_entry(15, 5, d)); // Popup
|
||||
.append(d => text_entry(10, 5, d)); // Popup
|
||||
// Set list of icons
|
||||
node_data
|
||||
.append(d => icon_list(17, 35, d));
|
||||
|
||||
.append(d => icon_list(12, 35, d));
|
||||
|
||||
node_group.select('.node_data').each(function(d){
|
||||
// set position of icons based of their length
|
||||
|
|
@ -450,8 +420,10 @@ function update(root, computed_node_width=0) {
|
|||
.attr('height', node_height + 5)
|
||||
.attr('width', selected_node_bbox_init.width + 50);
|
||||
|
||||
|
||||
// Set the width for all the nodes
|
||||
let selected_node_bbox = d3.select(this).select('rect').node().getBoundingClientRect(); // Required, as the node width need to include the rectangle
|
||||
d.node_width = selected_node_bbox.width;
|
||||
node_width = node_width > selected_node_bbox.width ? node_width : selected_node_bbox.width;
|
||||
|
||||
// Set Bookmark
|
||||
|
|
@ -501,6 +473,39 @@ function update(root, computed_node_width=0) {
|
|||
})
|
||||
.on('mouseout', (event, d) => d3.select('#tooltip').style('opacity', 0));
|
||||
};
|
||||
var thumbnail_size = 64;
|
||||
if (d.data.contains_rendered_urlnode) {
|
||||
d3.select(this).append("svg").append('rect')
|
||||
.attr('x', selected_node_bbox.width/3)
|
||||
.attr('y', node_height - 10)
|
||||
.attr('width', thumbnail_size)
|
||||
.attr('height', thumbnail_size)
|
||||
.attr('fill', 'white')
|
||||
.attr('stroke', 'black');
|
||||
|
||||
d3.select(this).append('image')
|
||||
.attr('x', selected_node_bbox.width/3)
|
||||
.attr('y', node_height - 10)
|
||||
.attr('id', 'screenshot_thumbnail')
|
||||
.attr("width", thumbnail_size)
|
||||
.attr("height", thumbnail_size)
|
||||
.attr("xlink:href", `data:image/png;base64,${screenshot_thumbnail}`)
|
||||
.attr('cursor', 'pointer')
|
||||
.on('mouseover', (event, d) => {
|
||||
d3.select('#tooltip')
|
||||
.style('opacity', 1)
|
||||
.style('left', `${event.pageX + 10}px`)
|
||||
.style('top', `${event.pageY + 10}px`)
|
||||
.text('Contains the URL rendered in the browser.');
|
||||
})
|
||||
.on('click', (event, d) => {
|
||||
$("#screenshotModal").modal('toggle');
|
||||
})
|
||||
.on('mouseout', (event, d) => {
|
||||
d3.select('#tooltip').style('opacity', 0)
|
||||
});
|
||||
};
|
||||
|
||||
const context_icon_size = 24;
|
||||
if (d.data.malicious) {
|
||||
// set bomb
|
||||
|
|
@ -515,7 +520,7 @@ function update(root, computed_node_width=0) {
|
|||
d3.select(this).append('image')
|
||||
.attr('x', selected_node_bbox.width - 22 - http_icon_size)
|
||||
.attr('y', selected_node_bbox.height - 13)
|
||||
.attr('id', 'insecure_image')
|
||||
.attr('id', 'malicious_image')
|
||||
.attr("width", context_icon_size)
|
||||
.attr("height", context_icon_size)
|
||||
.attr("xlink:href", '/static/bomb.svg')
|
||||
|
|
@ -540,7 +545,7 @@ function update(root, computed_node_width=0) {
|
|||
d3.select(this).append('image')
|
||||
.attr('x', selected_node_bbox.width - 22 - http_icon_size)
|
||||
.attr('y', selected_node_bbox.height - 13)
|
||||
.attr('id', 'insecure_image')
|
||||
.attr('id', 'known_image')
|
||||
.attr("width", context_icon_size)
|
||||
.attr("height", context_icon_size)
|
||||
.attr("xlink:href", '/static/check.svg')
|
||||
|
|
@ -565,7 +570,7 @@ function update(root, computed_node_width=0) {
|
|||
d3.select(this).append('image')
|
||||
.attr('x', selected_node_bbox.width - 22 - http_icon_size)
|
||||
.attr('y', selected_node_bbox.height - 13)
|
||||
.attr('id', 'insecure_image')
|
||||
.attr('id', 'empty_image')
|
||||
.attr("width", context_icon_size)
|
||||
.attr("height", context_icon_size)
|
||||
.attr("xlink:href", '/static/empty.svg')
|
||||
|
|
@ -578,6 +583,38 @@ function update(root, computed_node_width=0) {
|
|||
})
|
||||
.on('mouseout', (event, d) => d3.select('#tooltip').style('opacity', 0));
|
||||
};
|
||||
if (d.children || d._children) {
|
||||
d3.select(this)
|
||||
// Add Circle for the nodes
|
||||
.append('circle')
|
||||
.attr('class', 'node')
|
||||
.attr('r', 1e-6)
|
||||
.attr('cx', d => d.node_width)
|
||||
.attr('cy', d => node_height/2)
|
||||
.style("fill", d => d._children ? "lightsteelblue" : "#fff")
|
||||
.on('mouseover', (event, d) => {
|
||||
if (d.children || d._children) {
|
||||
d3.select('#tooltip')
|
||||
.style('opacity', 1)
|
||||
.style('left', `${event.pageX + 10}px`)
|
||||
.style('top', `${event.pageY + 10}px`)
|
||||
.text(d.children ? 'Collapse the childrens of this node.' : 'Expand the childrens of this node.');
|
||||
};
|
||||
}
|
||||
)
|
||||
.on('mouseout', (event, d) => {
|
||||
if (d.children || d._children) {
|
||||
d3.select('#tooltip').style('opacity', 0)
|
||||
};
|
||||
}
|
||||
)
|
||||
.on('click', (event, d) => {
|
||||
if (d.children || d._children) {
|
||||
toggle_children_collapse(event, d)
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
});
|
||||
|
||||
return node_group;
|
||||
|
|
@ -620,40 +657,19 @@ function update(root, computed_node_width=0) {
|
|||
const link = node_container.selectAll('path.link').data(links, d => d.id);
|
||||
|
||||
// Creates a curved (diagonal) path from parent to the child nodes
|
||||
let diagonal = (s, d) => {
|
||||
return `M ${s.y} ${s.x}
|
||||
C ${(s.y + d.y) / 2} ${s.x},
|
||||
${(s.y + d.y) / 2} ${d.x},
|
||||
${d.y} ${d.x}`
|
||||
};
|
||||
let diagonal = d3.linkHorizontal()
|
||||
.source(d => {return [d.y, d.x]})
|
||||
.target(d => {return [d.parent.y + d.parent.node_width, d.parent.x]});
|
||||
|
||||
link.join(
|
||||
enter => enter
|
||||
// Enter any new links at the parent's previous position.
|
||||
.insert('path', "g")
|
||||
.attr("class", "link")
|
||||
.attr('d', d => {
|
||||
let o = {
|
||||
x: d.x0,
|
||||
y: d.y0
|
||||
};
|
||||
return diagonal(o, o)
|
||||
}),
|
||||
.attr('d', diagonal),
|
||||
update => update,
|
||||
exit => exit
|
||||
.call(exit => exit
|
||||
.attr('d', d => {
|
||||
let o = {
|
||||
x: d.x0,
|
||||
y: d.y0
|
||||
};
|
||||
return diagonal(o, o)
|
||||
})
|
||||
.remove()
|
||||
)
|
||||
).call(link => link
|
||||
.attr('d', d => diagonal(d, d.parent))
|
||||
);
|
||||
exit => exit.call(exit => exit.attr('d', diagonal).remove())
|
||||
).call(link => link.attr('d', diagonal));
|
||||
|
||||
if (computed_node_width === 0) {
|
||||
update(root, node_width)
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@
|
|||
|
||||
<script>
|
||||
var treeUUID = "{{ tree_uuid }}";
|
||||
var screenshot_thumbnail = "{{ screenshot_thumbnail }}";
|
||||
var enable_bookmark = {{ enable_bookmark|tojson }};
|
||||
var treeData = {{ tree_json | safe }};
|
||||
</script>
|
||||
|
|
|
|||
Loading…
Reference in New Issue