new: Add permalink to lookyloo

Pipfile

@@ -21,6 +21,7 @@ flask-wtf = "*"
 gunicorn = {extras = ["gevent"],version = "*"}
 pyurlabuse = {editable = true,path = "./client"}
 pyfaup = {editable = true,git = "https://github.com/stricaud/faup.git/",subdirectory = "src/lib/bindings/python/"}
+pylookyloo = {editable = true,git = "https://github.com/CIRCL/lookyloo.git/",subdirectory = "client"}
 
 [requires]
 python_version = "3.6"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "21547085a908ef46ba32fe38cb88c8236645e1b778cbbeeb5d13c28e9055d858"
+            "sha256": "33ab0963bdc6c913e8d3ef25f35a88d726ef5a6fb882606e6cd5c0aa9f897415"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -218,7 +218,7 @@
         "pybgpranking": {
             "editable": true,
             "git": "https://github.com/D4-project/BGP-Ranking.git/",
-            "ref": "7a39941c235067127c98d8a3cdd2785882e1cbb1",
+            "ref": "7e698f87366e6f99b4d0d11852737db28e3ddc62",
             "subdirectory": "client"
         },
         "pyeupi": {
@@ -237,7 +237,13 @@
         "pyipasnhistory": {
             "editable": true,
             "git": "https://github.com/D4-project/IPASN-History.git/",
-            "ref": "2fb329beedb610788045482f36ddff0798398a27",
+            "ref": "e846cd36fe1ed6b22f60890bba89f84e61b62e59",
+            "subdirectory": "client"
+        },
+        "pylookyloo": {
+            "editable": true,
+            "git": "https://github.com/CIRCL/lookyloo.git/",
+            "ref": "608d8816a8ebc1a7fb379356344ee3167243eeb4",
             "subdirectory": "client"
         },
         "pypdns": {
@@ -267,11 +273,11 @@
         },
         "redis": {
             "hashes": [
-                "sha256:2100750629beff143b6a200a2ea8e719fcf26420adabb81402895e144c5083cf",
+                "sha256:74c892041cba46078ae1ef845241548baa3bd3634f9a6f0f952f006eb1619c71",
-                "sha256:8e0bdd2de02e829b6225b25646f9fb9daffea99a252610d040409a6738541f0a"
+                "sha256:7ba8612bbfd966dea8c62322543fed0095da2834dbd5a7c124afbc617a156aa7"
             ],
             "index": "pypi",
-            "version": "==3.0.1"
+            "version": "==3.1.0"
         },
         "requests": {
             "hashes": [
@@ -296,10 +302,10 @@
         },
         "soupsieve": {
             "hashes": [
-                "sha256:10687fc53eeb3518e01a0ac84d3d711da623d3298a3039459d3f649927c4a270",
+                "sha256:466910df7561796a60748826781ebe9a888f7a1668a636ae86783f44d10aae73",
-                "sha256:b23a0d7da0247200fe83c67c34de9d7599ad404106367313d8e65e04174d0b4b"
+                "sha256:87db12ae79194f0ff9808d2b1641c4f031ae39ffa3cab6b907ea7c1e5e5ed445"
             ],
-            "version": "==1.7.2"
+            "version": "==1.7.3"
         },
         "urllib3": {
             "hashes": [

bin/start_website.py

@@ -8,7 +8,7 @@ if __name__ == '__main__':
     website_dir = get_homedir() / 'website'
     Popen([f'{website_dir}/3drparty.sh'], cwd=website_dir)
     try:
-        Popen(['gunicorn', '--worker-class', 'gevent', '-w', '10', '-b', '0.0.0.0:5100', 'web:app'],
+        Popen(['gunicorn', '--worker-class', 'gevent', '-w', '10', '-b', '0.0.0.0:5200', 'web:app'],
               cwd=website_dir).communicate()
     except KeyboardInterrupt:
         print('Stopping gunicorn.')

urlabuse/urlabuse.py

@@ -21,7 +21,7 @@ import logging
 from pypdns import PyPDNS
 from pyipasnhistory import IPASNHistory
 from pybgpranking import BGPRanking
-
+from pylookyloo import Lookyloo
 
 from pypssl import PyPSSL
 from pyeupi import PyEUPI
@@ -471,6 +471,17 @@ class Query():
         self._cache_set(ip, to_return, 'bgpranking')
         return to_return
 
+    def lookyloo(self, url):
+        cached = self._cache_get(url, 'lookyloo')
+        if cached is not None:
+            return cached
+        lookyloo = Lookyloo()
+        lookyloo_perma_url = lookyloo.enqueue(url)
+        if lookyloo_perma_url:
+            self._cache_set(url, lookyloo_perma_url, 'lookyloo')
+            return lookyloo_perma_url
+        return None
+
     def _deserialize_cached(self, entry):
         to_return = {}
         redirects = []
@@ -537,6 +548,8 @@ class Query():
                 to_return += '\n{}\n'.format(url)
                 if 'whois' in info:
                     all_mails.update(info['whois'])
+                if 'lookyloo' in info:
+                    to_return += '\tLookyloo permanent URL: {}\n'.format(info['lookyloo'])
                 if 'vt' in info and len(info['vt']) == 4:
                     if info['vt'][2] is not None:
                         to_return += '\t{} out of {} positive detections in VT - {}\n'.format(

website/web/__init__.py

@@ -189,6 +189,12 @@ def run_query():
     return enqueue('is_valid_url', {'url': url})
 
 
+@app.route('/lookyloo', methods=['POST'])
+def lookyloo():
+    data = request.get_json(force=True)
+    return enqueue('lookyloo', {'url': data["url"]})
+
+
 @app.route('/urls', methods=['POST'])
 def urls():
     data = request.get_json(force=True)

website/web/static/main.js

@@ -78,6 +78,11 @@
 
       var userInput = $scope.input_url;
 
+      var lookyloo = function(jobID) {
+        $scope.poller(jobID, function(data){
+            $scope.lookyloo_url = data;
+        });
+      };
 
       var check_validity = function(jobID) {
         $scope.poller(jobID, function(data){
@@ -91,6 +96,7 @@
       };
 
       $scope.query('start', {"url": userInput}, check_validity);
+      $scope.query('lookyloo', {"url": userInput}, lookyloo);
     };
 
      $scope.submit_email = function() {

website/web/templates/index.html

@@ -42,6 +42,7 @@
       <center>
           <h1>Report</h1>
           <h3>{{ query_url }}</h3>
+          <h3><a href="{{ lookyloo_url }}" target="_blank">See on Lookyloo</a></h3>
           <div class="alert alert-danger ng-hide" role="alert" ng-show="error">
               {{ error }}
           </div>
@@ -54,7 +55,6 @@
       <div ng-show="urls">
         <br/>
         <center><a href ng-click="submit_email()">Send report to CIRCL</a></center>
-        <!-- <center><a href="submit/{{ query_url }}">Send report to CIRCL</a></center> -->
         <br/>
       </div>
       <div ng-repeat="url in urls">