From 66bad326326095d6f88062a567c9eff5d6111e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 2 Mar 2015 14:55:34 +0100 Subject: [PATCH] Initial commit --- .gitignore | 16 + LICENSE | 661 ++++++++++++++++++++++++++++++++++ README.md | 9 + config.ini.sample | 61 ++++ install.sh | 12 + requirements.txt | 6 + run_redis.sh | 9 + runapp.py | 13 + update_deps.sh | 17 + url_abuse_async.py | 401 +++++++++++++++++++++ web/__init__.py | 260 +++++++++++++ web/proxied.py | 32 ++ web/static/ajax-loader.gif | Bin 0 -> 673 bytes web/static/main.js | 274 ++++++++++++++ web/templates/404.html | 8 + web/templates/index.html | 66 ++++ web/templates/url-report.html | 46 +++ worker.py | 19 + 18 files changed, 1910 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 config.ini.sample create mode 100755 install.sh create mode 100644 requirements.txt create mode 100755 run_redis.sh create mode 100755 runapp.py create mode 100755 update_deps.sh create mode 100644 url_abuse_async.py create mode 100644 web/__init__.py create mode 100644 web/proxied.py create mode 100644 web/static/ajax-loader.gif create mode 100644 web/static/main.js create mode 100644 web/templates/404.html create mode 100644 web/templates/index.html create mode 100644 web/templates/url-report.html create mode 100755 worker.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..679fb36 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +*.pyc +*.log* + +# Configs +redis.conf +config.ini + +# Key files +*.key + +# Py libs +sphinxapi.py + +# JS libs +angular.min.js +ui-bootstrap-tpls.min.js diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..dba13ed --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c6ca593 --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +Add a new module +================ + +Look at the existings functions/modules. The changes will have to be made in the following files: + +* Add the function you want to execure in url\_abuse\_async.py +* Add a route in web/\_\_init\_\_.py. This route will do an async call to the function defined in url\_abuse\_async.py. The parameter of the function is sent in an POST object +* Add a statement in web/templates/url-report.html. The data option is the parameter to pass to the javascript directive +* Add a directive in web/static/main.js, it will take care of passing the parameter to the backend and regularly pull for the response of the async call diff --git a/config.ini.sample b/config.ini.sample new file mode 100644 index 0000000..1538371 --- /dev/null +++ b/config.ini.sample @@ -0,0 +1,61 @@ +[GLOBAL] +debug=False + +[WHOIS] +server=127.0.0.1 +port=4243 + +[SPHINX] +server=127.0.0.1 +port=9312 + +[ITS] +url=https://rt:8443/rt/RTIR/Display.html?id= + +[abuse] +ignore= + ripe.net$ + arin.net$ + apnic.net$ + idnic.net$ + peering@ + dns.lu$ + domreg@ + registrar-email + +fallback=set.this@invalid.tld + +[replacelist] +abuse@ispsystem.com=abuse@ispserver.com +abuse@ispsystem.net=abuse@ispserver.com +hostmaster@root.lu=abuse@as5577.net +noc@as5577.net=abuse@as5577.net +abuse@godaddy.com=abuse@godaddy.com,phishing@godaddy.com,malware@godaddy.com +ipadmin@websitewelcome.com=security@hostgator.com,ipadmin@websitewelcome.com + +[PHISHTANK] +url=http://checkurl.phishtank.com/checkurl/ + +[GOOGLESAFEBROWSING] +url=https://sb-ssl.google.com/safebrowsing/api/lookup?client=urlabuse&key={}&appver=1&pver=3.1 + +[VIRUSTOTAL] +url_upload=https://www.virustotal.com/vtapi/v2/url/scan +url_report=https://www.virustotal.com/vtapi/v2/url/report + +[PDNS_CIRCL] +url=https://www.circl.lu/pdns/query + +[PSSL_CIRCL] +url=https://www.circl.lu/pssl/query + +[URLQUERY] +url=https://uqapi.net/v3/json + +[domain] +ignore= + post.lu + pt.lu + netline.lu + apple.com + paypal.com diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..992c295 --- /dev/null +++ b/install.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e +set -x + +if [ ! -d virtenv ]; then + virtualenv virtenv +fi + +. ./virtenv/bin/activate + +pip install --upgrade -r requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b98e82a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +flask +flask-bootstrap +flask-wtf +rq +redis +pypssl diff --git a/run_redis.sh b/run_redis.sh new file mode 100755 index 0000000..d0bcc37 --- /dev/null +++ b/run_redis.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e +set -x + +REDIS_HOME='/change/me/' + +${REDIS_HOME}/redis-server ./redis.conf + diff --git a/runapp.py b/runapp.py new file mode 100755 index 0000000..e50bd8d --- /dev/null +++ b/runapp.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*-coding:utf-8 -* + +import os + +from web import create_app + +# create an app instance +abspath = os.path.abspath(__file__) +dname = os.path.dirname(abspath) +os.chdir(dname) +app = create_app() +app.run(host='0.0.0.0', port = 5100, debug=False, threaded=True) diff --git a/update_deps.sh b/update_deps.sh new file mode 100755 index 0000000..5714100 --- /dev/null +++ b/update_deps.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e +set -x + +DEST_DIR="web/static/" + +ANGULAR='1.3.14' +ANGULAR_BOOTSTRAP='0.12.1' + +wget https://ajax.googleapis.com/ajax/libs/angularjs/${ANGULAR}/angular.min.js -O ${DEST_DIR}/angular.min.js +wget https://angular-ui.github.io/bootstrap/ui-bootstrap-tpls-${ANGULAR_BOOTSTRAP}.min.js -O ${DEST_DIR}/ui-bootstrap-tpls.min.js + +wget https://sphinxsearch.googlecode.com/svn/trunk/api/sphinxapi.py -O sphinxapi.py + + + diff --git a/url_abuse_async.py b/url_abuse_async.py new file mode 100644 index 0000000..a1649a1 --- /dev/null +++ b/url_abuse_async.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python +# +# +# Copyright (C) 2014 Sascha Rommelfangen, Raphael Vinot +# Copyright (C) 2014 CIRCL Computer Incident Response Center Luxembourg (SMILE gie) +# + +import json +import redis +import urllib +from pyfaup.faup import Faup +import socket +import dns.resolver +import re +import sys +import logging +from pypdns import PyPDNS +import bgpranking_web +import urlquery +from pypssl import PyPSSL +import requests +from bs4 import BeautifulSoup + +try: + import sphinxapi + sphinx = True +except: + sphinx = False + +enable_cache = True +r_cache = None + + +def _cache_init(host='localhost', port=6334, db=1): + global r_cache + if enable_cache and r_cache is None: + r_cache = redis.Redis(host, port, db=db) + + +def _cache_set(key, value, field=None): + _cache_init() + if enable_cache: + if field is None: + r_cache.setex(key, json.dumps(value), 3600) + else: + r_cache.hset(key, field, json.dumps(value)) + r_cache.expire(key, 3600) + + +def _cache_get(key, field=None): + _cache_init() + if enable_cache: + if field is None: + value_json = r_cache.get(key) + else: + value_json = r_cache.hget(key, field) + if value_json is not None: + return json.loads(value_json) + return None + + +def to_bool(s): + """ + Converts the given string to a boolean. + """ + return s.lower() in ('1', 'true', 'yes', 'on') + + +def is_valid_url(url): + cached = _cache_get(url, 'valid') + if cached is not None: + return cached + fex = Faup() + if url.startswith('hxxp'): + url = 'http' + url[4:] + elif not url.startswith('http'): + url = 'http://' + url + logging.debug("Checking validity of URL: " + url) + fex.decode(url) + scheme = fex.get_scheme() + host = fex.get_host() + if scheme is None or host is None: + reason = "Not a valid http/https URL/URI" + return False, url, reason + _cache_set(url, (True, url, None), 'valid') + return True, url, None + + +def is_ip(host): + if ':' in host: + try: + socket.inet_pton(socket.AF_INET6, host) + return True + except: + pass + else: + try: + socket.inet_aton(host) + return True + except: + pass + return False + + +def try_resolve(fex, url): + fex.decode(url) + host = fex.get_host().lower() + if is_ip(host): + return True, None + try: + ipaddr = dns.resolver.query(host, 'A') + except Exception: + reason = "DNS server problem. Check resolver settings." + return False, reason + if not ipaddr: + reason = "Host " + host + " does not exist." + return False, reason + return True, None + + +def get_urls(url, depth=1): + if depth > 5: + print('Too many redirects.') + return + fex = Faup() + + def meta_redirect(content): + c = content.lower() + soup = BeautifulSoup(c) + for result in soup.find_all(attrs={'http-equiv': 'refresh'}): + if result: + out = result["content"].split(";") + if len(out) == 2: + wait, text = out + a, url = text.split('=', 1) + return url.strip() + return None + + resolve, reason = try_resolve(fex, url) + if not resolve: + # FIXME: inform that the domain does not resolve + yield url + return + + logging.debug("Making HTTP connection to " + url) + + headers = {'User-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:8.0) Gecko/20100101 Firefox/8.0'} + try: + response = requests.get(url, allow_redirects=True, headers=headers, + timeout=15, verify=False) + except: + # That one can fail (DNS for example) + # FIXME: inform that the get failed + yield url + return + if response.history is not None: + for h in response.history: + # Yeld the urls in the order we find them + yield h.url + + yield response.url + + meta_redir_url = meta_redirect(response.content) + if meta_redir_url is not None: + depth += 1 + if not meta_redir_url.startswith('http'): + fex.decode(url) + base = '{}://{}'.format(fex.get_scheme(), fex.get_host()) + port = fex.get_port() + if port is not None: + base += ':{}'.format(port) + if not base.endswith('/'): + base += '/' + meta_redir_url = base + meta_redir_url + for url in get_urls(meta_redir_url, depth): + yield url + + +def url_list(url): + cached = _cache_get(url, 'list') + if cached is not None: + return cached + list_urls = [u for u in get_urls(url) if u is not None] + _cache_set(url, list_urls, 'list') + return list_urls + + +def dns_resolve(url): + cached = _cache_get(url, 'dns') + if cached is not None: + return cached + fex = Faup() + fex.decode(url) + host = fex.get_host().lower() + ipv4 = None + ipv6 = None + if not is_ip(host): + try: + ipv4 = [str(ip) for ip in dns.resolver.query(host, 'A')] + except: + logging.debug("No IPv4 address assigned to: " + host) + try: + ipv6 = [str(ip) for ip in dns.resolver.query(host, 'AAAA')] + except: + logging.debug("No IPv6 address assigned to: " + host) + _cache_set(url, (ipv4, ipv6), 'dns') + return ipv4, ipv6 + + +def phish_query(url, key, query): + cached = _cache_get(query, 'phishtank') + if cached is not None: + return cached + postfields = {'url': urllib.quote(query), 'format': 'json', 'app_key': key} + response = requests.post(url, data=postfields) + res = response.json() + if res["meta"]["status"] == "success": + if res["results"]["in_database"]: + _cache_set(query, res["results"]["phish_detail_page"], 'phishtank') + return res["results"]["phish_detail_page"] + else: + # no information + pass + elif res["meta"]["status"] == 'error': + # Inform the user? + # errormsg = res["errortext"] + pass + return None + + +def sphinxsearch(server, port, url, query): + if not sphinx: + return None + cached = _cache_get(query, 'sphinx') + if cached is not None: + return cached + client = sphinxapi.SphinxClient() + client.SetServer(server, port) + client.SetMatchMode(2) + client.SetConnectTimeout(5.0) + result = [] + res = client.Query(query) + if res.get("matches") is not None: + for ticket in res["matches"]: + ticket_id = ticket["id"] + ticket_link = url + str(ticket_id) + result.append(ticket_link) + _cache_set(query, result, 'sphinx') + return result + + +def vt_query_url(url, url_up, key, query, upload=True): + cached = _cache_get(query, 'vt') + if cached is not None: + return cached + parameters = {"resource": query, "apikey": key} + if upload: + parameters['scan'] = 1 + response = requests.post(url, data=parameters) + if response.text is None or len(response.text) == 0: + return None + res = response.json() + msg = res["verbose_msg"] + link = res.get("permalink") + positives = res.get("positives") + total = res.get("total") + _cache_set(query, (msg, link, positives, total), 'vt') + return msg, link, positives, total + + +def gsb_query(url, query): + cached = _cache_get(query, 'gsb') + if cached is not None: + return cached + param = '1\n' + query + response = requests.post(url, data=param) + status = response.status_code + if status == 200: + _cache_set(query, response.text, 'gsb') + return response.text + + +def urlquery_query(url, key, query): + cached = _cache_get(query, 'urlquery') + if cached is not None: + return cached + try: + urlquery.url = url + urlquery.key = key + response = urlquery.search(query) + except: + return None + if response['_response_']['status'] == 'ok': + if response.get('reports') is not None: + total_alert_count = 0 + for r in response['reports']: + total_alert_count += r['urlquery_alert_count'] + total_alert_count += r['ids_alert_count'] + total_alert_count += r['blacklist_alert_count'] + _cache_set(query, total_alert_count, 'urlquery') + return total_alert_count + else: + return None + + +def process_emails(emails, ignorelist, replacelist): + to_return = list(set(emails)) + for mail in reversed(to_return): + for ignorelist_entry in ignorelist: + if re.search(ignorelist_entry, mail, re.I): + if mail in to_return: + to_return.remove(mail) + for k, v in replacelist.iteritems(): + if re.search(k, mail, re.I): + if k in to_return: + to_return.remove(k) + to_return += v + return to_return + + +def whois(server, port, domain, ignorelist, replacelist): + cached = _cache_get(domain, 'whois') + if cached is not None: + return cached + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(15) + try: + s.connect((server, port)) + except Exception: + print "Connection problems - check WHOIS server" + print "WHOIS request while problem occurred: " + domain + print "WHOIS server: {}:{}".format(server, port) + sys.exit(0) + if domain.startswith('http'): + fex = Faup() + fex.decode(domain) + domain = fex.get_domain().lower() + s.send(domain + "\r\n") + response = '' + while True: + d = s.recv(4096) + response += d + if d == '': + break + s.close() + match = re.findall(r'[\w\.-]+@[\w\.-]+', response) + emails = process_emails(match, ignorelist, replacelist) + if len(emails) == 0: + return None + list_mail = list(set(emails)) + _cache_set(domain, list_mail, 'whois') + return list_mail + + +def pdnscircl(url, user, passwd, q): + cached = _cache_get(q, 'pdns') + if cached is not None: + return cached + pdnscircl = PyPDNS(url, basic_auth=(user, passwd)) + response = pdnscircl.query(q) + all_uniq = [] + for e in reversed(response): + host = e['rrname'].lower() + if host in all_uniq: + continue + else: + all_uniq.append(host) + response = (len(all_uniq), all_uniq[:5]) + _cache_set(q, response, 'pdns') + return response + + +def psslcircl(url, user, passwd, q): + cached = _cache_get(q, 'pssl') + if cached is not None: + return cached + psslcircl = PyPSSL(url, basic_auth=(user, passwd)) + response = psslcircl.query(q) + if response.get(q) is not None: + entries = response[q] + _cache_set(q, entries, 'pssl') + return entries + return None + + +def bgpranking(ip): + cached = _cache_get(ip, 'bgp') + if cached is not None: + return cached + details = bgpranking_web.ip_lookup(ip, 7) + ptrr = details.get('ptrrecord') + if details.get('history') is None or len(details.get('history')) == 0: + return ptrr, None, None, None, None, None + asn = details['history'][0].get('asn') + rank_info = bgpranking_web.cached_daily_rank(asn) + position, total = bgpranking_web.cached_position(asn) + asn_descr = rank_info[1] + rank = rank_info[-1] + response = (ptrr, asn_descr, asn, int(position), int(total), float(rank)) + _cache_set(ip, response, 'bgp') + return response diff --git a/web/__init__.py b/web/__init__.py new file mode 100644 index 0000000..b1fb7d5 --- /dev/null +++ b/web/__init__.py @@ -0,0 +1,260 @@ +import json +import os + +from flask import Flask, render_template, request, Response, redirect, url_for +from flask_bootstrap import Bootstrap +from flask_wtf import Form +from wtforms import StringField, SubmitField +from wtforms.widgets import TextInput +from wtforms.validators import Required + +import logging +from logging.handlers import RotatingFileHandler +from logging import Formatter + +from rq import Queue +from rq.job import Job +from worker import conn + +import ConfigParser +# from pyfaup.faup import Faup +from proxied import ReverseProxied +from url_abuse_async import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \ + vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking + +config_path = 'config.ini' + + +class AngularTextInput(TextInput): + + def __call__(self, field, **kwargs): + kwargs['ng-model'] = 'input_url' + return super(AngularTextInput, self).__call__(field, **kwargs) + + +class URLForm(Form): + url = StringField('URL Field', + description='Enter the URL you want to lookup here.', + validators=[Required()], widget=AngularTextInput()) + + submit_button = SubmitField('Run lookup') + + +def make_dict(parser, section): + to_return = {} + entries = parser.items(section) + for k, v in entries: + to_return[k] = v.split(',') + return to_return + + +def prepare_auth(): + if not os.path.exists('users.key'): + return None + to_return = {} + with open('users.key', 'r') as f: + for l in f: + l = l.strip() + user, password = l.split('=') + to_return[user] = password + return to_return + + +def create_app(configfile=None): + app = Flask(__name__) + handler = RotatingFileHandler('urlabuse.log', maxBytes=10000, backupCount=5) + handler.setFormatter(Formatter('%(asctime)s %(message)s')) + app.wsgi_app = ReverseProxied(app.wsgi_app) + app.logger.addHandler(handler) + app.logger.setLevel(logging.INFO) + Bootstrap(app) + q = Queue(connection=conn) + + app.config['SECRET_KEY'] = 'devkey' + app.config['BOOTSTRAP_SERVE_LOCAL'] = True + app.config['configfile'] = config_path + + parser = ConfigParser.SafeConfigParser() + parser.read(app.config['configfile']) + + replacelist = make_dict(parser, 'replacelist') + auth_users = prepare_auth() + ignorelist = [i.strip() + for i in parser.get('abuse', 'ignore').split('\n') + if len(i.strip()) > 0] + + @app.route('/', methods=['GET', 'POST']) + def index(): + form = URLForm() + return render_template('index.html', form=form) + + @app.route('/urlreport', methods=['GET']) + def url_report(): + return render_template('url-report.html') + + @app.errorhandler(404) + def page_not_found(e): + ip = request.headers.get('X-Forwarded-For') + if ip is None: + ip = request.remote_addr + if request.path != '/_result/': + app.logger.info('404 of {} on {}'.format(ip, request.path)) + return render_template('404.html'), 404 + + def authenticate(): + """Sends a 401 response that enables basic auth""" + return Response('Could not verify your access level for that URL.\n' + 'You have to login with proper credentials', 401, + {'WWW-Authenticate': 'Basic realm="Login Required"'}) + + def check_auth(username, password): + """This function is called to check if a username / + password combination is valid. + """ + if auth_users is None: + return False + else: + db_pass = auth_users.get(username) + return db_pass == password + + @app.route('/login', methods=['GET', 'POST']) + def login(): + auth = request.authorization + if not auth or not check_auth(auth.username, auth.password): + return authenticate() + return redirect(url_for('index')) + + @app.route("/_result/", methods=['GET']) + def check_valid(job_key): + if job_key is None: + return json.dumps(None), 200 + job = Job.fetch(job_key, connection=conn) + if job.is_finished: + return json.dumps(job.result), 200 + else: + return json.dumps("Nay!"), 202 + + @app.route('/start', methods=['POST']) + def run_query(): + data = json.loads(request.data) + url = data["url"] + ip = request.headers.get('X-Forwarded-For') + if ip is None: + ip = request.remote_addr + app.logger.info('{} {}'.format(ip, url)) + is_valid = q.enqueue_call(func=is_valid_url, args=(url,), result_ttl=500) + return is_valid.get_id() + + @app.route('/urls', methods=['POST']) + def urls(): + data = json.loads(request.data) + url = data["url"] + u = q.enqueue_call(func=url_list, args=(url,), result_ttl=500) + return u.get_id() + + @app.route('/resolve', methods=['POST']) + def resolve(): + data = json.loads(request.data) + url = data["url"] + u = q.enqueue_call(func=dns_resolve, args=(url,), result_ttl=500) + return u.get_id() + + @app.route('/phishtank', methods=['POST']) + def phishtank(): + data = json.loads(request.data) + if not os.path.exists('phishtank.key'): + return None + url = parser.get("PHISHTANK", "url") + key = open('phishtank.key', 'r').readline().strip() + query = data["query"] + u = q.enqueue_call(func=phish_query, args=(url, key, query,), result_ttl=500) + return u.get_id() + + @app.route('/virustotal_report', methods=['POST']) + def vt(): + data = json.loads(request.data) + if not os.path.exists('virustotal.key'): + return None + url = parser.get("VIRUSTOTAL", "url_report") + url_up = parser.get("VIRUSTOTAL", "url_upload") + key = open('virustotal.key', 'r').readline().strip() + query = data["query"] + u = q.enqueue_call(func=vt_query_url, args=(url, url_up, key, query,), result_ttl=500) + return u.get_id() + + @app.route('/googlesafebrowsing', methods=['POST']) + def gsb(): + data = json.loads(request.data) + if not os.path.exists('googlesafebrowsing.key'): + return None + url = parser.get("GOOGLESAFEBROWSING", "url") + key = open('googlesafebrowsing.key', 'r').readline().strip() + url = url.format(key) + query = data["query"] + u = q.enqueue_call(func=gsb_query, args=(url, query,), result_ttl=500) + return u.get_id() + + @app.route('/urlquery', methods=['POST']) + def urlquery(): + data = json.loads(request.data) + if not os.path.exists('urlquery.key'): + return None + url = parser.get("URLQUERY", "url") + key = open('urlquery.key', 'r').readline().strip() + query = data["query"] + u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500) + return u.get_id() + + @app.route('/ticket', methods=['POST']) + def ticket(): + if not request.authorization: + return '' + data = json.loads(request.data) + server = parser.get("SPHINX", "server") + port = int(parser.get("SPHINX", "port")) + url = parser.get("ITS", "url") + query = data["query"] + u = q.enqueue_call(func=sphinxsearch, args=(server, port, url, query,), + result_ttl=500) + return u.get_id() + + @app.route('/whois', methods=['POST']) + def whoismail(): + if not request.authorization: + return '' + server = parser.get("WHOIS", "server") + port = parser.getint("WHOIS", "port") + data = json.loads(request.data) + query = data["query"] + u = q.enqueue_call(func=whois, args=(server, port, query, ignorelist, replacelist), + result_ttl=500) + return u.get_id() + + @app.route('/pdnscircl', methods=['POST']) + def dnscircl(): + url = parser.get("PDNS_CIRCL", "url") + user, password = open('pdnscircl.key', 'r').readlines() + data = json.loads(request.data) + query = data["query"] + u = q.enqueue_call(func=pdnscircl, args=(url, user.strip(), password.strip(), + query,), result_ttl=500) + return u.get_id() + + @app.route('/bgpranking', methods=['POST']) + def bgpr(): + data = json.loads(request.data) + query = data["query"] + u = q.enqueue_call(func=bgpranking, args=(query,), result_ttl=500) + return u.get_id() + + @app.route('/psslcircl', methods=['POST']) + def sslcircl(): + url = parser.get("PSSL_CIRCL", "url") + user, password = open('psslcircl.key', 'r').readlines() + data = json.loads(request.data) + query = data["query"] + u = q.enqueue_call(func=psslcircl, args=(url, user.strip(), password.strip(), + query,), result_ttl=500) + return u.get_id() + + return app diff --git a/web/proxied.py b/web/proxied.py new file mode 100644 index 0000000..88776a7 --- /dev/null +++ b/web/proxied.py @@ -0,0 +1,32 @@ +class ReverseProxied(object): + '''Wrap the application in this middleware and configure the + front-end server to add these headers, to let you quietly bind + this to a URL other than / and to an HTTP scheme that is + different than what is used locally. + + In nginx: + location /myprefix { + proxy_pass http://192.168.0.1:5001; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /myprefix; + } + + :param app: the WSGI application + ''' + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + script_name = environ.get('HTTP_X_SCRIPT_NAME', '') + if script_name: + environ['SCRIPT_NAME'] = script_name + path_info = environ['PATH_INFO'] + if path_info.startswith(script_name): + environ['PATH_INFO'] = path_info[len(script_name):] + + scheme = environ.get('HTTP_X_SCHEME', '') + if scheme: + environ['wsgi.url_scheme'] = scheme + return self.app(environ, start_response) diff --git a/web/static/ajax-loader.gif b/web/static/ajax-loader.gif new file mode 100644 index 0000000000000000000000000000000000000000..f2a1bc0c6f545e20e631a96e8e92f9822e75d046 GIT binary patch literal 673 zcmZ?wbhEHb6krfw_{6~Q|Nnmm28Kh24mmkF0U1e2Nli^nlO|14{3qpHl$uzQnxasi zS(2fUn3Y(Olb@KPmzkHA&!G5|g@FsGT=74*pKD04vtxj(k)8oFBTz^Oh=E26FfcG1 zbL_hF&)}42ws10s6^G;;cE1^EoUR)U5A70}d2pLv!jVIT7j&Z~EblI3x0K*v_sV|m z0W=b9G$XP(CLnYCdK49;TX=SFc-G}o=oA=|U?{1O;Nu!CwW3C5Yw7*Bi4yD$3fCnb zwK+>}QdQ9sf*QnxY>*kpE+b{_Q;sJloS71)&(@kO!}mqf@1v(v;*8Y=G9S3kY~Cw# zY=t&c z;3~JK4HxB^lY(MD+sYeQ=t%XSSW;x^1M?dTvN=W^yNcAcy`HCte31C;)5xP%b~qs> zDP&4(%TBqBNGHwnryK;BdMI$fEg xd0mc!C@j^ZpLxYv4HmnPfI0THYuv<%+6iSmMn&w3dPGDfL1|=LY008wP(boU~ literal 0 HcmV?d00001 diff --git a/web/static/main.js b/web/static/main.js new file mode 100644 index 0000000..f2bc9f8 --- /dev/null +++ b/web/static/main.js @@ -0,0 +1,274 @@ +(function () { + 'use strict'; + + var app = angular.module('URLabuseApp', ['ui.bootstrap']); + + app.factory('globFct', [ '$log', '$http', '$timeout', function($log, $http, $timeout){ + return { + poller: function myself(jobID, callback) { + var timeout = ""; + // fire another request + $http.get('_result/' + jobID). + success(function(data, status, headers, config) { + if(status === 202) { + $log.log(data, status); + } else if (status === 200){ + $log.log(data); + $timeout.cancel(timeout); + callback(angular.fromJson(data)); + return; + } + // continue to call the poller() function every 2 seconds + // until the timout is cancelled + timeout = $timeout(function() {myself(jobID, callback);}, 2000); + }); + }, + query: function(path, data, callback) { + $http.post(path, data). + success(callback). + error(function(error) { + $log.log(error); + }); + } + }; + }]); + + app.controller('URLabuseController', function($scope, $log, globFct) { + + $scope.poller = globFct.poller; + $scope.query = globFct.query; + + var get_redirects = function(jobID) { + $scope.poller(jobID, function(data){ + $log.log(data); + $scope.urls = data; + }); + }; + + + $scope.getResults = function() { + // get the URL from the input + $scope.query_url = ''; + $scope.urls = ''; + var userInput = $scope.input_url; + + + var check_validity = function(jobID) { + $scope.poller(jobID, function(data){ + $scope.query_url = data[1]; + if(data[0] === false){ + $scope.error = data[2]; + } else { + $scope.query('urls', {"url": data[1]}, get_redirects); + } + }); + }; + + $scope.query('start', {"url": userInput}, check_validity); + }; + }); + + app.directive('uqUrlreport', function(globFct) { + + return { + scope: { + url: '=uqUrlreport', + // status: {isFirstOpen: true, isFirstDisabled: false} + }, + link: function(scope, element, attrs) { + var get_ips = function(jobID) { + globFct.poller(jobID, function(data){ + scope.ipv4 = data[0]; + scope.ipv6 = data[1]; + }); + }; + globFct.query('resolve', {"url": scope.url}, get_ips); + }, + templateUrl: 'urlreport', + }; + + }); + + app.directive('uqPhishtank', function(globFct) { + return { + scope: { + query: '=data', + }, + link: function(scope, element, attrs) { + var get_response = function(jobID) { + globFct.poller(jobID, function(data){ + scope.response = data; + }); + }; + globFct.query('phishtank', {"query": scope.query}, get_response); + }, + template: function(elem, attr){ + return '
Known phishing website on Phishtank. More details.
';} + }; + }); + + app.directive('uqVirustotal', function(globFct) { + return { + scope: { + query: '=data', + }, + link: function(scope, element, attrs) { + var get_response = function(jobID) { + globFct.poller(jobID, function(data){ + scope.message = data[0]; + scope.link = data[1]; + scope.positives = data[2]; + scope.total = data[3]; + if(scope.link && scope.positives === null){ + scope.alert_val = "info"; + scope.message = "Scan request successfully queued, report available soon."; + } else if (scope.link && scope.positives === 0){ + scope.message = "None of the " + data[3] + " scanners know this URL as malicious."; + scope.alert_val = "success"; + } else if (scope.link && scope.positives < scope.total/3){ + scope.message = data[2] + " of the " + data[3] + " scanners know this URL as malicious."; + scope.alert_val = "warning"; + } else if (scope.link && scope.positives >= scope.total/3){ + scope.message = data[2] + " of the " + data[3] + " scanners know this URL as malicious."; + scope.alert_val = "danger"; + } + }); + }; + globFct.query('virustotal_report', {"query": scope.query}, get_response); + }, + template: function(elem, attr){ + return '
{{message}} More details.
';} + }; + }); + + app.directive('uqGooglesafebrowsing', function(globFct) { + return { + scope: { + query: '=data', + }, + link: function(scope, element, attrs) { + var get_response = function(jobID) { + globFct.poller(jobID, function(data){ + scope.response = data; + }); + }; + globFct.query('googlesafebrowsing', {"query": scope.query}, get_response); + }, + template: function(elem, attr){ + return '
Known {{response}} website on Google Safe Browsing. More details.
';} + }; + }); + + app.directive('uqUrlquery', function(globFct) { + return { + scope: { + query: '=data', + }, + link: function(scope, element, attrs) { + var get_response = function(jobID) { + globFct.poller(jobID, function(data){ + scope.response = data; + }); + }; + globFct.query('urlquery', {"query": scope.query}, get_response); + }, + template: function(elem, attr){ + return '
The total alert count on URLquery is {{response}}.
';} + }; + }); + + app.directive('uqTicket', function(globFct) { + return { + scope: { + query: '=data', + }, + link: function(scope, element, attrs) { + var get_response = function(jobID) { + globFct.poller(jobID, function(data){ + scope.response = data; + }); + }; + globFct.query('ticket', {"query": scope.query}, get_response); + }, + template: '
Tickets:
' + }; + }); + + app.directive('uqWhois', function(globFct) { + return { + scope: { + query: '=data', + }, + link: function(scope, element, attrs) { + var get_response = function(jobID) { + globFct.poller(jobID, function(data){ + scope.response = data.join(); + }); + }; + globFct.query('whois', {"query": scope.query}, get_response); + }, + template: '
Contact points from Whois: {{ response }}
' + }; + }); + app.directive('uqPdnscircl', function(globFct) { + return { + scope: { + query: '=data', + }, + link: function(scope, element, attrs) { + var get_response = function(jobID) { + globFct.poller(jobID, function(data){ + scope.nbentries = data[0]; + scope.lastentries = data[1]; + }); + }; + globFct.query('pdnscircl', {"query": scope.query}, get_response); + }, + template: '
Has {{nbentries}} unique entries in CIRCL Passive DNS. The {{lastentries.length}} most recent ones:
  • {{domain}}
' + }; + }); + app.directive('uqPsslcircl', function(globFct) { + return { + scope: { + query: '=data', + }, + link: function(scope, element, attrs) { + var get_response = function(jobID) { + globFct.poller(jobID, function(data){ + scope.entries = data; + }); + }; + globFct.query('psslcircl', {"query": scope.query}, get_response); + }, + template: '
SSL certificates related to this IP:
  • {{entry}}
' + }; + }); + app.directive('uqBgpranking', function(globFct) { + return { + scope: { + query: '=data', + }, + link: function(scope, element, attrs) { + var get_response = function(jobID) { + globFct.poller(jobID, function(data){ + scope.ptr = data[0]; + scope.asndesc = data[1]; + scope.asn = data[2]; + scope.position = data[3]; + scope.total = data[4]; + scope.value = data[5]; + if (scope.position < 100){ + scope.alert_val = "danger"; + } else if (scope.position < 1000){ + scope.alert_val = "warning"; + } else { + scope.alert_val = "info"; + } + }); + }; + globFct.query('bgpranking', {"query": scope.query}, get_response); + }, + template: '
Information from BGP Ranking:
  • PTR Resource Record: {{ptr}}
  • Announced by: {{asndesc}} ({{asn}})
  • This ASN is at the {{position}} in the list of {{total}} known ASNs ({{value}}).
' + }; + }); +}()); diff --git a/web/templates/404.html b/web/templates/404.html new file mode 100644 index 0000000..4d4967e --- /dev/null +++ b/web/templates/404.html @@ -0,0 +1,8 @@ +{% extends "index.html" %} +{% block title %}Page Not Found{% endblock %} +{% block body %} +

Page Not Found

+

What you were looking for is just not there. +

Back to index. +{% endblock %} + diff --git a/web/templates/index.html b/web/templates/index.html new file mode 100644 index 0000000..091e8da --- /dev/null +++ b/web/templates/index.html @@ -0,0 +1,66 @@ +{% extends "bootstrap/base.html" %} +{% import "bootstrap/wtf.html" as wtf %} +{% import "bootstrap/fixes.html" as fixes %} + +{% block title %}CIRCL URL Abuse{% endblock %} + +{% block navbar %} +

+
+
+{% endblock %} + +{% block html_attribs %} ng-app="URLabuseApp" {% endblock html_attribs %} + +{% block body_attribs %} ng-controller="URLabuseController" {% endblock body_attribs %} + +{% block content %} +
+

URL Abuse testing form

+

URL Abuse is a public CIRCL service to review URL.
For more information about the service

+
+
+
+ {{ form.hidden_tag() }} + {{ wtf.form_errors(form, hiddens="only") }} + {%- for field in form %} + {% if not bootstrap_is_hidden_field(field) -%} + {{ wtf.form_field(field, form_type=form_type, + horizontal_columns=horizontal_columns, + button_map={"submit_button": "primary"}) }} + {%- endif %} + {%- endfor %} +
+ + {% raw %} +
+
+

Report

+

{{ query_url }}

+ +
+
+ {% endraw %} +
+ {% raw %} +
+
+
+
+
+ {% endraw %} + + +
+{% endblock %} + +{% block head %} +{{super()}} +{{fixes.ie8()}} + + + +{% endblock %} + diff --git a/web/templates/url-report.html b/web/templates/url-report.html new file mode 100644 index 0000000..5c1fdbc --- /dev/null +++ b/web/templates/url-report.html @@ -0,0 +1,46 @@ +{% raw %} + + + + + {{url}} + + + + + + +
+ +
+ + + + + + + + + + + + +
+ +
+ + + + + + + + + + + +
+ + + +{% endraw %} diff --git a/worker.py b/worker.py new file mode 100755 index 0000000..634d3f9 --- /dev/null +++ b/worker.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os + +import redis +from rq import Worker, Queue, Connection + +listen = ['default'] + +redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6334') + +conn = redis.from_url(redis_url) + +if __name__ == '__main__': + with Connection(conn): + worker = Worker(map(Queue, listen)) + worker.work() +