Merge branch 'master' into master

pull/453/head
rmkml 2019-09-07 19:52:05 +02:00 committed by GitHub
commit 7c89cb308c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 14192 additions and 688 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
__pycache__

View File

@ -80,7 +80,17 @@
], ],
"uuid": "a4757e11-0837-42c0-958a-7490cff58687", "uuid": "a4757e11-0837-42c0-958a-7490cff58687",
"value": "SLUB" "value": "SLUB"
},
{
"description": "Since it first emerged in 2015, Asruex has been known for its backdoor capabilities and connection to the spyware DarkHotel. However, when we encountered Asruex in a PDF file, we found that a variant of the malware can also act as an infector particularly through the use of old vulnerabilities CVE-2012-0158 and CVE-2010-2883, which inject code in Word and PDF files respectively.",
"meta": {
"refs": [
"https://blog.trendmicro.com/trendlabs-security-intelligence/asruex-backdoor-variant-infects-word-documents-and-pdfs-through-old-ms-office-and-adobe-vulnerabilities/"
]
},
"uuid": "b7ad60a0-d648-4775-adec-c78b1a92fc34",
"value": "Asruex"
} }
], ],
"version": 5 "version": 6
} }

View File

@ -172,7 +172,17 @@
}, },
"uuid": "3434339f-ea87-472e-a330-62d2b5cf2c26", "uuid": "3434339f-ea87-472e-a330-62d2b5cf2c26",
"value": "SPOILER" "value": "SPOILER"
},
{
"description": "A wormable critical Remote Code Execution (RCE) vulnerability in Remote Desktop Services that could soon become the new go-to vector for spreading malware",
"meta": {
"refs": [
"https://www.welivesecurity.com/2019/05/22/patch-now-bluekeep-vulnerability/"
]
},
"uuid": "4f993170-f264-4c39-8c7f-58f9f2b9d105",
"value": "BlueKeep"
} }
], ],
"version": 3 "version": 4
} }

View File

@ -218,9 +218,6 @@
{ {
"description": "Taurus Builder is a tool used to generate malicious MS Word documents that contain macros. The kit is advertised on forums by the user \"badbullzvenom\". ", "description": "Taurus Builder is a tool used to generate malicious MS Word documents that contain macros. The kit is advertised on forums by the user \"badbullzvenom\". ",
"meta": { "meta": {
"refs": [
""
],
"status": "Active" "status": "Active"
}, },
"uuid": "63988ca2-46c8-4bda-be46-96a8670af357", "uuid": "63988ca2-46c8-4bda-be46-96a8670af357",

View File

@ -775,7 +775,7 @@
"meta": { "meta": {
"external_id": "T1452", "external_id": "T1452",
"kill_chain": [ "kill_chain": [
"mitre-mobile-attack:effects" "mitre-mobile-attack:impact"
], ],
"mitre_platforms": [ "mitre_platforms": [
"Android", "Android",
@ -2072,7 +2072,7 @@
"meta": { "meta": {
"external_id": "APP-28", "external_id": "APP-28",
"kill_chain": [ "kill_chain": [
"mitre-mobile-attack:effects" "mitre-mobile-attack:impact"
], ],
"mitre_platforms": [ "mitre_platforms": [
"Android", "Android",
@ -3648,7 +3648,7 @@
"meta": { "meta": {
"external_id": "T1472", "external_id": "T1472",
"kill_chain": [ "kill_chain": [
"mitre-mobile-attack:effects" "mitre-mobile-attack:impact"
], ],
"mitre_platforms": [ "mitre_platforms": [
"Android", "Android",
@ -3825,7 +3825,7 @@
"meta": { "meta": {
"external_id": "T1448", "external_id": "T1448",
"kill_chain": [ "kill_chain": [
"mitre-mobile-attack:effects" "mitre-mobile-attack:impact"
], ],
"mitre_platforms": [ "mitre_platforms": [
"Android" "Android"
@ -7096,7 +7096,7 @@
"meta": { "meta": {
"external_id": "T1447", "external_id": "T1447",
"kill_chain": [ "kill_chain": [
"mitre-mobile-attack:effects" "mitre-mobile-attack:impact"
], ],
"mitre_platforms": [ "mitre_platforms": [
"Android" "Android"
@ -9731,7 +9731,7 @@
"meta": { "meta": {
"external_id": "APP-28", "external_id": "APP-28",
"kill_chain": [ "kill_chain": [
"mitre-mobile-attack:effects" "mitre-mobile-attack:impact"
], ],
"mitre_platforms": [ "mitre_platforms": [
"Android" "Android"
@ -10263,7 +10263,7 @@
"value": "Repackaged Application - T1444" "value": "Repackaged Application - T1444"
}, },
{ {
"description": "Adversaries may destroy data data and files on specific systems or in large numbers on a network to interrupt availability to systems, services, and network resources. Data destruction is likely to render stored data irrecoverable by forensic techniques through overwriting files or data on local and remote drives.(Citation: Symantec Shamoon 2012)(Citation: FireEye Shamoon Nov 2016)(Citation: Palo Alto Shamoon Nov 2016)(Citation: Kaspersky StoneDrill 2017)(Citation: Unit 42 Shamoon3 2018)(Citation: Talos Olympic Destroyer 2018) Common operating system file deletion commands such as <code>del</code> and <code>rm</code> often only remove pointers to files without wiping the contents of the files themselves, making the files recoverable by proper forensic methodology. This behavior is distinct from [Disk Content Wipe](https://attack.mitre.org/techniques/T1488) and [Disk Structure Wipe](https://attack.mitre.org/techniques/T1487) because individual files are destroyed rather than sections of a storage disk or the disk's logical structure.\n\nAdversaries may attempt to overwrite files and directories with randomly generated data to make it irrecoverable.(Citation: Kaspersky StoneDrill 2017)(Citation: Unit 42 Shamoon3 2018) In some cases politically oriented image files have been used to overwrite data.(Citation: FireEye Shamoon Nov 2016)(Citation: Palo Alto Shamoon Nov 2016)(Citation: Kaspersky StoneDrill 2017)\n\nTo maximize impact on the target organization in operations where network-wide availability interruption is the goal, malware designed for destroying data may have worm-like features to propagate across a network by leveraging additional techniques like [Valid Accounts](https://attack.mitre.org/techniques/T1078), [Credential Dumping](https://attack.mitre.org/techniques/T1003), and [Windows Admin Shares](https://attack.mitre.org/techniques/T1077).(Citation: Symantec Shamoon 2012)(Citation: FireEye Shamoon Nov 2016)(Citation: Palo Alto Shamoon Nov 2016)(Citation: Kaspersky StoneDrill 2017)(Citation: Talos Olympic Destroyer 2018)", "description": "Adversaries may destroy data and files on specific systems or in large numbers on a network to interrupt availability to systems, services, and network resources. Data destruction is likely to render stored data irrecoverable by forensic techniques through overwriting files or data on local and remote drives.(Citation: Symantec Shamoon 2012)(Citation: FireEye Shamoon Nov 2016)(Citation: Palo Alto Shamoon Nov 2016)(Citation: Kaspersky StoneDrill 2017)(Citation: Unit 42 Shamoon3 2018)(Citation: Talos Olympic Destroyer 2018) Common operating system file deletion commands such as <code>del</code> and <code>rm</code> often only remove pointers to files without wiping the contents of the files themselves, making the files recoverable by proper forensic methodology. This behavior is distinct from [Disk Content Wipe](https://attack.mitre.org/techniques/T1488) and [Disk Structure Wipe](https://attack.mitre.org/techniques/T1487) because individual files are destroyed rather than sections of a storage disk or the disk's logical structure.\n\nAdversaries may attempt to overwrite files and directories with randomly generated data to make it irrecoverable.(Citation: Kaspersky StoneDrill 2017)(Citation: Unit 42 Shamoon3 2018) In some cases politically oriented image files have been used to overwrite data.(Citation: FireEye Shamoon Nov 2016)(Citation: Palo Alto Shamoon Nov 2016)(Citation: Kaspersky StoneDrill 2017)\n\nTo maximize impact on the target organization in operations where network-wide availability interruption is the goal, malware designed for destroying data may have worm-like features to propagate across a network by leveraging additional techniques like [Valid Accounts](https://attack.mitre.org/techniques/T1078), [Credential Dumping](https://attack.mitre.org/techniques/T1003), and [Windows Admin Shares](https://attack.mitre.org/techniques/T1077).(Citation: Symantec Shamoon 2012)(Citation: FireEye Shamoon Nov 2016)(Citation: Palo Alto Shamoon Nov 2016)(Citation: Kaspersky StoneDrill 2017)(Citation: Talos Olympic Destroyer 2018)",
"meta": { "meta": {
"external_id": "T1485", "external_id": "T1485",
"kill_chain": [ "kill_chain": [
@ -10637,7 +10637,7 @@
"value": "Masquerading - T1036" "value": "Masquerading - T1036"
}, },
{ {
"description": "Adversaries may use scripts to aid in operations and perform multiple actions that would otherwise be manual. Scripting is useful for speeding up operational tasks and reducing the time required to gain access to critical resources. Some scripting languages may be used to bypass process monitoring mechanisms by directly interacting with the operating system at an API level instead of calling other programs. Common scripting languages for Windows include VBScript and PowerShell but could also be in the form of command-line batch scripts.\n\nScripts can be embedded inside Office documents as macros that can be set to execute when files used in [Spearphishing Attachment](https://attack.mitre.org/techniques/T1193) and other types of spearphishing are opened. Malicious embedded macros are an alternative means of execution than software exploitation through [Exploitation for Client Execution](https://attack.mitre.org/techniques/T1203), where adversaries will rely on macros being allowed or that the user will accept to activate them.\n\nMany popular offensive frameworks exist which use forms of scripting for security testers and adversaries alike. (Citation: Metasploit) (Citation: Metasploit), (Citation: Veil) (Citation: Veil), and PowerSploit (Citation: Powersploit) are three examples that are popular among penetration testers for exploit and post-compromise operations and include many features for evading defenses. Some adversaries are known to use PowerShell. (Citation: Alperovitch 2014)", "description": "Adversaries may use scripts to aid in operations and perform multiple actions that would otherwise be manual. Scripting is useful for speeding up operational tasks and reducing the time required to gain access to critical resources. Some scripting languages may be used to bypass process monitoring mechanisms by directly interacting with the operating system at an API level instead of calling other programs. Common scripting languages for Windows include VBScript and PowerShell but could also be in the form of command-line batch scripts.\n\nScripts can be embedded inside Office documents as macros that can be set to execute when files used in [Spearphishing Attachment](https://attack.mitre.org/techniques/T1193) and other types of spearphishing are opened. Malicious embedded macros are an alternative means of execution than software exploitation through [Exploitation for Client Execution](https://attack.mitre.org/techniques/T1203), where adversaries will rely on macros being allowed or that the user will accept to activate them.\n\nMany popular offensive frameworks exist which use forms of scripting for security testers and adversaries alike. Metasploit (Citation: Metasploit_Ref), Veil (Citation: Veil_Ref), and PowerSploit (Citation: Powersploit) are three examples that are popular among penetration testers for exploit and post-compromise operations and include many features for evading defenses. Some adversaries are known to use PowerShell. (Citation: Alperovitch 2014)",
"meta": { "meta": {
"external_id": "T1064", "external_id": "T1064",
"kill_chain": [ "kill_chain": [
@ -11083,5 +11083,5 @@
"value": "DNSCalc - T1324" "value": "DNSCalc - T1324"
} }
], ],
"version": 9 "version": 10
} }

File diff suppressed because it is too large Load Diff

View File

@ -3672,5 +3672,5 @@
"value": "Security Software Discovery Mitigation - T1063" "value": "Security Software Discovery Mitigation - T1063"
} }
], ],
"version": 7 "version": 8
} }

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1670,5 +1670,5 @@
"value": "Malicious Software Development Tools - MOB-T1065" "value": "Malicious Software Development Tools - MOB-T1065"
} }
], ],
"version": 5 "version": 6
} }

View File

@ -274,6 +274,13 @@
"estimative-language:likelihood-probability=\"almost-certain\"" "estimative-language:likelihood-probability=\"almost-certain\""
], ],
"type": "mitigates" "type": "mitigates"
},
{
"dest-uuid": "2204c371-6100-4ae0-82f3-25c07c29772a",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
} }
], ],
"uuid": "649f7268-4c12-483b-ac84-4b7bca9fe2ee", "uuid": "649f7268-4c12-483b-ac84-4b7bca9fe2ee",
@ -304,5 +311,5 @@
"value": "Encrypt Network Traffic - MOB-M1009" "value": "Encrypt Network Traffic - MOB-M1009"
} }
], ],
"version": 6 "version": 7
} }

View File

@ -1117,5 +1117,5 @@
"value": "XcodeGhost - MOB-S0013" "value": "XcodeGhost - MOB-S0013"
} }
], ],
"version": 8 "version": 9
} }

View File

@ -2785,5 +2785,5 @@
"value": "Data Hiding - PRE-T1097" "value": "Data Hiding - PRE-T1097"
} }
], ],
"version": 6 "version": 7
} }

View File

@ -222,6 +222,13 @@
"estimative-language:likelihood-probability=\"almost-certain\"" "estimative-language:likelihood-probability=\"almost-certain\""
], ],
"type": "uses" "type": "uses"
},
{
"dest-uuid": "830c9528-df21-472c-8c14-a036bf17d665",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "uses"
} }
], ],
"uuid": "c47f937f-1022-4f42-8525-e7a4779a14cb", "uuid": "c47f937f-1022-4f42-8525-e7a4779a14cb",
@ -369,5 +376,5 @@
"value": "APT17 - G0025" "value": "APT17 - G0025"
} }
], ],
"version": 8 "version": 9
} }

View File

@ -2493,8 +2493,8 @@
"refs": [ "refs": [
"https://attack.mitre.org/software/S0262", "https://attack.mitre.org/software/S0262",
"https://github.com/quasar/QuasarRAT", "https://github.com/quasar/QuasarRAT",
"https://documents.trendmicro.com/assets/tech-brief-untangling-the-patchwork-cyberespionage-group.pdf", "https://www.volexity.com/blog/2018/06/07/patchwork-apt-group-targets-us-think-tanks/",
"https://www.volexity.com/blog/2018/06/07/patchwork-apt-group-targets-us-think-tanks/" "https://documents.trendmicro.com/assets/tech-brief-untangling-the-patchwork-cyberespionage-group.pdf"
], ],
"synonyms": [ "synonyms": [
"QuasarRAT", "QuasarRAT",
@ -3724,5 +3724,5 @@
"value": "Nltest - S0359" "value": "Nltest - S0359"
} }
], ],
"version": 13 "version": 15
} }

View File

@ -12889,8 +12889,7 @@
"read_me_for_recover_your_files.txt" "read_me_for_recover_your_files.txt"
], ],
"refs": [ "refs": [
"https://www.bleepingcomputer.com/news/security/the-week-in-ransomware-september-14th-2018-kraken-dharma-and-matrix/", "https://www.bleepingcomputer.com/news/security/the-week-in-ransomware-september-14th-2018-kraken-dharma-and-matrix/"
""
] ]
}, },
"uuid": "3675e50d-3f76-45f8-b3f3-4a645779e14d", "uuid": "3675e50d-3f76-45f8-b3f3-4a645779e14d",

View File

@ -119,6 +119,13 @@
"estimative-language:likelihood-probability=\"likely\"" "estimative-language:likelihood-probability=\"likely\""
], ],
"type": "similar" "type": "similar"
},
{
"dest-uuid": "c82c904f-b3b4-40a2-bf0d-008912953104",
"tags": [
"estimative-language:likelihood-probability=\"likely\""
],
"type": "used-by"
} }
], ],
"uuid": "4e104fef-8a2c-4679-b497-6e86d7d47db0", "uuid": "4e104fef-8a2c-4679-b497-6e86d7d47db0",
@ -674,6 +681,13 @@
"estimative-language:likelihood-probability=\"likely\"" "estimative-language:likelihood-probability=\"likely\""
], ],
"type": "similar" "type": "similar"
},
{
"dest-uuid": "c82c904f-b3b4-40a2-bf0d-008912953104",
"tags": [
"estimative-language:likelihood-probability=\"likely\""
],
"type": "used-by"
} }
], ],
"uuid": "255a59a7-db2d-44fc-9ca9-5859b65817c3", "uuid": "255a59a7-db2d-44fc-9ca9-5859b65817c3",
@ -3349,6 +3363,37 @@
"uuid": "1b6a066c-50ba-4aa6-a49b-823e94e110fe", "uuid": "1b6a066c-50ba-4aa6-a49b-823e94e110fe",
"value": "Caesar RAT" "value": "Caesar RAT"
}, },
{
"description": "During the month of October, Check Point researchers discovered a widespread malware campaign spreading a remote access trojan (dubbed “FlawedAmmy”) that allows attackers to take over victims computers and data. The campaign was the latest and most widespread delivering the FlawedAmmyy RAT, following a number of campaigns that have spread this malware in recent months. The Trojan allows attackers to gain full access to the machines camera and microphone, collect screen grabs, steal credentials and sensitive files, and intrusively monitor the victims actions. As a result, FlawedAmmy is the first RAT to enter the Global Threat Indexs top 10 ranking. ",
"meta": {
"refs": [
"https://www.helpnetsecurity.com/2018/11/14/flawedammy-most-wanted-malware-list/"
]
},
"uuid": "4b9b99f0-9c2d-4db5-aaff-09de88509c04",
"value": "FlawedAmmy"
},
{
"description": "The Zscaler ThreatLabZ team came across a new strain of infostealer Trojan called Felipe, which silently installs itself onto a users system and connects to a command-and-control (C&C) server to send system information from the compromised system. This malware is compiled for both 32-bit and 64-bit Windows operating systems. Felipe basically steals the victim's debit and credit card information and sends it, along with other personal information, to the remote C&C server. It also sets a date and time to perform other malicious activity upon successful infection of the victim machine.",
"meta": {
"refs": [
"https://www.zscaler.com/blogs/research/felipe-new-infostealer-trojan"
]
},
"uuid": "0f117f50-9657-11e9-8e2b-83e391e0ce57",
"value": "Felipe"
},
{
"description": "Amavaldo is banking trojan writen in Delphi and known to targeting Spanish or Portuguese speaking countries. It contains backdoor functionality and can work as multi stage. Amavaldo also abuses legitimate tools and softwares",
"meta": {
"date": "2019",
"refs": [
"https://www.welivesecurity.com/2019/08/01/banking-trojans-amavaldo/"
]
},
"uuid": "39c65b1d-7799-43d6-a963-4a058b1c756e",
"value": "Amavaldo Banking Trojan"
},
{ {
"description": "Open-Source Remote Administration Tool For Windows C# (RAT)", "description": "Open-Source Remote Administration Tool For Windows C# (RAT)",
"meta": { "meta": {
@ -3361,5 +3406,5 @@
"value": "AsyncRAT" "value": "AsyncRAT"
} }
], ],
"version": 29 "version": 30
} }

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -141,6 +141,13 @@
] ]
}, },
"related": [ "related": [
{
"dest-uuid": "c82c904f-b3b4-40a2-bf0d-008912953104",
"tags": [
"estimative-language:likelihood-probability=\"likely\""
],
"type": "used-by"
},
{ {
"dest-uuid": "4e104fef-8a2c-4679-b497-6e86d7d47db0", "dest-uuid": "4e104fef-8a2c-4679-b497-6e86d7d47db0",
"tags": [ "tags": [
@ -198,6 +205,15 @@
"Backdoor" "Backdoor"
] ]
}, },
"related": [
{
"dest-uuid": "c82c904f-b3b4-40a2-bf0d-008912953104",
"tags": [
"estimative-language:likelihood-probability=\"likely\""
],
"type": "used-by"
}
],
"uuid": "32a67552-3b31-47bb-8098-078099bbc813", "uuid": "32a67552-3b31-47bb-8098-078099bbc813",
"value": "Torn RAT" "value": "Torn RAT"
}, },
@ -1022,6 +1038,15 @@
"Gh0stRat, GhostRat" "Gh0stRat, GhostRat"
] ]
}, },
"related": [
{
"dest-uuid": "c82c904f-b3b4-40a2-bf0d-008912953104",
"tags": [
"estimative-language:likelihood-probability=\"likely\""
],
"type": "used-by"
}
],
"uuid": "cb8c8253-4024-4cc9-8989-b4a5f95f6c2f", "uuid": "cb8c8253-4024-4cc9-8989-b4a5f95f6c2f",
"value": "Gh0st Rat" "value": "Gh0st Rat"
}, },
@ -7762,7 +7787,28 @@
], ],
"uuid": "80365d3a-6d46-4195-a772-364749a6dc06", "uuid": "80365d3a-6d46-4195-a772-364749a6dc06",
"value": "SunOrcal" "value": "SunOrcal"
},
{
"description": "Threat actors have delivered Bookworm as a payload in attacks on targets in Thailand. Readers who are interested in this campaign should start with our first blog that lays out the overall functionality of the malware and introduces its many components.\n Unit 42 does not have detailed targeting information for all known Bookworm samples, but we are aware of attempted attacks on at least two branches of government in Thailand. We speculate that other attacks delivering Bookworm were also targeting organizations in Thailand based on the contents of the associated decoys documents, as well as several of the dynamic DNS domain names used to host C2 servers that contain the words “Thai” or “Thailand”. Analysis of compromised systems seen communicating with Bookworm C2 servers also confirms our speculation on targeting with a majority of systems existing within Thailand.",
"meta": {
"refs": [
"https://unit42.paloaltonetworks.com/attack-campaign-on-the-government-of-thailand-delivers-bookworm-trojan/",
"https://unit42.paloaltonetworks.com/bookworm-trojan-a-model-of-modular-architecture/"
]
},
"uuid": "9ff6e087-6755-447a-b537-8f06c7aa4a85",
"value": "Bookworm"
},
{
"description": "We named the malware family described in the rest of this blog post Amavaldo. This family is still in active development the latest version we have observed (10.7) has a compilation timestamp of June 10th, 2019.",
"meta": {
"refs": [
"https://www.welivesecurity.com/2019/08/01/banking-trojans-amavaldo/"
]
},
"uuid": "c72f8f57-fc2f-4ca2-afbe-ca5bfa5a1747",
"value": "Amavaldo"
} }
], ],
"version": 121 "version": 123
} }

View File

@ -1,6 +1,6 @@
{ {
"description": "Banking malware galaxy.", "description": "Banking malware galaxy.",
"icon": "usd", "icon": "dollar-sign",
"name": "Banker", "name": "Banker",
"namespace": "misp", "namespace": "misp",
"type": "banker", "type": "banker",

View File

@ -1,6 +1,6 @@
{ {
"description": "ATT&CK Mitigation", "description": "ATT&CK Mitigation",
"icon": "chain", "icon": "link",
"name": "Course of Action", "name": "Course of Action",
"namespace": "mitre-attack", "namespace": "mitre-attack",
"type": "mitre-course-of-action", "type": "mitre-course-of-action",

View File

@ -1,6 +1,6 @@
{ {
"description": "ATT&CK Mitigation", "description": "ATT&CK Mitigation",
"icon": "chain", "icon": "link",
"name": "Enterprise Attack - Course of Action", "name": "Enterprise Attack - Course of Action",
"namespace": "deprecated", "namespace": "deprecated",
"type": "mitre-enterprise-attack-course-of-action", "type": "mitre-enterprise-attack-course-of-action",

View File

@ -1,6 +1,6 @@
{ {
"description": "ATT&CK Mitigation", "description": "ATT&CK Mitigation",
"icon": "chain", "icon": "link",
"name": "Mobile Attack - Course of Action", "name": "Mobile Attack - Course of Action",
"namespace": "deprecated", "namespace": "deprecated",
"type": "mitre-mobile-attack-course-of-action", "type": "mitre-mobile-attack-course-of-action",

View File

@ -0,0 +1,9 @@
{
"description": "Description of targets of threat actors.",
"icon": "bullseye",
"name": "Target Information",
"namespace": "misp",
"type": "target-information",
"uuid": "709ed29c-aa00-11e9-82cd-67ac1a6ee3bc",
"version": 1
}

0
tools/__init__.py Normal file
View File

View File

@ -8,9 +8,19 @@ import os
import collections import collections
def loadjsons(path): def loadjsons(path, return_paths=False):
""" """
Find all Jsons and load them in a dict Find all Jsons and load them in a dict
Parameters:
path: string
return_names: boolean, if the name of the file should be returned,
default: False
Returns:
List of parsed file contents.
If return_paths is True, then every list item is a tuple of the
file name and the file content
""" """
files = [] files = []
data = [] data = []
@ -18,9 +28,14 @@ def loadjsons(path):
if os.path.isfile(os.path.join(path, name)) and name.endswith('.json'): if os.path.isfile(os.path.join(path, name)) and name.endswith('.json'):
files.append(name) files.append(name)
for jfile in files: for jfile in files:
data.append(json.load(open("%s/%s" % (path, jfile)))) filepath = os.path.join(path, jfile)
if return_paths:
data.append((filepath, json.load(open(filepath))))
else:
data.append(json.load(json.load(open(filepath))))
return data return data
if __name__ == '__main__': if __name__ == '__main__':
""" """
Iterate all name + synonyms Iterate all name + synonyms
@ -33,19 +48,19 @@ if __name__ == '__main__':
items = djson.get('values') items = djson.get('values')
for entry in items: for entry in items:
name = entry.get('value').strip().lower() name = entry.get('value').strip().lower()
counter[name]+=1 counter[name] += 1
namespace.append([name, djson.get('name')]) namespace.append([name, djson.get('name')])
try: try:
for synonym in entry.get('meta').get('synonyms'): for synonym in entry.get('meta').get('synonyms'):
name = synonym.strip().lower() name = synonym.strip().lower()
counter[name]+=1 counter[name] += 1
namespace.append([name, djson.get('name')]) namespace.append([name, djson.get('name')])
except (AttributeError, TypeError): except (AttributeError, TypeError):
pass pass
counter = dict(counter) counter = dict(counter)
for key, val in counter.items(): for key, val in counter.items():
if val>1: if val > 1:
print ("Warning duplicate %s" % key) print("Warning duplicate %s" % key)
for item in namespace: for item in namespace:
if item[0]==key: if item[0] == key:
print (item) print(item)

24
tools/chk_empty_strings.py Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env python3
# coding=utf-8
"""
Tools to find empty string entries in galaxies
"""
from .chk_dup import loadjsons
import sys
if __name__ == '__main__':
jsons = loadjsons("clusters", return_paths=True)
retval = 0
for clustername, djson in jsons:
items = djson.get('values')
for entry in items:
name = entry.get('value')
for key, value in entry.get('meta', {}).items():
if isinstance(value, list):
if '' in value:
retval = 1
print("Empty string found in Cluster %r: values/%s/meta/%s"
"" % (clustername, name, key),
file=sys.stderr)
sys.exit(retval)

View File

@ -84,3 +84,6 @@ do
fi fi
echo '' echo ''
done done
# check for empyt strings in clusters
python3 -m tools.chk_empty_strings