MISP
/
misp-galaxy
mirror of https://github.com/MISP/misp-galaxy
2
0
Fork 0
Code Issues Releases Wiki Activity

new: [tool] Generator for UK Health Security Agency Culture Collections

pull/955/head
Christophe Vandeplas 2024-03-29 14:43:41 +01:00 committed by GitHub
parent 2d4a03a553
commit 91827dbe83
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 133 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
tools/gen_ukhsa_culture_collections.py Normal file
View File

@ -0,0 +1,133 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# A simple convertor of the UK Health Security Agency Culture Collections
# to a MISP Galaxy datastructure.
# Copyright (C) 2024 MISP Project
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import json
import requests
import uuid
'''
From https://www.culturecollections.org.uk/search/?searchScope=Product&pageNumber=1&filter.collectionGroup=0&filter.collection=0&filter.sorting=DateCreated
JSON is loaded, needs to be paginated
Culturecollections.org.uk is published under the Open Government Licence, allowing the reproduction of information as
long as the license terms are obeyed. Material on this website is subject to Crown copyright protection unless otherwise
indicated. Users should be aware that information provided to third parties through feeds may be edited or cached, and
we do not guarantee the accuracy of such third-party products.
https://www.culturecollections.org.uk/training-and-support/policies/terms-and-conditions-of-use/
The Culture Collections represent deposits of cultures from world-wide sources. While every effort is made to ensure
details distributed by Culture Collections are accurate, Culture Collections cannot be held responsible for any
inaccuracies in the data supplied. References where quoted are mainly attributed to the establishment of the cell
culture and not for any specific property of the cell line, therefore further references should be obtained regarding
cell culture characteristics. Passage numbers where given act only as a guide and Culture Collections does not guarantee
the passage number stated will be the passage number received by the customer.
'''
def download_items():
data = {'items': [],
'collections': {},
'collection_groups': {}}
page_number = 1
page_number_max = None
while True:
url = 'https://www.culturecollections.org.uk/umbraco/api/searchApi/getSearchResults?searchParams={"searchText":"","searchScope":"Product","pageNumber":' + str(page_number) + ',"filter":{"collectionGroup":"0","collection":"0","facets":{},"sorting":"DateCreated"}}'
page_resp = requests.get(url)
page_resp.encoding = 'utf-8-sig'
page_data = page_resp.json()
page_number_max = page_data['pagination']['totalPages']
for c in page_data['filter']['collections']['aggregationItems']:
data['collections'][int(c['value'])] = c['title']
for cg in page_data['filter']['collectionGroups']['aggregationItems']:
data['collection_groups'][int(cg['value'])] = cg['title']
for item in page_data['items']:
item['collection'] = data['collections'][item['collectionId']]
data['items'].extend(page_data['items'])
print(f"Fetching page {page_number}/{page_number_max}: ", end="")
print(f"items size is now {len(data['items'])} as I extended with {len(page_data['items'])} items.")
if page_number >= page_number_max:
break
page_number += 1
return data
def save_items(d):
with open('items.json', 'w') as f:
json.dump(d, f, indent=2, sort_keys=True)
return True
def load_saved_items():
with open('items.json', 'r') as f:
d = json.load(f)
return d
data = download_items()
save_items(data)
# data = load_saved_items()
clusters = []
collections = set()
for item in data['items']:
cluster = {
'value': f"{item['name']}",
'uuid': str(uuid.uuid5(uuid.UUID("bbe11c06-1d6a-477e-88f1-cdda2d71de56"), item['catalogueNumber'])),
'meta': {
'refs': [item['url']],
'external_id': item['catalogueNumber']
}
}
for p in item['properties']:
if p['value']:
p_name = p['name'].lower().replace(' ', '_')
if p['name'] not in cluster['meta']:
cluster['meta'][p_name] = []
cluster['meta'][p_name].append(p['value'])
clusters.append(cluster)
json_galaxy = {
'icon': "virus",
'name': "UKHSA Culture Collections",
'description': "UK Health Security Agency Culture Collections represent deposits of cultures that consist of expertly preserved, authenticated cell lines and microbial strains of known provenance.",
'namespace': "gov.uk",
'type': "ukhsa-culture-collections",
'uuid': "bbe11c06-1d6a-477e-88f1-cdda2d71de56",
'version': 1
}
with open(os.path.join('', 'clusters', 'ukhsa-culture-collections.json'), 'r') as f: # FIXME add .. to path
json_cluster = json.load(f)
json_cluster['values'] = clusters
json_cluster['version'] += 1
# save the Galaxy and Cluster file
with open(os.path.join('', 'galaxies', 'ukhsa-culture-collections.json'), 'w') as f: # FIXME add .. to path
json.dump(json_galaxy, f, indent=2, sort_keys=True, ensure_ascii=False)
f.write('\n') # only needed for the beauty and to be compliant with jq_all_the_things
with open(os.path.join('', 'clusters', 'ukhsa-culture-collections.json'), 'w') as f: # FIXME add .. to path
json.dump(json_cluster, f, indent=2, sort_keys=True, ensure_ascii=False)
f.write('\n') # only needed for the beauty and to be compliant with jq_all_the_things
print("All done, please don't forget to ./jq_all_the_things.sh, commit, and then ./validate_all.sh.")