mirror of https://github.com/MISP/misp-galaxy
new: [tool] Generator for UK Health Security Agency Culture Collections
Christophe Vandeplas
GitHub
parent
2d4a03a553
commit
91827dbe83
|
|
@ -0,0 +1,133 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# A simple convertor of the UK Health Security Agency Culture Collections
|
||||
# to a MISP Galaxy datastructure.
|
||||
# Copyright (C) 2024 MISP Project
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as
|
||||
# published by the Free Software Foundation, either version 3 of the
|
||||
# License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import os
|
||||
import json
|
||||
import requests
|
||||
import uuid
|
||||
|
||||
'''
|
||||
From https://www.culturecollections.org.uk/search/?searchScope=Product&pageNumber=1&filter.collectionGroup=0&filter.collection=0&filter.sorting=DateCreated
|
||||
JSON is loaded, needs to be paginated
|
||||
|
||||
Culturecollections.org.uk is published under the Open Government Licence, allowing the reproduction of information as
|
||||
long as the license terms are obeyed. Material on this website is subject to Crown copyright protection unless otherwise
|
||||
indicated. Users should be aware that information provided to third parties through feeds may be edited or cached, and
|
||||
we do not guarantee the accuracy of such third-party products.
|
||||
https://www.culturecollections.org.uk/training-and-support/policies/terms-and-conditions-of-use/
|
||||
|
||||
The Culture Collections represent deposits of cultures from world-wide sources. While every effort is made to ensure
|
||||
details distributed by Culture Collections are accurate, Culture Collections cannot be held responsible for any
|
||||
inaccuracies in the data supplied. References where quoted are mainly attributed to the establishment of the cell
|
||||
culture and not for any specific property of the cell line, therefore further references should be obtained regarding
|
||||
cell culture characteristics. Passage numbers where given act only as a guide and Culture Collections does not guarantee
|
||||
the passage number stated will be the passage number received by the customer.
|
||||
'''
|
||||
|
||||
def download_items():
|
||||
data = {'items': [],
|
||||
'collections': {},
|
||||
'collection_groups': {}}
|
||||
page_number = 1
|
||||
page_number_max = None
|
||||
while True:
|
||||
url = 'https://www.culturecollections.org.uk/umbraco/api/searchApi/getSearchResults?searchParams={"searchText":"","searchScope":"Product","pageNumber":' + str(page_number) + ',"filter":{"collectionGroup":"0","collection":"0","facets":{},"sorting":"DateCreated"}}'
|
||||
page_resp = requests.get(url)
|
||||
page_resp.encoding = 'utf-8-sig'
|
||||
page_data = page_resp.json()
|
||||
page_number_max = page_data['pagination']['totalPages']
|
||||
|
||||
for c in page_data['filter']['collections']['aggregationItems']:
|
||||
data['collections'][int(c['value'])] = c['title']
|
||||
for cg in page_data['filter']['collectionGroups']['aggregationItems']:
|
||||
data['collection_groups'][int(cg['value'])] = cg['title']
|
||||
for item in page_data['items']:
|
||||
item['collection'] = data['collections'][item['collectionId']]
|
||||
data['items'].extend(page_data['items'])
|
||||
print(f"Fetching page {page_number}/{page_number_max}: ", end="")
|
||||
print(f"items size is now {len(data['items'])} as I extended with {len(page_data['items'])} items.")
|
||||
if page_number >= page_number_max:
|
||||
break
|
||||
page_number += 1
|
||||
return data
|
||||
|
||||
|
||||
def save_items(d):
|
||||
with open('items.json', 'w') as f:
|
||||
json.dump(d, f, indent=2, sort_keys=True)
|
||||
return True
|
||||
|
||||
def load_saved_items():
|
||||
with open('items.json', 'r') as f:
|
||||
d = json.load(f)
|
||||
return d
|
||||
|
||||
data = download_items()
|
||||
save_items(data)
|
||||
# data = load_saved_items()
|
||||
|
||||
clusters = []
|
||||
collections = set()
|
||||
for item in data['items']:
|
||||
cluster = {
|
||||
'value': f"{item['name']}",
|
||||
'uuid': str(uuid.uuid5(uuid.UUID("bbe11c06-1d6a-477e-88f1-cdda2d71de56"), item['catalogueNumber'])),
|
||||
'meta': {
|
||||
'refs': [item['url']],
|
||||
'external_id': item['catalogueNumber']
|
||||
}
|
||||
}
|
||||
for p in item['properties']:
|
||||
if p['value']:
|
||||
p_name = p['name'].lower().replace(' ', '_')
|
||||
if p['name'] not in cluster['meta']:
|
||||
cluster['meta'][p_name] = []
|
||||
cluster['meta'][p_name].append(p['value'])
|
||||
|
||||
clusters.append(cluster)
|
||||
|
||||
|
||||
|
||||
json_galaxy = {
|
||||
'icon': "virus",
|
||||
'name': "UKHSA Culture Collections",
|
||||
'description': "UK Health Security Agency Culture Collections represent deposits of cultures that consist of expertly preserved, authenticated cell lines and microbial strains of known provenance.",
|
||||
'namespace': "gov.uk",
|
||||
'type': "ukhsa-culture-collections",
|
||||
'uuid': "bbe11c06-1d6a-477e-88f1-cdda2d71de56",
|
||||
'version': 1
|
||||
}
|
||||
|
||||
with open(os.path.join('', 'clusters', 'ukhsa-culture-collections.json'), 'r') as f: # FIXME add .. to path
|
||||
json_cluster = json.load(f)
|
||||
json_cluster['values'] = clusters
|
||||
json_cluster['version'] += 1
|
||||
|
||||
# save the Galaxy and Cluster file
|
||||
with open(os.path.join('', 'galaxies', 'ukhsa-culture-collections.json'), 'w') as f: # FIXME add .. to path
|
||||
json.dump(json_galaxy, f, indent=2, sort_keys=True, ensure_ascii=False)
|
||||
f.write('\n') # only needed for the beauty and to be compliant with jq_all_the_things
|
||||
|
||||
with open(os.path.join('', 'clusters', 'ukhsa-culture-collections.json'), 'w') as f: # FIXME add .. to path
|
||||
json.dump(json_cluster, f, indent=2, sort_keys=True, ensure_ascii=False)
|
||||
f.write('\n') # only needed for the beauty and to be compliant with jq_all_the_things
|
||||
|
||||
print("All done, please don't forget to ./jq_all_the_things.sh, commit, and then ./validate_all.sh.")
|
||||
Loading…
Reference in New Issue