From fe77114b84b3c40620215fdaf971d825800bb382 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 13 Oct 2023 18:36:13 +0200 Subject: [PATCH] chg: [description_value] reprocess clusters to avoid duplicate on value --- tools/description_value.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 tools/description_value.py diff --git a/tools/description_value.py b/tools/description_value.py new file mode 100755 index 0000000..84c682d --- /dev/null +++ b/tools/description_value.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# coding=utf-8 +""" + Tool to remove duplicates in value +""" +import sys +import json + +with open(sys.argv[1], 'r') as f: + data = json.load(f) + +#for c in data['values']: +# c['value'] = f'{c["value"]} - {c["meta"]["description"]}' + +value_seen = [] +data_output = [] +for c in data['values']: + if c['value'] in value_seen: + continue + else: + data_output.append(c) + value_seen.append(c['value']) + +data['values'] = data_output +with open(sys.argv[1], 'w') as f: + json.dump(data, f) +