chg: [doc] updated Changelog.md to be more markdown friendly

chg: [tools] Changed the way gen_misp_install_docs.sh parseses the changelog
new: [tools] Added simple tool for git log sanitizing.

.gitchangelog.rc

@@ -14,7 +14,7 @@
 ##       'fix' is for bug fixes
 ##       'new' is for new features, big improvement
 ##
-##   AUDIENCE is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc'
+##   AUDIENCE is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc'|'docs'
 ##
 ##       Is WHO is concerned by the change.
 ##
@@ -82,13 +82,13 @@ ignore_regexps = [
 ##
 section_regexps = [
     ('New', [
-        r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+        r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n]*)$',
      ]),
     ('Changes', [
-        r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+        r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n]*)$',
      ]),
     ('Fix', [
-        r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+        r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n]*)$',
      ]),
 
     ('Other', None ## Match all lines
@@ -141,7 +141,7 @@ body_process = ReSub(r'((^|\n)[A-Z]\w+(-\w+)*: .*(\n\s+.*)*)+$', r'') | strip
 ##
 ## Available constructs are those listed in ``body_process`` doc.
 subject_process = (strip |
-    ReSub(r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$', r'\4') |
+    ReSub(r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$', r'\4') |
     SetIfEmpty("No commit message.") | ucfirst | final_dot)
 
 

tools/gen_misp_changelog.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+# Sanitize the gitchangelog output
+
+import shutil
+
+inputFile = "../docs/Changelog.md"
+outputFile = "/tmp/Changelog.tmp"
+
+previousLine = ""
+
+output_file = open(outputFile, "w")
+
+for line in open(inputFile, "r"):
+    if line == previousLine:
+        previousLine = line
+        continue
+    else:
+        output_file.write(line)
+        previousLine = line
+
+output_file.close()
+
+shutil.move(outputFile, inputFile)

tools/gen_misp_install_docs.sh

@@ -7,20 +7,56 @@ if [ -e "/usr/bin/virtualenv" ]; then
 	echo "Python virtualenv exists, continuing with mkdocs build"
 else
 	echo "NO virtualenv present, bye."
+  echo "sudo apt install virtualenv # Might help"
   exit 1
 fi
 
 if [ -z "$VIRTUAL_ENV" ]; then
-    virtualenv -p python3 mkdocs
+    virtualenv -p python3 mkdocs || echo "You probably having Python running somewhere."
     ${PWD}/mkdocs/bin/pip install mkdocs mkdocs-material markdown-include python-markdown-comments gitchangelog
 fi
 
-wget -O ../docs/Changelog.md https://www.misp-project.org/Changelog.txt
+# Fixing ASCII aborration introduced in: https://github.com/MISP/MISP/commit/1b028ee15a3bd2f209102cd6204e6c4bb519be97
+${PWD}/mkdocs/bin/gitchangelog |grep -v -e "  ,," -e "\.\.," > ../docs/Changelog.md
+# Removing consecutive dupe lines
+${PWD}/gen_misp_changelog.py
+
+# For local testing, gitchangelog on large repos takes time.
+#${PWD}/mkdocs/bin/gitchangelog > ../Changelog.txt
+#cat ../Changelog.txt |grep -v -e "  ,," -e "\.\.," > ../docs/Changelog.md
+
 # This search and replace is sub-optimal. It replaces 3 "~"s beginning of the line 
 # and then just replaces the remaining 2 following tildes in the document. 
 # This might change the sense of some commit messages...
 sed -i "s/^\~\~\~/---/" ../docs/Changelog.md
+sed -i "s/^- \#/- \\\#/" ../docs/Changelog.md
 sed -i "s/\~\~/--/g" ../docs/Changelog.md
+sed -i "s/%%version%%/v2.4 aka 2.4 for ever/g" ../docs/Changelog.md
+sed -i "s/\(unreleased\)/current changelog/g" ../docs/Changelog.md
+
+# Emojifying things
+sed -i "s/\/\!\\\/:warning:/g" ../docs/Changelog.md
+sed -i "s/WiP/:construction:/g" ../docs/Changelog.md
+sed -i "s/WIP/:construction:/g" ../docs/Changelog.md
+sed -i "s/Wip:/:construction:/g" ../docs/Changelog.md
+sed -i "s/\[security\]/:lock:/g" ../docs/Changelog.md
+
+## Other creative ways in sprinkling emoji goodness:
+### Source: https://gist.github.com/pocotan001/68f96bf86891db316f20
+#- :art:              when improving the format/structure of the code
+#- :rocket:           when improving performance
+#- :pencil2:          when writing docs
+#- :bulb:             new idea
+#- :construction:     work in progress
+#- :heavy_plus_sign:  when adding feature
+#- :heavy_minus_sign: when removing feature
+#- :speaker:          when adding logging
+#- :mute:             when reducing logging
+#- :bug:              when fixing a bug
+#- :white_check_mark: when adding tests
+#- :lock:             when dealing with security
+#- :arrow_up:         when upgrading dependencies
+#- :arrow_down:       when downgrading dependencies
 
 # Deploy mkdocs to gh-pages branch
 cd ../ ; ${PWD}/tools/mkdocs/bin/mkdocs gh-deploy