fix: [Global catch incomplete file error] fix #464

bin/Global.py

@@ -28,10 +28,21 @@ import os
 import sys
 import time
 import uuid
+
+import datetime
+import redis
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
 from pubsublogger import publisher
 
 from Helper import Process
 
+config_loader = ConfigLoader.ConfigLoader()
+r_stats = config_loader.get_redis_conn("ARDB_Statistics")
+config_loader = None
+
 def gunzip_bytes_obj(bytes_obj):
     in_ = io.BytesIO()
     in_.write(bytes_obj)
@@ -119,6 +130,14 @@ if __name__ == '__main__':
                         curr_file_content = f.read()
                 except EOFError:
                     publisher.warning('Global; Incomplete file: {}'.format(filename))
+                    # save daily stats
+                    r_stats.zincrby('module:Global:incomplete_file', datetime.datetime.now().strftime('%Y%m%d'), 1)
+                    # discard item
+                    continue
+                except OSError:
+                    publisher.warning('Global; Not a gzipped file: {}'.format(filename))
+                    # save daily stats
+                    r_stats.zincrby('module:Global:invalid_file', datetime.datetime.now().strftime('%Y%m%d'), 1)
                     # discard item
                     continue