Minor updates to alarms

elkserver/docker/redelk-base/redelkinstalldata/scripts/alarm.py

@@ -16,9 +16,13 @@
 
 from modules.helpers import *
 from config import alarms, notifications
+import config as localconfig
 import itertools
 
-LOG_LEVEL = logging.DEBUG
+if localconfig.DEBUG:
+    LOG_LEVEL = logging.DEBUG
+else:
+    LOG_LEVEL = logging.INFO
 
 if __name__ == '__main__':
     logging.basicConfig(
@@ -74,27 +78,28 @@
         if a in alarms and alarms[a]['enabled']:
             logger.debug('Alarm %s enabled, processing hits' % a)
             r = aD[a]['result']
+            alarm_name = aD[a]['info']['submodule']
             #logger.debug('Alarm results: %s' % aD[a]['result'])
             for rHit in r['hits']['hits']:
-                alarm_name = aD[a]['info']['submodule']
-                # Let's tag the doc with the alarm name
-                setTags(alarm_name, [rHit])
                 # First check if there is a mutation data to add
                 if rHit['_id'] in r['mutations']:
                     m = r['mutations'][rHit['_id']]
                 else:
                     m = {}
                 # And now, let's add mutations data to the doc and update back the hits
                 rHit = addAlarmData(rHit, m, alarm_name)
+            # Let's tag the doc with the alarm name
+            setTags(alarm_name, r['hits']['hits'])
+            logger.info('calling settags %s  (%d hits)' % (alarm_name, r['hits']['total']))
             # Needed as groupHits will change r['hits']['hits'] and different alarms might do different grouping
             r = copy.deepcopy(aD[a]['result'])
             for c in cD:
                 # connector will process ['hits']['hits'] which contains a list of 'jsons' looking like an ES line
                 # connector will report the fields in ['hits']['fields'] for each of the lines in the list
                 if c in notifications and notifications[c]['enabled']:
-                    logger.info('connector %s enabled, sending alarm (%d hits)' % (c, r['hits']['total']))
                     connector = cD[c]['m'].Module()
                     if r['hits']['total'] > 0:
+                        logger.info('connector %s enabled, sending alarm (%d hits)' % (c, r['hits']['total']))
                         # Group the hits before sending it to the alarm, based on the 'groubpby' array returned by the alarm
                         gb = list(r['groupby'])
                         r['hits']['hits'] = groupHits(r['hits']['hits'], gb)

elkserver/docker/redelk-base/redelkinstalldata/scripts/modules/alarm_dummy/module.py

@@ -27,7 +27,7 @@ def __init__(self):
     def run(self):
         ret = initial_alarm_result
         ret['info'] = info
-        ret['fields'] = ['@timestamp', 'host.name', 'user.name', 'ioc.type', 'file.name', 'file.hash.md5', 'ioc.domain', 'c2.message', 'alarm.alarm_filehash']
+        ret['fields'] = ['agent.hostname','@timestamp', 'host.name', 'user.name', 'ioc.type', 'file.name', 'file.hash.md5', 'ioc.domain', 'c2.message', 'alarm.alarm_filehash']
         ret['groupby'] = []
         self.logger.debug('Running dummy alarm')
         for r in self.alarm_dummy():
@@ -40,7 +40,7 @@ def run(self):
         return(ret)
 
     def alarm_dummy(self):
-        q = "c2.log.type:ioc AND NOT tags:%s"%(info['submodule'])
+        q = "c2.log.type:ioc AND NOT tags:alarm_*"
         report = {}
         report['alarm'] = False
         report['fname'] = "alarm_check2"

elkserver/docker/redelk-base/redelkinstalldata/scripts/modules/alarm_filehash/module.py

@@ -32,7 +32,7 @@ def __init__(self):
     def run(self):
         ret = initial_alarm_result
         ret['info'] = info
-        ret['fields'] = ['@timestamp', 'host.name', 'user.name', 'ioc.type', 'file.name', 'file.hash.md5', 'c2.message', 'alarm.alarm_filehash']
+        ret['fields'] = ['agent.hostname','@timestamp', 'host.name', 'user.name', 'ioc.type', 'file.name', 'file.hash.md5', 'c2.message', 'alarm.alarm_filehash']
         ret['groupby'] = ['file.hash.md5']
         try:
             report = self.alarm_check()

elkserver/docker/redelk-base/redelkinstalldata/scripts/modules/alarm_httptraffic/module.py

@@ -28,7 +28,7 @@ def __init__(self):
     def run(self):
         ret = initial_alarm_result
         ret['info'] = info
-        ret['fields'] = ['source.ip', 'source.nat.ip', 'source.geo.country_name', 'source.as.organization.name', 'redir.frontend.name', 'redir.backend.name', 'infra.attack_scenario', 'tags', 'redir.timestamp']
+        ret['fields'] = ['agent.hostname','source.ip', 'source.nat.ip', 'source.geo.country_name', 'source.as.organization.name', 'redir.frontend.name', 'redir.backend.name', 'infra.attack_scenario', 'tags', 'redir.timestamp']
         ret['groupby'] = ['source.ip']
         try:
             report = self.alarm_check()
@@ -45,7 +45,7 @@ def run(self):
 
     def alarm_check(self):
         # This check queries for IP's that aren't listed in any iplist* but do talk to c2* paths on redirectors\n
-        q = "NOT tags:iplist_* AND redir.backend.name:c2* AND tags:enriched_* AND NOT tags:%s"%(info['submodule'])
+        q = "NOT tags:iplist_* AND redir.backend.name:c2* AND NOT tags:alarm_httptraffic AND tags:enriched_*"
         i = countQuery(q)
         if i >= 10000:
             i = 10000

elkserver/docker/redelk-base/redelkinstalldata/scripts/modules/alarm_useragent/module.py

@@ -23,12 +23,13 @@
 class Module():
     def __init__(self):
         self.logger = logging.getLogger(info['submodule'])
+        #print("class init")
         pass
 
     def run(self):
         ret = initial_alarm_result
         ret['info'] = info
-        ret['fields'] = ['@timestamp','source.ip','http.headers.useragent','source.nat.ip','redir.frontend.name','redir.backend.name','infra.attack_scenario']
+        ret['fields'] = ['agent.hostname','@timestamp','source.ip','http.headers.useragent','source.nat.ip','redir.frontend.name','redir.backend.name','infra.attack_scenario']
         ret['groupby'] = ['source.ip','http.headers.useragent']
         try:
             report = self.alarm_check()
@@ -65,7 +66,7 @@ def alarm_check(self):
                 qSub = qSub + " OR http.headers.useragent:%s" % keyword
         qSub = qSub + ") "
         #q = "%s AND redir.backendname:c2* AND tags:enrich_* AND NOT tags:alarm_* "%qSub
-        q = "%s AND redir.backend.name:c2* AND NOT tags:%s"%(qSub,info['submodule'])
+        q = "%s AND redir.backend.name:c2* AND NOT tags:alarm_useragent" % qSub
         i = countQuery(q)
         if i >= 10000:
             i = 10000

elkserver/docker/redelk-base/redelkinstalldata/scripts/modules/helpers.py

@@ -78,7 +78,6 @@ def guiQueryWindow(q,start,end):
     return(q)
 
 def countQuery(query, index="redirtraffic-*"):
-    print('count')
     q3 = {'query': {'query_string': {'query': query}}}
     r3 = es.search(index=index, body=q3, size=0)
     return(r3['hits']['total']['value'])