issue:3563885:UFM Enterprise network reports

plugins/events_history_plugin/src/events_history/app.py

@@ -24,6 +24,8 @@
 from api.conf_api import EventsHistoryPluginConfigurationsAPI
 from mgr.events_history_configurations_mgr import EventsHistoryConfigParser
 from api.events_history_api import EventsHistoryApi
+from mgr.events_history_files_watcher import EventsHistoryFilesWatcher, EventsHistoryListener
+from constants.events_constants import EventsConstants
 
 def _init_logs(config_parser):
     # init logs configs
@@ -34,6 +36,15 @@ def _init_logs(config_parser):
     Logger.init_logs_config(logs_file_name, logs_level, max_log_file_size, log_file_backup_count)
 
 
+def _run_event_logs_watcher():
+    event_log_files = [os.path.join(EventsConstants.EVENT_LOGS_DIRECTORY, EventsConstants.EVENT_LOG)]
+    # Create watcher and listener.
+    e_listener = EventsHistoryListener()
+    r_critical_watcher = EventsHistoryFilesWatcher. \
+        getInstance(event_log_files)
+    r_critical_watcher.addListener(e_listener)
+
+
 if __name__ == '__main__':
     try:
 
@@ -56,6 +67,8 @@ def _init_logs(config_parser):
             "/events-history": EventsHistoryApi().application,
         }
 
+        _run_event_logs_watcher()
+
         app = BaseFlaskAPIApp(routes)
         run_api(app=app, port_number=int(plugin_port))
 

plugins/events_history_plugin/src/events_history/mgr/events_history_files_watcher.py

@@ -0,0 +1,74 @@
+from utils.filewatch.abstract_file_watcher import AbstractFileWatcher
+import pyinotify
+from utils.logger import Logger, LOG_LEVELS
+from mgr.events_history_mgr import EventsHistoryMgr
+from constants.events_constants import EventsConstants
+import os
+
+
+class EventsHistoryListener():
+    """
+    Listen for changes in files
+    """
+    # The last_position is used to read only the newly added lines in the log file by updating the pointer to that
+    # position each time the file is read.
+
+    def __init__(self):
+        self.last_position = self._get_last_position()
+        self.event_mgr = EventsHistoryMgr.getInstance()
+
+    def onChange(self, event):
+        """
+        Called whenever a modification has been monitored by the watcher
+        :param event: the data of the event that occurred(file name, mask name,..).
+        :return:
+        """
+        with open(event.pathname) as log_file:
+            log_file.seek(self.last_position)
+            new_lines = log_file.readlines()
+            self.last_position = log_file.tell()  # Update last position
+
+            for line in new_lines:
+                # Process each new line here
+                self.event_mgr.create_event(line)
+
+    def _get_last_position(self):
+        file_path = os.path.join(EventsConstants.EVENT_LOGS_DIRECTORY, EventsConstants.EVENT_LOG)
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                new_lines = f.readlines()
+                return f.tell()
+        except Exception as e:
+            Logger.log_message(f"Error while getting the last opsition in {file_path}: {e}", LOG_LEVELS.ERROR)
+
+
+class EventsHistoryFilesWatcher(AbstractFileWatcher):
+    """
+    This class is responsible for monitoring which files have been modified by listening to the IN_MODIFY events
+    provided by the pyinotify library.
+    """
+    def __init__(self, files_list, read_freq=0):
+        """
+        :param files_list: list, is the list of files that we want to listen to when a change
+        :param read_freq: int, is the interval between every time we request for the files that have been modified
+        """
+        events_list = [pyinotify.IN_MODIFY]
+        super(EventsHistoryFilesWatcher, self).__init__(files_list, events_list, read_freq=read_freq)
+
+    def notifyListeners(self, event):
+        """
+        This function is designed notify the listeners that a file has been modified
+        :param event: the data of the event that occurred(file name, mask name,..).
+        :return:
+        """
+        for listener in self._listener_dict.values():
+            listener.onChange(event)
+
+    def process_IN_MODIFY(self, event):
+        """
+        Notify listeners that a file has been modified.
+        This function will be called when the event of a file modification (IN_MODIFY event) occurs
+        :param event: the event that occurred.
+        """
+        Logger.log_message(f"event: {event}", LOG_LEVELS.DEBUG)
+        self.notifyListeners(event)

utils/filewatch/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# This software product is a proprietary product of Nvidia Corporation and its affiliates
+# (the "Company") and all right, title, and interest in and to the software
+# product, including all associated intellectual property rights, are and
+# shall remain exclusively with the Company.
+#
+# This software product is governed by the End User License Agreement
+# provided with the software product.
+# @author: Abeer Moghrabi
+# @date:   Aug 15, 2023
+#
+
+

utils/filewatch/abstract_file_watcher.py

@@ -0,0 +1,188 @@
+#
+# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# This software product is a proprietary product of Nvidia Corporation and its affiliates
+# (the "Company") and all right, title, and interest in and to the software
+# product, including all associated intellectual property rights, are and
+# shall remain exclusively with the Company.
+#
+# This software product is governed by the End User License Agreement
+# provided with the software product.
+# @author: Abeer Moghrabi
+# @date:   Aug 15, 2023
+#
+
+import os
+from threading import Lock
+import pyinotify
+from utils.singleton import Singleton
+from utils.logger import Logger, LOG_LEVELS
+
+
+class AbstractFileWatcher(Singleton):
+    """
+    Base class for File Watching ,
+    This Class should be extended and not used directly!
+    Each derived class should can watch one or more files.
+
+    Example of subclass can be found in infra.filewatch in file gvWatcher.py
+    Note: this class was copied from UFM project
+    """
+
+    def __init__(self, file_list, events, read_freq=300):
+        """
+        :param file_list: list, list of paths to files the watcher will listen on
+        :param events: list, list of events on the files that the watcher listens on taken from Events class (below this class)
+        :param read_freq: int, is the interval between every time we request for the files that have been modified
+        @ivar _watchedFileList: the list of files we want to watch
+        @ivar _watchedDirs: the list of directories we want to watch
+        @ivar _listener_list: the list of listeners to inform on file events
+        @ivar _listener_id_counter: the id that the next listener will get
+        @ivar _listener_data_lock: the lock being locked when adding/removing listeners
+        """
+        self._watchedFileList = set([os.path.abspath(f) for f in file_list])
+        self._watchedDirs = set([os.path.dirname(f) for f in self._watchedFileList])
+        self._listener_dict = {}
+        self._listener_id_counter = 0
+        self._listener_data_lock = Lock()
+        self._startWatching(events, read_freq=read_freq)
+
+    def _startWatching(self, events, read_freq):
+        """
+        Start file watching by the watch manager
+        :param events: list of events on the files that the watcher listens on taken from Events class (below this class)
+        """
+        # initialize bitmask with all zeros (no events).
+        mask = 0x00000000
+        # Iterate through the list of events and add each event to the bitmask.
+        # The '|' operator is used to perform a bitwise OR operation to combine event masks.
+        for event in events:
+            mask = mask | event
+        try :
+            wm = pyinotify.WatchManager()
+            eh = EventHandler(self)
+            self._notifier = pyinotify.ThreadedNotifier(wm, eh, read_freq=read_freq)
+
+            for path in self._watchedDirs:
+                wm.add_watch(path, mask)
+
+            self._notifier.daemon = True
+            self._notifier.start()
+
+        except Exception as e:
+            Logger.log_message("FileWatcher: Was NOT able to start, %s is NOT watched, the reason is : %s" % (self._watchedFileList, e), LOG_LEVELS.ERROR)
+            return
+        Logger.log_message("FileWatcher: %s is being watched " % self._watchedFileList, LOG_LEVELS.INFO)
+
+    def __del__(self):
+        """
+        Stop file watching by the watch manager
+        """
+        if hasattr(self, '_notifier'):
+            self._notifier.stop()
+            Logger.log_message("FileWatcher: %s is NO LONGER being watched" % self._watchedFileList, LOG_LEVELS.INFO)
+
+    def addListener(self, listener):
+        """
+        adds listener to the file watcher
+        :param listener: the listener to add to the listeners list
+        :return: the listener id
+        """
+        self._listener_data_lock.acquire()
+        id = self._listener_id_counter
+        self._listener_dict[id] = listener
+        self._listener_id_counter += 1
+        self._listener_data_lock.release()
+        return id
+
+    def removeListener(self, id):
+        """
+        removes listener from the file watcher by id given
+        :param id: the id of the listener to remove
+        :return: True if the listener was removed and false if no listener with the given id was found
+        """
+        try :
+            self._listener_dict.pop(id)
+
+        except :
+            return False
+
+        return True
+
+    def handleEvent(self, event):
+        if event.pathname in self._watchedFileList:
+            #print "==> ", event.maskname, ": ", event.pathname
+            # Invokes process_MASKNAME
+            meth = getattr(self, 'process_' + event.maskname, None)
+            if meth is not None:
+                meth(event)
+
+    """
+    The methods below should be implemented by subclasses, each method deals with different event type (on files).
+    for example, in this methods is where you will notify the listeners on the files of the event that happened.
+    ** If you are watching more then one file, you can use the event object to get the file that caused the event. **
+    """
+
+    def process_IN_ACCESS(self, event):
+        pass
+
+    def process_IN_MODIFY(self, event):
+        pass
+
+    def process_IN_ATTRIB(self, event):
+        pass
+
+    def process_IN_CLOSE_WRITE(self, event):
+        pass
+
+    def process_IN_CLOSE_NOWRITE(self, event):
+        pass
+
+    def process_IN_OPEN(self, event):
+        pass
+
+    def process_IN_MOVED_FROM(self, event):
+        pass
+
+    def process_IN_MOVED_TO(self, event):
+        pass
+
+    def process_IN_CREATE(self, event):
+        pass
+
+    def process_IN_DELETE(self, event):
+        pass
+
+    def process_IN_DELETE_SELF(self, event):
+        pass
+
+    def process_IN_MOVE_SELF(self, event):
+        pass
+
+    def process_default(self, event):
+        pass
+
+
+class EventHandler(pyinotify.ProcessEvent):
+    """
+    This class handle specific actions when filesystem events occur.
+    It's responsible for processing events such as file modifications, access, attribute changes, and more
+    !!! This class should not be touched !!!
+    """
+
+    def __init__(self, fileWatcher):
+        """
+        @ivar _fw :
+            The instance of the fileWatcher
+        """
+        self._fw = fileWatcher
+
+    def process_default(self, event):
+        """
+        when an event type that doesn't have a specific method override is encountered. In other words, it serves
+        as a catch-all method that gets called for any event type that doesn't have its own dedicated method defined
+        in your custom event handler class.
+        :param event: the data of the event that occurred(file name, mask name,..).
+        :return:
+        """
+        self._fw.handleEvent(event)