diff --git a/plugins/events_history_plugin/build/Dockerfile b/plugins/events_history_plugin/build/Dockerfile new file mode 100644 index 000000000..a2abbf5ec --- /dev/null +++ b/plugins/events_history_plugin/build/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:20.04 + +LABEL maintainer="abeerm@nvidia.com" + +ARG PLUGIN_NAME=events_history +ARG DEBIAN_FRONTEND=noninteractive +ARG BASE_PATH=/opt/ufm/ufm_plugin_${PLUGIN_NAME} +ARG SRC_BASE_DIR=${PLUGIN_NAME}_plugin + +COPY ${SRC_BASE_DIR}/ ${BASE_PATH}/${SRC_BASE_DIR}/ +COPY utils/ ${BASE_PATH}/utils/ + +COPY ${SRC_BASE_DIR}/conf/supervisord.conf /etc/supervisor/conf.d/ +COPY ${SRC_BASE_DIR}/scripts/init.sh ${SRC_BASE_DIR}/scripts/deinit.sh / + +RUN apt-get update && apt-get -y install supervisor python3 python3-pip vim curl sudo + +RUN python3 -m pip install -r ${BASE_PATH}/${SRC_BASE_DIR}/src/${PLUGIN_NAME}/requirements.txt + +CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/plugins/events_history_plugin/build/docker_build.sh b/plugins/events_history_plugin/build/docker_build.sh new file mode 100755 index 000000000..9affdccda --- /dev/null +++ b/plugins/events_history_plugin/build/docker_build.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +set -eE + +if [ "$EUID" -ne 0 ] + then echo "Please run the script as root" + exit +fi + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +PARENT_DIR=$(realpath "${SCRIPT_DIR}/../../../") + +PLUGIN_NAME=events_history +IMAGE_NAME="ufm-plugin-${PLUGIN_NAME}" +IMAGE_VERSION=$1 +OUT_DIR=$2 +RANDOM_HASH=$3 + +echo "RANDOM_HASH : [${RANDOM_HASH}]" +echo "SCRIPT_DIR : [${SCRIPT_DIR}]" +echo " " +echo "IMAGE_VERSION: [${IMAGE_VERSION}]" +echo "IMAGE_NAME : [${IMAGE_NAME}]" +echo "OUT_DIR : [${OUT_DIR}]" +echo " " + +if [ -z "${OUT_DIR}" ]; then + OUT_DIR="." +fi +if [ -z "${IMAGE_VERSION}" ]; then + IMAGE_VERSION="latest" +fi + +function create_out_dir() +{ + build_dir=$(mktemp --tmpdir -d ${IMAGE_NAME}_output_XXXXXXXX) + chmod 777 ${build_dir} + echo ${build_dir} +} + +function build_docker_image() +{ + build_dir=$1 + image_name=$2 + image_version=$3 + out_dir=$4 + random_hash=$5 + keep_image=$6 + prefix="mellanox" + + echo "build_docker_image" + echo " build_dir : [${build_dir}]" + echo " image_name : [${image_name}]" + echo " image_version : [${image_version}]" + echo " random_hash : [${random_hash}]" + echo " out_dir : [${out_dir}]" + echo " keep_image : [${keep_image}]" + echo " prefix : [${prefix}]" + echo " " + if [ "${IMAGE_VERSION}" == "0.0.00-0" ]; then + full_image_version="${image_name}_${image_version}-${random_hash}" + else + full_image_version="${image_name}_${image_version}" + fi + + echo " full_image_version : [${full_image_version}]" + + image_with_prefix="${prefix}/${image_name}" + image_with_prefix_and_version="${prefix}/${image_name}:${image_version}" + + pushd ${build_dir} + + echo "docker build --network host --no-cache --pull -t ${image_with_prefix_and_version} . --compress --build-arg PLUGIN_NAME=${PLUGIN_NAME}" + + docker build --network host --no-cache --pull -t ${image_with_prefix_and_version} . --compress --build-arg PLUGIN_NAME=${PLUGIN_NAME} + exit_code=$? + popd + if [ $exit_code -ne 0 ]; then + echo "Failed to build image" + return $exit_code + fi + + printf "\n\n\n" + echo "docker images | grep ${image_with_prefix}" + docker images | grep ${image_with_prefix} + printf "\n\n\n" + + echo "docker save ${image_with_prefix_and_version} | gzip > ${out_dir}/${full_image_version}-docker.img.gz" + docker save ${image_with_prefix_and_version} | gzip > ${out_dir}/${full_image_version}-docker.img.gz + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Failed to save image" + return $exit_code + fi + if [ "$keep_image" != "y" -a "$keep_image" != "Y" ]; then + docker image rm -f ${image_with_prefix_and_version} + fi + return 0 +} + + +pushd ${SCRIPT_DIR} + +echo ${IMAGE_VERSION} > ../../${PLUGIN_NAME}_plugin/version + +BUILD_DIR=$(create_out_dir) +cp Dockerfile ${BUILD_DIR} +cp -r ../../../utils ${BUILD_DIR} +cp -r ../../${PLUGIN_NAME}_plugin ${BUILD_DIR} + +echo "BUILD_DIR : [${BUILD_DIR}]" + +build_docker_image $BUILD_DIR $IMAGE_NAME $IMAGE_VERSION $OUT_DIR ${RANDOM_HASH} +exit_code=$? +rm -rf ${BUILD_DIR} +popd +exit $exit_code diff --git a/plugins/events_history_plugin/conf/events_history_httpd_proxy.conf b/plugins/events_history_plugin/conf/events_history_httpd_proxy.conf new file mode 100644 index 000000000..987231299 --- /dev/null +++ b/plugins/events_history_plugin/conf/events_history_httpd_proxy.conf @@ -0,0 +1 @@ +port=8686 \ No newline at end of file diff --git a/plugins/events_history_plugin/conf/events_history_plugin.conf b/plugins/events_history_plugin/conf/events_history_plugin.conf new file mode 100644 index 000000000..06628928a --- /dev/null +++ b/plugins/events_history_plugin/conf/events_history_plugin.conf @@ -0,0 +1,6 @@ +[logs-config] +logs_file_name = /log/events_history_plugin.log +logs_level = INFO +log_file_max_size = 10485760 +log_file_backup_count = 5 + diff --git a/plugins/events_history_plugin/conf/supervisord.conf b/plugins/events_history_plugin/conf/supervisord.conf new file mode 100644 index 000000000..398ba2f5e --- /dev/null +++ b/plugins/events_history_plugin/conf/supervisord.conf @@ -0,0 +1,32 @@ + +[unix_http_server] +username = dummy +password = dummy + +[supervisorctl] +username = dummy +password = dummy + +[supervisord] +user=root +nodaemon = true +environment = PLACEHOLDER=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:events_history_service] +directory=/opt/ufm/ufm_plugin_events_history +command=python3 /opt/ufm/ufm_plugin_events_history/events_history_plugin/src/events_history_plugin/app.py +autostart=true +autorestart=true +startretries=1 +startsecs=1 +user=root +killasgroup=true +stopasgroup=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/plugins/events_history_plugin/scripts/deinit.sh b/plugins/events_history_plugin/scripts/deinit.sh new file mode 100755 index 000000000..fd27bfd2a --- /dev/null +++ b/plugins/events_history_plugin/scripts/deinit.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 14, 2023 +# + +set -eE + +# removing log file +rm -rf /log/events_history_plugin*.log* + +exit 0 diff --git a/plugins/events_history_plugin/scripts/init.sh b/plugins/events_history_plugin/scripts/init.sh new file mode 100755 index 000000000..8f5f9ef64 --- /dev/null +++ b/plugins/events_history_plugin/scripts/init.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Copyright © 2013-2023 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 14, 2023 +# +# ================================================================ +# This script prepares and checks events_history docker container Environment +# ================================================================ + +set -eE +PLUGIN_NAME=events_history +SRC_DIR_PATH=/opt/ufm/ufm_plugin_${PLUGIN_NAME}/${PLUGIN_NAME}_plugin +CONFIG_PATH=/config + +touch ${CONFIG_PATH}/${PLUGIN_NAME}_shared_volumes.conf + +echo /opt/ufm/files/log/:/log > ${CONFIG_PATH}/${PLUGIN_NAME}_shared_volumes.conf + +# UFM version test +required_ufm_version=(6 12 0) +echo "Required UFM version: ${required_ufm_version[0]}.${required_ufm_version[1]}.${required_ufm_version[2]}" + +if [ "$1" == "-ufm_version" ]; then + actual_ufm_version_string=$2 + actual_ufm_version=(${actual_ufm_version_string//./ }) + echo "Actual UFM version: ${actual_ufm_version[0]}.${actual_ufm_version[1]}.${actual_ufm_version[2]}" + if [ ${actual_ufm_version[0]} -ge ${required_ufm_version[0]} ] \ + && [ ${actual_ufm_version[1]} -ge ${required_ufm_version[1]} ] \ + && [ ${actual_ufm_version[2]} -ge ${required_ufm_version[2]} ]; then + echo "UFM version meets the requirements" + exit 0 + else + echo "UFM version is older than required" + exit 1 + fi +else + exit 1 +fi + +exit 1 \ No newline at end of file diff --git a/plugins/events_history_plugin/src/events_history/api/__init__.py b/plugins/events_history_plugin/src/events_history/api/__init__.py new file mode 100644 index 000000000..38c004003 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/api/__init__.py @@ -0,0 +1,13 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# diff --git a/plugins/events_history_plugin/src/events_history/api/conf_api.py b/plugins/events_history_plugin/src/events_history/api/conf_api.py new file mode 100644 index 000000000..c96b55244 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/api/conf_api.py @@ -0,0 +1,77 @@ +# +# Copyright © 2013-2023 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# +from flask import make_response, request +from http import HTTPStatus + +from utils.flask_server.base_flask_api_server import BaseAPIApplication +from utils.config_parser import InvalidConfRequest +from utils.logger import Logger, LOG_LEVELS +from utils.json_schema_validator import validate_schema + +from mgr.events_history_configurations_mgr import EventsHistoryConfigParser + + +class EventsHistoryPluginConfigurationsAPI(BaseAPIApplication): + """ + This class was designed to support retrieving and editing config file. + """ + + def __init__(self): + super(EventsHistoryPluginConfigurationsAPI, self).__init__() + self.conf = EventsHistoryConfigParser.getInstance() + + # for debugging + # self.conf_schema_path = "plugins/events_history/src/events_history/schemas/set_conf.schema.json" + + # for production with docker + self.conf_schema_path = "events_history/src/events_history/schemas/set_conf.schema.json" + + def _get_error_handlers(self): + return [ + (InvalidConfRequest, + lambda e: (str(e), HTTPStatus.BAD_REQUEST)) + ] + + def _get_routes(self): + return { + self.get_conf: dict(urls=["/"], methods=["GET"]), + self.update_conf: dict(urls=["/"], methods=["PUT"]) + } + + def get_conf(self): + try: + _response = self.conf.conf_to_dict(self.conf_schema_path) + return make_response(_response) + except Exception as e: + Logger.log_message("Error occurred while getting the current plugin configurations: " + str(e), + LOG_LEVELS.ERROR) + raise e + + def update_conf(self): + Logger.log_message('Updating the plugin configurations', + LOG_LEVELS.DEBUG) + try: + request_data = request.json + # validate the new data + validate_schema(self.conf_schema_path, request_data) + # update the new values + self.conf.update_config_file_values(request_data) + self.conf.update_config_file(self.conf.config_file) + #### + _response = self.conf.conf_to_dict(self.conf_schema_path) + return make_response(_response) + except Exception as ex: + Logger.log_message(f'Updating the plugin configurations has been failed: {str(ex)}', + LOG_LEVELS.ERROR) + raise ex diff --git a/plugins/events_history_plugin/src/events_history/api/events_history_api.py b/plugins/events_history_plugin/src/events_history/api/events_history_api.py new file mode 100644 index 000000000..6f7eb327c --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/api/events_history_api.py @@ -0,0 +1,50 @@ +# +# Copyright © 2013-2023 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# +from flask import make_response, request +from http import HTTPStatus + +from utils.flask_server.base_flask_api_server import BaseAPIApplication +from utils.config_parser import InvalidConfRequest +from utils.logger import Logger, LOG_LEVELS +from mgr.events_history_mgr import EventsHistoryMgr + + +class EventsHistoryApi(BaseAPIApplication): + + def __init__(self): + super(EventsHistoryApi, self).__init__() + self.event_mgr = EventsHistoryMgr.getInstance() + + + def _get_error_handlers(self): + return [ + (InvalidConfRequest, + lambda e: (str(e), HTTPStatus.BAD_REQUEST)) + ] + + def _get_routes(self): + return { + self.get_events_history: dict(urls=["/"], methods=["GET"]), + } + + def get_events_history(self): + try: + # TODO implement get events history + return make_response({}) + except Exception as e: + Logger.log_message("Error occurred while getting events history: " + str(e), + LOG_LEVELS.ERROR) + raise e + + diff --git a/plugins/events_history_plugin/src/events_history/api/ui_files_api.py b/plugins/events_history_plugin/src/events_history/api/ui_files_api.py new file mode 100644 index 000000000..443a75da0 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/api/ui_files_api.py @@ -0,0 +1,34 @@ +# +# Copyright © 2013-2023 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# +from flask import send_from_directory +from utils.flask_server.base_flask_api_server import BaseAPIApplication + + +class EventsHistoryPluginUIFilesAPI(BaseAPIApplication): + """ + This class was designed to support retrieving UI files. + """ + + def __init__(self): + super(EventsHistoryPluginUIFilesAPI, self).__init__() + self.files_path = "/data/events_history_ui/" + + def _get_routes(self): + return { + self.get_file: dict(urls=["/"], methods=["GET"]) + } + + def get_file(self, file_name): + return send_from_directory(self.files_path, file_name) + diff --git a/plugins/events_history_plugin/src/events_history/app.py b/plugins/events_history_plugin/src/events_history/app.py new file mode 100644 index 000000000..5521d02d5 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/app.py @@ -0,0 +1,76 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# + +import os +import sys +sys.path.append(os.getcwd()) + +from utils.utils import Utils +from utils.flask_server import run_api +from utils.flask_server.base_flask_api_app import BaseFlaskAPIApp +from utils.logger import Logger, LOG_LEVELS +from api.ui_files_api import EventsHistoryPluginUIFilesAPI +from api.conf_api import EventsHistoryPluginConfigurationsAPI +from mgr.events_history_configurations_mgr import EventsHistoryConfigParser +from api.events_history_api import EventsHistoryApi +from mgr.events_history_files_watcher import EventsHistoryFilesWatcher, EventsHistoryListener +from constants.events_constants import EventsConstants + +def _init_logs(config_parser): + # init logs configs + logs_file_name = config_parser.get_logs_file_name() + logs_level = config_parser.get_logs_level() + max_log_file_size = config_parser.get_log_file_max_size() + log_file_backup_count = config_parser.get_log_file_backup_count() + Logger.init_logs_config(logs_file_name, logs_level, max_log_file_size, log_file_backup_count) + + +def _run_event_logs_watcher(): + event_log_files = [os.path.join(EventsConstants.EVENT_LOGS_DIRECTORY, EventsConstants.EVENT_LOG)] + # Create watcher and listener. + e_listener = EventsHistoryListener() + r_critical_watcher = EventsHistoryFilesWatcher. \ + getInstance(event_log_files) + r_critical_watcher.addListener(e_listener) + + +if __name__ == '__main__': + try: + + conf = EventsHistoryConfigParser.getInstance() + _init_logs(conf) + + except ValueError as ve: + Logger.log_message('Error occurred during the plugin logs initialization:' + + str(ve), LOG_LEVELS.ERROR) + except Exception as ex: + Logger.log_message(str(ex)) + try: + plugin_port = Utils.get_plugin_port( + port_conf_file='/config/events_history_httpd_proxy.conf', + default_port_value=8686) + + routes = { + "/conf": EventsHistoryPluginConfigurationsAPI().application, + "/files": EventsHistoryPluginUIFilesAPI().application, + "/events-history": EventsHistoryApi().application, + } + + _run_event_logs_watcher() + + app = BaseFlaskAPIApp(routes) + run_api(app=app, port_number=int(plugin_port)) + + except Exception as ex: + print(f'Failed to run the app due to: {str(ex)}') diff --git a/plugins/events_history_plugin/src/events_history/constants/__init__.py b/plugins/events_history_plugin/src/events_history/constants/__init__.py new file mode 100644 index 000000000..e6bad8910 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/constants/__init__.py @@ -0,0 +1,13 @@ +# +# Copyright © 2013-2023 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# \ No newline at end of file diff --git a/plugins/events_history_plugin/src/events_history/constants/events_constants.py b/plugins/events_history_plugin/src/events_history/constants/events_constants.py new file mode 100644 index 000000000..214771a10 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/constants/events_constants.py @@ -0,0 +1,73 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# + +from enum import Enum + + +class EventTypeEnum(Enum): + LINK_IS_UP = '328' + LINK_IS_DOWN = '329' + Node_IS_UP = '332' + Node_IS_DOWN = '331' + SWITCH_IS_UP = '908' + SWITCH_IS_DOWN = '907' + DIRECTOR_SWITCH_IS_UP = '910' + DIRECTOR_SWITCH_IS_DOWN = '909' + + +class EventsConstants: + ID = "id" + NAME = "name" + TYPE = "type" + EVENT_TYPE = "event_type" + SEVERITY = "severity" + TIMESTAMP = "timestamp" + COUNTER = "counter" + CATEGORY = "category" + OBJECT_NAME = "object_name" + OBJECT_PATH = "object_path" + WRITE_TO_SYSLOG = "write_to_syslog" + DESCRIPTION = "description" + EVENT_LOG_PATTERN = r"(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.*\d*) \[(?P\d+)\] \[(?P\d+)\] (?P\w+)( Site \[.+?\])? \[(?P.+?)\] (?P\w+) \[(?P.+?)\]( \[dev_id:?(?P.+?)\])?.*?: (?P.+)" + EVENT_LOGS_DIRECTORY = '/log' + EVENT_LOG = "event.log" + + EVENTS_INFO = { + EventTypeEnum.LINK_IS_UP.value: { + "name":"Link is Up", + }, + EventTypeEnum.LINK_IS_DOWN.value: { + "name": "Link is Down", + }, + EventTypeEnum.Node_IS_UP.value: { + "name": "Node is Up", + }, + EventTypeEnum.Node_IS_DOWN.value: { + "name": "Node is Down", + }, + EventTypeEnum.SWITCH_IS_UP.value: { + "name": "Switch is Up", + }, + EventTypeEnum.SWITCH_IS_DOWN.value: { + "name": "Switch is Down", + }, + EventTypeEnum.DIRECTOR_SWITCH_IS_UP.value: { + "name": "Director Switch is Up", + }, + EventTypeEnum.DIRECTOR_SWITCH_IS_DOWN.value: { + "name": "Director Switch is Down", + }, + + } + diff --git a/plugins/events_history_plugin/src/events_history/mgr/__init__.py b/plugins/events_history_plugin/src/events_history/mgr/__init__.py new file mode 100644 index 000000000..38c004003 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/mgr/__init__.py @@ -0,0 +1,13 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# diff --git a/plugins/events_history_plugin/src/events_history/mgr/events_history_configurations_mgr.py b/plugins/events_history_plugin/src/events_history/mgr/events_history_configurations_mgr.py new file mode 100644 index 000000000..9f47a6b3f --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/mgr/events_history_configurations_mgr.py @@ -0,0 +1,35 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# + +from utils.config_parser import ConfigParser +from utils.singleton import Singleton + + +class EventsHistoryConfigParser(ConfigParser, Singleton): + """ + This class was designed to manage 'events_history_plugin.conf' file + """ + + # for debugging + # config_file = "../../conf/events_history_plugin.conf" + + # for production with docker + config_file = "/config/events_history_plugin.conf" + + + def __init__(self): + super(EventsHistoryConfigParser, self).__init__(read_sdk_config=False) + self.sdk_config.read(self.config_file) + + diff --git a/plugins/events_history_plugin/src/events_history/mgr/events_history_files_watcher.py b/plugins/events_history_plugin/src/events_history/mgr/events_history_files_watcher.py new file mode 100644 index 000000000..90dcbf54c --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/mgr/events_history_files_watcher.py @@ -0,0 +1,74 @@ +from utils.filewatch.abstract_file_watcher import AbstractFileWatcher +import pyinotify +from utils.logger import Logger, LOG_LEVELS +from mgr.events_history_mgr import EventsHistoryMgr +from constants.events_constants import EventsConstants +import os + + +class EventsHistoryListener(): + """ + Listen for changes in files + """ + # The last_position is used to read only the newly added lines in the log file by updating the pointer to that + # position each time the file is read. + + def __init__(self): + self.last_position = self._get_last_position() + self.event_mgr = EventsHistoryMgr.getInstance() + + def onChange(self, event): + """ + Called whenever a modification has been monitored by the watcher + :param event: the data of the event that occurred(file name, mask name,..). + :return: + """ + with open(event.pathname) as log_file: + log_file.seek(self.last_position) + new_lines = log_file.readlines() + self.last_position = log_file.tell() # Update last position + + for line in new_lines: + # Process each new line here + self.event_mgr.create_event(line) + + def _get_last_position(self): + file_path = os.path.join(EventsConstants.EVENT_LOGS_DIRECTORY, EventsConstants.EVENT_LOG) + try: + with open(file_path, 'r', encoding='utf-8') as f: + new_lines = f.readlines() + return f.tell() + except Exception as e: + Logger.log_message(f"Error while getting the last opsition in {file_path}: {e}", LOG_LEVELS.ERROR) + + +class EventsHistoryFilesWatcher(AbstractFileWatcher): + """ + This class is responsible for monitoring which files have been modified by listening to the IN_MODIFY events + provided by the pyinotify library. + """ + def __init__(self, files_list, read_freq=0): + """ + :param files_list: list, is the list of files that we want to listen to when a change + :param read_freq: int, is the interval between every time we request for the files that have been modified + """ + events_list = [pyinotify.IN_MODIFY] + super(EventsHistoryFilesWatcher, self).__init__(files_list, events_list, read_freq=read_freq) + + def notifyListeners(self, event): + """ + This function is designed notify the listeners that a file has been modified + :param event: the data of the event that occurred(file name, mask name,..). + :return: + """ + for listener in self._listener_dict.values(): + listener.onChange(event) + + def process_IN_MODIFY(self, event): + """ + Notify listeners that a file has been modified. + This function will be called when the event of a file modification (IN_MODIFY event) occurs + :param event: the event that occurred. + """ + Logger.log_message(f"event: {event}", LOG_LEVELS.DEBUG) + self.notifyListeners(event) diff --git a/plugins/events_history_plugin/src/events_history/mgr/events_history_mgr.py b/plugins/events_history_plugin/src/events_history/mgr/events_history_mgr.py new file mode 100644 index 000000000..1b58ed2ba --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/mgr/events_history_mgr.py @@ -0,0 +1,151 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 9, 2023 +# + + +from utils.singleton import Singleton +import fnmatch +import os +import gzip +from utils.logger import Logger, LOG_LEVELS +from model.event import Event +from constants.events_constants import EventsConstants, EventTypeEnum +import re + + +class EventsHistoryMgr(Singleton): + """ + This class is designed to manage UFM events history + """ + + def __init__(self): + + self.events_collections = self._init_events_collections() + self.parse_event_log_files() + + def _init_events_collections(self): + """ + This function is designed to initialize an event collections map. Each supported event will have a corresponding + collection. This will increase filtering performance. + :return: + """ + events_collections = {} + for supported_event in EventTypeEnum: + events_collections[supported_event.value] = EventsCollection(Event) + return events_collections + + def openEventFile(self, fname, mode): + """ + Opens an event log file whether it is a compressed file or a regular one + @param fname: the event log file name to be opened + @param mode: a string indicating how the file is to be opened. + @return: a reference to the opened file + """ + try: + if fname.endswith(".gz"): + return gzip.open(fname, mode, encoding='utf8') + else: + return open(fname, mode) + except FileNotFoundError: + Logger.log_message(f"File {fname} not found.", + LOG_LEVELS.ERROR) + + except Exception as ex: + Logger.log_message(f"An error occurred while parsing {fname}:" + str(ex), + LOG_LEVELS.ERROR) + + def parse_event_log_files(self): + """ + This function is designed to parse the event log files and create an event if the event type is one of the topology change events. + """ + try: + # Gets all event log files + files = fnmatch.filter(os.listdir(EventsConstants.EVENT_LOGS_DIRECTORY), EventsConstants.EVENT_LOG + "*.gz") + files.append(EventsConstants.EVENT_LOG) + for file in files: + fname = os.path.join(EventsConstants.EVENT_LOGS_DIRECTORY, file) + Logger.log_message(f"Parsing file {fname}", LOG_LEVELS.INFO) + if os.path.exists(fname): + f = self.openEventFile(fname, 'rt') + if f: + for line in f: + self.create_event(line) + f.close() + Logger.log_message(f"Parsing file {fname} completed successfully", LOG_LEVELS.INFO) + except Exception as e: + Logger.log_message("Error occurred while parsing events logs files: " + str(e), + LOG_LEVELS.ERROR) + + def create_event(self, log_line): + """ + This function is designed to create an event object from the given log line; the only accepted event is a topology change event. + :param log_line: str, log line e.g. 2023-08-13 22:08:53.704 [45] [352] INFO [Logical_Model] Grid [Grid]: Network management is added + :return: + """ + try: + match = re.search(EventsConstants.EVENT_LOG_PATTERN, log_line) + if match: + # Extract the variables from the match object + time = match.group(EventsConstants.TIMESTAMP) + id = match.group(EventsConstants.ID) + event_type = match.group(EventsConstants.EVENT_TYPE) + severity = match.group(EventsConstants.SEVERITY) + category = match.group(EventsConstants.CATEGORY).replace('_', ' ') + type = match.group(EventsConstants.TYPE) + object_path = match.group(EventsConstants.OBJECT_PATH) + description = match.group(EventsConstants.DESCRIPTION) + # object_name not always exists + if match.group(EventsConstants.OBJECT_NAME): + object_name = match.group(EventsConstants.OBJECT_NAME) + else: + object_name = "N/A" + if event_type in [s_event.value for s_event in EventTypeEnum]: + event_name = EventsConstants.EVENTS_INFO[event_type]["name"] + event = Event(timestamp=time, id=id, event_type=event_type, severity=severity, + category=category, object_name=object_name, object_path=object_path, + description=description, name=event_name, type=type) + Logger.log_message(f"Event {event_name} with id={id} was created successfully", LOG_LEVELS.INFO) + self.events_collections[event_type].add(event) + except Exception as ex: + Logger.log_message(f"Error occurred while parsing event log line {log_line}: {str(ex)}", + LOG_LEVELS.ERROR) + + +class EventsCollection(dict): + """ + This class represents a collection of UFM events history. + """ + def __init__(self, typ): + self.typ = typ + + def __call__(self, id=0): + if id: + return list(self.values()[id:]) + else: + return list(self.values()) + + def count(self): + """return the number of element in the collection""" + return len(self) + + def add(self, obj): + """add event to the collection""" + self[obj.id] = obj + + def remove(self, obj): + """remove event from the collection""" + del self[obj.id] + + + + diff --git a/plugins/events_history_plugin/src/events_history/model/__init__.py b/plugins/events_history_plugin/src/events_history/model/__init__.py new file mode 100644 index 000000000..06258ad40 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/model/__init__.py @@ -0,0 +1,13 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 14, 2023 +# \ No newline at end of file diff --git a/plugins/events_history_plugin/src/events_history/model/event.py b/plugins/events_history_plugin/src/events_history/model/event.py new file mode 100644 index 000000000..afc91575c --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/model/event.py @@ -0,0 +1,17 @@ +class Event: + + def __init__(self, id, name, event_type, severity, timestamp, category, description, type, + object_path, write_to_syslog=False, object_name="N/A", counter="N/A"): + self.id = id + self.name = name + self.type = type + self.event_type = event_type + self.severity = severity + self.timestamp = timestamp + self.counter = counter + self.category = category + self.object_name = object_name + self.object_path = object_path + self.write_to_syslog = write_to_syslog + self.description = description + diff --git a/plugins/events_history_plugin/src/events_history/requirements.txt b/plugins/events_history_plugin/src/events_history/requirements.txt new file mode 100644 index 000000000..4ff79b86d --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/requirements.txt @@ -0,0 +1,8 @@ +requests +configparser +flask +flask_restful +twisted +service_identity +jsonschema + diff --git a/plugins/events_history_plugin/src/events_history/resources/__init__.py b/plugins/events_history_plugin/src/events_history/resources/__init__.py new file mode 100644 index 000000000..06258ad40 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/resources/__init__.py @@ -0,0 +1,13 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 14, 2023 +# \ No newline at end of file diff --git a/plugins/events_history_plugin/src/events_history/resources/base_resource.py b/plugins/events_history_plugin/src/events_history/resources/base_resource.py new file mode 100644 index 000000000..e42c47db3 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/resources/base_resource.py @@ -0,0 +1,28 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 13, 2023 +# + + +class BaseResource: + ATTRS = {} + + def __init__(self, obj): + if obj: + self.model_to_obj(obj) + + def model_to_obj(self, obj): + obj_dict = obj.__dict__ + for resource_attr, model_attr in self.ATTRS.items(): + value = obj_dict.get(model_attr) + setattr(self, resource_attr, value) + diff --git a/plugins/events_history_plugin/src/events_history/resources/event_resource.py b/plugins/events_history_plugin/src/events_history/resources/event_resource.py new file mode 100644 index 000000000..60bcee1d4 --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/resources/event_resource.py @@ -0,0 +1,23 @@ +from resources.base_resource import BaseResource +from constants.events_constants import EventsConstants + + +class EventResource(BaseResource): + BaseResource.ATTRS.update({ + EventsConstants.ID:EventsConstants.ID, + EventsConstants.NAME: EventsConstants.NAME, + EventsConstants.TYPE: EventsConstants.TYPE, + EventsConstants.EVENT_TYPE: EventsConstants.EVENT_TYPE, + EventsConstants.SEVERITY: EventsConstants.SEVERITY, + EventsConstants.TIMESTAMP: EventsConstants.TIMESTAMP, + EventsConstants.COUNTER: EventsConstants.COUNTER, + EventsConstants.CATEGORY: EventsConstants.CATEGORY, + EventsConstants.OBJECT_NAME: EventsConstants.OBJECT_NAME, + EventsConstants.OBJECT_PATH: EventsConstants.OBJECT_PATH, + EventsConstants.WRITE_TO_SYSLOG: EventsConstants.WRITE_TO_SYSLOG, + EventsConstants.DESCRIPTION: EventsConstants.DESCRIPTION + }) + + def __init__(self, obj): + super(EventResource, self).__init__(obj) + # TODO convert timestamp to UTC diff --git a/plugins/events_history_plugin/src/events_history/schemas/set_conf.schema.json b/plugins/events_history_plugin/src/events_history/schemas/set_conf.schema.json new file mode 100644 index 000000000..01fe0310d --- /dev/null +++ b/plugins/events_history_plugin/src/events_history/schemas/set_conf.schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Update events history configurations", + "type": "object", + "properties": { + "logs-config": { + "type": "object", + "properties": { + "logs_file_name": { + "type": "string" + }, + "logs_level": { + "type": "string", + "enum": [ + "FATAL", + "ERROR", + "WARNING", + "INFO", + "DEBUG", + "NOTSET" + ] + }, + "log_file_max_size": { + "type": "integer", + "minimum": 1, + "err_message": "[logs-config - log_file_max_size] attribute should be an integer greater than 0" + }, + "log_file_backup_count": { + "type": "integer", + "minimum": 1, + "err_message": "[logs-config - log_file_max_size] attribute should be an integer greater than 0" + } + } + } + }, + "additionalProperties": false +} diff --git a/plugins/events_history_plugin/version b/plugins/events_history_plugin/version new file mode 100644 index 000000000..82694261f --- /dev/null +++ b/plugins/events_history_plugin/version @@ -0,0 +1 @@ +1.0.0-1 diff --git a/utils/filewatch/__init__.py b/utils/filewatch/__init__.py new file mode 100644 index 000000000..888b84d75 --- /dev/null +++ b/utils/filewatch/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 15, 2023 +# + + diff --git a/utils/filewatch/abstract_file_watcher.py b/utils/filewatch/abstract_file_watcher.py new file mode 100644 index 000000000..f1a96949b --- /dev/null +++ b/utils/filewatch/abstract_file_watcher.py @@ -0,0 +1,188 @@ +# +# Copyright © 2013-2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# This software product is a proprietary product of Nvidia Corporation and its affiliates +# (the "Company") and all right, title, and interest in and to the software +# product, including all associated intellectual property rights, are and +# shall remain exclusively with the Company. +# +# This software product is governed by the End User License Agreement +# provided with the software product. +# @author: Abeer Moghrabi +# @date: Aug 15, 2023 +# + +import os +from threading import Lock +import pyinotify +from utils.singleton import Singleton +from utils.logger import Logger, LOG_LEVELS + + +class AbstractFileWatcher(Singleton): + """ + Base class for File Watching , + This Class should be extended and not used directly! + Each derived class should can watch one or more files. + + Example of subclass can be found in infra.filewatch in file gvWatcher.py + Note: this class was copied from UFM project + """ + + def __init__(self, file_list, events, read_freq=300): + """ + :param file_list: list, list of paths to files the watcher will listen on + :param events: list, list of events on the files that the watcher listens on taken from Events class (below this class) + :param read_freq: int, is the interval between every time we request for the files that have been modified + @ivar _watchedFileList: the list of files we want to watch + @ivar _watchedDirs: the list of directories we want to watch + @ivar _listener_list: the list of listeners to inform on file events + @ivar _listener_id_counter: the id that the next listener will get + @ivar _listener_data_lock: the lock being locked when adding/removing listeners + """ + self._watchedFileList = set([os.path.abspath(f) for f in file_list]) + self._watchedDirs = set([os.path.dirname(f) for f in self._watchedFileList]) + self._listener_dict = {} + self._listener_id_counter = 0 + self._listener_data_lock = Lock() + self._startWatching(events, read_freq=read_freq) + + def _startWatching(self, events, read_freq): + """ + Start file watching by the watch manager + :param events: list of events on the files that the watcher listens on taken from Events class (below this class) + """ + # initialize bitmask with all zeros (no events). + mask = 0x00000000 + # Iterate through the list of events and add each event to the bitmask. + # The '|' operator is used to perform a bitwise OR operation to combine event masks. + for event in events: + mask = mask | event + try : + wm = pyinotify.WatchManager() + eh = EventHandler(self) + self._notifier = pyinotify.ThreadedNotifier(wm, eh, read_freq=read_freq) + + for path in self._watchedDirs: + wm.add_watch(path, mask) + + self._notifier.daemon = True + self._notifier.start() + + except Exception as e: + Logger.log_message("FileWatcher: Was NOT able to start, %s is NOT watched, the reason is : %s" % (self._watchedFileList, e), LOG_LEVELS.ERROR) + return + Logger.log_message("FileWatcher: %s is being watched " % self._watchedFileList, LOG_LEVELS.INFO) + + def __del__(self): + """ + Stop file watching by the watch manager + """ + if hasattr(self, '_notifier'): + self._notifier.stop() + Logger.log_message("FileWatcher: %s is NO LONGER being watched" % self._watchedFileList, LOG_LEVELS.INFO) + + def addListener(self, listener): + """ + adds listener to the file watcher + :param listener: the listener to add to the listeners list + :return: the listener id + """ + self._listener_data_lock.acquire() + id = self._listener_id_counter + self._listener_dict[id] = listener + self._listener_id_counter += 1 + self._listener_data_lock.release() + return id + + def removeListener(self, id): + """ + removes listener from the file watcher by id given + :param id: the id of the listener to remove + :return: True if the listener was removed and false if no listener with the given id was found + """ + try : + self._listener_dict.pop(id) + + except : + return False + + return True + + def handleEvent(self, event): + if event.pathname in self._watchedFileList: + #print "==> ", event.maskname, ": ", event.pathname + # Invokes process_MASKNAME + meth = getattr(self, 'process_' + event.maskname, None) + if meth is not None: + meth(event) + + """ + The methods below should be implemented by subclasses, each method deals with different event type (on files). + for example, in this methods is where you will notify the listeners on the files of the event that happened. + ** If you are watching more then one file, you can use the event object to get the file that caused the event. ** + """ + + def process_IN_ACCESS(self, event): + pass + + def process_IN_MODIFY(self, event): + pass + + def process_IN_ATTRIB(self, event): + pass + + def process_IN_CLOSE_WRITE(self, event): + pass + + def process_IN_CLOSE_NOWRITE(self, event): + pass + + def process_IN_OPEN(self, event): + pass + + def process_IN_MOVED_FROM(self, event): + pass + + def process_IN_MOVED_TO(self, event): + pass + + def process_IN_CREATE(self, event): + pass + + def process_IN_DELETE(self, event): + pass + + def process_IN_DELETE_SELF(self, event): + pass + + def process_IN_MOVE_SELF(self, event): + pass + + def process_default(self, event): + pass + + +class EventHandler(pyinotify.ProcessEvent): + """ + This class handle specific actions when filesystem events occur. + It's responsible for processing events such as file modifications, access, attribute changes, and more + !!! This class should not be touched !!! + """ + + def __init__(self, fileWatcher): + """ + @ivar _fw : + The instance of the fileWatcher + """ + self._fw = fileWatcher + + def process_default(self, event): + """ + when an event type that doesn't have a specific method override is encountered. In other words, it serves + as a catch-all method that gets called for any event type that doesn't have its own dedicated method defined + in your custom event handler class. + :param event: the data of the event that occurred(file name, mask name,..). + :return: + """ + self._fw.handleEvent(event)