import base64
import json
import re
import threading
import traceback
from copy import deepcopy
from datetime import datetime

import requests

from scripts.config import app_configuration as config
from scripts.config.app_constants import *
from scripts.config.db_connection_obj import ConnectionObj
from scripts.core.handler.pipeline.CreateConfig import FlowConfigBuilder

from templates.pipeline_templates.pipeline_template import CLASSNAME
from scripts.core.handler.rule_configurations.alarm_configuration_handler import AlarmConfigurationHandler
from scripts.core.handler.rule_configurations.rule_configuration_handler import RuleConfigurationHandler
from scripts.utils.data_security_util import DataSecurity
from scripts.utils.delete_util import DeleteUtil
from scripts.utils.get_new_id import GetNewId
from scripts.logging.logger import logger
from scripts.utils.mongo_utility import MongoConnect
from scripts.utils.common_utils import CommonUtils


def background(f):
    """
    a threading decorator
    use @background above the function you want to run in the background
    """

    def bg_f(*a, **kw):
        threading.Thread(target=f, args=a, kwargs=kw).start()

    return bg_f


class PipelineHandler:
    def __init__(self):
        try:
            if ConnectionObj.mongo_connection_obj is None:
                ConnectionObj.mongo_connection_obj = MongoConnect()
            self.metadata = DBMapping.mongo_db_name
            self.flow_constant = MongoMetadata.flow_constant
            self.data_security_obj = DataSecurity()
            self.users = MongoMetadata.user
            self.custom_node = MongoMetadata.custom_node
            self.job_list = MongoMetadata.job_list
            self.pipe_category = MongoMetadata.pipeline_type
            self.job_list_header_content = Pipeline.job_list_header_content
            self.del_obj = DeleteUtil()
            self.message = CLASSNAME(config, logger)
            self.pipeline_info_collection = MongoMetadata.pipeline_info_collection
            self.pipeline_instance_collection = MongoMetadata.pipeline_instance_info_collection
            self.component_category_collection = MongoMetadata.component_category_collection
            self.associate_pp_device_collection = MongoMetadata.associate_pp_device
        except Exception as e:
            logger.exception(str(e))
            traceback.print_exc()

    def create_record(self, input_data):
        """
        Definition for create and update the flow records
        :param input_data:
        :return:
        """
        final_json = {"status": "failed",
                      "message": "failed to save the changes", "pipeline_id": ""}
        try:
            logger.info("Creating Pipeline")
            logger.debug("Input data for pipeline: {}".format(input_data))

            # Fetch user details
            user_json = self.data_security_obj.get_user_id({})
            user_id = user_json["user_id"]

            # validate mandatory fields
            mandatory_fields = [
                'action', Pipeline.key_pipeline_id, Pipeline.key_pipeline_id]
            for each_key in mandatory_fields:
                if each_key not in input_data:
                    logger.error(
                        "'{}' not found in the input".format(each_key))
                    raise Exception("'{}' not found".format(each_key))

            if input_data["action"] == "create":
                try:
                    logger.info("Creating new pipeline")
                    if self.check_if_pipeline_exists(input_data[Pipeline.key_pipeline_name]):
                        logger.error(
                            "A pipeline with same name already exists")
                        raise Exception(
                            "A pipeline with same name already exists")

                    main_insert_json = deepcopy(Pipeline.default_json)
                    try:
                        for i in main_insert_json:
                            if i in input_data:
                                main_insert_json[i] = input_data[i]
                    except Exception as e:
                        logger.error(
                            "Error occurred while creating pipeline : {}".format(str(e)))
                        raise Exception("Failed to create pipeline")

                    pipeline_id = self.create_new_pipeline_id()
                    main_insert_json[Pipeline.key_pipeline_id] = pipeline_id
                    main_insert_json['meta'] = {"created_by": user_id,
                                                "created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                                                "last_edited_by": user_id,
                                                "last_edited_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                                                }
                    main_insert_json["version"] = 1
                    try:
                        pipeline_instance_id, version = self.save_pipeline_instance(
                            main_insert_json)
                        main_insert_json[Pipeline.key_pipeline_instance_id] = pipeline_instance_id
                        ConnectionObj.mongo_connection_obj.database_insertion(self.metadata,
                                                                              self.pipeline_info_collection,
                                                                              main_insert_json)
                        final_json["status"] = "success"
                        final_json["message"] = "Pipeline Created Successfully"
                        if "pipeline_category" in input_data:
                            final_json['pipeline_category'] = input_data['pipeline_category']
                        final_json['version'] = {Pipeline.key_pipeline_instance_id: pipeline_instance_id,
                                                 "pipeline_version": "v" + str(version),
                                                 "build_status": "pending",
                                                 "message": ""}
                        final_json[Pipeline.key_pipeline_id] = pipeline_id
                        final_json[Pipeline.key_pipeline_instance_id] = pipeline_instance_id
                        final_json["pipeline_version"] = version
                    except Exception as e:
                        logger.error(
                            "Error occurred while inserting the record into db : {}".format(str(e)))
                        raise Exception("Failed to create pipeline")
                except Exception as e:
                    logger.error(str(e))
                    raise Exception(str(e))

            elif input_data["action"] in ["rename", "save"]:
                try:
                    logger.debug("Update pipeline")
                    query = {Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                             Pipeline.key_project_id: input_data[Pipeline.key_project_id]}

                    main_insert_json = ConnectionObj.mongo_connection_obj. \
                        find_one(db_name=self.metadata, collection_name=self.pipeline_info_collection, query=query,
                                 search_json={"_id": 0})

                    if not len(list(main_insert_json.keys())):
                        logger.error("Pipeline not found to update")
                        raise Exception("Pipeline not found to update")

                    main_insert_json["meta"]["last_edited_by"] = user_id
                    main_insert_json["meta"]['last_edited_at'] = datetime.now().strftime(
                        "%Y-%m-%d %H:%M:%S")

                    if input_data["action"] == "rename":
                        main_insert_json["pipeline_name"] = input_data["pipeline_name"]
                        main_insert_json["pipeline_description"] = input_data["pipeline_description"]

                    elif input_data["action"] == "save":
                        main_insert_json["version"] = main_insert_json.get(
                            "version", 1) + 1
                        main_insert_json["counter"] = input_data["flow_data"]["counter"]
                        main_insert_json["flow"] = input_data["flow_data"]["flow"]
                        main_insert_json["pipeline"] = input_data["flow_data"]["pipeline"]
                        pipeline_instance_id = main_insert_json.get(
                            Pipeline.key_pipeline_instance_id, None)
                        # print("Pipeline instance id -->", pipeline_instance_id)
                        pipeline_instance_id, version = \
                            self.save_pipeline_instance(pipeline_json=main_insert_json,
                                                        pipeline_instance_id=pipeline_instance_id)
                        main_insert_json['pipeline_instance_id'] = pipeline_instance_id

                        del main_insert_json['flow']
                        del main_insert_json['pipeline']

                    data_to_update = {"$set": main_insert_json}
                    ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                                                                  collection_name=self.pipeline_info_collection,
                                                                  query=query, set_json=data_to_update)

                    final_json["pipeline_id"] = main_insert_json["pipeline_id"]
                    final_json['pipeline_category'] = input_data.get(
                        'pipeline_category', "")
                    final_json["status"] = "success"
                    final_json["message"] = "Pipeline Updated Successfully"
                except Exception as e:
                    logger.error(str(e))
                    raise Exception(str(e))

            elif "action" in input_data and input_data["action"] in ["start", "stop", "upgrade"]:
                try:
                    logger.debug("Deploy pipeline")

                    query = {Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                             Pipeline.key_project_id: input_data[Pipeline.key_project_id]}

                    main_insert_json = ConnectionObj.mongo_connection_obj. \
                        find_one(db_name=self.metadata, collection_name=self.pipeline_info_collection, query=query,
                                 search_json={"_id": 0})
                    main_insert_json['thread'] = input_data.get('thread', 1)

                    try:
                        build_info = self.check_job_status_and_get_image(job_id=input_data['job_id'],
                                                                         pipeline_id=input_data['pipeline_id'])
                    except Exception as ae:
                        logger.error(
                            "Error occurred while fetching building info for the pipeline : {}".format(str(ae)))
                        raise Exception(
                            "Failed to fetch build info for the pipeline")

                    request_json = {"container_id": main_insert_json["container_id"],
                                    "pipeline_id": main_insert_json["pipeline_id"],
                                    "thread": main_insert_json["thread"],
                                    "data": main_insert_json["pipeline"],
                                    "job_id": input_data['job_id'],
                                    "image": build_info['image'],
                                    "pipeline_internal": config.PIPELINE_INTERNAL_CONFIGURATION}

                    if input_data["action"] == "start":
                        try:
                            logger.info("Launch container")
                            logger.debug(
                                "Request json for launching container : {}".format(request_json))

                            if not main_insert_json["container_id"]:
                                r = requests.post(config.CONTAINER_URL + Pipeline.launch_container,
                                                  json=request_json, verify=False)
                                status_json = json.loads(r.text)
                                logger.debug(
                                    "Status for starting container --> : {}".format(status_json))
                            else:
                                r = requests.post(config.CONTAINER_URL + Pipeline.restart_container,
                                                  json=request_json,
                                                  verify=False)
                                status_code = r.status_code
                                logger.debug(
                                    "Status code for starting container --> {}".format(status_code))

                                if status_code != 200:
                                    raise Exception(
                                        "Failed to deploy pipeline")

                                status_json = json.loads(r.text)
                                logger.debug(
                                    "Status for starting container --> {}".format(status_json))

                            if "status" in status_json and status_json["status"].lower() == "success":
                                main_insert_json["container_id"] = status_json["container_id"]
                                main_insert_json["status"] = "start"
                                final_json["message"] = "Pipeline deployed Successfully"
                                final_json["status"] = "success"
                                if "deployment_config" in status_json:
                                    self.update_job_deployment_config(job_id=input_data['job_id'],
                                                                      deployment_config=status_json[
                                                                          'deployment_config'])
                            else:
                                raise Exception("Failed to deploy pipeline")
                        except Exception as e:
                            logger.error(
                                "Error occurred deploying pipeline : {}".format(str(e)))
                            raise Exception("Failed to deploy pipeline")

                    elif input_data["action"] == "stop":
                        r = requests.post(config.CONTAINER_URL + Pipeline.stop_container,
                                          json=request_json,
                                          verify=False)
                        status_code = r.status_code
                        logger.debug(
                            "Status code for stopping the pipeline --> {}".format(status_code))
                        status_json = json.loads(r.text)
                        logger.debug(
                            "Status for stopping the container --> {}".format(status_json))

                        if status_code != 200:
                            raise Exception("Failed to stop pipeline")

                        if "status" in status_json and status_json["status"] == "success":
                            main_insert_json["status"] = "stop"
                            final_json["message"] = "Pipeline Stopped Successfully"
                            final_json["status"] = "success"
                        else:
                            raise Exception('Failed to stop pipeline')

                    elif input_data["action"] == "upgrade":
                        r = requests.post(config.CONTAINER_URL + Pipeline.stop_container,
                                          json=request_json,
                                          verify=False)
                        status_code = r.status_code
                        logger.debug(
                            "Status code for update the pipeline --> {}".format(status_code))

                        if status_code != 200:
                            raise Exception("Failed to upgrade pipeline")

                        status_json = json.loads(r.text)
                        if "status" in status_json and status_json["status"] == "success":
                            d_r = requests.post(config.CONTAINER_URL + Pipeline.delete_container,
                                                json=request_json,
                                                verify=False)
                            delete_json = json.loads(d_r.text)
                            if "status" in delete_json and delete_json["status"] == "success":
                                l_r = requests.post(config.CONTAINER_URL + Pipeline.launch_container,
                                                    json=request_json,
                                                    verify=False)
                                launch_json = json.loads(l_r.text)
                                if "status" in launch_json and launch_json["status"] == "success":
                                    main_insert_json["status"] = "start"
                                    main_insert_json["container_id"] = launch_json["container_id"]
                                    final_json["message"] = "Pipeline Upgraded Successfully"
                                    final_json["status"] = "success"
                                    if "deployment_config" in status_json:
                                        self.update_job_deployment_config(job_id=input_data['job_id'],
                                                                          deployment_config=status_json[
                                                                              'deployment_config'])
                        else:
                            raise Exception('Failed to upgrate pipeline')

                    main_insert_json["meta"]["last_edited_by"] = user_id
                    main_insert_json["meta"]['last_edited_at'] = datetime.now().strftime(
                        "%Y-%m-%d %H:%M:%S")
                    new_values = {"$set": main_insert_json}
                    ConnectionObj.mongo_connection_obj.update_one(self.metadata, self.pipeline_info_collection, query,
                                                                  new_values)
                    final_json[Pipeline.key_pipeline_id] = input_data[Pipeline.key_pipeline_id]
                except Exception as e:
                    logger.exception(str(e))

        except Exception as e:
            logger.error(str(e))
        return final_json

    def save_pipeline_instance(self, pipeline_json, pipeline_instance_id=None):
        """
        This method is to save the pipeline instance
        """
        try:
            data_to_mongo = dict()
            logger.info("Save pipeline instance for the pipeline")
            data_to_mongo[Pipeline.key_pipeline_id] = pipeline_json[Pipeline.key_pipeline_id]
            data_to_mongo["flow"] = pipeline_json["flow"]
            data_to_mongo["pipeline"] = pipeline_json["pipeline"]
            data_to_mongo['version'] = pipeline_json['version']
            data_to_mongo['counter'] = pipeline_json['counter']
            data_to_mongo['meta'] = pipeline_json['meta']
            data_to_mongo[Pipeline.key_pipeline_category] = pipeline_json[Pipeline.key_pipeline_category]

            if pipeline_instance_id is None:
                pipeline_instance_id = self.create_new_pipeline_instance_id()
                data_to_mongo[Pipeline.key_pipeline_instance_id] = pipeline_instance_id
                data_to_mongo['pipeline_version'] = self.get_pipeline_version(
                    data_to_mongo[Pipeline.key_pipeline_id])
                ConnectionObj.mongo_connection_obj.database_insertion(db_name=self.metadata,
                                                                      collection_name=self.pipeline_instance_collection,
                                                                      query_json=data_to_mongo)
                return pipeline_instance_id, data_to_mongo['pipeline_version']
            else:
                query = {Pipeline.key_pipeline_id: pipeline_json[Pipeline.key_pipeline_id],
                         Pipeline.key_pipeline_instance_id: pipeline_instance_id}
                pipeline_instance_info = ConnectionObj.mongo_connection_obj. \
                    find_with_condition(json_data=query, database_name=self.metadata,
                                        collection_name=self.pipeline_instance_collection)
                if pipeline_instance_info.count():
                    pipeline_instance_info = pipeline_instance_info[0]
                    pipeline_instance_info['flow'] = pipeline_json['flow']
                    pipeline_instance_info['pipeline'] = pipeline_json['pipeline']
                    pipeline_instance_info['version'] = pipeline_json['version']
                    pipeline_instance_info['counter'] = pipeline_json['counter']
                    pipeline_instance_info['meta']['last_edited_by'] = datetime.now().strftime(
                        "%Y-%m-%d %H:%M:%S")

                    if "_id" in pipeline_instance_info:
                        del pipeline_instance_info['_id']

                    ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                                                                  collection_name=self.pipeline_instance_collection,
                                                                  query=query,
                                                                  set_json={"$set": pipeline_instance_info})

                    # if "build_status" not in pipeline_instance_info or \
                    #         pipeline_instance_info['build_status'].lower() == 'failed':
                    #     pipeline_instance_info.update(data_to_mongo)
                    #     ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                    #                                                   collection_name=self.pipeline_instance_collection,
                    #                                                   query=query,
                    #                                                   set_json={"$set": pipeline_instance_info})
                    # elif "build_status" in pipeline_instance_info and \
                    #         pipeline_instance_info['build_status'].lower() == 'success':
                    #     print("Found new version --->")
                    #     pipeline_instance_id = self.create_new_pipeline_instance_id()
                    #     print(100 * "*")
                    #     print(pipeline_instance_id)
                    #     data_to_mongo[FlowURL.key_pipeline_instance_id] = pipeline_instance_id
                    #     data_to_mongo['pipeline_version'] = self.get_pipeline_version(
                    #         data_to_mongo[FlowURL.key_pipeline_id])
                    #     ConnectionObj.mongo_connection_obj. \
                    #         database_insertion(db_name=self.metadata,
                    #         collection_name=self.pipeline_instance_collection,
                    #                            query_json=data_to_mongo)

                return pipeline_instance_id, pipeline_instance_info['pipeline_version']
        except Exception as e:
            logger.error(
                "Error occurred while saving the pipeline instance : {}".format(str(e)))
            raise Exception("Failed to save the pipeline instance")

    def get_record_list(self, input_data):
        """
        Definition for getting all the records
        :return:
        """
        final_json = {"status": "failed",
                      "message": "Failed to list pipelines", "data": {}}
        logger.info("Fetch pipelines list")
        logger.debug(
            "Input data for fetching pipeline list : {}".format(input_data))
        try:
            logger.debug("Fetching pipelines list")
            final_json["data"]["headerContent"] = Pipeline.header_content
            final_json["data"]["bodyContent"] = []
            user_mapping = self.fetch_user_mapping()
            status_json = self.container_status()

            query = {
                "isdeleted": "false",
                Pipeline.key_project_id: input_data[Pipeline.key_project_id]
            }
            all_data = ConnectionObj.mongo_connection_obj. \
                find_with_condition(json_data=query, database_name=self.metadata,
                                    collection_name=self.pipeline_info_collection)
            count = 1

            if all_data:
                for each_data in all_data:
                    try:
                        if "_id" in each_data:
                            del each_data['_id']

                        each_data["meta"]['created_by'] = user_mapping.get(
                            each_data["meta"]['created_by'], "")
                        each_data["meta"]["last_edited_by"] = user_mapping.get(
                            each_data["meta"]['last_edited_by'], "")

                        running = False
                        status = each_data["status"]

                        try:
                            if each_data['pipeline_id'] in status_json:
                                running = status_json[each_data["pipeline_id"]]["Running"]
                                status = status_json[each_data["pipeline_id"]]["Status"].title(
                                )
                            else:
                                status = 'Pending'
                        except Exception as e:
                            logger.error(
                                "Failed to find pipeline status : {}".format(str(e)))

                        latest_build_job_id = ""
                        query_pipeline_instance = {
                            "pipeline_instance_id": each_data.get('pipeline_instance_id')
                        }
                        pipeline_instance_info = ConnectionObj.mongo_connection_obj. \
                            find_with_condition(json_data=query_pipeline_instance, database_name=self.metadata,
                                                collection_name=self.pipeline_instance_collection)
                        if pipeline_instance_info.count():
                            pipeline_instance_info = pipeline_instance_info[0]
                            latest_build_job_id = pipeline_instance_info.get(
                                'latest_build_job_id', '')
                        try:
                            environment = each_data["pipeline_build_configurations"]["environment"]
                        except Exception as e:
                            logger.error(str(e))
                            environment = ""

                        temp_json = {"sno": str(count),
                                     Pipeline.key_pipeline_name: each_data[Pipeline.key_pipeline_name],
                                     Pipeline.key_pipeline_id: each_data[Pipeline.key_pipeline_id],
                                     Pipeline.key_pipeline_desc: each_data[Pipeline.key_pipeline_desc],
                                     "created_by": each_data["meta"]["created_by"],
                                     "created_at": each_data["meta"]["created_at"],
                                     "last_edited_by": each_data["meta"]['created_by'],
                                     "last_edited_at": each_data["meta"]['last_edited_at'],
                                     "status": status,
                                     "running": running,
                                     "environment": environment,
                                     "job_id": latest_build_job_id,
                                     "versions": self.get_pipe_version_details(each_data[Pipeline.key_pipeline_id])
                                     }

                        # Label mapping for device type
                        try:
                            if each_data['target_device_type'] == 'edgeDevices':
                                temp_json['target_device_type'] = "Edge Device"
                            elif each_data['target_device_type'] == 'edgeAppliances':
                                temp_json['target_device_type'] = 'Edge Appliance'
                            else:
                                temp_json['target_device_type'] = each_data['target_device_type']
                        except KeyError:
                            temp_json['target_device_type'] = ""

                        try:
                            if each_data["pipeline_type"].lower() == 'stream':
                                temp_json['pipeline_type'] = "Stream"
                            elif each_data["pipeline_type"].lower() == 'batch':
                                temp_json['pipeline_type'] = "Batch"
                        except KeyError:
                            temp_json['pipeline_type'] = ""

                        temp_json['pipeline_category'] = each_data.get(
                            'pipeline_category', '')

                        final_json["data"]["bodyContent"].append(temp_json)
                        count += 1

                        # Added to sort the pipelines
                        final_json['data']['bodyContent'] = sorted(final_json['data']['bodyContent'],
                                                                   key=lambda k: datetime.strptime(k['last_edited_at'],
                                                                                                   "%Y-%m-%d %H:%M:%S")
                                                                   if "last_edited_at" in k else datetime.strptime(
                                                                       k['created_at'], "%Y-%m-%d %H:%M:%S"),
                                                                   reverse=True)
                        if len(final_json['data']['bodyContent']):
                            for index, value in enumerate(final_json['data']['bodyContent']):
                                value['sno'] = index + 1

                    except Exception as e:
                        logger.exception(str(e))
                        continue
            final_json["status"] = "success"
            final_json["message"] = "Successfully fetched pipeline list"
        except Exception as e:
            logger.exception(str(e))
            traceback.print_exc()
        return final_json

    @staticmethod
    def container_status():
        """
        Get container status of pipeline
        """
        status_json = dict()
        try:
            if Pipeline.is_pipeline_deployment_on_premise:
                try:
                    r = requests.post(config.CONTAINER_URL + Pipeline.get_status,
                                      data=base64.b64encode(json.dumps({}).encode('utf-8')), verify=False)
                    status_json = json.loads(r.text)
                except Exception as e:
                    logger.error(
                        "Failed to fetch pipeline container status : {}".format(str(e)))
        except Exception as e:
            logger.error(
                "Error occurred while fetching container status : {}".format(str(e)))
        return status_json

    def get_record(self, input_data):
        """
        This method is to retrieve pipeline info
        """
        final_json = {"status": "failed",
                      "message": "Failed to fetch pipeline data"}
        logger.info("Fetch pipeline info")
        try:
            logger.debug("Fetching pipeline info")
            logger.debug(
                "Input data for fetching pipeline info :{}".format(input_data))

            if Pipeline.key_pipeline_id not in input_data:
                logger.error("'{}' not found in th input".format(
                    Pipeline.key_pipeline_id))
                raise Exception("'{}' not found in th input".format(
                    Pipeline.key_pipeline_id))

            pipeline_id = input_data[Pipeline.key_pipeline_id]
            pipeline_instance_id = input_data['version'][Pipeline.key_pipeline_instance_id]
            pipeline_version = input_data['version']['pipeline_version']

            flow_record = ConnectionObj.mongo_connection_obj. \
                find_one(db_name=self.metadata, collection_name=self.pipeline_info_collection,
                         query={Pipeline.key_pipeline_id: pipeline_id,
                                Pipeline.key_project_id: input_data[Pipeline.key_project_id]}, search_json={"_id": 0})

            if flow_record:
                query_pipe_instance_details = {
                    Pipeline.key_pipeline_instance_id: pipeline_instance_id,
                    Pipeline.key_pipeline_id: pipeline_id
                }
                pipeline_instance_details = ConnectionObj.mongo_connection_obj. \
                    find_one(db_name=self.metadata, collection_name=self.pipeline_instance_collection,
                             query=query_pipe_instance_details, search_json={"_id": 0})
                if pipeline_instance_details:
                    final_json.update({"data": {}})
                    final_json["data"]["flow_data"] = {"counter": pipeline_instance_details["counter"],
                                                       "flow": pipeline_instance_details["flow"],
                                                       "pipeline": pipeline_instance_details["pipeline"]}

                    final_json["data"]["pipeline_build_configurations"] = pipeline_instance_details.get(
                        "pipeline_build_configurations",
                        {})
                    if "latest_build_job_id" in pipeline_instance_details:
                        final_json['data']['job_id'] = pipeline_instance_details['latest_build_job_id']

                    final_json["status"] = "success"
                    final_json["message"] = "Successfully fetched pipeline info"
        except Exception as e:
            traceback.print_exc()
            logger.error(
                "Error occurred while fetching pipeline info : {}".format(str(e)))
        return final_json

    def load_nodes(self, input_data):
        """
        Definition for fetching the nodes
        :return:
        """
        final_json = {"status": "failed", "data": []}
        try:
            logger.info("Load components")
            logger.debug(
                "Input data for loading components : {}".format(input_data))
            fetch_data = ConnectionObj.mongo_connection_obj.fetch_all(
                self.metadata, self.flow_constant)
            project_id = input_data[Pipeline.key_project_id]
            category_json = self.fetch_component_category_info(project_id)
            pipeline_category = input_data.get(
                Pipeline.key_pipeline_category, None)
            if pipeline_category is None:
                raise Exception(f'Pipeline category not found in input!')
            if input_data[Pipeline.key_pipeline_category].lower() == Pipeline.category_normal or \
                    input_data[Pipeline.key_pipeline_category].lower() == Pipeline.category_acquisition:
                category_list = []
                custom_node_list = []
                # alarm_list = self.get_alarms()
                alarm_list = []
                rule_list = self.get_rules()
                node_info = dict()
                query = {Pipeline.key_node_type: Pipeline.component_type_code,
                         Pipeline.key_project_id: project_id, "isdeleted": "false"}
                logger.debug(f"Query for custom code nodes: {query}")
                node_data = ConnectionObj.mongo_connection_obj.search_record_by_query2(db_name=self.metadata,
                                                                                       collection_name=self.custom_node,
                                                                                       query_json=query,
                                                                                       search_option={"_id": 0,
                                                                                                      "code": 0}
                                                                                       )
                logger.debug(f"Single container node data: {node_data}")
                for each_node in node_data:
                    if Pipeline.key_node_type in each_node and each_node[Pipeline.key_node_type].lower() == \
                            Pipeline.component_type_code:
                        if Pipeline.key_pipeline_category_id in each_node and \
                                each_node[Pipeline.key_pipeline_category_id] != "":
                            if each_node[Pipeline.key_pipeline_category_id] not in node_info:
                                node_info[each_node[Pipeline.key_pipeline_category_id]] = [
                                    each_node]
                            else:
                                node_info[each_node[Pipeline.key_pipeline_category_id]].append(
                                    each_node)
                        else:
                            custom_node_list.append(each_node)

                for each_cat in node_info:
                    if each_cat in category_json:
                        category_list.append(
                            {'folder_name': category_json[each_cat], "node_section_name": category_json[each_cat],
                             "node_groups": node_info[each_cat], "section_type": "custom_comp"})
                if fetch_data:
                    for each_record in fetch_data:
                        try:
                            del each_record["_id"]
                            data = each_record["data"]
                            for each_index in range(len(data)):

                                if data[each_index]["folder_name"] == "Custom":
                                    data[each_index]["node_groups"] = custom_node_list

                                elif data[each_index]["folder_name"] == "Alarm_Rules":
                                    data[each_index]["node_groups"][0]["node_form_data"]["headerContent"][0]["data"][1][
                                        "options"] = alarm_list
                                    data[each_index]["node_groups"][1]["node_form_data"]["headerContent"][0]["data"][1][
                                        "options"] = rule_list

                            final_json["data"] = data + category_list
                            break
                        except Exception as e:
                            logger.exception(str(e))
                            continue
                    final_json["status"] = "success"
            elif input_data[Pipeline.key_pipeline_category].lower() == Pipeline.category_ai:
                node_info = dict()
                category_list = []
                query = {Pipeline.key_node_type: Pipeline.component_type_docker,
                         Pipeline.key_project_id: project_id, "isdeleted": "false"}
                logger.debug(f"Query for docker image nodes: {query}")
                node_data = ConnectionObj.mongo_connection_obj. \
                    search_record_by_query2(db_name=self.metadata, collection_name=self.custom_node,
                                            query_json=query,
                                            search_option={"_id": 0, "code": 0})
                logger.debug(f"Multi container node data: {node_data}")
                for each_node in node_data:
                    if Pipeline.key_pipeline_category_id in \
                            each_node and each_node[Pipeline.key_pipeline_category_id] != "":
                        if each_node[Pipeline.key_pipeline_category_id] not in node_info:
                            node_info[each_node[Pipeline.key_pipeline_category_id]] = [
                                each_node]
                        else:
                            node_info[each_node[Pipeline.key_pipeline_category_id]].append(
                                each_node)

                for each_cat in node_info:
                    if each_cat in category_json:
                        category_list.append(
                            {'folder_name': category_json[each_cat],
                             "node_section_name": category_json[each_cat],
                             "node_groups": node_info[each_cat],
                             "section_type": "custom_comp"})
                final_json["data"] = category_list
                final_json['status'] = 'success'
                final_json['message'] = 'Successfully loaded components'
            else:
                raise Exception(
                    f"Unknown pipeline category {pipeline_category} requested")
        except Exception as e:
            logger.error("Failed to load components : {}".format(str(e)))
        return final_json

    def delete_pipeline(self, input_data, request):
        """
        This method is to delete the pipeline
        """
        final_json = {"status": "failed", "message": "failed to delete"}
        try:
            logger.debug("inside the delete flow config")
            logger.debug("Input json for deleting pipeline : {}".format(
                json.dumps(input_data)))
            query = {Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                     Pipeline.key_project_id: input_data[Pipeline.key_project_id]}
            main_insert_json = ConnectionObj.mongo_connection_obj.find_one(self.metadata, self.pipeline_info_collection,
                                                                           query, {"_id": 0})
            user_id = request.cookies.get("user_id")
            if main_insert_json["container_id"]:
                request_json = {"container_id": main_insert_json["container_id"],
                                FlowURL.key_pipeline_id: main_insert_json[FlowURL.key_pipeline_id],
                                "thread": main_insert_json["thread"],
                                "data": main_insert_json["pipeline"]}
                r = requests.post(config.CONTAINER_URL + FlowURL.delete_container,
                                  json=request_json,
                                  verify=False)
                status_json = json.loads(r.text)
                logger.debug(status_json)
                if "status" in status_json and status_json["status"] == "success":
                    return_json = self.del_obj.delete_record(service_name=self.pipeline_info_collection,
                                                             query={FlowURL.key_pipeline_id: input_data[
                                                                 FlowURL.key_pipeline_id],
                                                                    Pipeline.key_project_id: input_data[
                                                                        Pipeline.key_project_id]})

                    if "status" in return_json and return_json["status"] == "success":
                        final_json["status"] = "success"
                        final_json["message"] = "Record Deleted Successfully"

                    else:
                        final_json['status'] = return_json['status']
                        final_json['message'] = return_json['message']
            else:
                return_json = self.del_obj.delete_record(service_name=self.pipeline_info_collection,
                                                         query={FlowURL.key_pipeline_id: input_data[
                                                             FlowURL.key_pipeline_id],
                                                                Pipeline.key_project_id: input_data[
                                                                    Pipeline.key_project_id]})
                if "status" in return_json and return_json["status"] == "success":
                    final_json["status"] = "success"
                    final_json["message"] = "Record Deleted Successfully"
        except Exception as e:
            logger.error(
                "Error occurred while deleting pipeline : {}".format(str(e)))
        return final_json

    def create_nodes(self, input_data):
        """
        Definition for creating the nodes
        :return:
        """
        final_json = {"status": "failed"}
        try:
            logger.debug("Creating component")
            user_json = self.data_security_obj.get_user_id({})

            if "user_id" not in user_json:
                final_json["message"] = " User not found"
                return final_json

            if "action" in input_data and input_data["action"] == "update":
                node_id = input_data['custom_node_id']
                node_name = input_data['node_name']

                if self.check_if_node_name_already_exists(node_name=node_name, node_id=node_id):
                    return {"status": "failed",
                            "message": "Component '{}' already exists".format(input_data["node_name"])}

                if not self.validate_name(node_name):
                    return {"status": "failed",
                            "message": "Invalid component name '{}'. "
                                       "Component name cannot contain "
                                       "spaces, numbers and special characters "
                                       "except '_' ".format(input_data["node_name"])}

                job_id = "job_" + GetNewId.get_next_id("job")
                job_details = {"job_id": job_id,
                               "status": "Pending",
                               Pipeline.key_project_id: input_data[Pipeline.key_project_id],
                               "job_name": "Update component : '{}' ".format(input_data["node_name"]),
                               # "job_name": input_data["node_name"],
                               "custom_node_id": input_data["custom_node_id"],
                               "start_time": "",
                               "end_time": "",
                               "owner": user_json["user_id"],
                               "job_type": "node_creation",
                               "action": "edit",
                               "remark": ""}
                if Pipeline.key_pipeline_category_id in input_data:
                    cat_info = ConnectionObj.mongo_connection_obj. \
                        find_one(db_name=self.metadata, collection_name=self.pipe_category,
                                 query={
                                     Pipeline.key_pipeline_category_id: input_data[
                                         Pipeline.key_pipeline_category_id]})
                    input_data[Pipeline.key_pipeline_category_name] = cat_info.get(
                        Pipeline.key_pipeline_category_name,
                        '')
                else:
                    input_data[Pipeline.key_pipeline_category_id] = ""
                    input_data[Pipeline.key_pipeline_category_name] = ""

                meta_info = self.get_metadata_for_category()
                input_data["last_edited_by"] = meta_info["last_updated_by_id"]
                input_data["last_edited_at"] = meta_info["last_updated_at"]
                input_data["last_edited_user"] = meta_info["last_updated_by_name"]
                ConnectionObj.mongo_connection_obj.database_insertion(db_name=self.metadata,
                                                                      collection_name=self.job_list,
                                                                      query_json=job_details)
                ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                                                              collection_name=self.custom_node,
                                                              query={
                                                                  "node_unique_id": input_data["node_unique_id"]},
                                                              set_json={"$set": input_data})
                final_json["status"] = "success"
                final_json["message"] = f"Component update initiated successfully. Please check '{job_id}'" \
                                        f" for detailed progress"
                final_json['job_id'] = job_id

            if "action" in input_data and input_data["action"] == "create":

                if not self.validate_name(input_data["node_name"]):
                    return {"status": "failed",
                            "message": "Invalid component name '{}'. "
                                       "Component name cannot contain "
                                       "spaces, numbers and special characters "
                                       "except '_' ".format(input_data["node_name"])}

                if self.check_if_node_name_already_exists(node_name=input_data["node_name"]):
                    return {"status": "failed",
                            "message": "Component '{}' already exists".format(input_data["node_name"])}

                new_node_id = "pipeline_node_" + \
                              GetNewId.get_next_id("pipeline_node")
                job_id = "job_" + GetNewId.get_next_id("job")
                input_data["node_unique_id"] = input_data["node_name"]
                input_data["custom_node_id"] = new_node_id
                input_data["job_id"] = job_id
                input_data["created_by"] = user_json["user_id"]
                input_data["status"] = "inactive"
                if Pipeline.key_pipeline_category_id in input_data:
                    cat_info = ConnectionObj.mongo_connection_obj. \
                        find_one(db_name=self.metadata, collection_name=self.pipe_category,
                                 query={
                                     Pipeline.key_pipeline_category_id: input_data[
                                         Pipeline.key_pipeline_category_id]})
                    input_data[Pipeline.key_pipeline_category_name] = cat_info.get(
                        Pipeline.key_pipeline_category_name,
                        "")
                else:
                    input_data[Pipeline.key_pipeline_category_id] = ""
                    input_data[Pipeline.key_pipeline_category_name] = ""
                meta_info = self.get_metadata_for_category()
                input_data["last_edited_by"] = meta_info["last_updated_by_id"]
                input_data["last_edited_at"] = meta_info["last_updated_at"]
                input_data["last_edited_user"] = meta_info["last_updated_by_name"]
                input_data["isdeleted"] = "false"
                ConnectionObj.mongo_connection_obj.database_insertion(db_name=self.metadata,
                                                                      collection_name=self.custom_node,
                                                                      query_json=input_data)
                # initially job status will be pending
                # start_time and end_time will be updated by engine
                job_details = {"job_id": input_data["job_id"],
                               "status": "Pending",
                               Pipeline.key_project_id: input_data[Pipeline.key_project_id],
                               "job_name": "Add component : '{}' ".format(input_data["node_name"]),
                               # "job_name": input_data["node_name"],
                               "custom_node_id": input_data["custom_node_id"],
                               "start_time": "",
                               "end_time": "",
                               "owner": user_json["user_id"],
                               "job_type": "node_creation",
                               "action": "create",
                               "remark": ""}
                ConnectionObj.mongo_connection_obj.database_insertion(db_name=self.metadata,
                                                                      collection_name=self.job_list,
                                                                      query_json=job_details)
                final_json["status"] = "success"
                final_json["node_unique_id"] = input_data["node_unique_id"]
                final_json[
                    "message"] = f"Component creation initiated successfully. Please check '{job_id}'" \
                                 f" for detailed progress"
                final_json['job_id'] = job_id
                # inserting data to custom component inside flow_model_static_content collection

            self.add_node_to_components(final_json)
        except Exception as e:
            traceback.print_exc()
            logger.exception("exception while creating a node:" + str(e))
        return final_json

    def check_if_node_name_already_exists(self, node_name, node_id=None):
        """
        This method is to check if the component name already exists
        """
        try:
            logger.info("Check if the component name already exists")

            if node_id is not None:
                query_json = {"node_name": node_name, "custom_node_id": {
                    "$ne": node_id}, "isdeleted": "false"}
            else:
                query_json = {"node_name": node_name, "isdeleted": "false"}
            logger.debug(
                "Query used to check node name : {}".format(query_json))
            resp = ConnectionObj.mongo_connection_obj.find_record_count(db_name=self.metadata,
                                                                        collection_name=self.custom_node,
                                                                        query_json=query_json)
            if resp > 0:
                return True
            else:
                return False

        except Exception as e:
            logger.error(
                "Error occurred while checking if the component already exists : {}".format(str(e)))
            raise Exception(
                "Failed to check if the component name already exists")

    def save_custom_node_configurations(self, input_data):
        """
        This method is to save the individual node configurations
        """
        status_json = {"status": "failed",
                       "message": "Failed to save node configuration", "data": {}}
        try:
            logger.info("Save node configurations")
            logger.debug(
                "Input data for saving the node configurations : {}".format(input_data))
            user_json = self.data_security_obj.get_user_id({})
            mandatory_field_validation = [
                Pipeline.key_pipeline_id, Pipeline.key_pipeline_instance_id, 'version']

            for each_filed in mandatory_field_validation:
                if each_filed not in mandatory_field_validation:
                    logger.error(" '{}' not found ")
                    raise Exception("'{}' not found in the input")

            query_pipeline = {
                Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id]}
            pipeline_instance_id = input_data["version"][Pipeline.key_pipeline_instance_id]
            pipeline_version = int(
                input_data['version']['pipeline_version'].split('v')[1])
            query_pipeline_instance = {Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                                       Pipeline.key_pipeline_instance_id: pipeline_instance_id}
            pipe_inst_resp = ConnectionObj.mongo_connection_obj. \
                find_one(db_name=self.metadata, collection_name=self.pipeline_instance_collection,
                         query=query_pipeline_instance)
            if pipe_inst_resp:
                for each_node in pipe_inst_resp['pipeline']['nodes']:

                    if input_data['node_playground_id'] == each_node:
                        pipe_inst_resp['pipeline']['nodes'][each_node]['node_configuration'] = input_data[
                            'node_configuration']
                        if "_id" in pipe_inst_resp:
                            del pipe_inst_resp['_id']
                        data_to_update = {"$set": pipe_inst_resp}
                        ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                                                                      collection_name=self.pipeline_instance_collection,
                                                                      query=query_pipeline_instance,
                                                                      set_json=data_to_update)
                        ConnectionObj.mongo_connection_obj. \
                            update_one(db_name=self.metadata,
                                       collection_name=self.pipeline_instance_collection,
                                       query=query_pipeline,
                                       set_json={
                                           '$set': {'meta.last_edited_at': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                                                    "meta.last_edited_by": user_json.get('user_id', '')}, })

                        status_json['status'] = 'success'
                        status_json['message'] = 'Node configuration saved successfully'
                        break
            else:
                logger.error("Pipeline not found")
                raise Exception("Pipeline not found")
        except Exception as e:
            logger.error("Error occurred while saving the node configuration : {}".format(
                str(e)), exc_info=True)
        return status_json

    def delete_node(self, input_data):
        """
        Definition for deleting the nodes
        :return:
        """
        final_json = {"status": "failed"}
        try:
            user_json = self.data_security_obj.get_user_id({})
            if "user_id" not in user_json:
                final_json["message"] = " User not found"
                return final_json

            query = {"custom_node_id": input_data.get("node_unique_id")}

            component_info = list(ConnectionObj.mongo_connection_obj.
                                  find_with_condition(json_data=query,
                                                      collection_name=self.custom_node,
                                                      database_name=self.metadata))
            if len(component_info):
                component_info = component_info[0]
                component_name = component_info['node_name']
                component_unique_id = component_info['custom_node_id']

                job_id = "job_" + GetNewId.get_next_id("job")
                job_details = {"job_id": job_id,
                               "status": "Pending",
                               Pipeline.key_project_id: input_data[Pipeline.key_project_id],
                               "job_name": "Delete component: '{}' ".format(component_name),
                               "custom_node_id": component_unique_id,
                               "start_time": "",
                               "end_time": "",
                               "owner": user_json["user_id"],
                               "job_type": "node_deletion",
                               "action": "delete",
                               "remark": ""}
                ConnectionObj.mongo_connection_obj.database_insertion(db_name=self.metadata,
                                                                      collection_name=self.job_list,
                                                                      query_json=job_details)
                final_json["status"] = "success"
                final_json["message"] = \
                    "Component delete initiated successfully. Please check '{job_id}' for detailed progress".format(
                        job_id=job_id)

                self.create_component_delete_job(job_details)
            else:
                logger.error("Component not found to delete")
                raise Exception('Component not found to delete')
        except Exception as e:
            logger.exception("exception while deleting a node:" + str(e))
            final_json['message'] = str(e)
        return final_json

    @background
    def create_component_delete_job(self, input_data):
        """
        This method is to add the node to the components
        """
        try:
            logger.info("Delete component")
            if "_id" in input_data:
                del input_data['_id']
            request_url = config.CONTAINER_URL + Pipeline.delete_component
            response = requests.post(url=request_url, json=input_data)
            logger.debug("Status code for triggering the delete component is : {}".format(
                response.status_code))
            if response.status_code == 200:
                resp = response.json()
                logger.debug(
                    "Save node response : {}".format(json.dumps(resp)))
            else:
                logger.error("Failed to initiate delete component ")
        except Exception as e:
            traceback.print_exc()
            logger.error('Error occurred while deleting the component : {}'.format(
                str(e)), exc_info=True)
            raise Exception("Failed to delete the component")

    def get_template(self, input_data):
        """
        Definition for deleting the nodes
        :return:
        """
        final_json = {"status": "success", "data": {}}
        node_id = None
        if "node_name" not in input_data:
            logger.error("Node name not found")
            return {"status": "failed", "message": "Node name not found in the input"}

        if "node_unique_id" in input_data and input_data['node_unique_id'] != "":
            node_id = input_data['node_unique_id']

        if self.check_if_node_name_already_exists(node_name=input_data["node_name"], node_id=node_id):
            return {"status": "failed",
                    "message": "Component '{}' already exists".format(input_data["node_name"])}

        template_path = 'templates/pipeline_templates/pipeline_template.py'
        if "node_component_type" in input_data and input_data['node_component_type'].lower() == 'input':
            template_path = 'templates/pipeline_templates/input_pipeline_template.py'
        try:
            with open(template_path, "r") as f:
                code = f.read()
                code = code.replace("CLASSNAME", input_data["node_name"])
            final_json["data"]["code"] = code
        except Exception as e:
            logger.exception("exception while fetching a template:" + str(e))
        return final_json

    def fetch_nodes(self, input_data):
        """
        Definition for creating the nodes
        :return:
        """
        final_json = {"status": "failed", "data": {}}
        try:
            if "node_unique_id" in input_data:
                if "pipeline_node_" in input_data["node_unique_id"]:
                    node_record = ConnectionObj.mongo_connection_obj.find_one(db_name=self.metadata,
                                                                              collection_name=self.custom_node,
                                                                              query={"custom_node_id": input_data[
                                                                                  "node_unique_id"]},
                                                                              search_json={"_id": 0})
                else:
                    node_record = ConnectionObj.mongo_connection_obj.find_one(db_name=self.metadata,
                                                                              collection_name=self.custom_node,
                                                                              query={"node_unique_id": input_data[
                                                                                  "node_unique_id"]},
                                                                              search_json={"_id": 0})
                final_json["data"] = node_record
                final_json["status"] = "success"
                return final_json
            else:
                return {"status": "failed", "message": "node_unique_id doesn't exists in the input payload"}
        except Exception as e:
            logger.exception("exception while creating a node:" + str(e))
        return final_json

    def all_operation_on_categories(self, input_data):
        operation = input_data["action"]

        try:
            # user_id = self.data_security_obj.get_user_id()["user_id"]
            user_id = "user_6515"
            input_data["data"]["user_id"] = user_id

            if Pipeline.key_project_id not in input_data:
                logger.error("Missing mandatory field : {}".format(
                    Pipeline.key_project_id))
                raise Exception(" '{}' is not present in the input ".format(
                    Pipeline.key_project_id))

            input_data['data'][Pipeline.key_project_id] = input_data[Pipeline.key_project_id]

            if input_data["action"] == "create":

                if not self.validate_name(input_data['data'][FlowURL.key_pipeline_category_name]):
                    return {"status": "failed",
                            "message": "Invalid category '{}'. Category name cannot contain "
                                       "spaces, numbers and special characters except '_' ".format(
                                input_data['data'][FlowURL.key_pipeline_category_name])}

                if self.check_if_category_already_exists(input_data['data'][FlowURL.key_pipeline_category_name]):
                    return {"status": "failed", "message": "Category '{}' already exists".format(
                        input_data['data'][FlowURL.key_pipeline_category_name])}

                cat_id = "category_" + GetNewId.get_next_id("category")
                input_data["data"][Pipeline.key_pipeline_category_id] = cat_id
                input_data["data"]["meta"] = self.get_metadata_for_category()
                input_data["data"]["isdeleted"] = "false"
                in_data = CommonUtils().get_user_meta(
                    input_data["data"], check_flag=True)
                ConnectionObj.mongo_connection_obj.insert_one(json_data=in_data,
                                                              database_name=self.metadata,
                                                              collection_name=self.pipe_category)
                return_json = {"status": "success", "message": "Category created successfully",
                               Pipeline.key_pipeline_category_id: cat_id}
                return return_json

            elif input_data["action"] == "edit_fetch":
                category_id = input_data["data"][Pipeline.key_pipeline_category_id]

                query = {Pipeline.key_pipeline_category_id: category_id,
                         Pipeline.key_project_id: input_data['data'][Pipeline.key_project_id]}

                cat_data = ConnectionObj.mongo_connection_obj.find_one(db_name=self.metadata,
                                                                       collection_name=self.pipe_category,
                                                                       query=query,
                                                                       search_json={"_id": 0})
                cat_data.pop("created_by")
                cat_data.pop("meta")
                if cat_data is not None:
                    return_json = {"status": "success", "data": cat_data,
                                   "message": "Category fetched successfully"}
                    return return_json
                else:
                    return_json = {"status": "success", "data": {},
                                   "message": "Category Unavailable"}
                    return return_json

            elif input_data["action"] == "update":
                if not self.validate_name(input_data['data'][FlowURL.key_pipeline_category_name]):
                    return {"status": "failed",
                            "message": "Invalid category '{}'. Category name cannot contain "
                                       "spaces, numbers and special characters except '_' ".format(
                                input_data['data'][FlowURL.key_pipeline_category_name])}

                category_id = input_data["data"][Pipeline.key_pipeline_category_id]
                if self.check_if_category_already_exists(input_data['data'][FlowURL.key_pipeline_category_name],
                                                         category_id=category_id):
                    return {"status": "failed", "message": "Category '{}' already exists".format(
                        input_data['data'][FlowURL.key_pipeline_category_name])}
                query = {Pipeline.key_pipeline_category_id: category_id}
                input_data["data"]["meta"] = self.get_metadata_for_category()
                in_data = CommonUtils().get_user_meta(
                    input_data["data"], check_flag=False)
                ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                                                              collection_name=self.pipe_category,
                                                              query=query,
                                                              set_json=in_data, upsert=True)
                return_json = dict(status="success", message="Category updated successfully",
                                   category_id=category_id)
                return return_json
            elif input_data["action"] == "delete":
                category_id = input_data["data"][Pipeline.key_pipeline_category_id]

                query = {Pipeline.key_pipeline_category_id: category_id,
                         Pipeline.key_project_id: input_data['data'][Pipeline.key_project_id]}
                cat_data = ConnectionObj.mongo_connection_obj.find_one(db_name=self.metadata,
                                                                       collection_name=self.pipe_category,
                                                                       query=query,
                                                                       search_json={"_id": 0})
                if not len(list(cat_data.keys())):
                    logger.error("Category not found to delete")
                    return {'status': "failed", "message": "Category not found to delete"}
                else:
                    category_name = cat_data['category_name']

                    job_id = "job_" + GetNewId.get_next_id("job")
                    job_details = {"job_id": job_id,
                                   "status": "Pending",
                                   Pipeline.key_project_id: input_data[Pipeline.key_project_id],
                                   "job_name": "Delete category: '{}' ".format(category_name),
                                   "category_id": input_data["data"][FlowURL.key_pipeline_category_id],
                                   "start_time": "",
                                   "end_time": "",
                                   "owner": user_id,
                                   "job_type": "category_deletion",
                                   "action": "delete",
                                   "remark": ""}
                    ConnectionObj.mongo_connection_obj.database_insertion(db_name=self.metadata,
                                                                          collection_name=self.job_list,
                                                                          query_json=job_details)
                    self.create_component_delete_job(job_details)

                # new_values = {"$set": {"isdeleted": "true"}}
                #
                # # Updating the category
                # ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                # collection_name=self.pipe_category,
                #                                               query=query, set_json=new_values)
                # # Updating the components in that category
                # ConnectionObj.mongo_connection_obj.update_many_values(db_name=self.metadata,
                #                                                       collection_name=self.custom_node,
                #                                                       query=query, set_json=new_values)
                return_json = dict(status="success", message="Category deleted successfully",
                                   category_id=category_id)
                return return_json
            elif input_data["action"] == "make_public":
                category_id = input_data["data"][Pipeline.key_pipeline_category_id]

                query = {Pipeline.key_pipeline_category_id: category_id,
                         Pipeline.key_project_id: input_data['data'][Pipeline.key_project_id]}

                ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                                                              collection_name=self.pipe_category,
                                                              query=query,
                                                              set_json=input_data["data"], upsert=True)
                return_json = dict(status="success", message="Category updated successfully",
                                   category_id=category_id)
                return return_json
        except Exception as e:
            logger.exception("Exception while {op} : {ex}".format(
                op=operation, ex=str(e)))
            return {"status": "failed", "message": "{op} error".format(op=operation)}

    def check_if_category_already_exists(self, category_name, category_id=None):
        """
        This method is to check if the category name already exists
        """
        try:
            logger.info("Checking if the category name already exists ")

            if category_id is not None:
                query = {FlowURL.key_pipeline_category_name: category_name,
                         FlowURL.key_pipeline_category_id: {"$ne": category_id},
                         "isdeleted": "false"
                         }
            else:
                query = {
                    FlowURL.key_pipeline_category_name: category_name, "isdeleted": "false"}

            resp = list(
                ConnectionObj.mongo_connection_obj.find_with_condition(json_data=query, database_name=self.metadata,
                                                                       collection_name=self.pipe_category))

            if len(resp):
                return True
            else:
                logger.debug(
                    "Checking if the component category exists in the built in components")
                built_in_categories = self.fetch_built_in_component_categories()
                if category_name in built_in_categories:
                    logger.debug(
                        "Component : '{}' found in the build in components ".format(category_name))
                    return True
                else:
                    return False
        except Exception as e:
            logger.error(
                "Error occurred while checking if the category name already exists : {}".format(str(e)))
            raise Exception("Failed to check if the category already exists")

    def fetch_built_in_component_categories(self):
        """
        This method is to fetch the list of built in components
        """
        try:
            logger.info("Fetch built in component categories")
            category_list = []
            resp = list(
                ConnectionObj.mongo_connection_obj.read(json_data={}, database_name=self.metadata,
                                                        collection_name=self.flow_constant))
            if len(resp):
                resp = resp[0]
                if "data" in resp:
                    category_list = [item['folder_name']
                                     for item in resp['data'] if len(resp['data'])]
            return category_list
        except Exception as e:
            logger.error(
                "Error occurred while fetching the built in component categories : {}".format(str(e)))
            raise Exception("Failed to fetch component categories ")

    @staticmethod
    def validate_name(name):
        """
        This method is to validate the component/category name
        """
        try:
            logger.info("Validate component/category name ")
            # Regex is to ensure the component/category name does not contain
            # any special characters and empty spaces
            reg_exp = r"[ +-@!#$%^&*()<>?/\|}{~:0-9]"
            reg_comp = re.compile(reg_exp)
            if reg_comp.search(name) is None:
                return True
            else:
                return False
        except Exception as e:
            logger.error(
                "Error occurred while validating the component/category name : {}".format(str(e)))
            raise Exception("Failed to validate the name : {}".format(str(e)))

    def list_pipeline_categories(self, input_data):
        try:
            final_json = {"status": "success",
                          "data": {
                              "headerContent": [
                                  {
                                      "label": "Category Name",
                                      "value": "category_name"
                                  },
                                  {
                                      "label": "Category Description",
                                      "value": "category_description"
                                  },
                                  {
                                      "label": "Pipeline Category",
                                      "value": "pipeline_category_display"
                                  },
                                  {
                                      "label": "Last Updated By",
                                      "value": "last_updated_by"
                                  },
                                  {
                                      "label": "Last Updated Time",
                                      "value": "last_updated_time"
                                  }
                              ],
                              "bodyContent": []}}
            if input_data["action"] == "list":
                query = {
                    "$or": [{Pipeline.key_project_id: input_data[Pipeline.key_project_id], "isdeleted": "false"},
                            {"default": True}]}
            else:
                query = {}
            # query = {Pipeline.key_project_id: input_data[Pipeline.key_project_id],"isdeleted": "false"}
            cat_data = list(ConnectionObj.mongo_connection_obj.read(json_data=query,
                                                                    database_name=self.metadata,
                                                                    collection_name=self.pipe_category))
            for each_category in cat_data:
                each_category.pop("_id")
                if "default" not in each_category:
                    each_category["default"] = False
                each_category.pop("created_by")
                each_category["last_updated_by"] = each_category["meta"]["last_updated_by_name"]
                each_category["last_updated_time"] = each_category["meta"]["last_updated_at"]
                each_category.pop("meta")
                final_json["data"]["bodyContent"].append(each_category)
            return final_json

        except Exception as e:
            logger.exception("Exception while listing categories :" + str(e))
            return {"status": "failed", "message": "Failed to list categories"}

    def list_pipeline_components(self, input_data):
        try:
            final_json = {
                "status": "success",
                "data": {
                    "headerContent": Pipeline.component_list_header,
                    "bodyContent": []}}
            if input_data["action"] == "list":
                user_id = self.data_security_obj.get_user_id()["user_id"]
                logger.debug(
                    "Component list fetched by user : {}".format(user_id))
                # query = {Pipeline.key_project_id: input_data[Pipeline.key_project_id]}
                query = {"$or": [{Pipeline.key_project_id: input_data[Pipeline.key_project_id], "isdeleted": "false"},
                                 {"default": True}]}
                model_list = list(
                    ConnectionObj.mongo_connection_obj.read(json_data=query, database_name=self.metadata,
                                                            collection_name=self.custom_node))
                for each_model in model_list:
                    try:
                        temp_json = dict()
                        temp_json["component_id"] = each_model["custom_node_id"]
                        temp_json["component_name"] = each_model["node_name"]
                        temp_json["component_category"] = "Default"
                        if "last_edited_user" not in each_model:
                            temp_json["last_updated_by"] = \
                                ConnectionObj.mongo_connection_obj.find_one(db_name=self.metadata,
                                                                            collection_name=self.users,
                                                                            query={
                                                                                "user_id": each_model[
                                                                                    "last_edited_by"]})[
                                    "username"]
                        else:
                            temp_json["last_updated_by"] = each_model["last_edited_user"]
                        temp_json["last_updated_time"] = each_model["last_edited_at"]
                        if "category_name" in each_model and each_model["category_name"] != "":
                            temp_json["component_category"] = each_model["category_name"]
                        if "show_to_public" in each_model:
                            temp_json["show_to_public"] = each_model["show_to_public"]
                        else:
                            temp_json["show_to_public"] = False
                        if "default" in each_model:
                            temp_json["default"] = each_model["default"]
                        else:
                            temp_json["default"] = False
                        # if each_model["created_by"] == user_id:
                        #     temp_json["is_owner"] = True
                        # else:
                        #     temp_json["is_owner"] = False
                        final_json["data"]["bodyContent"].append(temp_json)
                    except Exception as e:
                        logger.error(str(e))
                        continue
            return final_json
        except Exception as e:
            logger.exception("Exception while listing components :" + str(e))
            return {"status": "failed", "message": "Failed to list components"}

    def make_component_public(self, input_data):
        try:
            if input_data["action"] == "show_to_public":
                comp_id = input_data["data"]["component_id"]
                temp_json = {
                    "show_to_public": input_data["data"]["show_to_public"]}
                query = {"custom_node_id": comp_id,
                         Pipeline.key_project_id: input_data[Pipeline.key_project_id]}
                ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                                                              collection_name=self.custom_node,
                                                              query=query,
                                                              set_json=temp_json, upsert=True
                                                              )
                return_json = {"status": "success",
                               "message": "", "component_id": comp_id}
                if temp_json["show_to_public"]:
                    return_json["message"] = "Component enabled for public access"
                else:
                    return_json["message"] = "Component disabled for public access"
                return return_json
        except Exception as e:
            logger.exception(
                "Exception while making component public:" + str(e))
            return {"status": "failed", "message": "Failed to make component public"}

    def jobs_list(self, input_data):
        """
        Definition for creating the nodes
        :return:
        """
        final_json = {
            "status": "success",
            "tableData": {
                "headerContent": self.job_list_header_content,
                "bodyContent": []
            }
        }

        try:
            logger.debug("inside the job list definition")
            if input_data["type"] == "all":
                query_json = {
                    Pipeline.key_project_id: input_data[Pipeline.key_project_id]}
            else:
                query_json = {"status": input_data["type"],
                              Pipeline.key_project_id: input_data[Pipeline.key_project_id]}
            logger.debug("query ".format(query_json))
            jobs_data = ConnectionObj.mongo_connection_obj.search_record_by_query2(db_name=self.metadata,
                                                                                   collection_name=self.job_list,
                                                                                   query_json=query_json,
                                                                                   search_option={'_id': 0})
            users_data = ConnectionObj.mongo_connection_obj.search_record_by_query2(db_name=self.metadata,
                                                                                    collection_name=self.users,
                                                                                    query_json={},
                                                                                    search_option={"_id": 0})
            logger.debug("user data".format(users_data))
            user_id_name_mapping = self.fetch_user_mapping()
            for each_job in jobs_data:
                try:
                    each_job["owner"] = user_id_name_mapping.get(
                        each_job["owner"], "")
                    final_json["tableData"]["bodyContent"].append(each_job)
                except Exception as e:
                    traceback.print_exc()
                    logger.debug(
                        " exception while iterating each job:" + str(e))
        except Exception as e:
            traceback.print_exc()
            logger.exception("exception while creating a node:" + str(e))
        if len(final_json['tableData']['bodyContent']):
            final_json["tableData"]["bodyContent"] = sorted(final_json['tableData']['bodyContent'],
                                                            key=lambda k: int(
                                                                k["job_id"].lstrip("job_")),
                                                            reverse=True)
        return final_json

    def get_job_log(self, input_data):
        """
        Definition for deleting the nodes
        :return:
        """
        final_json = {"status": "failed", "data": {},
                      "message": "Failed to fetch job details"}
        try:
            if "job_id" in input_data:
                job_info = self.get_job_info(job_id=input_data['job_id'])
                logger.debug(" job id:" + str(input_data["job_id"]))
                request_json = {"job_id": input_data["job_id"]}

                if "job_type" in input_data:
                    request_json['job_type'] = input_data['job_type']
                else:
                    request_json['job_type'] = ""

                request_json[FlowURL.key_pipeline_id] = job_info.get(
                    FlowURL.key_pipeline_id)
                request_json[FlowURL.key_pipeline_instance_id] = job_info.get(
                    FlowURL.key_pipeline_instance_id)
                user_details = self.fetch_user_mapping()
                user_id = job_info["data"]["owner"]
                job_info["data"]["owner"] = user_details[user_id]
                response = requests.post(config.CONTAINER_URL + Pipeline.job_log_url,
                                         json=request_json,
                                         verify=False)

                status_json = json.loads(response.text)
                if status_json["status"] == "success":
                    final_json["status"] = "success"
                    final_json["message"] = "success"
                    final_json["data"]["log_data"] = status_json["log_data"]
                    final_json['data']['job_data'] = job_info
                else:
                    logger.debug("Failed to fetch job details")
                    final_json['message'] = 'Failed to fetch job details'
            else:
                return {"status": "failed", "message": "job_id is missing in request payload", "data": {}}
        except Exception as e:
            logger.exception("exception while fetching a template:" + str(e))
        return final_json

    def get_deployment_job_log(self, input_data):
        """
        Definition for deleting the nodes
        :return:
        """
        final_json = {"status": "failed", "data": {}}
        try:
            if "job_id" in input_data:
                job_info = self.get_job_info(job_id=input_data['job_id'])
                logger.debug(" job id:" + str(input_data["job_id"]))
                request_json = {"job_id": input_data["job_id"]}

                if "job_type" in input_data:
                    request_json['job_type'] = input_data['job_type']
                else:
                    request_json['job_type'] = ""

                request_json[Pipeline.key_pipeline_id] = input_data[Pipeline.key_pipeline_id, '']
                response = requests.post(config.CONTAINER_URL + Pipeline.deployment_job_url,
                                         json=request_json,
                                         verify=False)

                status_json = json.loads(response.text)
                if status_json["status"] == "success":
                    final_json["status"] = "success"
                    final_json["data"]["log_data"] = status_json["log_data"]
                    final_json['data']['job_data'] = job_info
                else:
                    logger.debug(
                        " got status failed from the flow parser get job api")
            else:
                return {"status": "failed", "message": "job_id is missing in request payload", "data": {}}
        except Exception as e:
            logger.exception("exception while fetching a template:" + str(e))
        return final_json

    def get_topic_mapping_for_a_pipeline_nodes(self, input_data):
        """
        This method is to get the topic mapping for each node
        """
        final_response = {
            'status': "failed", "message": "Failed to fetch pipeline deployment configurations"}
        try:
            logger.info("Get topic mapping for each node")
            if "pipeline_id" not in input_data:
                logger.error(
                    "Key : '{}' not found in the input".format('pipeline_id'))
                raise Exception(
                    "Key : '{}' not found in the input".format('pipeline_id'))
            if "job_id" not in input_data:
                logger.error(
                    "Key : '{}' not found in the input".format('job_id'))
                raise Exception(
                    "Key : '{}' not found in the input".format('job_id'))
            job_info = ConnectionObj.mongo_connection_obj.find_one(self.metadata, self.job_list,
                                                                   {'job_id': input_data['job_id']})

            if job_info is not None and len(job_info.keys()):
                if "deployment_config" in job_info and 'topic_mapping' in job_info['deployment_config']:
                    final_response['status'] = "success"
                    final_response['message'] = 'Successfully fetched the pipeline deployment configuration'
                    final_response['data'] = {}
                    final_response['data']['topic_mapping'] = job_info['deployment_config']['topic_mapping']
                    final_response['data']["meta"] = config.PIPELINE_INTERNAL_CONFIGURATION
            else:
                logger.error(
                    "pipeline deployment config not found for the pipeline")
                raise Exception(
                    "Failed to fetch the pipeline deployment configurations")
        except Exception as e:
            logger.error("Error : {}".format(str(e)))
            raise Exception(str(e))
        return final_response

    def fetch_component_category_info(self, project_id):
        """
        This method is to fetch the category info
        """
        try:
            logger.info("Fetch component category info")
            category_json = dict()
            query_project = {
                Pipeline.key_project_id: project_id, "isdeleted": "false"}
            category_info = list(ConnectionObj.mongo_connection_obj.
                                 find_with_condition(database_name=self.metadata,
                                                     collection_name=self.component_category_collection,
                                                     json_data=query_project))
            if len(category_info):
                for each_category in category_info:
                    if Pipeline.key_pipeline_category_id in each_category:
                        category_json[each_category[Pipeline.key_pipeline_category_id]] = each_category[
                            Pipeline.key_pipeline_category_name]
            return category_json
        except Exception as e:
            traceback.print_exc()
            logger.error(
                "Error occurred while fetching the category info : {}".format(str(e)))

    def create_new_pipeline_instance(self, input_data):
        """
        This method is to create a new pipeline version
        """
        final_response = {
            'status': 'failed', 'message': "Failed to create new instance for the pipeline"}
        try:
            logger.info("Create new pipeline version")
            data_to_update = dict()
            mandatory_fields = [Pipeline.key_pipeline_id,
                                Pipeline.key_pipeline_instance_id, 'pipeline_version']

            # Fetch user data
            user_json = self.data_security_obj.get_user_id({})
            user_id = user_json["user_id"]
            # user_id = 'user_6501'
            logger.debug("Pipeline updated by user : {}".format(user_id))

            # Mandatory field validation
            for each_field in mandatory_fields:
                if each_field not in input_data:
                    logger.error("'{}' not found")
                    raise Exception("'{}' not found")

            # Fetch pipeline instance info
            pipe_instance_info_query = {
                Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                Pipeline.key_pipeline_instance_id: input_data[Pipeline.key_pipeline_instance_id],
            }
            pipeline_instance_info = ConnectionObj.mongo_connection_obj.find_with_condition(
                json_data=pipe_instance_info_query, database_name=self.metadata,
                collection_name=self.pipeline_instance_collection)

            if pipeline_instance_info.count():
                pipeline_instance_info = pipeline_instance_info[0]

                data_to_update[Pipeline.key_pipeline_id] = input_data[Pipeline.key_pipeline_id]
                data_to_update['flow'] = pipeline_instance_info['flow']
                data_to_update['pipeline'] = pipeline_instance_info['pipeline']
                data_to_update['version'] = pipeline_instance_info['version']
                data_to_update['counter'] = pipeline_instance_info['counter']
                data_to_update["meta"] = pipeline_instance_info['meta']
                data_to_update['product_encrypted'] = pipeline_instance_info['product_encrypted']
                data_to_update['meta']['last_edited_by'] = user_id
                data_to_update['meta']['last_edited_at'] = datetime.now().strftime(
                    "%Y-%m-%d %H:%M:%S")
                data_to_update['created_from'] = {
                    Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                    Pipeline.key_pipeline_instance_id: input_data[Pipeline.key_pipeline_instance_id],
                    'pipeline_version': int(input_data['pipeline_version'].split('v')[1])
                }
                data_to_update[Pipeline.key_pipeline_instance_id] = self.create_new_pipeline_instance_id(
                )
                data_to_update['pipeline_version'] = self.get_pipeline_version(
                    data_to_update[Pipeline.key_pipeline_id])
                ConnectionObj.mongo_connection_obj.database_insertion(query_json=data_to_update,
                                                                      db_name=self.metadata,
                                                                      collection_name=self.pipeline_instance_collection)
                logger.debug("New pipeline instance created successfully")
                pipeline_info_to_update = {
                    Pipeline.key_pipeline_instance_id: data_to_update[Pipeline.key_pipeline_instance_id],
                    "meta.last_edited_by": user_id,
                    "meta.last_edited_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                }

                ConnectionObj.mongo_connection_obj.update_one(
                    query={Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id]}, db_name=self.metadata,
                    collection_name=self.pipeline_info_collection, set_json={"$set": pipeline_info_to_update})
                logger.debug("Pipeline info updated successfully")

                final_response['status'] = "success"
                final_response['message'] = 'Successfully create new version for pipeline'
                final_response['data'] = {
                    Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                    Pipeline.key_pipeline_instance_id: data_to_update[Pipeline.key_pipeline_instance_id],
                    "pipeline_version": "v" + str(data_to_update['pipeline_version'])
                }
            else:
                final_response['message'] = 'Failed to find the selected version of pipelinen'
        except Exception as e:
            traceback.print_exc()
            logger.error(
                "Error occurred while creating a new pipeline version : {}".format(str(e)))
        return final_response

    @staticmethod
    def get_alarms():
        """
        This method is to fetch the configured alarms
        """
        alarm_list = []
        try:
            logger.debug("alarm list")
            all_list = AlarmConfigurationHandler().get_alarm_list()
            if "status" in all_list and all_list["status"] == "success" and all_list["body_content"]:
                for each_alarm in all_list["body_content"]:
                    try:
                        alarm_list.append(
                            {"value": each_alarm["id"], "label": each_alarm["alarmName"]})
                    except Exception as e:
                        logger.exception(str(e))
                        continue
        except Exception as e:
            logger.error("Failed to list alarms : {}".format(str(e)))
        return alarm_list

    @staticmethod
    def get_rules():
        rule_list = []
        try:
            logger.debug("rule list")
            all_rules = RuleConfigurationHandler().get_rule_engine_list({})
            if "status" in all_rules and all_rules["status"] == "success" and all_rules["bodyContent"]:
                for each_rule in all_rules["bodyContent"]:
                    try:
                        rule_list.append(
                            {"value": each_rule["rule_engine_id"], "label": each_rule["rule_name"]})
                    except Exception as e:
                        logger.exception(str(e))
                        continue
        except Exception as e:
            logger.error("Failed to list rules : {}".format(str(e)))
        return rule_list

    def create_build(self, input_data):
        """
        This method is to initiate build for pipeline
        :param input_data:
        :return:
        """
        # status, label, node_name = self.validate_mandatory_fields(input_data)
        # if not status:
        #     raise Exception(f"Mandatory Configuration '{label}' not filled for the component '{node_name}'")

        logger.info("Build pipeline")
        try:
            logger.debug("Initiate build for pipeline")
            logger.debug(
                "Input data for building pipeline : {}".format(input_data))

            user_json = self.data_security_obj.get_user_id({})
            if "user_id" not in user_json:
                final_json = {"status": "success"}
                final_json["message"] += " (User not found)"
                return final_json
            user_id = user_json["user_id"]

            if Pipeline.key_pipeline_id not in input_data:
                return {"status": "failed", "message": "Pipeline not found"}

            query_pipe_info = {
                Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id]}
            pipeline_details = ConnectionObj.mongo_connection_obj. \
                find_one(db_name=self.metadata, collection_name=self.pipeline_info_collection,
                         query=query_pipe_info,
                         search_json={"_id": 0})
            pipeline_category = pipeline_details.get(
                Pipeline.key_pipeline_category, "")
            # pipeline_instance_id = input_data['version'][Pipeline.key_pipeline_instance_id]
            pipeline_instance_id = pipeline_details[Pipeline.key_pipeline_instance_id]
            pipeline_version = input_data['version']['pipeline_version']
            # logger.debug("Building for version : {}".format(pipeline_version))
            logger.debug("Pipeline instance id : {}".format(
                pipeline_instance_id))
            query_pipe_instance_info = {Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                                        Pipeline.key_pipeline_instance_id: pipeline_instance_id}

            pipe_instance_details = ConnectionObj.mongo_connection_obj. \
                find_one(db_name=self.metadata, collection_name=self.pipeline_instance_collection,
                         query=query_pipe_instance_info,
                         search_json={"_id": 0})

            if "build_status" in pipe_instance_details and pipe_instance_details['build_status'].lower() == 'success':
                request_json = {Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                                Pipeline.key_pipeline_instance_id: pipeline_instance_id,
                                'pipeline_version': "v" + str(pipe_instance_details['pipeline_version'])}
                response = self.create_new_pipeline_instance(request_json)
                logger.debug(100 * "+")
                logger.debug(response)
                pipeline_instance_id = response['data'][Pipeline.key_pipeline_instance_id]
                new_version = response['data']['pipeline_version']
                logger.debug('NEW PIPELINE INSTANCE : {}'.format(
                    pipeline_instance_id))
                logger.debug("New PIPELINE VERSION : {}".format(new_version))
                query_pipe_instance_info = {Pipeline.key_pipeline_id: input_data[Pipeline.key_pipeline_id],
                                            Pipeline.key_pipeline_instance_id: pipeline_instance_id}
            if pipeline_category.lower() == Pipeline.category_normal or \
                    pipeline_category.lower() == Pipeline.category_acquisition:

                # framing job details to insert record into job list
                job_id = "job_" + GetNewId.get_next_id("job")
                job_details = {
                    "job_id": job_id,
                    "status": "Pending",
                    Pipeline.key_project_id: input_data[Pipeline.key_project_id],
                    "job_name": "pipeline build",
                    "custom_node_id": input_data[Pipeline.key_pipeline_id],
                    "pipeline_id": input_data[Pipeline.key_pipeline_id],
                    Pipeline.key_pipeline_instance_id: pipeline_instance_id,
                    "start_time": datetime.now().timestamp(),
                    "end_time": "",
                    "owner": user_id,
                    "job_type": "flow_creation",
                    "action": "edit",
                    "remark": ""
                }
                ConnectionObj.mongo_connection_obj.database_insertion(db_name=self.metadata,
                                                                      collection_name=self.job_list,
                                                                      query_json=job_details)

                pipe_instance_details = ConnectionObj.mongo_connection_obj. \
                    find_one(db_name=self.metadata, collection_name=self.pipeline_instance_collection,
                             query=query_pipe_instance_info,
                             search_json={"_id": 0})

                # mapping input payload to existing pipeline details
                pipe_instance_details["counter"] = input_data["flow_data"]["counter"]
                pipe_instance_details["flow"] = input_data["flow_data"]["flow"]
                pipe_instance_details["pipeline"] = input_data["flow_data"]["pipeline"]
                pipe_instance_details["meta"]["last_edited_by"] = user_id
                pipe_instance_details['meta']['last_edited_at'] = datetime.now().strftime(
                    "%Y-%m-%d %H:%M:%S")

                if "pipeline_build_configurations" in input_data:
                    pipe_instance_details['pipeline_build_configurations'] = input_data[
                        'pipeline_build_configurations']
                    pipe_instance_details['pipeline_build_configurations'][
                        'download_url'] = Endpoints.api_download_win_build
                else:
                    pipe_instance_details['pipeline_build_configurations'] = {}

                pipe_instance_details['latest_build_job_id'] = job_id
                pipe_instance_details['enable_for_deployment'] = input_data.get(
                    'enable_for_deployment', False)
                update_pipe_instance_details = {"$set": pipe_instance_details}

                ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                                                              collection_name=self.pipeline_instance_collection,
                                                              query=query_pipe_instance_info,
                                                              set_json=update_pipe_instance_details,
                                                              upsert=True)
                job_info_json = {"job_id": job_id, "build_type": input_data.get(
                    'pipeline_build_configurations').get('build_type', "container")}
                self.initiate_build_for_pipeline(job_info_json)
                return {"status": "success",
                        "message": f"Pipeline build initiated successfully. "
                                   f"Please check '{job_id}' for detailed progress", "job_id": job_id}
            elif pipeline_category.lower() == Pipeline.category_ai:
                update_instance_details = {'meta.last_edited_by': user_id,
                                           'meta.last_edited_at': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                                           "enable_for_deployment": input_data.get('enable_for_deployment', False),
                                           "build_status": "success"}
                ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata,
                                                              collection_name=self.pipeline_instance_collection,
                                                              query=query_pipe_instance_info,
                                                              set_json={
                                                                  "$set": update_instance_details},
                                                              )
        except Exception as e:
            traceback.print_exc()
            logger.exception(
                "exception while creating the build for pipeline:" + str(e))
            return {"status": "failed", "message": "Failed to initiate pipeline build"}

    @background
    def initiate_build_for_pipeline(self, input_data):
        """
        This method is to initiate the build for the pipeline
        """
        try:
            post_data = {"job_id": input_data.get('job_id')}
            logger.info("Initiate container build for the pipeline ")
            build_type = input_data.get('build_type')
            request_url = config.CONTAINER_URL + Pipeline.build_pipeline

            if build_type == 'container':
                request_url = config.CONTAINER_URL + Pipeline.build_pipeline
            if build_type == 'windows':
                request_url = config.CONTAINER_URL + Pipeline.windows_build

            response = requests.post(url=request_url, json=post_data)
            logger.debug(
                "Status code for initiating the container build for the pipeline : {}".format(response.status_code))
            if response.status_code == 200:
                resp = response.json()
                logger.debug(
                    "Initiate container build for pipeline response : {}".format(json.dumps(resp)))
            else:
                logger.error(
                    "Failed to initiate container build for the pipeline")
        except Exception as e:
            logger.error("Error occurred while initiating the container build for the pipeline : {}".format(str(e)),
                         exc_info=True)
            raise Exception(
                "Failed to initiate container build for the pipeline")

    def fetch_user_mapping(self):
        """
        Fetch user mapping
        """
        user_mapping = dict()
        try:
            logger.debug("Fetching user mapping")
            all_users = ConnectionObj.mongo_connection_obj.fetch_all(
                self.metadata, self.users)
            if all_users:
                for i in all_users:
                    try:
                        user_mapping[i["user_id"]] = i["username"]
                    except Exception as e:
                        logger.error(str(e))
                        continue
            return user_mapping
        except Exception as e:
            logger.error(
                "Error occurred while creating user mapping : {}".format(str(e)))
            raise Exception("Failed to create user mapping")

    def check_if_pipeline_exists(self, pipeline_name):
        """
        This method is to check if the pipeline already exists
        """
        try:
            logger.info("Checking if the pipeline already exists")
            query = {"pipeline_name": {"$regex": "^{}$".format(
                pipeline_name), "$options": "i"}, "isdeleted": "false"}
            resp = list(
                ConnectionObj.mongo_connection_obj.find_with_condition(json_data=query, database_name=self.metadata,
                                                                       collection_name=self.pipeline_info_collection))
            if len(resp):
                return True
            else:
                return False
        except Exception as e:
            logger.error(
                "Error occurred while checking the pipeline existence : {}".format(str(e)))
            raise Exception("Failed to check if the pipeline exists")

    @staticmethod
    def create_new_pipeline_id():
        """
        This method is to crete a new pipeline id
        """
        try:
            logger.info("Create new pipeline id")
            return "pipeline_" + GetNewId.get_next_id("pipeline")
        except Exception as e:
            logger.error(
                "Error occurred while creating a new pipeline id : {}".format(str(e)))
            raise Exception("Failed to create a new pipeline id")

    @staticmethod
    def create_new_pipeline_instance_id():
        """
        This method is to crete a new pipeline id
        """
        try:
            logger.info("Create new pipeline id")
            return "pipeline_instance_" + GetNewId.get_next_id("pipeline_instance")
        except Exception as e:
            logger.error(
                "Error occurred while creating a new pipeline instance id : {}".format(str(e)))
            raise Exception("Failed to create a new pipeline instance id")

    @staticmethod
    def get_pipeline_version(pipeline_id):
        """
        This method is to fetch the latest version for the pipeline
        """
        try:
            logger.info("Create new pipeline version")
            return int(GetNewId.get_next_id(pipeline_id))
        except Exception as e:
            logger.error(
                "Error occurred while creating a new pipeline instance id : {}".format(str(e)))
            raise Exception("Failed to create a new pipeline instance id")

    def check_job_status_and_get_image(self, pipeline_id, job_id):
        """
        This method is to check the build status of the pipeline and get the built image
        """
        final_json = {'status': False, "image": ""}
        image = None
        try:
            logger.info("Check job status and fetch built image")
            logger.debug(
                "Check job status and fetch built image for the job : {}".format(job_id))
            job_info = ConnectionObj.mongo_connection_obj.find_one(db_name=self.metadata,
                                                                   collection_name=self.job_list,
                                                                   query={"job_id": job_id})
            if job_info and len(list(job_info.keys())):
                try:
                    if job_info['status'].lower() == 'success':
                        pipe_info = ConnectionObj.mongo_connection_obj. \
                            find_one(db_name=self.metadata, collection_name=self.pipeline_info_collection,
                                     query={Pipeline.key_pipeline_id: pipeline_id})
                        if "pipeline_build_info" in pipe_info and len(pipe_info['pipeline_build_info']):
                            for each_job in pipe_info['pipeline_build_info']:
                                if each_job['job_id'] == job_id:
                                    image = each_job['image']
                                    break
                        if image:
                            final_json['status'] = 'success'
                            final_json['image'] = image
                except Exception as e:
                    logger.error(
                        "Failed to fetch the build status for the pipeline : {}".format(str(e)))
                    raise Exception("Failed to get build status of pipeline")
            else:
                logger.error(
                    "Build status not status not found for the pipeline")
                raise Exception("Pipeline build not successful")
        except Exception as e:
            logger.error(
                "Error occurred while checking job status and fetch built image to deploy : {}".format(str(e)))
            raise Exception(str(e))
        return final_json

    def update_job_deployment_config(self, job_id, deployment_config):
        """
        This method is to update the job deployment config
        """
        try:
            logger.info("Update pipeline deployment config")
            job_info = ConnectionObj.mongo_connection_obj.find_one(
                self.metadata, self.job_list, {'job_id': job_id})
            if job_info is not None and len(job_info.keys()):
                if "_id" in job_info:
                    del job_info['_id']
                job_info['deployment_config'] = deployment_config
                job_info['deployed_at'] = datetime.now().strftime(
                    "%Y-%m-%d %H:%M:%S")
                ConnectionObj.mongo_connection_obj.update_one(db_name=self.metadata, collection_name=self.job_list,
                                                              query={'job_id': job_id}, set_json={"$set": job_info},
                                                              upsert=True)
        except Exception as e:
            logger.error("Error occurred while updating the pipeline deployment config : %s", str(
                e), exc_info=True)
            raise Exception(
                "Failed to update the deployment configurations for the pipeline")

    def get_metadata_for_category(self):
        meta_info = {
            "last_updated_by_id": "",
            "last_updated_at": "",
            "last_updated_by_name": ""
        }
        try:
            updated_by = self.data_security_obj.get_user_id()["user_id"]
            user_details = ConnectionObj.mongo_connection_obj.find_one(db_name=self.metadata,
                                                                       collection_name=self.users,
                                                                       query={"user_id": updated_by})
            if user_details:
                user_name = user_details["username"]
            else:
                user_name = updated_by
            timestamp = (datetime.now()).strftime("%d-%b-%Y %H:%M:%S")
            meta_info["last_updated_by_id"] = updated_by
            meta_info["last_updated_by_name"] = user_name
            meta_info["last_updated_at"] = timestamp
            return meta_info
        except Exception as e:
            logger.exception("Exception while fetching metadata :" + str(e))
            return meta_info

    @background
    def add_node_to_components(self, input_data):
        """
        This method is to add the node to the components
        """
        try:
            logger.info("Add node to the custom component group")
            request_url = config.CONTAINER_URL + Pipeline.save_node
            response = requests.post(url=request_url, json=input_data)
            logger.debug("Status code for triggering the save node is : {}".format(
                response.status_code))
            if response.status_code == 200:
                resp = response.json()
                logger.debug(
                    "Save node response : {}".format(json.dumps(resp)))
            else:
                logger.error("Failed to initiate save node ")
        except Exception as e:
            traceback.print_exc()
            logger.error('Error occurred while adding the node : {}'.format(
                str(e)), exc_info=True)
            raise Exception("Failed to add the node")

    def get_job_info(self, job_id):
        """
        This method is to fetch the job data for the selected job id
        """
        final_json = {
            "order": [
                "job_id",
                "job_name",
                "owner",
                "start_time",
                "end_time",
                # "remark",
                "status"
            ],
            "mapping": {
                "job_id": "Job Id",
                "job_name": "Job Name",
                "owner": "Owner",
                "start_time": "Start Time",
                "end_time": "End Time",
                "status": "Status"
                # "remark": "Remark"
            },
            "data": {}
        }
        try:
            logger.info("Fetch job info for the selected job")
            logger.info("Fetch job info for the job : {}".format(job_id))
            query_json = {'job_id': job_id}
            job_info = ConnectionObj.mongo_connection_obj.find_one(db_name=self.metadata,
                                                                   collection_name=self.job_list,
                                                                   query=query_json)

            if job_info is not None and len(list(job_info.keys())):
                if "_id" in job_info:
                    del job_info['_id']
                if "remark" in job_info:
                    del job_info['remark']
                final_json['data'] = job_info
                return final_json
            else:
                logger.error("Job not found")
                return final_json
        except Exception as e:
            logger.error("Error occurred while fetching job info : {}".format(
                str(e)), exc_info=True)
            raise Exception("Failed to fetch the job info for the job")

    def get_pipe_version_details(self, pipeline_id):
        """
        This method is to fetch the list of all the available versions for the pipeline
        """
        try:
            temp_list = []
            logger.info("Fetch available version for the pipeline ")
            query_pipeline_info = {Pipeline.key_pipeline_id: pipeline_id}
            pipeline_info = ConnectionObj.mongo_connection_obj. \
                find_with_condition(json_data=query_pipeline_info, database_name=self.metadata,
                                    collection_name=self.pipeline_instance_collection)
            for each_item in pipeline_info:
                temp_json = dict()
                temp_json[Pipeline.key_pipeline_instance_id] = each_item[Pipeline.key_pipeline_instance_id]
                temp_json["pipeline_version"] = "v" + \
                                                str(each_item['pipeline_version'])
                temp_json['build_status'] = each_item.get(
                    'build_status', 'pending')
                if temp_json['build_status'].lower() == 'success':
                    temp_json[
                        'message'] = 'Build successful for the pipeline. Would you like a create a new version ' \
                                     'or continue editing the same version'
                else:
                    temp_json['message'] = ""
                temp_list.append(temp_json)
            return temp_list
        except Exception as e:
            logger.error(
                f"Error occurred while fetching the available version list for the pipeline: {e}")
            raise Exception("Failed to fetch available version list")

    @staticmethod
    def get_component_details(component_id):
        """
        This method is to fetch the component details
        """
        try:
            logger.info("Fetch component details give in the pipeline")
            query_node_details = {'custom_node_id': component_id,
                                  Pipeline.key_node_type: Pipeline.component_type_docker}
            logger.debug("Query to fetch node registry details : {}".format(
                query_node_details))
            component_info = ConnectionObj.mongo_connection_obj.find_with_condition(query_node_details,
                                                                                    DBMapping.mongo_db_name,
                                                                                    collection_name="custom_node")
            if component_info.count():
                component_info = component_info[0]
                if 'node_registry_details' in component_info:
                    return component_info['node_registry_details']
        except Exception as e:
            logger.error(
                "Error occurred while fetching the component details : {}".format(str(e)))
            raise Exception("Failed to fetch the component details ")

    @staticmethod
    def get_component_types(component_info):
        """
        This method is to fetch the component types
        """
        component_type_info = {}
        try:
            logger.info("Fetch component types : {}".format(component_info))
            for each_node in component_info:
                if "node_component_type" in component_info[each_node]:
                    component_type_info[each_node] = component_info[each_node]['node_component_type']
                else:
                    component_type_info[each_node] = 'built_in'
            logger.debug('COMPONENT TYPES ---------------<>')
            logger.debug(component_type_info)
            return component_type_info
        except Exception as e:
            logger.error(
                "Error occurred while fetching component types : {}".format(str(e)))
            raise Exception("Failed to fetch component types")

    def fetch_pipeline_details(self, pipeline_id, pipeline_instance_id, pipeline_category, device_info,
                               override_config=False, pipeline_override_config=None):
        try:
            temp_dict = dict()

            query_pipe_instance = {
                Pipeline.key_pipeline_id: pipeline_id,
                Pipeline.key_pipeline_instance_id: pipeline_instance_id
            }
            pipeline_info = ConnectionObj.mongo_connection_obj.find_with_condition(
                json_data=query_pipe_instance,
                database_name=self.metadata,
                collection_name=MongoMetadata.pipeline_instance_info_collection
            )
            _fcp_ = FlowConfigBuilder(
                pipeline_info["pipeline"], device_info=device_info)
            if pipeline_info.count():
                pipeline_info = pipeline_info[0]

                temp_dict['pipeline_category'] = pipeline_category
                temp_dict["pipeline_id"] = pipeline_id
                temp_dict["thread"] = pipeline_info.get("thread", 1)
                temp_dict["version"] = pipeline_info["pipeline_version"]
                temp_dict["pipeline_internal"]: config.PIPELINE_INTERNAL_CONFIGURATION

                # todo: convert job to pipeline version
                if pipeline_category.lower() == Pipeline.category_normal:
                    temp_dict['image'] = pipeline_info['pipeline_build_info'][-1]['image']
                    temp_dict['job_id'] = pipeline_info['pipeline_build_info'][-1]['job_id']
                    temp_dict['pipeline_build_configurations'] = pipeline_info[
                        "pipeline_build_configurations"]
                elif pipeline_category.lower() == Pipeline.category_acquisition:
                    # temp_dict['download_url'] = pipeline_info['pipeline_build_info'][-1]['download_url']
                    temp_dict['pipeline_deployment_type'] = 'process'
                    temp_dict['job_id'] = pipeline_info['pipeline_build_info'][-1]['job_id']
                    temp_dict['pipeline_build_configurations'] = pipeline_info[
                        "pipeline_build_configurations"]

                elif pipeline_category.lower() == Pipeline.category_ai:
                    temp_dict['job_id'] = pipeline_info['pipeline_version']
                    temp_dict['pipeline_build_configurations'] = {}
                    temp_dict["pipeline_deployment_type"] = "docker"
                    temp_dict["docker_deployment_type"] = "multi"
                    temp_dict['image'] = []

                    for each_component in pipeline_info['pipeline']['nodes']:
                        logger.debug(
                            "each component ---> {}".format(each_component))
                        node_info = self.get_component_details(
                            pipeline_info['pipeline']['nodes'][each_component]['custom_node_id'])
                        if node_info is not None:
                            node_info['node_unique_id'] = each_component
                            temp_dict['image'].append(node_info)
                            command_dict = dict(metadata=_fcp_.get_commands(node_unique_id=each_component,
                                                                            image_name=node_info['image_name']),
                                                node_unique_id=each_component)
                            temp_dict['commands'].append(command_dict)

                if pipeline_override_config is None:
                    pipeline_override_config = dict()

                deployment_config = _fcp_.create_config(
                    pipeline_id=temp_dict["pipeline_id"],
                    pipeline_type=pipeline_category,
                    component_types=self.get_component_types(
                        pipeline_info['pipeline']['nodes']),
                    job_id=temp_dict['job_id'],
                    thread=temp_dict["thread"],
                    override_config=override_config,
                    pipeline_override_config=pipeline_override_config)
                temp_dict['config'] = deployment_config
                return temp_dict
        except Exception as e:
            logger.error(
                "Server faced a problem when fetching all pipeline details: {}".format(str(e)))
            raise Exception(
                "Server faced a problem when fetching all pipeline details: {}".format(str(e)))

    @staticmethod
    def soft_delete_associated_pipelines(device_id, pipeline_category,
                                         associated_pipelines, user_associated_pipelines, pipeline_builder_type=None):
        try:
            logger.debug("Soft deleting associated pipelines")
            deleted_pipelines = [
                value for value in associated_pipelines if value not in user_associated_pipelines]
            for pipeline in deleted_pipelines:
                query = {"ilens_device_id": device_id,
                         "pipeline_id": pipeline,
                         "pipeline_category": pipeline_category}
                if pipeline_builder_type is not None:
                    query.update({"pipeline_builder_type": pipeline_builder_type})
                ConnectionObj.mongo_connection_obj. \
                    update_one(query=query,
                               db_name=DBMapping.mongo_db_name,
                               collection_name=MongoMetadata.associate_pp_device,
                               set_json={"deploymentStatus": "DELETED"})
            logger.debug("Pipelines have been soft deleted successfully")
            return True
        except Exception as e:
            logger.error("Server faced a problem when performing a soft delete of associated pipelines: {}".
                         format(str(e)))
            raise Exception("Server faced a problem when performing a new association of pipeline: {}".
                            format(str(e)))

    def update_associated_pipeline(self, pipeline, pipeline_category, user_associated_pipeline,
                                   device_info, device_id, deployment_type, associated_pipeline,
                                   cpu, memory, run_time, gpu,
                                   override_config, pipeline_override_config, pipeline_builder_type=None):

        pipeline_info = self. \
            fetch_pipeline_details(pipeline_id=pipeline,
                                   pipeline_category=pipeline_category,
                                   pipeline_instance_id=user_associated_pipeline[
                                       "pipeline_instance_id"], device_info=device_info,
                                   override_config=override_config,
                                   pipeline_override_config=pipeline_override_config)
        pipeline_info["old_version"] = associated_pipeline["version"]
        pipeline_info["ilens_device_id"] = device_id
        pipeline_info["deploymentType"] = deployment_type
        pipeline_info["cpu"] = cpu
        pipeline_info["memory"] = memory
        pipeline_info["run_time"] = run_time
        pipeline_info["gpu"] = gpu

        pipeline_info["associated_at"] = datetime.now().strftime(
            AppTimeFormats.USER_META_TIME_FORMAT)
        pipeline_info["deploymentStatus"] = "UPDATED"
        query = {"ilens_device_id": device_id,
                 "pipeline_id": pipeline,
                 "pipeline_category": pipeline_category}
        if pipeline_builder_type is not None:
            query.update({"pipeline_builder_type": pipeline_builder_type})
        ConnectionObj.mongo_connection_obj. \
            update_one(query=query,
                       db_name=DBMapping.mongo_db_name,
                       collection_name=MongoMetadata.associate_pp_device,
                       set_json=pipeline_info)

    def update_associated_pipelines(self, device_id, pipeline_category,
                                    associated_pipelines, user_associated_pipelines,
                                    deployment_type, user_input, associated_pipeline_meta, device_info,
                                    pipeline_builder_type=None):
        try:
            logger.debug("Updating associated pipelines")
            existant_pipelines = [
                value for value in associated_pipeline_meta if value in user_associated_pipelines]
            for pipeline in existant_pipelines:
                for associated_pipeline in associated_pipelines:
                    for user_associated_pipeline in user_input:
                        logger.debug(
                            f'User associated pipelines: {user_associated_pipeline}')
                        if associated_pipeline["pipeline_id"] == pipeline and \
                                user_associated_pipeline["pipeline_id"] == pipeline:

                            associated_version = str(
                                associated_pipeline["version"])
                            pipeline_override_config = user_associated_pipeline.get(
                                "component_config", dict())
                            logger.debug(pipeline_override_config)

                            cpu = user_associated_pipeline.get("cpu")
                            memory = user_associated_pipeline.get("memory")
                            run_time = user_associated_pipeline.get("run_time")
                            gpu = user_associated_pipeline.get("gpu")
                            update_key = user_associated_pipeline.get("update")
                            logger.debug(
                                f'User associated pipelines: cpu: {cpu}')
                            if "v" not in associated_version:
                                associated_version = "v" + \
                                                     str(associated_version)

                            # if user_associated_pipeline.get("pipeline_version") != associated_version:

                            # adding new pipeline
                            if update_key:
                                self.update_associated_pipeline(pipeline, pipeline_category, user_associated_pipeline,
                                                                device_info, device_id, deployment_type,
                                                                associated_pipeline,
                                                                cpu,
                                                                memory,
                                                                run_time,
                                                                gpu,
                                                                override_config=True,
                                                                pipeline_override_config=pipeline_override_config,
                                                                pipeline_builder_type=pipeline_builder_type)
                                logger.debug(
                                    "entered update for the first pipeline")

                            # else:
                            #     query = {
                            #         "ilens_device_id": device_id,
                            #         "physical_device_group_id": device_info["physical_device_group_id"],
                            #         "eventType": "deploy",
                            #         "pipeline_deployment_info.pipeline_id": user_associated_pipeline["pipeline_id"]}
                            #     logger.debug(f'Fetching all deploy events from DB {query}')
                            #
                            #     events = ConnectionObj.mongo_connection_obj.find_with_sort_and_skip(
                            #         db_name=app_constants.DBMapping.mongo_db_name,
                            #         collection_name="events_collection",
                            #         query_json=query,
                            #         skip=0,
                            #         sort_json=[("eventTime", -1)],
                            #         limit=0)
                            #
                            #     for each_event in events:
                            #         if each_event["event_status"] != "FAILED" and "v" + str(
                            #                 each_event["pipeline_deployment_info"]["version"]) == \
                            #                 user_associated_pipeline["pipeline_version"]:
                            #             logger.debug("Entered when status is not failed")
                            #             self.save_overrided_config(pipeline,
                            #             pipeline_category, user_associated_pipeline,
                            #                              device_info, device_id, deployment_type,
                            #                              associated_pipeline,
                            #                              user_input,
                            #                              override_config=True,
                            #                              pipeline_override_config=pipeline_override_config)
                            #
                            #             logger.debug("Updated pipelines for already existing pipelines")
                            #             break
                            #         elif each_event["event_status"] == "FAILED" and "v" + str(
                            #                 each_event["pipeline_deployment_info"]["version"]) == \
                            #                 user_associated_pipeline["pipeline_version"]:
                            #             logger.debug("Entered when status is failed")
                            #             update_json = {"deploymentStatus": "UPDATED"}
                            #             ConnectionObj.mongo_connection_obj. \
                            #                 update_one(query={"ilens_device_id": device_id,
                            #                                   "pipeline_id": user_associated_pipeline["pipeline_id"],
                            #                                   "pipeline_category": pipeline_category},
                            #                            db_name=DBMapping.mongo_db_name,
                            #                            collection_name=MongoMetadata.associate_pp_device,
                            #                            set_json=update_json)

            logger.debug("Pipelines have been updated successfully")
            return True
        except Exception as e:
            logger.error("Server faced a problem when performing an update of associated pipelines: {}".
                         format(str(e)), exc_info=True)
            raise Exception("Server faced a problem when performing a new association of pipeline: {}".
                            format(str(e)))

    def add_associated_pipelines(self, device_id, pipeline_category,
                                 associated_pipelines, user_associated_pipelines,
                                 deployment_type, user_input, device_info, pipeline_builder_type=None):
        try:
            logger.debug("Adding new associated pipelines")
            new_pipelines = [
                value for value in user_associated_pipelines if value not in associated_pipelines]
            for pipeline in new_pipelines:
                for user_associated_pipeline in user_input:
                    pipeline_override_config = user_associated_pipeline.get(
                        'component_config', dict())
                    cpu = user_associated_pipeline.get("cpu")
                    memory = user_associated_pipeline.get("memory")
                    run_time = user_associated_pipeline.get("run_time")
                    gpu = user_associated_pipeline.get("gpu")

                    if user_associated_pipeline["pipeline_id"] == pipeline:
                        pipeline_info = self.fetch_pipeline_details(pipeline_id=pipeline,
                                                                    pipeline_category=pipeline_category,
                                                                    pipeline_instance_id=user_associated_pipeline[
                                                                        "pipeline_instance_id"],
                                                                    device_info=device_info,
                                                                    override_config=True,
                                                                    pipeline_override_config=pipeline_override_config)
                        pipeline_info["pipeline_instance_id"] = user_associated_pipeline["pipeline_instance_id"]
                        pipeline_info["ilens_device_id"] = device_id
                        pipeline_info["deploymentType"] = deployment_type
                        pipeline_info["cpu"] = cpu
                        pipeline_info["memory"] = memory
                        pipeline_info["run_time"] = run_time
                        pipeline_info["gpu"] = gpu
                        pipeline_info["associated_at"] = datetime.now().strftime(
                            AppTimeFormats.USER_META_TIME_FORMAT)
                        pipeline_info["deploymentStatus"] = "ASSOCIATED"

                        pipeline_info["pipeline_builder_type"] = pipeline_builder_type
                        ConnectionObj.mongo_connection_obj. \
                            insert_one(json_data=pipeline_info,
                                       database_name=DBMapping.mongo_db_name,
                                       collection_name=MongoMetadata.associate_pp_device)
            logger.debug("Pipelines have been associated successfully")
            return True
        except Exception as e:
            logger.error("Server faced a problem when performing a new association of pipeline: {}".
                         format(str(e)), exc_info=True)
            raise Exception("Server faced a problem when performing a new association of pipeline: {}".
                            format(str(e)))

    def associate_pipelines(self, input_json, pipeline_category, pipeline_builder_type=None):
        try:
            for each_device in input_json["selected_devices"]:
                device_id = each_device["ilens_device_id"]
                query = {"ilens_device_id": device_id,
                         "pipeline_category": pipeline_category}
                if pipeline_builder_type is not None:
                    query.update({"pipeline_builder_type": pipeline_builder_type})
                associated_pipelines = list(ConnectionObj.mongo_connection_obj.
                                            find_with_condition(json_data=query,
                                                                database_name=DBMapping.mongo_db_name,
                                                                collection_name=MongoMetadata.associate_pp_device))

                device_info = list(
                    ConnectionObj.mongo_connection_obj.find_with_condition(
                        {"ilens_device_id": device_id},
                        database_name=DBMapping.mongo_db_name,
                        collection_name=DeviceConfiguration.DEVICE_INFO_COLLECTION))
                if len(device_info):
                    device_info = device_info[0]

                    if "_id" in device_info:
                        del device_info['_id']
                else:
                    logger.error("Device not found to associate pipeline")
                    raise Exception("Device not found to associate pipeline")

                associated_pipelines_meta = list()
                user_associated_pipelines = list()

                for associated_pipeline in associated_pipelines:
                    associated_pipelines_meta.append(
                        associated_pipeline["pipeline_id"])

                if 'pipelines' in input_json["bodyContent"][device_id] and len(
                        input_json["bodyContent"][device_id]["pipelines"]):
                    for each_pipeline in input_json["bodyContent"][device_id]["pipelines"]:
                        user_associated_pipelines.append(
                            each_pipeline["pipeline_id"])
                if not self.soft_delete_associated_pipelines(device_id=device_id,
                                                             pipeline_category=pipeline_category,
                                                             associated_pipelines=associated_pipelines_meta,
                                                             user_associated_pipelines=user_associated_pipelines,
                                                             pipeline_builder_type=pipeline_builder_type):
                    logger.critical("Soft delete of associated pipeline failed. "
                                    "This will not remove the pipeline from the remote device")
                if not self.update_associated_pipelines(device_id=device_id, pipeline_category=pipeline_category,
                                                        associated_pipelines=associated_pipelines,
                                                        user_associated_pipelines=user_associated_pipelines,
                                                        user_input=input_json["bodyContent"][device_id][
                                                            "pipelines"],
                                                        deployment_type=input_json["bodyContent"][device_id][
                                                            "deploymentType"],
                                                        associated_pipeline_meta=associated_pipelines_meta,
                                                        device_info=device_info,
                                                        pipeline_builder_type=pipeline_builder_type):
                    logger.critical("Update of associated pipeline failed. "
                                    "This will not update the pipeline on the remote device")
                if not self.add_associated_pipelines(device_id=device_id, pipeline_category=pipeline_category,
                                                     associated_pipelines=associated_pipelines_meta,
                                                     user_associated_pipelines=user_associated_pipelines,
                                                     deployment_type=input_json["bodyContent"][device_id][
                                                         "deploymentType"],
                                                     user_input=input_json["bodyContent"][device_id]["pipelines"],
                                                     device_info=device_info,
                                                     pipeline_builder_type=pipeline_builder_type):
                    logger.critical("Associating a new pipeline failed. "
                                    "This will not run the pipeline on the remote device")
            return True
        except Exception as e:
            logger.error(
                "Server faced a problem when associating docker pipelines with device --> {}".format(str(e)),
                exc_info=True)
            return False

    def publish_summary(self, input_data, device_info, pipeline_category, pipeline_builder_type=None):
        try:
            deployment_summary = list()
            device_id = input_data.get("ilens_device_id", None)
            device_name = device_info.get("device_name", None)
            # Summary of all the delete pipeline events
            logger.debug("Fetching pipelines to be deleted")
            query = {"ilens_device_id": device_id,
                     "pipeline_category": pipeline_category,
                     "deploymentStatus": "DELETED"}
            if pipeline_builder_type is not None:
                query.update({"pipeline_builder_type": pipeline_builder_type})
            associate_pp_device_delete = ConnectionObj.mongo_connection_obj. \
                find_many(db_name=self.metadata, collection_name=self.associate_pp_device_collection,
                          query_json=query)
            logger.debug("DEL" + str(query))
            logger.debug(associate_pp_device_delete)
            for associate_pp_device in associate_pp_device_delete:
                del associate_pp_device["_id"]
                pipeline_details = ConnectionObj.mongo_connection_obj.find_one(
                    query={"pipeline_id": associate_pp_device["pipeline_id"]},
                    db_name=self.metadata,
                    collection_name=self.pipeline_info_collection)
                deployment_summary.append({
                    "assign_ref": [],
                    "ilens_device_name": device_name,
                    "ilens_device_id": device_id,
                    "pipeline_name": pipeline_details["pipeline_name"],
                    "pipeline_execution_type": pipeline_details.get('pipeline_type', None),
                    "pipeline_id": associate_pp_device["pipeline_id"],
                    "pipeline_category": pipeline_category,
                    "update_key": associate_pp_device.get("update_key"),
                    "deploymentStatus": "DELETED",
                    "pipeline_builder_type": pipeline_builder_type,
                    "version": {
                        "associated_version": 'N/A',
                        "latest_version": associate_pp_device["version"]
                    },
                    "pipeline_event_config": associate_pp_device,
                    "pp_associated": True})

            # Summary of all the update pipeline events
            logger.debug("Fetching pipelines to be updated")
            query = {"ilens_device_id": device_id,
                     "pipeline_category": pipeline_category,
                     "deploymentStatus": "UPDATED"}
            if pipeline_builder_type is not None:
                query.update({"pipeline_builder_type": pipeline_builder_type})
            associate_pp_device_updated = ConnectionObj.mongo_connection_obj. \
                find_many(db_name=self.metadata, collection_name=self.associate_pp_device_collection,
                          query_json=query)
            logger.debug("UPD" + str(query))
            logger.debug(associate_pp_device_updated)
            for associate_pp_device in associate_pp_device_updated:
                del associate_pp_device["_id"]
                pipeline_details = ConnectionObj.mongo_connection_obj.find_one(
                    query={"pipeline_id": associate_pp_device["pipeline_id"]},
                    db_name=self.metadata,
                    collection_name=self.pipeline_info_collection)
                deployment_summary.append({
                    "assign_ref": [],
                    "ilens_device_name": device_name,
                    "ilens_device_id": device_id,
                    "pipeline_name": pipeline_details["pipeline_name"],
                    "pipeline_execution_type": pipeline_details.get('pipeline_type', None),
                    "pipeline_id": associate_pp_device["pipeline_id"],
                    "pipeline_category": pipeline_category,
                    "update_key": associate_pp_device.get("update_key"),
                    "deploymentStatus": "UPDATED",
                    "pipeline_builder_type": pipeline_builder_type,
                    "version": {
                        "associated_version": associate_pp_device["version"],
                        "latest_version": associate_pp_device["old_version"]
                    },
                    "pipeline_event_config": associate_pp_device,
                    "pp_associated": True})

            # Summary of all the new pipeline deployments
            logger.debug("Fetching pipelines to be associated")
            query = {"ilens_device_id": device_id,
                     "pipeline_category": pipeline_category,
                     "deploymentStatus": "ASSOCIATED"}
            if pipeline_builder_type is not None:
                query.update({"pipeline_builder_type": pipeline_builder_type})
            associate_pp_device_add = ConnectionObj.mongo_connection_obj. \
                find_many(db_name=self.metadata, collection_name=self.associate_pp_device_collection,
                          query_json=query)
            logger.debug("ADD" + str(query))
            logger.debug(associate_pp_device_add)
            for associate_pp_device in associate_pp_device_add:
                del associate_pp_device["_id"]
                pipeline_details = ConnectionObj.mongo_connection_obj.find_one(
                    query={"pipeline_id": associate_pp_device["pipeline_id"]},
                    db_name=self.metadata,
                    collection_name=self.pipeline_info_collection)
                deployment_summary.append({
                    "assign_ref": [],
                    "ilens_device_name": device_name,
                    "ilens_device_id": device_id,
                    "pipeline_name": pipeline_details["pipeline_name"],
                    "pipeline_execution_type": pipeline_details.get('pipeline_type', None),
                    "pipeline_id": associate_pp_device["pipeline_id"],
                    "pipeline_category": pipeline_category,
                    "update_key": associate_pp_device.get("update_key"),
                    "deploymentStatus": "ASSOCIATED",
                    "pipeline_builder_type": pipeline_builder_type,
                    "version": {
                        "associated_version": associate_pp_device["version"],
                        "latest_version": "N/A"
                    },
                    "pipeline_event_config": associate_pp_device,
                    "pp_associated": True})
            logger.debug(">>>" + str(deployment_summary))
            return deployment_summary
        except Exception as e:
            logger.error("Server faced a problem when fetching the summary of events fot device {}: {}".
                         format(input_data["ilens_device_id"], str(e)))
            return False

    def update_associate_pp_after_event(self, associate_pp_event):
        try:
            if associate_pp_event["deploymentStatus"] == "DELETED":
                ConnectionObj.mongo_connection_obj.delete_one_record(
                    query_json={"ilens_device_id": associate_pp_event["ilens_device_id"],
                                "pipeline_category": associate_pp_event["pipeline_category"],
                                "deploymentStatus": associate_pp_event["deploymentStatus"]},
                    db_name=self.metadata,
                    collection_name=self.associate_pp_device_collection
                )
            elif associate_pp_event["deploymentStatus"] == "ASSOCIATED" or \
                    associate_pp_event["deploymentStatus"] == "UPDATED":
                ConnectionObj.mongo_connection_obj.update_one(
                    query={"ilens_device_id": associate_pp_event["ilens_device_id"],
                           "pipeline_category": associate_pp_event["pipeline_category"],
                           "deploymentStatus": associate_pp_event["deploymentStatus"]},
                    set_json={"deploymentStatus": "DEPLOYED"},
                    db_name=self.metadata,
                    collection_name=self.associate_pp_device_collection)
            else:
                raise Exception("Unknown deployment status to handle")
        except Exception as e:
            logger.error("Server was unable to handle the associated pipeline record after event push: {}".
                         format(str(e)))
            raise Exception("Server was unable to handle the associated pipeline record after event push: {}".
                            format(str(e)))

    @staticmethod
    def validate_pipeline(input_json):
        try:
            device_id = list(input_json.get("bodyContent").keys())[0]
            pipelines = input_json.get("bodyContent").get(
                device_id).get("pipelines")
            pipeline_info = list()
            for each_pipeline in pipelines:
                if each_pipeline.get("pipeline_id") not in pipeline_info:
                    pipeline_info.append(each_pipeline.get("pipeline_id"))
                else:
                    logger.error(
                        "associate pipeline has failed, trying to associate same pipelines")
                    return False
            return True
        except Exception as e:
            logger.error(
                "Server faced a problem when associating docker pipelines with device --> {}".format(str(e)))
            return False

    @staticmethod
    def validate_mandatory_fields(input_data):
        try:
            components = input_data.get(
                "flow_data").get("pipeline").get("nodes")
            for each_component in components.values():
                component_name = each_component.get("node_form_data")
                if component_name:
                    mandatory_elements = component_name.get("headerContent")[
                        0].get("data")
                    for each_key in mandatory_elements:
                        key = each_key.get("key")
                        if each_key.get("required"):
                            value = each_key.get(key, False)
                            if not value or value == "":
                                node_name = each_component.get("node_name")
                                label = each_key.get("label")
                                return False, label, node_name
            return True, None, None
        except Exception as e:
            traceback.print_exc()
            logger.exception(
                "exception while creating the build for pipeline:" + str(e))
            return {"status": "failed", "message": "Failed to initiate pipeline build"}

    def get_pipeline_category(self, input_data):
        """
        This method is to fetch the pipeline types
        """
        response = {'status': 'failed',
                    'message': "Failed to list pipeline categories", "data": {}}
        try:
            logger.info("Get pipeline categories")

            if input_data['type'].lower() == 'device_manager':
                query = {"type": "pipeline_type_device_manager"}

            elif input_data['type'].lower() == 'pipeline':
                query = {"type": "pipeline_type"}

            else:
                raise Exception(
                    "Incorrect type : {}".format(input_data['type']))

            pipeline_options_data = ConnectionObj.mongo_connection_obj. \
                find_one(db_name=self.metadata, collection_name=DBMapping.constants, query=query,
                         search_json={"_id": 0})
            response["data"] = pipeline_options_data["data"]
            response['status'] = 'success'
            response['message'] = 'Successfully listed pipeline categories'
        except Exception as e:
            logger.error(
                "Error occurred while fetching pipeline categories : {}".format(str(e)))
        return response

    def create_pipeline_options(self):
        final_json = {"status": "failed", "data": {}}
        try:
            logger.debug("Inside the create pipeline options definition")
            query = {"type": "pipeline_create"}
            pipeline_options_data = ConnectionObj.mongo_connection_obj. \
                find_one(db_name=self.metadata, collection_name=DBMapping.constants, query=query,
                         search_json={"_id": 0})
            final_json["data"] = pipeline_options_data["data"]
            final_json["status"] = "success"
        except Exception as e:
            logger.exception(
                "Exception while fetching pipeline options->%s" % str(e))
        return final_json

    def save_overrided_config(self, pipeline, pipeline_category, user_associated_pipeline,
                              device_info, device_id, deployment_type, associated_pipeline, user_input,
                              override_config, pipeline_override_config):
        logger.debug(f'entered save_config {user_input}')
        cc = list()
        for each in deepcopy(user_input):
            logger.debug(f'user_ put {user_input}')
            cc.append(each.get("component_config"))

        pipeline_info = self. \
            fetch_pipeline_details(pipeline_id=pipeline,
                                   pipeline_category=pipeline_category,
                                   pipeline_instance_id=user_associated_pipeline[
                                       "pipeline_instance_id"], device_info=device_info,
                                   override_config=override_config,
                                   pipeline_override_config=pipeline_override_config)
        pipeline_info["old_version"] = associated_pipeline["version"]
        pipeline_info["ilens_device_id"] = device_id
        pipeline_info["deploymentType"] = deployment_type
        pipeline_info["cpu"] = user_associated_pipeline.get("cpu")
        pipeline_info["memory"] = user_associated_pipeline.get("memory")
        pipeline_info["run_time"] = user_associated_pipeline.get("run_time")
        pipeline_info["gpu"] = user_associated_pipeline.get("gpu")

        get_event_consumer = pipeline_info["config"].get("event_consumer")

        # for each_component in user_associated_pipeline["component_config"]:
        #     if each_component == get_event_consumer:
        #         for each_cc in cc:
        #             if each_cc.get(each_component):
        #                 event_consumer_config = each_cc.get(each_component)
        #                 pipeline_info["config"]["pipe_blocks"]["EventConsumer"][0] = event_consumer_config
        #     else:
        #         for each_otopic in pipeline_info["config"]["topic_mapping"][each_component]["output_topic"]:
        #             x = each_otopic
        #         if pipeline_info["config"]["pipe_blocks"].get("Writer"):
        #             for each_writer in pipeline_info["config"]["pipe_blocks"]["Writer"]:
        #                 y = each_writer.get("topic")
        #             if x == y:
        #                 for each_cc in cc:
        #                     if each_cc.get(each_component):
        #                         writer_config = each_cc.get(each_component)
        #                         pipeline_info["config"]["pipe_blocks"]["Writer"][0] = writer_config

        pipeline_info["associated_at"] = datetime.now().strftime(
            AppTimeFormats.USER_META_TIME_FORMAT)
        pipeline_info["deploymentStatus"] = "UPDATED"
        ConnectionObj.mongo_connection_obj. \
            update_one(query={"ilens_device_id": device_id,
                              "pipeline_id": pipeline,
                              "pipeline_category": pipeline_category},
                       db_name=DBMapping.mongo_db_name,
                       collection_name=MongoMetadata.associate_pp_device,
                       set_json=pipeline_info)
