From edfa48ddcfe22e66f120f424ed0b462b6478dea4 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Mon, 26 Oct 2020 16:35:57 +0100 Subject: [PATCH] initial commit ALL WORK IN PROGRESS --- .gitignore | 14 + README.md | 3 + module/__init__.py | 2 + module/common/__init__.py | 3 + module/common/cli_parser.py | 47 +++ module/common/configuration.py | 98 +++++ module/common/logging.py | 83 ++++ module/common/misc.py | 103 +++++ module/common/support.py | 34 ++ module/netbox/__init__.py | 0 module/netbox/connection.py | 365 ++++++++++++++++ module/netbox/inventory.py | 241 +++++++++++ module/netbox/object_classes.py | 624 ++++++++++++++++++++++++++++ module/sources/__init__.py | 67 +++ module/sources/vmware/connection.py | 578 ++++++++++++++++++++++++++ netbox-sync.py | 140 +++++++ requirements.txt | 13 + settings-example.ini | 62 +++ 18 files changed, 2477 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 module/__init__.py create mode 100644 module/common/__init__.py create mode 100644 module/common/cli_parser.py create mode 100644 module/common/configuration.py create mode 100644 module/common/logging.py create mode 100644 module/common/misc.py create mode 100644 module/common/support.py create mode 100644 module/netbox/__init__.py create mode 100644 module/netbox/connection.py create mode 100644 module/netbox/inventory.py create mode 100644 module/netbox/object_classes.py create mode 100644 module/sources/__init__.py create mode 100644 module/sources/vmware/connection.py create mode 100755 netbox-sync.py create mode 100644 requirements.txt create mode 100644 settings-example.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6c644c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# Exclude all hidden files +.* + +# Except those related to git +!.git* + +# python cache +__pycache__ + +# log dir +log/* + +# settings file +settings.ini diff --git a/README.md b/README.md new file mode 100644 index 0000000..c7b82d7 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +This Scrypt syncs data from various sources to NetBox + +WIP \ No newline at end of file diff --git a/module/__init__.py b/module/__init__.py new file mode 100644 index 0000000..fbb2930 --- /dev/null +++ b/module/__init__.py @@ -0,0 +1,2 @@ + +plural = lambda x: "s" if x != 1 else "" \ No newline at end of file diff --git a/module/common/__init__.py b/module/common/__init__.py new file mode 100644 index 0000000..b28b04f --- /dev/null +++ b/module/common/__init__.py @@ -0,0 +1,3 @@ + + + diff --git a/module/common/cli_parser.py b/module/common/cli_parser.py new file mode 100644 index 0000000..b28b147 --- /dev/null +++ b/module/common/cli_parser.py @@ -0,0 +1,47 @@ + +import os +from os.path import realpath + +from argparse import ArgumentParser, RawDescriptionHelpFormatter + +from module.common.logging import valid_log_levels + + +def parse_command_line(version=None, + self_description=None, + version_date=None, + default_config_file_path=None, + ): + """parse command line arguments + Also add current version and version date to description + """ + + # define command line options + description = f"{self_description}\nVersion: {version} ({version_date})" + + parser = ArgumentParser( + description=description, + formatter_class=RawDescriptionHelpFormatter) + + parser.add_argument("-c", "--config", default=default_config_file_path, dest="config_file", + help="points to the config file to read config data from " + + "which is not installed under the default path '" + + default_config_file_path + "'", + metavar="settings.ini") + + parser.add_argument("-l", "--log_level", choices=valid_log_levels, dest="log_level", + help="set log level (overrides config)") + + parser.add_argument("-p", "--purge", action="store_true", + help="Remove (almost) all synced objects which were create by this script. " + "This is helpful if you want to start fresh or stop using this script.") + + args = parser.parse_args() + + # fix supplied config file path + if args.config_file != default_config_file_path and args.config_file[0] != "/": + args.config_file = realpath(os.getcwd() + "/" + args.config_file) + + return args + +# EOF diff --git a/module/common/configuration.py b/module/common/configuration.py new file mode 100644 index 0000000..d8d67c7 --- /dev/null +++ b/module/common/configuration.py @@ -0,0 +1,98 @@ + + +import configparser +from os.path import realpath + +from module.common.misc import grab, do_error_exit +from module.common.logging import get_logger + +log = get_logger() + + +def get_config_file(config_file): + + if config_file is None or config_file == "": + do_error_exit("ERROR: Config file not defined.") + + base_dir = "/".join(__file__.split("/")[0:-3]) + if config_file[0] != "/": + config_file = f"{base_dir}/{config_file}" + + return realpath(config_file) + + +def open_config_file(config_file): + + if config_file is None or config_file == "": + do_error_exit("ERROR: Config file not defined.") + + # setup config parser and read config + config_handler = configparser.ConfigParser(strict=True, allow_no_value=True) + + # noinspection PyBroadException + try: + config_handler.read_file(open(config_file)) + except configparser.Error as e: + do_error_exit(f"ERROR: Problem while config file parsing: {e}") + # noinspection PyBroadException + except Exception: + do_error_exit(f"ERROR: Unable to open file '{config_file}'") + + return config_handler + + +def get_config(config_handler=None, section=None, valid_settings=None): + """parsing and basic validation of own config file + Parameters + ---------- + args : ArgumentParser object + default_log_level: str + default log level if log level is not set in config + Returns + ------- + dict + a dictionary with all config options parsed from the config file + """ + + def get_config_option(section, item, default=None): + + if isinstance(default, bool): + value = config_handler.getboolean(section, item, fallback=default) + else: + value = config_handler.get(section, item, fallback=default) + + if value == "": + value = None + + config_dict[item] = value + + # take care of logging sensitive data + for sensitive_item in ["token", "pass"]: + + if sensitive_item.lower() in item.lower(): + value = value[0:3] + "***" + + log.debug(f"Config: {section}.{item} = {value}") + + + config_dict = {} + + config_error = False + + if valid_settings is None: + log.error("No valid settings passed to config parser!") + + # read specified section section + if section is not None: + if section not in config_handler.sections(): + log.error("Section '{section}' not found in config_file") + config_error = True + else: + for config_item, default_value in valid_settings.items(): + get_config_option(section, config_item, default=default_value) + + + return config_dict + +# EOF + diff --git a/module/common/logging.py b/module/common/logging.py new file mode 100644 index 0000000..16ef8eb --- /dev/null +++ b/module/common/logging.py @@ -0,0 +1,83 @@ + +import logging +from logging.handlers import RotatingFileHandler + +from module.common.misc import do_error_exit + + +# define DEBUG2 and DEBUG3 log levels +DEBUG2 = 6 # extended messages + +# define valid log levels +valid_log_levels = [ "DEBUG2", "DEBUG", "INFO", "WARNING", "ERROR"] + +# add log level DEBUG2 +logging.addLevelName(DEBUG2, "DEBUG2") +def debug2(self, message, *args, **kws): + if self.isEnabledFor(DEBUG2): + # Yes, logger takes its '*args' as 'args'. + self._log(DEBUG2, message, args, **kws) +logging.Logger.debug2 = debug2 + + +def get_logger(): + + return logging.getLogger("Netbox-Sync") + +def setup_logging(log_level=None, log_file=None): + """Setup logging + + Parameters + ---------- + args : ArgumentParser object + + default_log_level: str + default log level if args.log_level is not set + + """ + + if log_level is None or log_level == "": + do_error_exit("ERROR: log level undefined or empty. Check config please.") + + # check set log level against self defined log level array + if not log_level.upper() in valid_log_levels: + do_error_exit(f"ERROR: Invalid log level: {log_level}") + + # check the provided log level + if log_level == "DEBUG2": + numeric_log_level = DEBUG2 + else: + numeric_log_level = getattr(logging, log_level.upper(), None) + + log_format = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s') + + # create logger instance + logger = get_logger() + + logger.setLevel(numeric_log_level) + + # setup stream handler + log_stream = logging.StreamHandler() + log_stream.setFormatter(log_format) + logger.addHandler(log_stream) + + # setup log file handler + if log_file is not None: + # base directory is three levels up + base_dir = "/".join(__file__.split("/")[0:-3]) + if log_file[0] != "/": + log_file = f"{base_dir}/{log_file}" + + try: + log_file_handler = RotatingFileHandler( + filename=log_file, + maxBytes=10 * 1024 * 1024, # Bytes to Megabytes + backupCount=5 + ) + except Exception as e: + do_error_exit(f"ERROR: Problems setting up log file: {e}") + + log_file_handler.setFormatter(log_format) + logger.addHandler(log_file_handler) + + return logger diff --git a/module/common/misc.py b/module/common/misc.py new file mode 100644 index 0000000..13f3cae --- /dev/null +++ b/module/common/misc.py @@ -0,0 +1,103 @@ + +import sys + + +def grab(structure=None, path=None, separator=".", fallback=None): + """ + get data from a complex object/json structure with a + "." separated path information. If a part of a path + is not not present then this function returns the + value of fallback (default: "None"). + + example structure: + data_structure = { + "rows": [{ + "elements": [{ + "distance": { + "text": "94.6 mi", + "value": 152193 + }, + "status": "OK" + }] + }] + } + example path: + "rows.0.elements.0.distance.value" + example return value: + 15193 + + Parameters + ---------- + structure: dict, list, object + object structure to extract data from + path: str + nested path to extract + separator: str + path separator to use. Helpful if a path element + contains the default (.) separator. + fallback: dict, list, str, int + data to return if no match was found. + + Returns + ------- + str, dict, list + the desired path element if found, otherwise None + """ + + max_recursion_level = 100 + + current_level = 0 + levels = len(path.split(separator)) + + if structure is None or path is None: + return fallback + + # noinspection PyBroadException + def traverse(r_structure, r_path): + nonlocal current_level + current_level += 1 + + if current_level > max_recursion_level: + return fallback + + for attribute in r_path.split(separator): + if isinstance(r_structure, dict): + r_structure = {k.lower(): v for k, v in r_structure.items()} + + try: + if isinstance(r_structure, list): + data = r_structure[int(attribute)] + elif isinstance(r_structure, dict): + data = r_structure.get(attribute.lower()) + else: + data = getattr(r_structure, attribute) + + except Exception: + return fallback + + if current_level == levels: + return data if data is not None else fallback + else: + return traverse(data, separator.join(r_path.split(separator)[1:])) + + return traverse(structure, path) + + +def dump(obj): + for attr in dir(obj): + if hasattr( obj, attr ): + print( "obj.%s = %s" % (attr, getattr(obj, attr))) + + +def do_error_exit(log_text): + """log an error and exit with return code 1 + Parameters + ---------- + log_text : str + the text to log as error + """ + + print(log_text, file=sys.stderr) + exit(1) + +# EOF diff --git a/module/common/support.py b/module/common/support.py new file mode 100644 index 0000000..d26f016 --- /dev/null +++ b/module/common/support.py @@ -0,0 +1,34 @@ + +from ipaddress import ip_network, ip_interface +import aiodns +import logging + + + +def format_ip(ip_addr): + """ + Formats IPv4 addresses and subnet to IP with CIDR standard notation. + + :param ip_addr: IP address with subnet; example `192.168.0.0/255.255.255.0` + :type ip_addr: str + :return: IP address with CIDR notation; example `192.168.0.0/24` + :rtype: str + """ + try: + return ip_interface(ip_addr).compressed + except Exception: + return None + +def normalize_mac_address(mac_address=None): + + if mac_address is None: + return None + + mac_address = mac_address.upper() + + # add colons to interface address + if ":" not in mac_address: + mac_address = ':'.join(mac_address[i:i+2] for i in range(0,len(mac_address),2)) + + return mac_address + diff --git a/module/netbox/__init__.py b/module/netbox/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/module/netbox/connection.py b/module/netbox/connection.py new file mode 100644 index 0000000..0646d43 --- /dev/null +++ b/module/netbox/connection.py @@ -0,0 +1,365 @@ + +import requests +import json +import logging + +import pickle + +from packaging import version + +import pprint + +from module import plural +from module.common.misc import grab, do_error_exit, dump +from module.netbox.object_classes import * +from module.common.logging import get_logger + +log = get_logger() + +# ToDo: +# * compare version +# * reset debugging level for requests + +class NetBoxHandler: + """ + Handles NetBox connection state and interaction with API + + + """ + + # connection defaults + verify_tls = True + timeout = 30 + max_retry_attempts = 4 + + default_netbox_result_limit = 200 + + minimum_api_version = "2.9" + + settings = { + "api_token": None, + "host_fqdn": None, + "port": None, + "disable_tls": False, + "validate_tls_certs": True, + "prune_enabled": False, + "prune_delay_in_days": 30 + } + + primary_tag = "NetBox-synced" + orphaned_tag = f"{primary_tag}: Orphaned" + + inventory = None + + instance_tags = None + instance_interfaces = {} + instance_virtual_interfaces = {} + + def __init__(self, cli_args=None, settings=None, inventory=None): + + self.settings = settings + self.inventory = inventory + + # set primary tag + setattr(self.inventory, "primary_tag", self.primary_tag) + + proto = "https" + if bool(settings.get("disable_tls", False)) is True: + proto = "http" + + port = "" + if settings.get("port", None) is not None: + port = ":{}".format(settings.get("port")) + + self.url = f"{proto}://%s{port}/api/" % settings.get("host_fqdn") + + self.verify_tls = bool(self.settings.get("validate_tls_certs")) + + self.session = self.create_session() + + # check for minimum version + if version.parse(self.get_api_version()) < version.parse(self.minimum_api_version): + do_error_exit(f"Netbox API version '{self.api_version}' not supported. " + f"Minimum API version: {self.minimum_api_version}") + + def create_session(self): + """ + Creates a session with NetBox + + :return: `True` if session created else `False` + :rtype: bool + """ + header = {"Authorization": "Token {}".format(self.settings.get("api_token"))} + + session = requests.Session() + session.headers.update(header) + + log.debug("Created new HTTP Session for NetBox.") + return session + + def get_api_version(self): + """ + Determines the current NetBox API Version + + :return: NetBox API version + :rtype: float + """ + response = None + try: + response = self.session.get( + self.url, + timeout=self.timeout, + verify=self.verify_tls) + except Exception as e: + do_error_exit(str(e)) + + result = str(response.headers["API-Version"]) + + log.debug(f"Detected NetBox API v{result}.") + + return result + + def request(self, object_class, req_type="GET", data=None, params=None, nb_id=None): + + + result = None + + request_url = f"{self.url}{object_class.api_path}/" + + # append NetBox ID + if nb_id is not None: + request_url += f"{nb_id}/" + + if params is None: + params = dict() + + params["limit"] = self.default_netbox_result_limit + + # prepare request + this_request = self.session.prepare_request( + requests.Request(req_type, request_url, params=params, json=data) + ) + + # issue request + response = self.single_request(this_request) + + log.debug2("Received HTTP Status %s.", response.status_code) + + try: + result = response.json() + except json.decoder.JSONDecodeError: + pass + + if response.status_code == 200: + + # retrieve paginated results + #""" pagination disabled + if this_request.method == "GET" and result is not None: + while response.json().get("next") is not None: + this_request.url = response.json().get("next") + log.debug2("NetBox results are paginated. Getting next page") + + response = self.single_request(this_request) + result["results"].extend(response.json().get("results")) + #""" + elif response.status_code in [201, 204]: + + action = "created" if response.status_code == 201 else "deleted" + + # ToDo: + # * switch to object naming schema based on primary keys + log.info( + f"NetBox successfully {action} {object_class.name} object '%s'." % (result.get(object_class.primary_key)) + ) + + # token issues + elif response.status_code == 403: + + do_error_exit("NetBox returned: %s: %s" % (response.reason, grab(result, "detail"))) + + # we screw up something else + elif response.status_code >= 400 and response.status_code < 500: + + log.error(f"NetBox returned: {this_request.method} {this_request.path_url} {response.reason}") + log.debug(f"NetBox returned body: {result}") + result = None + + elif response.status_code >= 500: + + do_error_exit(f"NetBox returned: {response.status_code} {response.reason}") + + return result + + def single_request(self, this_request): + + req = None + + for _ in range(self.max_retry_attempts): + + log_message = f"Sending {this_request.method} to '{this_request.url}'" + + if this_request.body is not None: + log_message += f" with data '{this_request.body}'." + + log.debug2(log_message) + + try: + req = self.session.send(this_request, + timeout=self.timeout, verify=self.verify_tls) + + except (ConnectionError, requests.exceptions.ConnectionError, + requests.exceptions.ReadTimeout): + log.warning(f"Request failed, trying again: {log_message}") + continue + else: + break + else: + do_error_exit(f"Giving up after {self.max_retry_attempts} retries.") + + return req + + def query_current_data(self, netbox_objects_to_query=None): + + if netbox_objects_to_query is None: + raise AttributeError(f"Argument netbox_objects_to_query is: '{netbox_objects_to_query}'") + + # query all dependencies + for nb_object_class in netbox_objects_to_query: + + if nb_object_class not in NetBoxObject.__subclasses__(): + raise AttributeError(f"Class '{nb_object_class.__name__}' must be a subclass of '{NetBoxObject.__name__}'") + + cached_nb_data = None + try: + cached_nb_data = pickle.load( open( f"cache/{nb_object_class.__name__}.cache", "rb" ) ) + #pprint.pprint(cached_nb_data) + except Exception: + pass + + nb_data = dict() + if cached_nb_data is None: + # get all objects of this class + log.debug(f"Requesting {nb_object_class.name}s from NetBox") + nb_data = self.request(nb_object_class) + + pickle.dump(nb_data.get("results"), open( f"cache/{nb_object_class.__name__}.cache", "wb" ) ) + else: + nb_data["results"] = cached_nb_data + + + if nb_data.get("results") is None: + log.warning(f"Result data from NetBox for object {nb_object_class.__name__} missing!") + continue + + log.debug(f"Processing %s returned {nb_object_class.name}%s" % (len(nb_data.get("results")),plural(len(nb_data.get("results"))))) + + for object_data in nb_data.get("results"): + self.inventory.add_item_from_netbox(nb_object_class, data=object_data) + + return + + def inizialize_basic_data(self): + + log.debug("Checking/Adding NetBox Sync dependencies") + + self.inventory.add_update_object(NBTags, data = { + "name": self.orphaned_tag, + "color": "607d8b", + "description": "The source which has previously " + "provided the object no longer " + "states it exists.{}".format( + " An object with the 'Orphaned' tag will " + "remain in this state until it ages out " + "and is automatically removed." + ) if bool(self.settings.get("prune_enabled", False)) else "" + }) + + self.inventory.add_update_object(NBTags, data = { + "name": self.primary_tag, + "description": "Created and used by NetBox Sync Script " + "to keep track of created items." + }) + + def update_object(self, nb_object_sub_class): + + for object in self.inventory.get_all_items(nb_object_sub_class): + + # resolve dependencies + for dependency in object.get_dependencies(): + if dependency not in self.inventory.resolved_dependencies: + log.debug2("Resolving dependency: %s" % (dependency.name)) + self.update_object(dependency) + + + returned_object_data = None + + patch_issue = False + data_to_patch = dict() + + if object.is_new is True: + object.updated_items = object.data.keys() + + for key, value in object.data.items(): + if key in object.updated_items: + + object_type = object.data_model.get(key) + + if object_type == NBTags: + data_to_patch[key] = [{"name": d.get_display_name()} for d in value] + + elif object_type in NetBoxObject.__subclasses__(): + data_to_patch[key] = value.get_nb_reference() + + if value.nb_id == 0: + log.error(f"Unable to find a NetBox reference to {value.name} '{value.get_display_name()}'. Might be a dependency issue.") + patch_issue = True + + else: + data_to_patch[key] = value + + if patch_issue == True: + continue + + issued_request = False + if object.is_new is True: + log.info("Creating new NetBox '%s' object: %s" % (object.name, object.get_display_name())) + + returned_object_data = self.request(nb_object_sub_class, req_type="POST", data=data_to_patch) + + issued_request = True + + if object.is_new is False and len(object.updated_items) > 0: + + log.info("Updating NetBox '%s' object '%s' with data: %s" % (object.name, object.get_display_name(), data_to_patch)) + + returned_object_data = self.request(nb_object_sub_class, req_type="PATCH", data=data_to_patch, nb_id=object.nb_id) + + issued_request = True + + if returned_object_data is not None: + + object.update(data = returned_object_data, read_from_netbox=True) + + elif issued_request is True: + log.error(f"Request Failed for {nb_object_sub_class.name}. Used data: {data_to_patch}") + pprint.pprint(object.to_dict()) + + # add class to resolved dependencies + self.inventory.resolved_dependencies = list(set(self.inventory.resolved_dependencies + [nb_object_sub_class] )) + + def update_instance(self): + + log.info("Updating changed data in NetBox") + + # update all items in NetBox accordingly + for nb_object_sub_class in NetBoxObject.__subclasses__(): + self.update_object(nb_object_sub_class) + + # prune objects + #self.prune_instance() + + + #print(self.inventory) + + + return diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py new file mode 100644 index 0000000..72912ee --- /dev/null +++ b/module/netbox/inventory.py @@ -0,0 +1,241 @@ + +import logging +import json + +import pprint + +from module.netbox.object_classes import * +from module.common.logging import get_logger + +log = get_logger() + +class NetBoxInventorySearchResult: + members = list() + +class NetBoxInventory: + + base_structure = dict() + resolved_dependencies = list() + + primary_tag = None + + def __init__(self): + for object_type in NetBoxObject.__subclasses__(): + + self.base_structure[object_type.name] = list() + + + def get_by_id(self, object_type, id=None): + + if object_type not in NetBoxObject.__subclasses__(): + raise AttributeError("'%s' object must be a sub class of '%s'." % + (object_type.__name__, NetBoxObject.__name__)) + + if id is None or self.base_structure[object_type.name] is None: + return None + + for object in self.base_structure[object_type.name]: + + if object.nb_id == id: + return object + + + def get_by_data(self, object_type, data=None): + + if object_type not in NetBoxObject.__subclasses__(): + raise AttributeError("'%s' object must be a sub class of '%s'." % + (object_type.__name__, NetBoxObject.__name__)) + + + if data is None: + return None + + if self.base_structure[object_type.name] is None: + return None + + if not isinstance(data, dict): + # ToDo: + # * proper handling + log.error("data is not dict") + pprint.pprint(data) + exit(0) + + # shortcut if data contains valid id + data_id = data.get("id") + if data_id is not None and data_id != 0: + return self.get_by_id(object_type, id=data_id) + + # try to find by name + if data.get(object_type.primary_key) is not None: + object_name_to_find = None + results = list() + for object in self.base_structure[object_type.name]: + + # Todo: + # * try to compare second key if present. + + if object_name_to_find is None: + object_name_to_find = object.get_display_name(data) + #print(f"get_by_data(): Object Display Name: {object_name_to_find}") + + if object_name_to_find == object.get_display_name(): + results.append(object) + + # found exactly one match + # ToDo: + # * add force secondary key if one object has a secondary key + + if len(results) == 1: + #print(f"found exact match: {object_name_to_find}") + return results[0] + + # compare secondary key + elif len(results) > 1: + + object_name_to_find = None + for object in results: + + if object_name_to_find is None: + object_name_to_find = object.get_display_name(data, including_second_key=True) + #print(f"get_by_data(): Object Display Name: {object_name_to_find}") + + if object_name_to_find == object.get_display_name(including_second_key=True): + return object + + # try to match all data attributes + else: + + for object in self.base_structure[object_type.name]: + all_items_match = True + for attr_name, attr_value in data.items(): + + if object.data.get(attr_name) != attr_value: + all_items_match = False + break + + if all_items_match == True: + return object + + """ + if data.get(object_type.primary_key) is not None and \ + object.resolve_attribute(object_type.primary_key) == object.resolve_attribute(object_type.primary_key, data=data): + + # object type has a secondary key, lets check if it matches + if getattr(object_type, "secondary_key", None) is not None and data.get(object_type.secondary_key) is not None: + + if object.resolve_attribute(object_type.secondary_key) == object.resolve_attribute(object_type.secondary_key, data=data): + return_data.append(object) + + # object has no secondary key but the same name, add to list + else: + return_data.append(object) + """ + return None + + def add_item_from_netbox(self, object_type, data=None): + """ + only to be used if data is read from NetBox and added to inventory + """ + + # create new object + new_object = object_type(data, read_from_netbox=True, inventory=self) + + # add to inventory + self.base_structure[object_type.name].append(new_object) + + return + + def add_update_object(self, object_type, data=None, read_from_netbox=False, source=None): + + if data is None: + # ToDo: + # * proper error handling + log.error("NO DATA") + return + + this_object = self.get_by_data(object_type, data=data) + + if this_object is None: + this_object = object_type(data, read_from_netbox=read_from_netbox, inventory=self, source=source) + self.base_structure[object_type.name].append(this_object) + if read_from_netbox is False: + log.debug(f"Created new {this_object.name} object: {this_object.get_display_name()}") + + else: + # ToDo: + # * resolve relations if updated from netbox + this_object.update(data, read_from_netbox=read_from_netbox, source=source) + log.debug("Updated %s object: %s" % (this_object.name, this_object.get_display_name())) + + return this_object + + def resolve_relations(self): + + log.debug("Start resolving relations") + for object_type in NetBoxObject.__subclasses__(): + + for object in self.base_structure.get(object_type.name, list()): + + object.resolve_relations() + + log.debug("Finished resolving relations") + + def get_all_items(self, object_type): + + if object_type not in NetBoxObject.__subclasses__(): + raise AttributeError("'%s' object must be a sub class of '%s'." % + (object_type.__name__, NetBoxObject.__name__)) + + return self.base_structure.get(object_type.name, list()) + + + def tag_all_the_things(self, sources, netbox_handler): + + # ToDo: + # * DONE: add main tag to all objects retrieved from a source + # * Done: add source tag all objects of that source + # * check for orphaned objects + # * DONE: objects tagged by a source but not present in source anymore (add) + # * DONE: objects tagged as orphaned but are present again (remove) + + source_tags = [x.source_tag for x in sources] + + for object_type in NetBoxObject.__subclasses__(): + + if self.base_structure[object_type.name] is None: + continue + + for object in self.base_structure[object_type.name]: + + # if object was found in source + if object.source is not None: + object.add_tags([netbox_handler.primary_tag, object.source.source_tag]) + + # if object was orphaned remove tag again + if netbox_handler.orphaned_tag in object.get_tags(): + object.remove_tags(netbox_handler.orphaned_tag) + + # if object was tagged by a source in previous runs but is not present + # anymore then add the orphaned tag + else: + for source_tag in source_tags: + if source_tag in object.get_tags(): + object.add_tags(netbox_handler.orphaned_tag) + + def to_dict(self): + + output = dict() + for nb_object_class in NetBoxObject.__subclasses__(): + + output[nb_object_class.name] = list() + + for object in self.base_structure[nb_object_class.name]: + output[nb_object_class.name].append(object.to_dict()) + + return output + + def __str__(self): + + return json.dumps(self.to_dict(), sort_keys=True, indent=4) + +# EOF diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py new file mode 100644 index 0000000..3491934 --- /dev/null +++ b/module/netbox/object_classes.py @@ -0,0 +1,624 @@ + +import json +import logging + +import pprint + +from module.common.misc import grab, do_error_exit, dump +from module.common.logging import get_logger + +log = get_logger() + +class NetBoxObject(): + default_attributes = { + "data": None, + "is_new": True, + "nb_id": 0, + "updated_items": list(), + "is_present_in_source": False, + "source": None, + } + + # keep handle to inventory instance to append objects on demand + inventory = None + + def __init__(self, data=None, read_from_netbox=False, inventory=None, source=None): + + # inherit and create default attributes from parent + for attr_key, attr_value in self.default_attributes.items(): + if isinstance(attr_value, (list, dict, set)): + setattr(self, attr_key, attr_value.copy()) + else: + setattr(self, attr_key, attr_value) + + # store provided inventory handle + self.inventory = inventory + + # store source handle + if source is not None: + self.source = source + + self.update(data=data, read_from_netbox=read_from_netbox) + + def __repr__(self): + return "<%s instance '%s' at %s>" % (self.__class__.__name__, self.get_display_name(), id(self)) + + def to_dict(self): + + out = dict() + + for key in dir(self): + value = getattr(self, key) + if "__" in key: + continue + if callable(value) is True: + continue + if key in ["inventory", "default_attributes"]: + continue + if key == "data_model": + + data_model = dict() + for dkey, dvalue in value.items(): + # if value is class name then print class name + if type(dvalue) == type: + dvalue = str(dvalue) + + data_model[dkey] = dvalue + + value = data_model + + if key == "data": + + data = dict() + for dkey, dvalue in value.items(): + # if value is class name then print class name + if isinstance(dvalue, NetBoxObject): + dvalue = repr(dvalue) + + if dkey == "tags": + dvalue = [x.get_display_name() for x in dvalue] + + data[dkey] = dvalue + + value = data + + out[key] = value + + return out + + def __str__(self): + return json.dumps(self.to_dict(), sort_keys=True, indent=4) + + def __iter__(self): + for key, value in self.to_dict(): + yield (key, value) + + @staticmethod + def format_slug(text=None, max_len=50): + """ + Format string to comply to NetBox slug acceptable pattern and max length. + + :param text: Text to be formatted into an acceptable slug + :type text: str + :return: Slug of allowed characters [-a-zA-Z0-9_] with max length of 50 + :rtype: str + """ + + if text is None or len(text) == 0: + raise AttributeError("Argument 'text' can't be None or empty!") + + permitted_chars = ( + "abcdefghijklmnopqrstuvwxyz" # alphabet + "0123456789" # numbers + "_-" # symbols + ) + + # Replace separators with dash + for sep in [" ", ",", "."]: + text = text.replace(sep, "-") + + # Strip unacceptable characters + text = "".join([c for c in text.lower() if c in permitted_chars]) + + # Enforce max length + return text[0:max_len] + + def update(self, data=None, read_from_netbox=False, source=None): + + if data is None: + return + + if not isinstance(data, dict): + raise AttributeError("Argument 'data' needs to be a dict!") + + if data.get("id") is not None: + self.nb_id = data.get("id") + + if read_from_netbox is True: + self.is_new = False + self.data = data + self.updated_items = list() + + return + + if source is not None: + self.source = source + + display_name = self.get_display_name(data) + + log.debug(f"Parsing '{self.name}' data structure: {display_name}") + + parsed_data = dict() + for key, value in data.items(): + + if key not in self.data_model.keys(): + log.error(f"Found undefined data model key '{key}' for object '{self.__class__.__name__}'") + continue + + # skip unset values + if value is None: + log.info(f"Found unset/empty key: {key}") + continue + + # check data model to see how we have to parse the value + defined_value_type = self.data_model.get(key) + + # value must be a string witch a certain max length + if isinstance(defined_value_type, int): + if not isinstance(value, str): + log.error(f"Invalid data type for '{self.__class__.__name__}.{key}' (must be str), got: '{value}'") + continue + + value = value[0:defined_value_type] + + if key == "slug": + value = self.format_slug(text=value, max_len=defined_value_type) + else: + value = value[0:defined_value_type] + + if isinstance(defined_value_type, list): + + if value not in defined_value_type: + log.error(f"Invalid data type for '{key}' (must be one of {defined_value_type}), got: '{value}'") + continue + + # just check the type of the value + type_check_faild = False + for valid_type in [bool, str, int]: + + if defined_value_type == valid_type and not isinstance(value, valid_type): + log.error(f"Invalid data type for '{key}' (must be {valid_type.__name__}), got: '{value}'") + type_check_faild = True + break + + if type_check_faild is True: + continue + + # this is meant to be reference to a different object + if defined_value_type in NetBoxObject.__subclasses__(): + + # tags need to be treated as list of dictionaries + if defined_value_type == NBTags: + self.add_tags(value) + continue + + if not isinstance(value, NetBoxObject): + # try to find object. + value = self.inventory.add_update_object(defined_value_type, data=value) + + # add to parsed data dict + parsed_data[key] = value + + # add/update slug + # if data model contains a slug we need to handle it + # so far slug is always referenced to "name" + if "slug" in self.data_model.keys() and data.get("slug") is None and data.get(self.primary_key) is not None: + + parsed_data["slug"] = self.format_slug(text=parsed_data.get(self.primary_key), max_len=self.data_model.get("slug")) + + # this is a new set of data + if self.data is None: + self.data = parsed_data + + # add empty tag list if not tags were provided + if "tags" in self.data_model.keys() and data.get("tags") is None: + self.data["tags"] = list() + + # see if data just got updated and mark it as such. + else: + + for key, new_value in parsed_data.items(): + + # nothing changed, continue with next key + current_value = self.data.get(key) + if current_value == new_value: + continue + + if self.data_model.get(key) in NetBoxObject.__subclasses__(): + if isinstance(current_value, NetBoxObject): + current_value_str = current_value.get_display_name() + else: + current_value_str = str(current_value) + new_value_str = new_value.get_display_name() + + # if data model is a list then we need to read the netbox data value + elif isinstance(self.data_model.get(key), list) and isinstance(current_value, dict): + current_value_str = str(current_value.get("value")) + new_value_str = str(new_value) + + else: + current_value_str = str(current_value).replace("\r","") + new_value_str = str(new_value).replace("\r","") + + # just check again if values might match now + if current_value_str == new_value_str: + continue + + self.data[key] = new_value + self.updated_items.append(key) + + log.info(f"{self.name.capitalize()} '{display_name}' attribute '{key}' changed from '{current_value_str}' to '{new_value_str}'") + + self.resolve_relations() + + + def get_display_name(self, data=None, including_second_key=False): + + this_data_set = data + if data is None: + this_data_set = self.data + + if this_data_set is None: + return None + + my_name = this_data_set.get(self.primary_key) + + secondary_key = getattr(self, "secondary_key", None) + enforce_secondary_key = getattr(self, "enforce_secondary_key", False) + + if secondary_key is not None and (enforce_secondary_key is True or including_second_key is True): + + secondary_key_value = this_data_set.get(secondary_key) + + if isinstance(secondary_key_value, NetBoxObject): + secondary_key_value = secondary_key_value.get_display_name() + + if secondary_key_value is not None: + #import pprint + #pprint.pprint(this_data_set) + + my_name = f"{my_name} ({secondary_key_value})" + + return my_name + + + def resolve_relations(self): + + for key, value in self.data_model.items(): + + # continue if value is not an NetBox object + if value not in NetBoxObject.__subclasses__(): + continue + + data_value = self.data.get(key) + + resolved_data = None + if value == NBTags: + + resolved_tag_list = list() + for tag in data_value: + + if isinstance(tag, NetBoxObject): + tag_object = tag + else: + tag_object = self.inventory.get_by_data(value, data=tag) + + if tag_object is not None: + resolved_tag_list.append(tag_object) + + resolved_data = resolved_tag_list + else: + if data_value is None: + continue + + if isinstance(data_value, NetBoxObject): + resolved_data = data_value + else: + data_to_find = None + if isinstance(data_value, int): + data_to_find = {"id": data_value} + elif isinstance(data_value, dict): + data_to_find = data_value + + resolved_data = self.inventory.get_by_data(value, data=data_to_find) + + if resolved_data is not None: + self.data[key] = resolved_data + else: + log.error(f"Problems resolving relation '{key}' for object '%s' and value '%s'" % (self.get_display_name(), data_value)) + + def raw(self): + + return self.data + + + def get_dependencies(self): + + return [x for x in self.data_model.values() if x in NetBoxObject.__subclasses__()] + + def get_tags(self): + + return [x.get_display_name() for x in self.data.get("tags", list())] + + def update_tags(self, tags, remove=False): + + if tags is None or NBTags not in self.data_model.values(): + return + + action = "Adding" if remove is False else "Removing" + + log.debug(f"{action} Tag: {tags}") + new_tags = list() + + def extract_tags(this_tags): + if isinstance(this_tags, str): + new_tags.append(this_tags) + elif isinstance(this_tags, dict) and this_tags.get("name") is not None: + new_tags.append(this_tags.get("name")) + + if isinstance(tags, list): + for tag in tags: + extract_tags(tag) + else: + extract_tags(tags) + + + log.debug(f"Tag list: {new_tags}") + + current_tags = self.get_tags() + + for tag_name in new_tags: + + if tag_name not in current_tags and remove == False: + # add tag + + tag = self.inventory.add_update_object(NBTags, data={"name": tag_name}) + + self.data["tags"].append(tag) + if self.is_new is False: + self.updated_items.append("tags") + + if tag_name in current_tags and remove == True: + + tag = self.inventory.get_by_data(NBTags, data={"name": tag_name}) + + self.data["tags"].remove(tag) + if self.is_new is False: + self.updated_items.append("tags") + + new_tags = self.get_tags() + log.info(f"{self.name.capitalize()} '{self.get_display_name()}' attribute 'tags' changed from '{current_tags}' to '{new_tags}'") + + def add_tags(self, tags_to_add): + self.update_tags(tags_to_add) + + def remove_tags(self, tags_to_remove): + self.update_tags(tags_to_remove, remove=True) + + + def get_nb_reference(self): + """ + Default class to return reference of how this object is usually referenced. + + default: return NetBox ID + """ + + if self.nb_id == 0: + return self + + return self.nb_id + + +class NBTags(NetBoxObject): + name = "tag" + api_path = "extras/tags" + primary_key = "name" + data_model = { + "name": 100, + "slug": 100, + "color": 6, + "description": 200 + } + + +class NBManufacturers(NetBoxObject): + name = "manufacturer" + api_path = "dcim/manufacturers" + primary_key = "name" + data_model = { + "name": 50, + "slug": 50, + "description": 200 + } + + +class NBDeviceTypes(NetBoxObject): + name ="device type" + api_path = "dcim/device-types" + primary_key = "model" + data_model = { + "model": 50, + "slug": 50, + "part_number": 50, + "description": 200, + "manufacturer": NBManufacturers, + "tags": NBTags + } + +class NBPlatforms(NetBoxObject): + name = "platform" + api_path = "dcim/platforms" + primary_key = "name" + data_model = { + "name": 100, + "slug": 100, + "manufacturer": NBManufacturers, + "description": 200 + } + +class NBClusterTypes(NetBoxObject): + name = "cluster type" + api_path = "virtualization/cluster-types" + primary_key = "name" + data_model = { + "name": 50, + "slug": 50, + "description": 200 + } + +class NBClusterGroups(NetBoxObject): + name = "cluster group" + api_path = "virtualization/cluster-groups" + primary_key = "name" + data_model = { + "name": 50, + "slug": 50, + "description": 200 + } + +class NBDeviceRoles(NetBoxObject): + name = "device role" + api_path = "dcim/device-roles" + primary_key = "name" + data_model = { + "name": 50, + "slug": 50, + "color": 6, + "description": 200, + "vm_role": bool + } + +class NBSites(NetBoxObject): + name = "site" + api_path = "dcim/sites" + primary_key = "name" + data_model = { + "name": 50, + "slug": 50, + "comments": str, + "tags": NBTags + } + + +class NBClusters(NetBoxObject): + name = "cluster" + api_path = "virtualization/clusters" + primary_key = "name" + data_model = { + "name": 100, + "comments": str, + "type": NBClusterTypes, + "group": NBClusterGroups, + "site": NBSites, + "tags": NBTags + } + + +class NBDevices(NetBoxObject): + name = "device" + api_path = "dcim/devices" + primary_key = "name" + secondary_key = "site" + data_model = { + "name": 64, + "device_type": NBDeviceTypes, + "device_role": NBDeviceRoles, + "platform": NBPlatforms, + "serial": 50, + "site": NBSites, + "status": [ "offline", "active", "planned", "staged", "failed", "inventory", "decommissioning" ], + "cluster": NBClusters, + "asset_tag": 50, + "tags": NBTags + } + + +class NBVMs(NetBoxObject): + name = "virtual machine" + api_path = "virtualization/virtual-machines" + primary_key = "name" + secondary_key = "cluster" + data_model = { + "name": 64, + "status": [ "offline", "active", "planned", "staged", "failed", "decommissioning" ], + "cluster": NBClusters, + "role": NBDeviceRoles, + "platform": NBPlatforms, + "vcpus": int, + "memory": int, + "disk": int, + "comments": str, + "tags": NBTags + } + +class NBVMInterfaces(NetBoxObject): + name = "virtual machine interface" + api_path = "virtualization/interfaces" + primary_key = "name" + secondary_key = "virtual_machine" + enforce_secondary_key = True + data_model = { + "name": 64, + "virtual_machine": NBVMs, + "enabled": bool, + "mac_address": str, + "description": 200, + "tags": NBTags + } + +class NBInterfaces(NetBoxObject): + name = "interface" + api_path = "dcim/interfaces" + primary_key = "name" + secondary_key = "device" + enforce_secondary_key = True + data_model = { + "name": 64, + "device": NBDevices, + "label": 64, + "type": [ "virtual", "100base-tx", "1000base-t", "10gbase-t", "25gbase-x-sfp28", "40gbase-x-qsfpp", "other" ], + "enabled": bool, + "mac_address": str, + "mgmt_only": bool, + "mtu": int, + "description": 200, + "connection_status": bool, + "tags": NBTags + } + + +class NBIPAddresses(NetBoxObject): + name = "IP address" + api_path = "ipam/ip-addresses" + primary_key = "address" + data_model = { + "address": str, + "assigned_object_type": str, + "assigned_object_id": int, + "description": 200, + "tags": NBTags + } + + +class NBPrefixes(NetBoxObject): + name = "IP prefix" + api_path = "ipam/prefixes" + primary_key = "prefix" + data_model = { + "prefix": str, + "site": NBSites, + "description": 200, + "tags": NBTags + } + +# EOF diff --git a/module/sources/__init__.py b/module/sources/__init__.py new file mode 100644 index 0000000..dbd4e0f --- /dev/null +++ b/module/sources/__init__.py @@ -0,0 +1,67 @@ + +# define all available sources here +from .vmware.connection import VMWareHandler + +# list of valid sources +valid_sources = [ VMWareHandler ] + +############### +from module.common.configuration import get_config +from module.common.logging import get_logger + + +def instanciate_sources(config_handler=None, inventory=None): + + log = get_logger() + + if config_handler is None: + raise Exception("No config handler defined!") + + if inventory is None: + raise Exception("No inventory defined!") + + sources = list() + + # iterate over sources and validate them + for source_section in config_handler.sections(): + + # a source section needs to start with "source/" + if not source_section.startswith("source/"): + continue + + # get type of source + source_type = config_handler.get(source_section, "type", fallback=None) + + if source_type is None: + log.error(f"Source {source_section} option 'type' is undefined") + config_error = True + + source_class = None + for possible_source_class in valid_sources: + source_class_type = getattr(possible_source_class, "source_type", None) + if source_class_type is None: + raise AttributeError("'%s' class attribute 'source_type' not defined." % + (source_class_type.__name__)) + + if source_class_type == source_type: + source_class = possible_source_class + break + + if source_class is None: + log.error(f"Unknown source type '{source_type}' defined for '{source_section}'") + config_error = True + continue + + source_config = get_config(config_handler, section=source_section, valid_settings=source_class.settings) + + source_handler = source_class(name=source_section.replace("source/",""), + inventory=inventory, + config=source_config) + + # add to list of source handlers + if source_handler.init_successfull is True: + sources.append(source_handler) + + return sources + +# EOF diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py new file mode 100644 index 0000000..7b1622a --- /dev/null +++ b/module/sources/vmware/connection.py @@ -0,0 +1,578 @@ + +import atexit +from socket import gaierror +from ipaddress import ip_address, ip_network, ip_interface +import re + +import pprint + +from pyVim.connect import SmartConnectNoSSL, Disconnect +from pyVmomi import vim + +from module.netbox.object_classes import * +from module.common.misc import grab, do_error_exit, dump +from module.common.support import normalize_mac_address, format_ip +from module import plural +from module.common.logging import get_logger + +log = get_logger() + +class VMWareHandler(): + + dependend_netbox_objects = [ + NBTags, + NBManufacturers, + NBDeviceTypes, + NBPlatforms, + NBClusterTypes, + NBClusterGroups, + NBDeviceRoles, + NBSites, + NBClusters, + #] + #""" + NBDevices, + NBVMs, + NBVMInterfaces, + NBInterfaces, + NBIPAddresses, + + ] #""" + + session = None + inventory = None + + init_successfull = False + + source_type = "vmware" + + source_tag = None + site_name = None + + networks = dict() + standalone_hosts = list() + + settings = { + "host_fqdn": None, + "port": 443, + "username": None, + "password": None, + "host_exclude_filter": None, + "host_include_filter": None, + "vm_exclude_filter": None, + "vm_include_filter": None, + "netbox_host_device_role": "Server", + "netbox_vm_device_role": "Server", + "permitted_subnets": None, + "collect_hardware_asset_tag": True + } + + def __init__(self, name=None, config=None, inventory=None): + + if name is None: + raise ValueError("Invalid value for attribute 'name': '{name}'.") + + self.inventory = inventory + self.name = name + + self.parse_config(config) + + self.create_session() + + self.source_tag = f"Source: {name}" + self.site_name = f"vCenter: {name}" + + if self.session is not None: + self.init_successfull = True + + + def parse_config(self, config): + + validation_failed = False + for setting in ["host_fqdn", "port", "username", "password" ]: + if config.get(setting) is None: + log.error(f"Config option '{setting}' in 'source/{self.name}' can't be empty/undefined") + validation_failed = True + + # check permitted ip subnets + if config.get("permitted_subnets") is None: + log.info(f"Config option 'permitted_subnets' in 'source/{self.name}' is undefined. No IP addresses will be populated to Netbox!") + else: + config["permitted_subnets"] = [x.strip() for x in config.get("permitted_subnets").split(",") if x.strip() != ""] + + permitted_subnets = list() + for permitted_subnet in config["permitted_subnets"]: + try: + permitted_subnets.append(ip_network(permitted_subnet)) + except Exception as e: + log.error(f"Problem parsing permitted subnet: {e}") + validation_failed = True + + config["permitted_subnets"] = permitted_subnets + + # check include and exclude filter expressions + for setting in [x for x in config.keys() if "filter" in x]: + if config.get(setting) is None or config.get(setting).strip() == "": + continue + + re_compiled = None + try: + re_compiled = re.compile(config.get(setting)) + except Exception as e: + log.error(f"Problem parsing parsing regular expression for '{setting}': {e}") + validation_failed = True + + config[setting] = re_compiled + + if validation_failed is True: + do_error_exit("Config validation failed. Exit!") + + for setting in self.settings.keys(): + setattr(self, setting, config.get(setting)) + + def create_session(self): + + if self.session is not None: + return True + + log.info(f"Starting vCenter connection to '{self.host_fqdn}'") + + try: + instance = SmartConnectNoSSL( + host=self.host_fqdn, + port=self.port, + user=self.username, + pwd=self.password + ) + atexit.register(Disconnect, instance) + self.session = instance.RetrieveContent() + + except (gaierror, vim.fault.InvalidLogin, OSError) as e: + + log.error( + f"Unable to connect to vCenter instance '{self.host_fqdn}' on port {self.port}. " + f"Reason: {e}" + ) + + return False + + log.info(f"Successfully connected to vCenter '{self.host_fqdn}'") + + return True + + def apply(self): + + self.inizialize_basic_data() + + log.info(f"Query data from vCenter: '{self.host_fqdn}'") + + # Mapping of object type keywords to view types and handlers + object_mapping = { + "datacenter": { + "view_type": vim.Datacenter, + "view_handler": self.add_datacenter + }, + "cluster": { + "view_type": vim.ClusterComputeResource, + "view_handler": self.add_cluster + }, + "network": { + "view_type": vim.Network, + "view_handler": self.add_network + }, + "host": { + "view_type": vim.HostSystem, + "view_handler": self.add_host + }, + "virtual machine": { + "view_type": vim.VirtualMachine, + "view_handler": self.add_virtual_machine + } + } + + for view_name, view_details in object_mapping.items(): + + if self.session is None: + log.info("No existing vCenter session found.") + self.create_session() + + view_data = { + "container": self.session.rootFolder, + "type": [view_details.get("view_type")], + "recursive": True + } + + try: + container_view = self.session.viewManager.CreateContainerView(**view_data) + except Exception as e: + log.error(f"Problem creating vCenter view for '{view_name}s': {e}") + continue + + view_objects = grab(container_view, "view") + + if view_objects is None: + log.error(f"Creating vCenter view for '{view_name}s' failed!") + continue + + log.info("vCenter returned '%d' %s%s" % (len(view_objects), view_name, plural(len(view_objects)))) + + for obj in view_objects: + + view_details.get("view_handler")(obj) + + container_view.Destroy() + + def add_datacenter(self, obj): + + if grab(obj, "name") is None: + return + + self.inventory.add_update_object(NBClusterGroups, + data = { "name": obj.name }, source=self) + + def add_cluster(self, obj): + + if grab(obj, "name") is None or grab(obj, "parent.parent.name") is None: + return + + self.inventory.add_update_object(NBClusters, + data = { + "name": obj.name, + "type": { "name": "VMware ESXi" }, + "group": { "name": obj.parent.parent.name } + }, + source=self) + + def add_network(self, obj): + + if grab(obj, "key") is None or grab(obj, "name") is None: + return + + self.networks[obj.key] = obj.name + + def add_host(self, obj): + + # ToDo: + # * find Host based on device mac addresses + + name = grab(obj, "name") + + # parse data + log.debug2(f"Parsing vCenter host: {name}") + + # filter hosts + # first includes + if self.host_include_filter is not None: + if not self.host_include_filter.match(name): + log.debug(f"Host '{name}' did not match include filter '{self.host_include_filter.pattern}'. Skipping") + return + + # second excludes + if self.host_exclude_filter is not None: + if self.host_exclude_filter.match(name): + log.debug(f"Host '{name}' matched exclude filter '{self.host_exclude_filter.pattern}'. Skipping") + return + + manufacturer = grab(obj, "summary.hardware.vendor") + model = grab(obj, "summary.hardware.model") + platform = "{} {}".format(grab(obj, "config.product.name"), grab(obj, "config.product.version")) + + cluster = grab(obj, "parent.name") + status = "active" if grab(obj, "summary.runtime.connectionState") == "connected" else "offline" + + # prepare identifiers + identifiers = grab(obj, "summary.hardware.otherIdentifyingInfo") + identifier_dict = dict() + if identifiers is not None: + for item in identifiers: + value = grab(item, "identifierValue", fallback="") + if len(str(value).strip()) > 0: + identifier_dict[grab(item, "identifierType.key")] = str(value).strip() + + # try to find serial + serial = None + + for serial_num_key in [ "EnclosureSerialNumberTag", "SerialNumberTag", "ServiceTag"]: + if serial_num_key in identifier_dict.keys(): + serial = identifier_dict.get(serial_num_key) + break + + # add asset tag if desired and present + asset_tag = None + + if bool(self.collect_hardware_asset_tag) is True and "AssetTag" in identifier_dict.keys(): + + banned_tags = [ "Default string", "NA", "N/A", "None", "Null", "oem", "o.e.m", + "to be filled by o.e.m.", "Unknown" ] + + this_asset_tag = identifier_dict.get("AssetTag") + + if this_asset_tag.lower() not in [x.lower() for x in banned_tags]: + asset_tag = this_asset_tag + + # handle standalone hosts + if cluster == name: + # Store the host so that we can check VMs against it + self.standalone_hosts.append(cluster) + cluster = "Standalone ESXi Host" + + data={ + "name": name, + "device_role": {"name": self.netbox_host_device_role}, + "device_type": { + "model": model, + "manufacturer": { + "name": manufacturer + } + }, + "platform": {"name": platform}, + "site": {"name": self.site_name}, + "cluster": {"name": cluster}, + "status": status + } + + if serial is not None: + data["serial"]: serial + if asset_tag is not None: + data["asset_tag"]: asset_tag + + host_object = self.inventory.add_update_object(NBDevices, data=data, source=self) + + for pnic in grab(obj, "config.network.pnic", fallback=list()): + + log.debug2("Parsing {}: {}".format(grab(pnic, "_wsdlName"), grab(pnic, "device"))) + + pnic_link_speed = grab(pnic, "linkSpeed.speedMb") + if pnic_link_speed is None: + pnic_link_speed = grab(pnic, "spec.linkSpeed.speedMb") + if pnic_link_speed is None: + pnic_link_speed = grab(pnic, "validLinkSpecification.0.speedMb") + + pnic_link_speed_text = f"{pnic_link_speed}Mbps " if pnic_link_speed is not None else "" + + pnic_speed_type_mapping = { + 100: "100base-tx", + 1000: "1000base-t", + 10000: "10gbase-t", + 25000: "25gbase-x-sfp28", + 40000: "40gbase-x-qsfpp" + } + + pnic_data = { + "name": grab(pnic, "device"), + "device": host_object, + "mac_address": normalize_mac_address(grab(pnic, "mac")), + "enabled": bool(grab(pnic, "spec.linkSpeed")), + "description": f"{pnic_link_speed_text}Physical Interface", + "type": pnic_speed_type_mapping.get(pnic_link_speed, "other") + } + + self.inventory.add_update_object(NBInterfaces, data=pnic_data, source=self) + + + for vnic in grab(obj, "config.network.vnic", fallback=list()): + + log.debug2("Parsing {}: {}".format(grab(vnic, "_wsdlName"), grab(vnic, "device"))) + + vnic_data = { + "name": grab(vnic, "device"), + "device": host_object, + "mac_address": normalize_mac_address(grab(vnic, "spec.mac")), + "mtu": grab(vnic, "spec.mtu"), + "description": grab(vnic, "portgroup"), + "type": "virtual" + } + + vnic_object = self.inventory.add_update_object(NBInterfaces, data=vnic_data, source=self) + + vnic_ip = "{}/{}".format(grab(vnic, "spec.ip.ipAddress"), grab(vnic, "spec.ip.subnetMask")) + + if format_ip(vnic_ip) is None: + logging.error(f"IP address '{vnic_ip}' for {vnic_object.get_display_name()} invalid!") + continue + + ip_permitted = False + + ip_address_object = ip_address(grab(vnic, "spec.ip.ipAddress")) + for permitted_subnet in self.permitted_subnets: + if ip_address_object in permitted_subnet: + ip_permitted = True + break + + if ip_permitted is False: + log.debug(f"IP address {vnic_ip} not part of any permitted subnet. Skipping.") + continue + + vnic_ip_data = { + "address": format_ip(vnic_ip), + "assigned_object_id": vnic_object.nb_id, + "assigned_object_type": "dcim.interface" + } + + self.inventory.add_update_object(NBIPAddresses, data=vnic_ip_data, source=self) + + + def add_virtual_machine(self, obj): + + # ToDo: + # * find VM based on device mac addresses + + name = grab(obj, "name") + + log.debug2(f"Parsing vCenter host: {name}") + + # filter VMs + # first includes + if self.vm_include_filter is not None: + if not self.vm_include_filter.match(name): + log.debug(f"Virtual machine '{name}' did not match include filter '{self.vm_include_filter.pattern}'. Skipping") + return + + # second excludes + if self.vm_exclude_filter is not None: + if self.vm_exclude_filter.match(name): + log.debug(f"Virtual Machine '{name}' matched exclude filter '{self.vm_exclude_filter.pattern}'. Skipping") + return + + cluster = grab(obj, "runtime.host.parent.name") + if cluster in self.standalone_hosts: + cluster = "Standalone ESXi Host" + + platform = grab(obj, "config.guestFullName") + platform = grab(obj, "guest.guestFullName", fallback=platform) + + status = "active" if grab(obj, "runtime.powerState") == "poweredOn" else "offline" + + hardware_devices = grab(obj, "config.hardware.device", fallback=list()) + + disk = int(sum([ getattr(comp, "capacityInKB", 0) for comp in hardware_devices + if isinstance(comp, vim.vm.device.VirtualDisk) + ]) / 1024 / 1024) + + data = { + "name": grab(obj, "name"), + "role": {"name": self.settings.get("netbox_vm_device_role")}, + "status": status, + "memory": grab(obj, "config.hardware.memoryMB"), + "vcpus": grab(obj, "config.hardware.numCPU"), + "disk": disk, + "comments": grab(obj, "config.annotation") + } + + if cluster is not None: + data["cluster"] = {"name": cluster} + if platform is not None: + data["platform"] = {"name": platform} + + vm_object = self.inventory.add_update_object(NBVMs, data=data, source=self) + + # ToDo: + # * get current interfaces and compare description (primary key in vCenter) + + # get vm interfaces + for vm_device in hardware_devices: + + int_mac = normalize_mac_address(grab(vm_device, "macAddress")) + + # not a network interface + if int_mac is None: + continue + + log.debug2("Parsing device {}: {}".format(grab(vm_device, "_wsdlName"), grab(vm_device, "macAddress"))) + + int_network_name = self.networks.get(grab(vm_device, "backing.port.portgroupKey")) + int_connected = grab(vm_device, "connectable.connected") + int_label = grab(vm_device, "deviceInfo.label", fallback="") + + int_name = "vNIC {}".format(int_label.split(" ")[-1]) + + int_ip_addresses = list() + + for guest_nic in grab(obj, "guest.net", fallback=list()): + + if int_mac != normalize_mac_address(grab(guest_nic, "macAddress")): + continue + + int_network_name = grab(guest_nic, "network", fallback=int_network_name) + int_connected = grab(guest_nic, "connected", fallback=int_connected) + + for ip in grab(guest_nic, "ipConfig.ipAddress", fallback=list()): + int_ip_addresses.append(f"{ip.ipAddress}/{ip.prefixLength}") + + + if int_network_name is not None: + int_name = f"{int_name} ({int_network_name})" + + vm_nic_data = { + "name": int_name, + "virtual_machine": vm_object, + "mac_address": int_mac, + "description": int_label, + "enabled": int_connected, + } + + vm_nic_object = self.inventory.get_by_data(NBVMInterfaces, data={"mac_address": int_mac}) + + if vm_nic_object is not None: + vm_nic_object.update(data=vm_nic_data, source=self) + else: + vm_nic_object = self.inventory.add_update_object(NBVMInterfaces, data=vm_nic_data, source=self) + + for int_ip_address in int_ip_addresses: + + if format_ip(int_ip_address) is None: + logging.error(f"IP address '{int_ip_address}' for {vm_nic_object.get_display_name()} invalid!") + continue + + ip_permitted = False + + ip_address_object = ip_address(int_ip_address.split("/")[0]) + for permitted_subnet in self.permitted_subnets: + if ip_address_object in permitted_subnet: + ip_permitted = True + break + + if ip_permitted is False: + log.debug(f"IP address {int_ip_address} not part of any permitted subnet. Skipping.") + continue + + # apply ip filter + vm_nic_ip_data = { + "address": format_ip(int_ip_address), + "assigned_object_id": vm_nic_object.nb_id, + "assigned_object_type": "virtualization.vminterface" + } + + self.inventory.add_update_object(NBIPAddresses, data=vm_nic_ip_data, source=self) + + def inizialize_basic_data(self): + + # add source identification tag + self.inventory.add_update_object(NBTags, data={ + "name": self.source_tag, + "description": f"Marks sources synced from vCenter {self.name} " + f"({self.host_fqdn}) to this NetBox Instance." + }) + + self.inventory.add_update_object(NBSites, data={ + "name": self.site_name, + "comments": f"A default virtual site created to house objects " + "that have been synced from this vCenter instance." + }) + + self.inventory.add_update_object(NBClusters, data={ + "name": "Standalone ESXi Host", + "type": {"name": "VMware ESXi"}, + "comments": "A default cluster created to house standalone " + "ESXi hosts and VMs that have been synced from " + "vCenter." + }) + + self.inventory.add_update_object(NBDeviceRoles, data={ + "name": "Server", + "color": "9e9e9e", + "vm_role": True + }) + + +# EOF diff --git a/netbox-sync.py b/netbox-sync.py new file mode 100755 index 0000000..6fcf598 --- /dev/null +++ b/netbox-sync.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +self_description = \ +""" +Sync objects from various sources to Netbox +""" + + +from datetime import date, datetime + +from module.common.misc import grab +from module.common.cli_parser import parse_command_line +from module.common.logging import setup_logging +from module.common.configuration import get_config_file, open_config_file, get_config +from module.netbox.connection import NetBoxHandler +from module.netbox.inventory import NetBoxInventory +#from module.netbox.object_classes import * + +from module.sources import * + + +import pprint + +__version__ = "0.0.1" +__version_date__ = "2020-10-01" +__author__ = "Ricardo Bartels " +__description__ = "NetBox Sync" +__license__ = "MIT" +__url__ = "https://github.com/bb-Ricardo/unknown" + + +default_log_level = "WARNING" +default_config_file_path = "./settings.ini" + + +""" +ToDo: +* host "Management" interface is Primary +* return more then one object if found more then one and add somehow to returned objects. Maybe related? +* Add purge option +""" + +def main(): + + start_time = datetime.now() + + sources = list() + + # parse command line + args = parse_command_line(self_description=self_description, + version=__version__, + version_date=__version_date__, + default_config_file_path=default_config_file_path) + + # get config file path + config_file = get_config_file(args.config_file) + + # get config handler + config_handler = open_config_file(config_file) + + # get logging configuration + + # set log level + log_level = default_log_level + # config overwrites default + log_level = config_handler.get("common", "log_level", fallback=log_level) + # cli option overwrites config file + if grab(args, "log_level") is not None: + log_level = grab(args, "log_level") + + log_file = None + if bool(config_handler.getboolean("common", "log_to_file", fallback=False)) is True: + log_file = config_handler.get("common", "log_file", fallback=None) + + # setup logging + log = setup_logging(log_level, log_file) + + + # now we are ready to go + log.info("Starting " + __description__) + log.debug(f"Using config file: {config_file}") + + # initialize an empty inventory which will be used to hold and reference all objects + inventory = NetBoxInventory() + + # get config for netbox handler + netbox_settings = get_config(config_handler, section="netbox", valid_settings=NetBoxHandler.settings) + + # establish NetBox connection + NB_handler = NetBoxHandler(cli_args=args, settings=netbox_settings, inventory=inventory) + + # instantiate source handlers and get attributes + sources = instanciate_sources(config_handler, inventory) + + # all sources are unavailable + if len(sources) == 0: + do_error_exit("No working sources found. Exit.") + + # retrieve all dependent object classes + netbox_objects_to_query = list() + for source in sources: + netbox_objects_to_query.extend(source.dependend_netbox_objects) + + # request NetBox data + NB_handler.query_current_data(list(set(netbox_objects_to_query))) + + # resolve object relations within the initial inventory + inventory.resolve_relations() + + # initialize basic data needed for syncing + NB_handler.inizialize_basic_data() + + # for object in inventory.get_all_items(NBIPAddresses): + # pprint.pprint(object.__dict__) + # exit(0) + # loop over sources and patch netbox data + for source in sources: + source.apply() + + # add/remove tags to/from all inventory items + inventory.tag_all_the_things(sources, NB_handler) + + #for object in inventory.get_all_items(NBVMs): + # print(object.get_display_name()) + """ + nb.set_primary_ips() + # Optional tasks + if settings.POPULATE_DNS_NAME: + nb.set_dns_names() + """ + + # update data in NetBox + NB_handler.update_instance() + + # finish + + log.info("Completed NetBox Sync! Total execution time %s." % (datetime.now() - start_time)) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..76c9260 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +aiodns==2.0.0 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +idna==2.8 +pycares==3.1.1 +pycparser==2.19 +pyvmomi==6.7.3 +requests==2.24.0 +six==1.13.0 +typing==3.7.4.1 +urllib3==1.25.7 +packaging diff --git a/settings-example.ini b/settings-example.ini new file mode 100644 index 0000000..bf8c492 --- /dev/null +++ b/settings-example.ini @@ -0,0 +1,62 @@ + + +[common] + +log_level = INFO + +# Places all logs in a rotating file if True +log_to_file = True + +log_file = log/netbox_sync.log + +# define different sources +[source/my-example] + +# currently supported +# * vmware : VMware vcenter +type = vmware + +host_fqdn = vcenter.example.com + +port = 443 +username = vcenteruser +password = supersecret + +host_exclude_filter = +host_include_filter = + +vm_exclude_filter = +vm_include_filter = + +netbox_host_device_role = Server +netbox_vm_device_role = Server + +permitted_subnets = 172.16.0.0/12, 10.0.0.0/8, 192.168.0.0/16, fe80::/64 + +# Attempt to collect asset tags from vCenter hosts +collect_hardware_asset_tag = True + +# ToDo: +# * add following options + +dns_name_lookup = True + +custom_dns_servers = 192.168.1.11, 192.168.1.12 + + +[netbox] + +api_token = XYZ + +disable_tls = true +validate_tls_certs = true + +host_fqdn = netbox.example.com + +port = 443 + +prune_enabled = true + +prune_delay_in_days = 30 + +# EOF