# -*- coding: utf-8 -*- # Copyright (c) 2020 - 2022 Ricardo Bartels. All rights reserved. # # netbox-sync.py # # This work is licensed under the terms of the MIT license. # For a copy, see file LICENSE.txt included in this # repository or visit: . import atexit import datetime import pprint import re import ssl from ipaddress import ip_address, ip_network, ip_interface from urllib.parse import unquote import urllib3 import requests # noinspection PyUnresolvedReferences from pyVim import connect # noinspection PyUnresolvedReferences from pyVmomi import vim # noinspection PyUnresolvedReferences from pyVmomi.VmomiSupport import VmomiJSONEncoder from module.sources.common.source_base import SourceBase from module.common.logging import get_logger, DEBUG3 from module.common.misc import grab, dump, get_string_or_none, plural, quoted_split from module.common.support import normalize_mac_address, ip_valid_to_add_to_netbox from module.netbox.object_classes import ( NetBoxInterfaceType, NBTag, NBManufacturer, NBDeviceType, NBPlatform, NBClusterType, NBClusterGroup, NBDeviceRole, NBSite, NBCluster, NBDevice, NBVM, NBVMInterface, NBInterface, NBIPAddress, NBPrefix, NBTenant, NBVRF, NBVLAN, NBCustomField ) vsphere_automation_sdk_available = True try: # noinspection PyUnresolvedReferences from com.vmware.vapi.std_client import DynamicID # noinspection PyUnresolvedReferences from vmware.vapi.vsphere.client import create_vsphere_client except ImportError: vsphere_automation_sdk_available = False log = get_logger() # noinspection PyTypeChecker class VMWareHandler(SourceBase): """ Source class to import data from a vCenter instance and add/update NetBox objects based on gathered information """ dependent_netbox_objects = [ NBTag, NBManufacturer, NBDeviceType, NBPlatform, NBClusterType, NBClusterGroup, NBDeviceRole, NBSite, NBCluster, NBDevice, NBVM, NBVMInterface, NBInterface, NBIPAddress, NBPrefix, NBTenant, NBVRF, NBVLAN, NBCustomField ] settings = { "enabled": True, "host_fqdn": None, "port": 443, "username": None, "password": None, "validate_tls_certs": False, "proxy_host": None, "proxy_port": None, "cluster_exclude_filter": None, "cluster_include_filter": None, "host_exclude_filter": None, "host_include_filter": None, "vm_exclude_filter": None, "vm_include_filter": None, "permitted_subnets": None, "collect_hardware_asset_tag": True, "match_host_by_serial": True, "cluster_site_relation": None, "cluster_tag_relation": None, "cluster_tenant_relation": None, "cluster_tag_source": None, "host_role_relation": None, "host_site_relation": None, "host_tag_relation": None, "host_tenant_relation": None, "host_tag_source": None, "vm_platform_relation": None, "vm_role_relation": None, "vm_tag_relation": None, "vm_tenant_relation": None, "vm_tag_source": None, "dns_name_lookup": False, "custom_dns_servers": None, "set_primary_ip": "when-undefined", "skip_vm_comments": False, "skip_vm_templates": True, "skip_offline_vms": False, "skip_srm_placeholder_vms": False, "strip_host_domain_name": False, "strip_vm_domain_name": False, "sync_custom_attributes": False, "host_custom_object_attributes": None, "vm_custom_object_attributes": None, "set_source_name_as_cluster_group": False, "sync_vm_dummy_interfaces": False, "disable_vlan_sync": False, "host_management_interface_match": "management, mgmt" } deprecated_settings = {} removed_settings = { "netbox_host_device_role": "host_role_relation", "netbox_vm_device_role": "vm_role_relation", "sync_tags": "host_tag_source', 'vm_tag_source' or 'cluster_tag_source", "sync_parent_tags": "host_tag_source', 'vm_tag_source' or 'cluster_tag_source" } init_successful = False inventory = None name = None source_tag = None source_type = "vmware" recursion_level = 0 # internal vars session = None tag_session = None site_name = None def __init__(self, name=None, settings=None, inventory=None): if name is None: raise ValueError(f"Invalid value for attribute 'name': '{name}'.") self.inventory = inventory self.name = name self.parse_config_settings(settings) self.source_tag = f"Source: {name}" self.site_name = f"vCenter: {name}" if self.enabled is False: log.info(f"Source '{name}' is currently disabled. Skipping") return self.create_sdk_session() if self.session is None: log.info(f"Source '{name}' is currently unavailable. Skipping") return self.create_api_session() self.init_successful = True # instantiate source specific vars self.network_data = { "vswitch": dict(), "pswitch": dict(), "host_pgroup": dict(), "dpgroup": dict(), "dpgroup_ports": dict() } self.processed_host_names = dict() self.processed_vm_names = dict() self.processed_vm_uuid = list() self.object_cache = dict() self.parsing_vms_the_first_time = True def parse_config_settings(self, config_settings): """ Validate parsed settings from config file Parameters ---------- config_settings: dict dict of config settings """ validation_failed = False for setting in ["host_fqdn", "port", "username", "password"]: if config_settings.get(setting) is None: log.error(f"Config option '{setting}' in 'source/{self.name}' can't be empty/undefined") validation_failed = True # check permitted ip subnets if config_settings.get("permitted_subnets") is None: log.info(f"Config option 'permitted_subnets' in 'source/{self.name}' is undefined. " f"No IP addresses will be populated to NetBox!") else: config_settings["permitted_subnets"] = \ [x.strip() for x in config_settings.get("permitted_subnets").split(",") if x.strip() != ""] permitted_subnets = list() excluded_subnets = list() # add "invisible" config option self.settings["excluded_subnets"] = None for subnet in config_settings["permitted_subnets"]: excluded = False if subnet[0] == "!": excluded = True subnet = subnet[1:].strip() try: if excluded is True: excluded_subnets.append(ip_network(subnet)) else: permitted_subnets.append(ip_network(subnet)) except Exception as e: log.error(f"Problem parsing permitted subnet: {e}") validation_failed = True config_settings["permitted_subnets"] = permitted_subnets config_settings["excluded_subnets"] = excluded_subnets # check include and exclude filter expressions for setting in [x for x in config_settings.keys() if "filter" in x]: if config_settings.get(setting) is None or config_settings.get(setting).strip() == "": continue re_compiled = None try: re_compiled = re.compile(config_settings.get(setting)) except Exception as e: log.error(f"Problem parsing regular expression for '{setting}': {e}") validation_failed = True config_settings[setting] = re_compiled for relation_option in [x for x in self.settings.keys() if "relation" in x]: if config_settings.get(relation_option) is None: continue relation_data = list() relation_type = relation_option.split("_")[1] for relation in quoted_split(config_settings.get(relation_option)): object_name = relation.split("=")[0].strip(' "') relation_name = relation.split("=")[1].strip(' "') if len(object_name) == 0 or len(relation_name) == 0: log.error(f"Config option '{relation}' malformed got '{object_name}' for " f"object name and '{relation_name}' for {relation_type} name.") validation_failed = True try: re_compiled = re.compile(object_name) except Exception as e: log.error(f"Problem parsing regular expression '{object_name}' for '{relation}': {e}") validation_failed = True continue relation_data.append({ "object_regex": re_compiled, f"assigned_name": relation_name }) config_settings[relation_option] = relation_data for custom_object_option in [x for x in self.settings.keys() if "custom_object_attributes" in x]: if config_settings.get(custom_object_option) is None: continue config_settings[custom_object_option] = quoted_split(config_settings.get(custom_object_option)) for tag_source in [x for x in self.settings.keys() if "tag_source" in x]: config_settings[tag_source] = quoted_split(config_settings.get(tag_source)) valid_tag_sources = [ "object", "parent_folder_1", "parent_folder_2", "cluster", "datacenter" ] for tag_source_option in config_settings[tag_source]: if tag_source_option not in valid_tag_sources: log.error(f"Tag source '{tag_source_option}' for '{tag_source}' option invalid.") validation_failed = True continue if config_settings.get("dns_name_lookup") is True and config_settings.get("custom_dns_servers") is not None: custom_dns_servers = \ [x.strip() for x in config_settings.get("custom_dns_servers").split(",") if x.strip() != ""] tested_custom_dns_servers = list() for custom_dns_server in custom_dns_servers: try: tested_custom_dns_servers.append(str(ip_address(custom_dns_server))) except ValueError: log.error(f"Config option 'custom_dns_servers' value '{custom_dns_server}' " f"does not appear to be an IP address.") validation_failed = True config_settings["custom_dns_servers"] = tested_custom_dns_servers if len(config_settings.get("host_management_interface_match") or "") > 0: host_management_interface_match = config_settings.get("host_management_interface_match") else: host_management_interface_match = self.settings.get("host_management_interface_match") config_settings["host_management_interface_match"] = \ [x.strip() for x in host_management_interface_match.split(",")] if validation_failed is True: log.error("Config validation failed. Exit!") exit(1) for setting in self.settings.keys(): setattr(self, setting, config_settings.get(setting)) def create_sdk_session(self): """ Initialize SDK session with vCenter Returns ------- bool: if initialization was successful or not """ if self.session is not None: return True log.debug(f"Starting vCenter SDK connection to '{self.host_fqdn}'") ssl_context = ssl.create_default_context() if bool(self.validate_tls_certs) is False: ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE connection_params = dict( host=self.host_fqdn, port=self.port, sslContext=ssl_context ) # uses connect.SmartStubAdapter if self.proxy_host is not None and self.proxy_port is not None: connection_params.update( httpProxyHost=self.proxy_host, httpProxyPort=int(self.proxy_port), ) # uses connect.SmartConnect else: connection_params.update( user=self.username, pwd=self.password, ) def_exception_text = f"Unable to connect to vCenter instance '{self.host_fqdn}' on port {self.port}." try: if self.proxy_host is not None and self.proxy_port is not None: smart_stub = connect.SmartStubAdapter(**connection_params) instance = vim.ServiceInstance('ServiceInstance', smart_stub) content = instance.RetrieveContent() content.sessionManager.Login(self.username, self.password, None) else: instance = connect.SmartConnect(**connection_params) atexit.register(connect.Disconnect, instance) self.session = instance.RetrieveContent() except vim.fault.InvalidLogin as e: log.error(f"{def_exception_text} {e.msg}") return False except vim.fault.NoPermission as e: log.error(f"{def_exception_text} User {self.username} does not have required permission. {e.msg}") return False except Exception as e: log.error(f"{def_exception_text} Reason: {e}") return False log.info(f"Successfully connected to vCenter SDK '{self.host_fqdn}'") return True def create_api_session(self): """ Initialize API session with vCenter Returns ------- bool: if initialization was successful or not """ if self.tag_session is not None: return True if len(self.cluster_tag_source) + len(self.host_tag_source) + len(self.vm_tag_source) == 0: return False log.debug(f"Starting vCenter API connection to '{self.host_fqdn}'") if len(self.cluster_tag_source) > 0 or len(self.host_tag_source) > 0 or len(self.vm_tag_source) > 0: if vsphere_automation_sdk_available is False: self.__setattr__("cluster_tag_source", list()) self.__setattr__("host_tag_source", list()) self.__setattr__("vm_tag_source", list()) log.warning(f"Unable to import Python 'vsphere-automation-sdk'. Tag syncing will be disabled.") return False # create a requests session to enable/disable TLS verification session = requests.session() session.verify = bool(self.validate_tls_certs) # disable TLS insecure warnings if user explicitly switched off validation if bool(self.validate_tls_certs) is False: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # adds proxy to the session if self.proxy_host is not None and self.proxy_port is not None: session.proxies.update({ "http": f"http://{self.proxy_host}:{self.proxy_port}", "https": f"http://{self.proxy_host}:{self.proxy_port}", }) try: self.tag_session = create_vsphere_client( server=f"{self.host_fqdn}:{self.port}", username=self.username, password=self.password, session=session) except Exception as e: self.__setattr__("cluster_tag_source", list()) self.__setattr__("host_tag_source", list()) self.__setattr__("vm_tag_source", list()) log.warning(f"Unable to connect to vCenter API instance '{self.host_fqdn}' on port {self.port}: {e}") log.warning("Tag syncing will be disabled.") return False log.info(f"Successfully connected to vCenter API '{self.host_fqdn}'") return True def apply(self): """ Main source handler method. This method is called for each source from "main" program to retrieve data from it source and apply it to the NetBox inventory. Every update of new/existing objects fot this source has to happen here. """ log.info(f"Query data from vCenter: '{self.host_fqdn}'") """ Mapping of object type keywords to view types and handlers iterate over all VMs twice. To handle VMs with the same name in a cluster we first iterate over all VMs and look only at the active ones and sync these first. Then we iterate a second time to catch the rest. This has been implemented to support migration scenarios where you create the same machines with a different setup like a new version or something. This way NetBox will be updated primarily with the actual active VM data. # disabled, no useful information at this moment "virtual switch": { "view_type": vim.DistributedVirtualSwitch, "view_handler": self.add_virtual_switch }, """ object_mapping = { "datacenter": { "view_type": vim.Datacenter, "view_handler": self.add_datacenter }, "cluster": { "view_type": vim.ClusterComputeResource, "view_handler": self.add_cluster }, "network": { "view_type": vim.dvs.DistributedVirtualPortgroup, "view_handler": self.add_port_group }, "host": { "view_type": vim.HostSystem, "view_handler": self.add_host }, "virtual machine": { "view_type": vim.VirtualMachine, "view_handler": self.add_virtual_machine }, "offline virtual machine": { "view_type": vim.VirtualMachine, "view_handler": self.add_virtual_machine } } # skip virtual machines which are reported offline if self.skip_offline_vms is True: log.info("Skipping offline VMs") del object_mapping["offline virtual machine"] for view_name, view_details in object_mapping.items(): # test if session is still alive try: self.session.sessionManager.currentSession.key except (vim.fault.NotAuthenticated, AttributeError): log.info("No existing vCenter session found.") self.session = None self.tag_session = None self.create_sdk_session() self.create_api_session() if self.session is None: log.error("Recreating session failed") break view_data = { "container": self.session.rootFolder, "type": [view_details.get("view_type")], "recursive": True } try: container_view = self.session.viewManager.CreateContainerView(**view_data) except Exception as e: log.error(f"Problem creating vCenter view for '{view_name}s': {e}") continue view_objects = grab(container_view, "view") if view_objects is None: log.error(f"Creating vCenter view for '{view_name}s' failed!") continue if view_name != "offline virtual machine": log.debug("vCenter returned '%d' %s%s" % (len(view_objects), view_name, plural(len(view_objects)))) else: self.parsing_vms_the_first_time = False log.debug("Iterating over all virtual machines a second time ") for obj in view_objects: if log.level == DEBUG3: try: dump(obj) except Exception as e: log.error(e) # noinspection PyArgumentList view_details.get("view_handler")(obj) container_view.Destroy() self.update_basic_data() @staticmethod def passes_filter(name, include_filter, exclude_filter): """ checks if object name passes a defined object filter. Parameters ---------- name: str name of the object to check include_filter: regex object regex object of include filter exclude_filter: regex object regex object of exclude filter Returns ------- bool: True if all filter passed, otherwise False """ # first includes if include_filter is not None and not include_filter.match(name): log.debug(f"Object '{name}' did not match include filter '{include_filter.pattern}'. Skipping") return False # second excludes if exclude_filter is not None and exclude_filter.match(name): log.debug(f"Object '{name}' matched exclude filter '{exclude_filter.pattern}'. Skipping") return False return True def get_site_name(self, object_type, object_name, cluster_name=""): """ Return a site name for a NBCluster or NBDevice depending on config options host_site_relation and cluster_site_relation Parameters ---------- object_type: (NBCluster, NBDevice) object type to check site relation for object_name: str object name to check site relation for cluster_name: str cluster name of NBDevice to check for site name Returns ------- str: site name if a relation was found """ if object_type not in [NBCluster, NBDevice]: raise ValueError(f"Object must be a '{NBCluster.name}' or '{NBDevice.name}'.") log.debug2(f"Trying to find site name for {object_type.name} '{object_name}'") # check if site was provided in config relation_name = "host_site_relation" if object_type == NBDevice else "cluster_site_relation" site_name = self.get_object_relation(object_name, relation_name) if object_type == NBDevice and site_name is None: site_name = self.get_site_name(NBCluster, cluster_name) if site_name is not None: log.debug2(f"Found a matching cluster site for {object_name}, using site '{site_name}'") # set default site name if site_name is None: site_name = self.site_name log.debug(f"No site relation for '{object_name}' found, using default site '{site_name}'") return site_name def get_object_based_on_macs(self, object_type, mac_list=None): """ Try to find a NetBox object based on list of MAC addresses. Iterate over all interfaces of this object type and compare MAC address with list of desired MAC addresses. If match was found store related machine object and count every correct match. If exactly one machine with matching interfaces was found then this one will be returned. If two or more machines with matching MACs are found compare the two machines with the highest amount of matching interfaces. If the ration of matching interfaces exceeds 2.0 then the top matching machine is chosen as desired object. If the ration is below 2.0 then None will be returned. The probability is to low that this one is the correct one. None will also be returned if no machine was found at all. Parameters ---------- object_type: (NBDevice, NBVM) type of NetBox device to find in inventory mac_list: list list of MAC addresses to compare against NetBox interface objects Returns ------- (NBDevice, NBVM, None): object instance of found device, otherwise None """ object_to_return = None if object_type not in [NBDevice, NBVM]: raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") if mac_list is None or not isinstance(mac_list, list) or len(mac_list) == 0: return interface_typ = NBInterface if object_type == NBDevice else NBVMInterface objects_with_matching_macs = dict() matching_object = None for interface in self.inventory.get_all_items(interface_typ): if grab(interface, "data.mac_address") in mac_list: matching_object = grab(interface, f"data.{interface.secondary_key}") if not isinstance(matching_object, (NBDevice, NBVM)): continue log.debug2("Found matching MAC '%s' on %s '%s'" % (grab(interface, "data.mac_address"), object_type.name, matching_object.get_display_name(including_second_key=True))) if objects_with_matching_macs.get(matching_object) is None: objects_with_matching_macs[matching_object] = 1 else: objects_with_matching_macs[matching_object] += 1 # try to find object based on amount of matching MAC addresses num_devices_witch_matching_macs = len(objects_with_matching_macs.keys()) if num_devices_witch_matching_macs == 1 and isinstance(matching_object, (NBDevice, NBVM)): log.debug2("Found one %s '%s' based on MAC addresses and using it" % (object_type.name, matching_object.get_display_name(including_second_key=True))) object_to_return = list(objects_with_matching_macs.keys())[0] elif num_devices_witch_matching_macs > 1: log.debug2(f"Found {num_devices_witch_matching_macs} {object_type.name}s with matching MAC addresses") # now select the two top matches first_choice, second_choice = \ sorted(objects_with_matching_macs, key=objects_with_matching_macs.get, reverse=True)[0:2] first_choice_matches = objects_with_matching_macs.get(first_choice) second_choice_matches = objects_with_matching_macs.get(second_choice) log.debug2(f"The top candidate {first_choice.get_display_name()} with {first_choice_matches} matches") log.debug2(f"The second candidate {second_choice.get_display_name()} with {second_choice_matches} matches") # get ratio between matching_ration = first_choice_matches / second_choice_matches # only pick the first one if the ration exceeds 2 if matching_ration >= 2.0: log.debug2(f"The matching ratio of {matching_ration} is high enough " f"to select {first_choice.get_display_name()} as desired {object_type.name}") object_to_return = first_choice else: log.debug2("Both candidates have a similar amount of " "matching interface MAC addresses. Using NONE of them!") return object_to_return def get_object_based_on_primary_ip(self, object_type, primary_ip4=None, primary_ip6=None): """ Try to find a NBDevice or NBVM based on the primary IP address. If an exact match was found the device/vm object will be returned immediately without checking of the other primary IP address (if defined). Parameters ---------- object_type: (NBDevice, NBVM) object type to look for primary_ip4: str primary IPv4 address of object to find primary_ip6: str primary IPv6 address of object to find Returns ------- """ def _matches_device_primary_ip(device_primary_ip, ip_needle): ip = None if device_primary_ip is not None and ip_needle is not None: if isinstance(device_primary_ip, dict): ip = grab(device_primary_ip, "address") elif isinstance(device_primary_ip, int): ip = self.inventory.get_by_id(NBIPAddress, nb_id=device_primary_ip) ip = grab(ip, "data.address") if ip is not None and ip.split("/")[0] == ip_needle: return True return False if object_type not in [NBDevice, NBVM]: raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") if primary_ip4 is None and primary_ip6 is None: return if primary_ip4 is not None: primary_ip4 = str(primary_ip4).split("/")[0] if primary_ip6 is not None: primary_ip6 = str(primary_ip6).split("/")[0] for device in self.inventory.get_all_items(object_type): if _matches_device_primary_ip(grab(device, "data.primary_ip4"), primary_ip4) is True: log.debug2(f"Found existing host '{device.get_display_name()}' " f"based on the primary IPv4 '{primary_ip4}'") return device if _matches_device_primary_ip(grab(device, "data.primary_ip6"), primary_ip6) is True: log.debug2(f"Found existing host '{device.get_display_name()}' " f"based on the primary IPv6 '{primary_ip6}'") return device def get_vmware_object_tags(self, obj): """ Get tags from vCenter for submitted object. Parameters ---------- obj pyvmomi object to retrieve tags for Returns ------- tag_list: list list of NBTag objets retrieved from vCenter for this object """ if obj is None: return tag_list = list() if vsphere_automation_sdk_available is True: if len(self.cluster_tag_source) + len(self.host_tag_source) + len(self.vm_tag_source) > 0: # noinspection PyBroadException try: object_tag_ids = self.tag_session.tagging.TagAssociation.list_attached_tags( DynamicID(type=grab(obj, "_wsdlName"), id=grab(obj, "_moId"))) except Exception as e: log.error(f"Unable to retrieve vCenter tags for '{obj.name}': {e}") object_tag_ids = list() for tag_id in object_tag_ids: # noinspection PyBroadException try: tag_name = self.tag_session.tagging.Tag.get(tag_id).name tag_description = self.tag_session.tagging.Tag.get(tag_id).description except Exception as e: log.error(f"Unable to retrieve vCenter tag '{tag_id}' for '{obj.name}': {e}") continue tag_list.append(self.inventory.add_update_object(NBTag, data={ "name": tag_name, "description": tag_description })) return tag_list def collect_object_tags(self, obj): """ collect tags from object based on the config settings Parameters ---------- obj pyvmomi object to retrieve tags for Returns ------- tag_list: list list of NBTag objets retrieved from vCenter for this object """ if obj is None: return tag_list = list() if isinstance(obj, vim.ClusterComputeResource): tag_source = self.cluster_tag_source elif isinstance(obj, vim.HostSystem): tag_source = self.host_tag_source elif isinstance(obj, vim.VirtualMachine): tag_source = self.vm_tag_source else: raise ValueError(f"Tags for '{grab(obj, '_wsdlName')}' are not supported") if len(tag_source) == 0 or vsphere_automation_sdk_available is False: return tag_list log.debug2(f"Collecting tags for {obj.name}") if "object" in tag_source: tag_list.extend(self.get_vmware_object_tags(obj)) if "parent_folder_1" in tag_source or "parent_folder_2" in tag_source: parent_folder_1 = self.get_parent_object_by_class(obj, vim.Folder) if parent_folder_1 is not None: if "parent_folder_1" in tag_source: tag_list.extend(self.get_vmware_object_tags(parent_folder_1)) if "parent_folder_2" in tag_source: parent_folder_2 = self.get_parent_object_by_class(obj, vim.Folder) if parent_folder_2 is not None: tag_list.extend(self.get_vmware_object_tags(parent_folder_2)) if not isinstance(obj, vim.ClusterComputeResource) and "cluster" in tag_source: cluster = self.get_parent_object_by_class(obj, vim.ClusterComputeResource) if cluster is not None: tag_list.extend(self.get_vmware_object_tags(cluster)) if "datacenter" in tag_source: datacenter = self.get_parent_object_by_class(obj, vim.Datacenter) if datacenter is not None: tag_list.extend(self.get_vmware_object_tags(datacenter)) return tag_list def get_object_custom_fields(self, obj): """ Get custom attributes from vCenter for submitted object and as NetBox custom fields Parameters ---------- obj pyvmomi object to retrieve custom attributes from Returns ------- custom_fields: dict dictionary with assigned custom fields """ return_custom_fields = dict() custom_value = list() if self.sync_custom_attributes is True: custom_value = grab(obj, "customValue", fallback=list()) if grab(obj, "_wsdlName") == "VirtualMachine": content_type = "virtualization.virtualmachine" custom_object_attributes = self.vm_custom_object_attributes or list() object_attribute_prefix = "vm" else: content_type = "dcim.device" custom_object_attributes = self.host_custom_object_attributes or list() object_attribute_prefix = "host" # add basic host data to device if content_type == "dcim.device": num_cpu_cores = grab(obj, "summary.hardware.numCpuCores") cpu_model = grab(obj, "summary.hardware.cpuModel") memory_size = grab(obj, "summary.hardware.memorySize") if num_cpu_cores is not None: custom_field = self.add_update_custom_field({ "name": "vcsa_host_cpu_cores", "label": "Physical CPU Cores", "content_types": [content_type], "type": "text", "description": f"vCenter '{self.name}' reported Host CPU cores" }) return_custom_fields[grab(custom_field, "data.name")] = f"{num_cpu_cores} {cpu_model}" if isinstance(memory_size, int): custom_field = self.add_update_custom_field({ "name": "vcsa_host_memory", "label": "Memory", "content_types": [content_type], "type": "text", "description": f"vCenter '{self.name}' reported Memory" }) memory_size = round(memory_size / 1024 ** 3) memory_unit = "GB" if memory_size >= 1024: memory_size = memory_size / 1024 memory_unit = "TB" return_custom_fields[grab(custom_field, "data.name")] = f"{memory_size} {memory_unit}" field_definition = {grab(k, "key"): grab(k, "name") for k in grab(obj, "availableField", fallback=list())} for obj_custom_field in custom_value: key = grab(obj_custom_field, "key") value = grab(obj_custom_field, "value") if key is None or value is None: continue label = field_definition.get(key) if label is None: continue custom_field = self.add_update_custom_field({ "name": f"vcsa_{label}", "label": label, "content_types": [content_type], "type": "text", "description": f"vCenter '{self.name}' synced custom attribute '{label}'" }) return_custom_fields[grab(custom_field, "data.name")] = value for custom_object_attribute in custom_object_attributes: attribute_data = grab(obj, custom_object_attribute, fallback="NOT FOUND") if attribute_data == "NOT FOUND": log.warning(f"This object has no attribute '{custom_object_attribute}' or attribute is undefined.") continue if isinstance(attribute_data, datetime.datetime): custom_field_type = "text" attribute_data = attribute_data.strftime("%Y-%m-%dT%H:%M:%S%z") elif isinstance(attribute_data, bool): custom_field_type = "boolean" elif isinstance(attribute_data, int): custom_field_type = "integer" elif isinstance(attribute_data, str): custom_field_type = "text" else: import json # noinspection PyBroadException try: attribute_data = json.loads(json.dumps(attribute_data, cls=VmomiJSONEncoder, sort_keys=True)) except Exception: attribute_data = json.loads(json.dumps(str(attribute_data))) custom_field_type = "json" custom_field = self.add_update_custom_field({ "name": f"vcsa_{object_attribute_prefix}_{custom_object_attribute}", "label": custom_object_attribute, "content_types": [content_type], "type": custom_field_type, "description": f"vCenter '{self.name}' synced object attribute '{custom_object_attribute}'" }) return_custom_fields[grab(custom_field, "data.name")] = attribute_data return return_custom_fields def get_object_relation(self, name, relation, fallback=None): """ Parameters ---------- name: str name of the object to find a relation for relation: str name of the config variable relation (i.e: vm_tag_relation) fallback: str fallback string if no relation matched Returns ------- data: str, list, None string of matching relation or list of matching tags """ resolved_list = list() for single_relation in grab(self, relation, fallback=list()): object_regex = single_relation.get("object_regex") match_found = False if object_regex.match(name): resolved_name = single_relation.get("assigned_name") log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) for {name}") resolved_list.append(resolved_name) match_found = True # special cluster condition if match_found is False and grab(f"{relation}".split("_"), "0") == "cluster": stripped_name = "/".join(name.split("/")[1:]) if object_regex.match(stripped_name): resolved_name = single_relation.get("assigned_name") log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) " f"for {stripped_name}") resolved_list.append(resolved_name) if grab(f"{relation}".split("_"), "1") == "tag": return resolved_list else: resolved_name = fallback if len(resolved_list) >= 1: resolved_name = resolved_list[0] if len(resolved_list) > 1: log.debug(f"Found {len(resolved_list)} matches for {name} in {relation}." f" Using first on: {resolved_name}") return resolved_name def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, vnic_data=None, nic_ips=None, p_ipv4=None, p_ipv6=None): """ Add/update device/VM object in inventory based on gathered data. Try to find object first based on the object data, interface MAC addresses and primary IPs. 1. try to find by name and cluster/site 2. try to find by mac addresses interfaces 3. try to find by serial number (1st) or asset tag (2nd) (ESXi host) 4. try to find by primary IP IP addresses for each interface are added here as well. First they will be checked and added if all checks pass. For each IP address a matching IP prefix will be searched for. First we look for longest matching IP Prefix in the same site. If this failed we try to find the longest matching global IP Prefix. If a IP Prefix was found then we try to get the VRF and VLAN for this prefix. Now we compare if interface VLAN and prefix VLAN match up and warn if they don't. Then we try to add data to the IP address if not already set: add prefix VRF if VRF for this IP is undefined add tenant if tenant for this IP is undefined 1. try prefix tenant 2. if prefix tenant is undefined try VLAN tenant And we also set primary IP4/6 for this object depending on the "set_primary_ip" setting. If a IP address is set as primary IP for another device then using this IP on another device will be rejected by NetBox. Setting "always": check all NBDevice and NBVM objects if this IP address is set as primary IP to any other object then this one. If we found another object, then we unset the primary_ip* for the found object and assign it to this object. This setting will also reset the primary IP if it has been changed in NetBox Setting "when-undefined": Will set the primary IP for this object if primary_ip4/6 is undefined. Will cause a NetBox error if IP has been assigned to a different object as well Setting "never": Well, the attribute primary_ip4/6 will never be touched/changed. Parameters ---------- object_type: (NBDevice, NBVM) NetBoxObject sub class of object to add object_data: dict data of object to add/update pnic_data: dict data of physical interfaces of this object, interface name as key vnic_data: dict data of virtual interfaces of this object, interface name as key nic_ips: dict dict of ips per interface of this object, interface name as key p_ipv4: str primary IPv4 as string including netmask/prefix p_ipv6: str primary IPv6 as string including netmask/prefix """ if object_type not in [NBDevice, NBVM]: raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") if log.level == DEBUG3: log.debug3("function: add_device_vm_to_inventory") log.debug3(f"Object type {object_type}") pprint.pprint(object_data) pprint.pprint(pnic_data) pprint.pprint(vnic_data) pprint.pprint(nic_ips) pprint.pprint(p_ipv4) pprint.pprint(p_ipv6) # check existing Devices for matches log.debug2(f"Trying to find a {object_type.name} based on the collected name, cluster, IP and MAC addresses") device_vm_object = self.inventory.get_by_data(object_type, data=object_data) if device_vm_object is not None: log.debug2("Found a exact matching %s object: %s" % (object_type.name, device_vm_object.get_display_name(including_second_key=True))) # keep searching if no exact match was found else: log.debug2(f"No exact match found. Trying to find {object_type.name} based on MAC addresses") # on VMs vnic data is used, on physical devices pnic data is used mac_source_data = vnic_data if object_type == NBVM else pnic_data nic_macs = [x.get("mac_address") for x in mac_source_data.values()] device_vm_object = self.get_object_based_on_macs(object_type, nic_macs) # look for devices with same serial or asset tag if object_type == NBDevice: if device_vm_object is None and object_data.get("serial") is not None and \ bool(self.match_host_by_serial) is True: log.debug2(f"No match found. Trying to find {object_type.name} based on serial number") device_vm_object = self.inventory.get_by_data(object_type, data={"serial": object_data.get("serial")}) if device_vm_object is None and object_data.get("asset_tag") is not None: log.debug2(f"No match found. Trying to find {object_type.name} based on asset tag") device_vm_object = self.inventory.get_by_data(object_type, data={"asset_tag": object_data.get("asset_tag")}) if device_vm_object is not None: log.debug2("Found a matching %s object: %s" % (object_type.name, device_vm_object.get_display_name(including_second_key=True))) # keep looking for devices with the same primary IP else: log.debug2(f"No match found. Trying to find {object_type.name} based on primary IP addresses") device_vm_object = self.get_object_based_on_primary_ip(object_type, p_ipv4, p_ipv6) if device_vm_object is None: object_name = object_data.get(object_type.primary_key) log.debug(f"No existing {object_type.name} object for {object_name}. Creating a new {object_type.name}.") device_vm_object = self.inventory.add_object(object_type, data=object_data, source=self) else: device_vm_object.update(data=object_data, source=self) # update role according to config settings object_name = object_data.get(object_type.primary_key) role_name = self.get_object_relation(object_name, "host_role_relation" if object_type == NBDevice else "vm_role_relation", fallback="Server") if object_type == NBDevice: device_vm_object.update(data={"device_role": {"name": role_name}}) if object_type == NBVM: device_vm_object.update(data={"role": {"name": role_name}}) # compile all nic data into one dictionary if object_type == NBVM: nic_data = vnic_data else: nic_data = {**pnic_data, **vnic_data} # map interfaces of existing object with discovered interfaces nic_object_dict = self.map_object_interfaces_to_current_interfaces(device_vm_object, nic_data) if object_data.get("status", "") == "active" and (nic_ips is None or len(nic_ips.keys()) == 0): log.debug(f"No IP addresses for '{object_name}' found!") primary_ipv4_object = None primary_ipv6_object = None if p_ipv4 is not None: try: primary_ipv4_object = ip_interface(p_ipv4) except ValueError: log.error(f"Primary IPv4 ({p_ipv4}) does not appear to be a valid IP address (needs included suffix).") if p_ipv6 is not None: try: primary_ipv6_object = ip_interface(p_ipv6) except ValueError: log.error(f"Primary IPv6 ({p_ipv6}) does not appear to be a valid IP address (needs included suffix).") for int_name, int_data in nic_data.items(): # add/update interface with retrieved data nic_object, ip_address_objects = self.add_update_interface(nic_object_dict.get(int_name), device_vm_object, int_data, nic_ips.get(int_name, list()), self.disable_vlan_sync) # add all interface IPs for ip_object in ip_address_objects: ip_interface_object = ip_interface(grab(ip_object, "data.address")) if ip_object is None: continue # continue if address is not a primary IP if ip_interface_object not in [primary_ipv4_object, primary_ipv6_object]: continue # set/update/remove primary IP addresses set_this_primary_ip = False ip_version = ip_interface_object.ip.version if self.set_primary_ip == "always": for object_type in [NBDevice, NBVM]: # new IPs don't need to be removed from other devices/VMs if ip_object.is_new is True: break for devices_vms in self.inventory.get_all_items(object_type): # device has no primary IP of this version this_primary_ip = grab(devices_vms, f"data.primary_ip{ip_version}") # we found this exact object if devices_vms == device_vm_object: continue # device has the same object assigned if this_primary_ip == ip_object: devices_vms.unset_attribute(f"primary_ip{ip_version}") set_this_primary_ip = True elif self.set_primary_ip != "never" and grab(device_vm_object, f"data.primary_ip{ip_version}") is None: set_this_primary_ip = True if set_this_primary_ip is True: log.debug(f"Setting IP '{grab(ip_object, 'data.address')}' as primary IPv{ip_version} for " f"'{device_vm_object.get_display_name()}'") device_vm_object.update(data={f"primary_ip{ip_version}": ip_object}) return def get_parent_object_by_class(self, obj, object_class_to_find): if obj is None or object_class_to_find is None: return if isinstance(obj, object_class_to_find): self.recursion_level = 0 return obj max_recursion = 20 while True: if self.recursion_level >= max_recursion: self.recursion_level = 0 return None # noinspection PyBroadException try: parent = obj.parent except Exception: self.recursion_level = 0 return None if isinstance(parent, object_class_to_find): self.recursion_level = 0 return parent self.recursion_level += 1 return self.get_parent_object_by_class(parent, object_class_to_find) def add_object_to_cache(self, vm_object, netbox_object): if None in [vm_object, netbox_object]: return # noinspection PyBroadException try: vm_class_name = vm_object.__class__.__name__ # noinspection PyProtectedMember vm_object_id = vm_object._GetMoId() except Exception: return if self.object_cache.get(vm_class_name) is None: self.object_cache[vm_class_name] = dict() self.object_cache[vm_class_name][vm_object_id] = netbox_object def get_object_from_cache(self, vm_object): if vm_object is None: return # noinspection PyBroadException try: vm_class_name = vm_object.__class__.__name__ # noinspection PyProtectedMember vm_object_id = vm_object._GetMoId() except Exception: return if self.object_cache.get(vm_class_name) is None: return return self.object_cache[vm_class_name].get(vm_object_id) def add_datacenter(self, obj): """ Add a vCenter datacenter as a NBClusterGroup to NetBox Parameters ---------- obj: vim.Datacenter datacenter object """ if self.set_source_name_as_cluster_group is True: name = self.name else: name = get_string_or_none(grab(obj, "name")) if name is None: return log.debug(f"Parsing vCenter datacenter: {name}") object_data = {"name": name} if self.set_source_name_as_cluster_group is True: label = "Datacenter Name" custom_field = self.add_update_custom_field({ "name": f"vcsa_{label}", "label": label, "content_types": ["virtualization.clustergroup"], "type": "text", "description": f"vCenter '{self.name}' synced custom attribute '{label}'" }) object_data["custom_fields"] = { grab(custom_field, "data.name"): get_string_or_none(grab(obj, "name")) } self.add_object_to_cache(obj, self.inventory.add_update_object(NBClusterGroup, data=object_data, source=self)) def add_cluster(self, obj): """ Add a vCenter cluster as a NBCluster to NetBox. Cluster name is checked against cluster_include_filter and cluster_exclude_filter config setting. Parameters ---------- obj: vim.ClusterComputeResource cluster to add """ name = get_string_or_none(grab(obj, "name")) if self.set_source_name_as_cluster_group is True: group = self.inventory.get_by_data(NBClusterGroup, data={"name": self.name}) else: group = self.get_object_from_cache(self.get_parent_object_by_class(obj, vim.Datacenter)) if name is None or group is None: return group_name = grab(group, "data.name") full_cluster_name = f"{group_name}/{name}" log.debug(f"Parsing vCenter cluster: {full_cluster_name}") # check for full name and then for cluster name only if self.passes_filter(full_cluster_name, self.cluster_include_filter, self.cluster_exclude_filter) is False \ and self.passes_filter(name, self.cluster_include_filter, self.cluster_exclude_filter) is False: return site_name = self.get_site_name(NBCluster, full_cluster_name) data = { "name": name, "type": {"name": "VMware ESXi"}, "group": group, "site": {"name": site_name} } tenant_name = self.get_object_relation(full_cluster_name, "cluster_tenant_relation") if tenant_name is not None: data["tenant"] = {"name": tenant_name} cluster_tags = self.get_object_relation(full_cluster_name, "cluster_tag_relation") cluster_tags.extend(self.collect_object_tags(obj)) if len(cluster_tags) > 0: data["tags"] = cluster_tags # try to find cluster including cluster group log.debug2("Trying to find a matching existing cluster") cluster_object = None fallback_cluster_object = None for cluster_candidate in self.inventory.get_all_items(NBCluster): if grab(cluster_candidate, "data.name") != name: continue # try to find a cluster with matching site if cluster_candidate.get_site_name() == site_name: cluster_object = cluster_candidate log.debug2("Found an existing cluster where 'name' and 'site' are matching") break if grab(cluster_candidate, "data.group") is not None and \ grab(cluster_candidate, "data.group.data.name") == group_name: cluster_object = cluster_candidate log.debug2("Found an existing cluster where 'name' and 'cluster group' are matching") break if grab(cluster_candidate, "data.tenant") is not None and \ tenant_name is not None and \ grab(cluster_candidate, "data.tenant.data.name") == tenant_name: cluster_object = cluster_candidate log.debug2("Found an existing cluster where 'name' and 'tenant' are matching") break # if only the name matches and there are multiple cluster with the same name we choose the first # cluster returned from netbox. This needs to be done to not ignore possible matches in one of # the next iterations if fallback_cluster_object is None: fallback_cluster_object = cluster_candidate if cluster_object is None and fallback_cluster_object is not None: log.debug2(f"Found an existing cluster where 'name' " f"matches (NetBox id: {fallback_cluster_object.get_nb_reference()})") cluster_object = fallback_cluster_object if cluster_object is not None: cluster_object.update(data=data, source=self) else: cluster_object = self.inventory.add_update_object(NBCluster, data=data, source=self) self.add_object_to_cache(obj, cluster_object) def add_virtual_switch(self, obj): """ CURRENTLY UNUSED Parses port data of each distributed virtual switch. Parameters ---------- obj: vim.DistributedVirtualSwitch dvs to retrieve port data from """ uuid = get_string_or_none(grab(obj, "uuid")) name = get_string_or_none(grab(obj, "name")) if uuid is None or name is None: return log.debug(f"Parsing vCenter virtual switch: {name}") # add ports self.network_data["dpgroup_ports"][uuid] = dict() criteria = vim.dvs.PortCriteria() ports = obj.FetchDVPorts(criteria) log.debug2(f"Found {len(ports)} vCenter virtual switch ports") for port in ports: self.network_data["dpgroup_ports"][uuid][port.key] = port def add_port_group(self, obj): """ Parse distributed virtual port group to extract VLAN IDs from each port group Parameters ---------- obj: vim.dvs.DistributedVirtualPortgroup portgroup to parse """ key = get_string_or_none(grab(obj, "key")) name = get_string_or_none(grab(obj, "name")) private = False vlan_ids = list() vlan_id_ranges = list() if key is None or name is None: return log.debug(f"Parsing vCenter port group: {name}") vlan_info = grab(obj, "config.defaultPortConfig.vlan") if isinstance(vlan_info, vim.dvs.VmwareDistributedVirtualSwitch.TrunkVlanSpec): for item in grab(vlan_info, "vlanId", fallback=list()): if item.start == item.end: vlan_ids.append(item.start) vlan_id_ranges.append(str(item.start)) elif item.start == 0 and item.end == 4094: vlan_ids.append(4095) vlan_id_ranges.append(f"{item.start}-{item.end}") else: vlan_ids.extend(range(item.start, item.end+1)) vlan_id_ranges.append(f"{item.start}-{item.end}") elif isinstance(vlan_info, vim.dvs.VmwareDistributedVirtualSwitch.PvlanSpec): vlan_ids.append(grab(vlan_info, "pvlanId")) private = True else: vlan_ids.append(grab(vlan_info, "vlanId")) self.network_data["dpgroup"][key] = { "name": name, "vlan_ids": vlan_ids, "vlan_id_ranges": vlan_id_ranges, "private": private } def add_host(self, obj): """ Parse a vCenter host (ESXi) add to NetBox once all data is gathered. First host is filtered: host has a cluster and is it permitted was host with same name and site already parsed does the host pass the host_include_filter and host_exclude_filter Then all necessary host data will be collected. host model, manufacturer, serial, physical interfaces, virtual interfaces, virtual switches, proxy switches, host port groups, interface VLANs, IP addresses Primary IPv4/6 will be determined by 1. if the interface port group name contains "management" or "mngt" 2. interface is the default route of this host Parameters ---------- obj: vim.HostSystem host object to parse """ name = get_string_or_none(grab(obj, "name")) if name is not None and self.strip_host_domain_name is True: name = name.split(".")[0] # parse data log.debug(f"Parsing vCenter host: {name}") # # Filtering # # manage site and cluster cluster_object = self.get_parent_object_by_class(obj, vim.ClusterComputeResource) cluster_name = get_string_or_none(grab(cluster_object, "name")) if cluster_object is None: log.error(f"Requesting cluster for host '{name}' failed. Skipping.") return # get cluster object nb_cluster_object = self.get_object_from_cache(cluster_object) if log.level == DEBUG3: try: log.info("Cluster data") dump(cluster_object) except Exception as e: log.error(e) # handle standalone hosts if cluster_name == name or (self.strip_host_domain_name is True and cluster_name.split(".")[0] == name): # apply strip_domain_name to cluster as well if activated if self.strip_host_domain_name is True: cluster_name = cluster_name.split(".")[0] log.debug2(f"Host name and cluster name are equal '{cluster_name}'. " f"Assuming this host is a 'standalone' host.") elif nb_cluster_object is None: log.debug(f"Host '{name}' is not part of a permitted cluster. Skipping") return # get a site for this host if self.set_source_name_as_cluster_group is True: group = self.inventory.get_by_data(NBClusterGroup, data={"name": self.name}) else: group = self.get_object_from_cache(self.get_parent_object_by_class(obj, vim.Datacenter)) group_name = grab(group, "data.name") site_name = self.get_site_name(NBDevice, name, f"{group_name}/{cluster_name}") if name in self.processed_host_names.get(site_name, list()): log.warning(f"Host '{name}' for site '{site_name}' already parsed. " "Make sure to use unique host names. Skipping") return # add host to processed list if self.processed_host_names.get(site_name) is None: self.processed_host_names[site_name] = list() self.processed_host_names[site_name].append(name) # filter hosts by name if self.passes_filter(name, self.host_include_filter, self.host_exclude_filter) is False: return # add host as single cluster to cluster list if cluster_name == name: # add cluster to NetBox cluster_data = { "name": cluster_name, "type": {"name": "VMware ESXi"}, "group": group, "site": {"name": site_name} } nb_cluster_object = self.inventory.add_update_object(NBCluster, data=cluster_data, source=self) self.add_object_to_cache(cluster_object, nb_cluster_object) # # Collecting data # # collect all necessary data manufacturer = get_string_or_none(grab(obj, "summary.hardware.vendor")) model = get_string_or_none(grab(obj, "summary.hardware.model")) product_name = get_string_or_none(grab(obj, "summary.config.product.name")) product_version = get_string_or_none(grab(obj, "summary.config.product.version")) platform = f"{product_name} {product_version}" # if the device vendor/model cannot be retrieved (due to problem on the host), # set a dummy value so the host still gets synced if manufacturer is None: manufacturer = "Generic Vendor" if model is None: model = "Generic Model" # get status status = "offline" if get_string_or_none(grab(obj, "summary.runtime.connectionState")) == "connected": status = "active" # prepare identifiers to find asset tag and serial number identifiers = grab(obj, "summary.hardware.otherIdentifyingInfo", fallback=list()) identifier_dict = dict() for item in identifiers: value = grab(item, "identifierValue", fallback="") if len(str(value).strip()) > 0: identifier_dict[grab(item, "identifierType.key")] = str(value).strip() # try to find serial serial = None for serial_num_key in ["SerialNumberTag", "ServiceTag", "EnclosureSerialNumberTag"]: if serial_num_key in identifier_dict.keys(): log.debug2(f"Found {serial_num_key}: {get_string_or_none(identifier_dict.get(serial_num_key))}") if serial is None: serial = get_string_or_none(identifier_dict.get(serial_num_key)) # add asset tag if desired and present asset_tag = None if bool(self.collect_hardware_asset_tag) is True and "AssetTag" in identifier_dict.keys(): banned_tags = ["Default string", "NA", "N/A", "None", "Null", "oem", "o.e.m", "to be filled by o.e.m.", "Unknown"] this_asset_tag = identifier_dict.get("AssetTag") if this_asset_tag.lower() not in [x.lower() for x in banned_tags]: asset_tag = this_asset_tag # get host_tenant_relation tenant_name = self.get_object_relation(name, "host_tenant_relation") # get host_tag_relation host_tags = self.get_object_relation(name, "host_tag_relation") # get vCenter tags host_tags.extend(self.collect_object_tags(obj)) # prepare host data model host_data = { "name": name, "device_type": { "model": model, "manufacturer": { "name": manufacturer } }, "site": {"name": site_name}, "cluster": nb_cluster_object, "status": status } # add data if present if serial is not None: host_data["serial"] = serial if asset_tag is not None: host_data["asset_tag"] = asset_tag if platform is not None: host_data["platform"] = {"name": platform} if tenant_name is not None: host_data["tenant"] = {"name": tenant_name} if len(host_tags) > 0: host_data["tags"] = host_tags # add custom fields if present and configured host_custom_fields = self.get_object_custom_fields(obj) if len(host_custom_fields) > 0: host_data["custom_fields"] = host_custom_fields # iterate over hosts virtual switches, needed to enrich data on physical interfaces self.network_data["vswitch"][name] = dict() for vswitch in grab(obj, "config.network.vswitch", fallback=list()): vswitch_name = unquote(grab(vswitch, "name")) vswitch_pnics = [str(x) for x in grab(vswitch, "pnic", fallback=list())] if vswitch_name is not None: log.debug2(f"Found host vSwitch {vswitch_name}") self.network_data["vswitch"][name][vswitch_name] = { "mtu": grab(vswitch, "mtu"), "pnics": vswitch_pnics } # iterate over hosts proxy switches, needed to enrich data on physical interfaces # also stores data on proxy switch configured mtu which is used for VM interfaces self.network_data["pswitch"][name] = dict() for pswitch in grab(obj, "config.network.proxySwitch", fallback=list()): pswitch_uuid = grab(pswitch, "dvsUuid") pswitch_name = unquote(grab(pswitch, "dvsName")) pswitch_pnics = [str(x) for x in grab(pswitch, "pnic", fallback=list())] if pswitch_uuid is not None: log.debug2(f"Found host proxySwitch {pswitch_name}") self.network_data["pswitch"][name][pswitch_uuid] = { "name": pswitch_name, "mtu": grab(pswitch, "mtu"), "pnics": pswitch_pnics } # iterate over hosts port groups, needed to enrich data on physical interfaces self.network_data["host_pgroup"][name] = dict() for pgroup in grab(obj, "config.network.portgroup", fallback=list()): pgroup_name = grab(pgroup, "spec.name") if pgroup_name is not None: log.debug2(f"Found host portGroup {pgroup_name}") nic_order = grab(pgroup, "computedPolicy.nicTeaming.nicOrder") pgroup_nics = list() if grab(nic_order, "activeNic") is not None: pgroup_nics += nic_order.activeNic if grab(nic_order, "standbyNic") is not None: pgroup_nics += nic_order.standbyNic self.network_data["host_pgroup"][name][pgroup_name] = { "vlan_id": grab(pgroup, "spec.vlanId"), "vswitch": unquote(grab(pgroup, "spec.vswitchName")), "nics": pgroup_nics } # now iterate over all physical interfaces and collect data pnic_data_dict = dict() for pnic in grab(obj, "config.network.pnic", fallback=list()): pnic_name = grab(pnic, "device") pnic_key = grab(pnic, "key") log.debug2("Parsing {}: {}".format(grab(pnic, "_wsdlName"), pnic_name)) pnic_link_speed = grab(pnic, "linkSpeed.speedMb") if pnic_link_speed is None: pnic_link_speed = grab(pnic, "spec.linkSpeed.speedMb") if pnic_link_speed is None: pnic_link_speed = grab(pnic, "validLinkSpecification.0.speedMb") # determine link speed text pnic_description = "" if pnic_link_speed is not None: if pnic_link_speed >= 1000: pnic_description = "%iGb/s " % int(pnic_link_speed / 1000) else: pnic_description = f"{pnic_link_speed}Mb/s " pnic_description = f"{pnic_description} pNIC" pnic_mtu = None pnic_mode = None # check virtual switches for interface data for vs_name, vs_data in self.network_data["vswitch"][name].items(): if pnic_key in vs_data.get("pnics", list()): pnic_description = f"{pnic_description} ({vs_name})" pnic_mtu = vs_data.get("mtu") # check proxy switches for interface data for ps_uuid, ps_data in self.network_data["pswitch"][name].items(): if pnic_key in ps_data.get("pnics", list()): ps_name = ps_data.get("name") pnic_description = f"{pnic_description} ({ps_name})" pnic_mtu = ps_data.get("mtu") pnic_mode = "tagged-all" # check vlans on this pnic pnic_vlans = list() for pg_name, pg_data in self.network_data["host_pgroup"][name].items(): if pnic_name in pg_data.get("nics", list()): pnic_vlans.append({ "name": pg_name, "vid": pg_data.get("vlan_id") }) pnic_data = { "name": unquote(pnic_name), "device": None, # will be set once we found the correct device "mac_address": normalize_mac_address(grab(pnic, "mac")), "enabled": bool(grab(pnic, "linkSpeed")), "description": unquote(pnic_description), "type": NetBoxInterfaceType(pnic_link_speed).get_this_netbox_type() } if pnic_mtu is not None: pnic_data["mtu"] = pnic_mtu if pnic_mode is not None: pnic_data["mode"] = pnic_mode # determine interface mode for non VM traffic NICs if len(pnic_vlans) > 0: vlan_ids = list(set([x.get("vid") for x in pnic_vlans])) if len(vlan_ids) == 1 and vlan_ids[0] == 0: pnic_data["mode"] = "access" elif 4095 in vlan_ids: pnic_data["mode"] = "tagged-all" else: pnic_data["mode"] = "tagged" tagged_vlan_list = list() for pnic_vlan in pnic_vlans: # only add VLANs if port is tagged if pnic_data.get("mode") != "tagged": break # ignore VLAN ID 0 if pnic_vlan.get("vid") == 0: continue tagged_vlan_list.append({ "name": pnic_vlan.get("name"), "vid": pnic_vlan.get("vid"), "site": { "name": site_name } }) if len(tagged_vlan_list) > 0: pnic_data["tagged_vlans"] = tagged_vlan_list pnic_data_dict[pnic_name] = pnic_data host_primary_ip4 = None host_primary_ip6 = None # now iterate over all virtual interfaces and collect data vnic_data_dict = dict() vnic_ips = dict() for vnic in grab(obj, "config.network.vnic", fallback=list()): vnic_name = grab(vnic, "device") log.debug2("Parsing {}: {}".format(grab(vnic, "_wsdlName"), vnic_name)) vnic_portgroup = grab(vnic, "portgroup") vnic_portgroup_data = self.network_data["host_pgroup"][name].get(vnic_portgroup) vnic_portgroup_vlan_id = 0 vnic_dv_portgroup_key = grab(vnic, "spec.distributedVirtualPort.portgroupKey") vnic_dv_portgroup_data = self.network_data["dpgroup"].get(vnic_dv_portgroup_key) vnic_dv_portgroup_data_vlan_ids = list() vnic_description = None vnic_mode = None # get data from local port group if vnic_portgroup_data is not None: vnic_portgroup_vlan_id = vnic_portgroup_data.get("vlan_id") vnic_vswitch = vnic_portgroup_data.get("vswitch") vnic_description = f"{vnic_portgroup} ({vnic_vswitch}, vlan ID: {vnic_portgroup_vlan_id})" vnic_mode = "access" # get data from distributed port group elif vnic_dv_portgroup_data is not None: vnic_description = vnic_dv_portgroup_data.get("name") vnic_dv_portgroup_data_vlan_ids = vnic_dv_portgroup_data.get("vlan_ids") if len(vnic_dv_portgroup_data_vlan_ids) == 1 and vnic_dv_portgroup_data_vlan_ids[0] == 4095: vlan_description = "all vlans" vnic_mode = "tagged-all" else: if len(vnic_dv_portgroup_data.get("vlan_id_ranges")) > 0: vlan_description = "vlan IDs: %s" % ", ".join(vnic_dv_portgroup_data.get("vlan_id_ranges")) else: vlan_description = f"vlan ID: {vnic_dv_portgroup_data_vlan_ids[0]}" if len(vnic_dv_portgroup_data_vlan_ids) == 1 and vnic_dv_portgroup_data_vlan_ids[0] == 0: vnic_mode = "access" else: vnic_mode = "tagged" vnic_dv_portgroup_dswitch_uuid = grab(vnic, "spec.distributedVirtualPort.switchUuid", fallback="NONE") vnic_vswitch = grab(self.network_data, f"pswitch|{name}|{vnic_dv_portgroup_dswitch_uuid}|name", separator="|") if vnic_vswitch is not None: vnic_description = f"{vnic_description} ({vnic_vswitch}, {vlan_description})" # add data vnic_data = { "name": unquote(vnic_name), "device": None, # will be set once we found the correct device "mac_address": normalize_mac_address(grab(vnic, "spec.mac")), "enabled": True, # ESXi vmk interface is enabled by default "mtu": grab(vnic, "spec.mtu"), "type": "virtual" } if vnic_mode is not None: vnic_data["mode"] = vnic_mode if vnic_description is not None: vnic_data["description"] = unquote(vnic_description) else: vnic_description = "" if vnic_portgroup_data is not None and vnic_portgroup_vlan_id != 0: vnic_data["untagged_vlan"] = { "name": unquote(f"ESXi {vnic_portgroup} (ID: {vnic_portgroup_vlan_id}) ({site_name})"), "vid": vnic_portgroup_vlan_id, "site": { "name": site_name } } elif vnic_dv_portgroup_data is not None: tagged_vlan_list = list() for vnic_dv_portgroup_data_vlan_id in vnic_dv_portgroup_data_vlan_ids: if vnic_mode != "tagged": break if vnic_dv_portgroup_data_vlan_id == 0: continue tagged_vlan_list.append({ "name": unquote(f"{vnic_dv_portgroup_data.get('name')}-{vnic_dv_portgroup_data_vlan_id}"), "vid": vnic_dv_portgroup_data_vlan_id, "site": { "name": site_name } }) if len(tagged_vlan_list) > 0: vnic_data["tagged_vlans"] = tagged_vlan_list vnic_data_dict[vnic_name] = vnic_data # check if interface has the default route or is described as management interface vnic_is_primary = False for mgnt_match in self.host_management_interface_match: if mgnt_match in vnic_description.lower(): vnic_is_primary = True if grab(vnic, "spec.ipRouteSpec") is not None: vnic_is_primary = True if vnic_ips.get(vnic_name) is None: vnic_ips[vnic_name] = list() int_v4 = "{}/{}".format(grab(vnic, "spec.ip.ipAddress"), grab(vnic, "spec.ip.subnetMask")) if ip_valid_to_add_to_netbox(int_v4, self.permitted_subnets, self.excluded_subnets, vnic_name) is True: vnic_ips[vnic_name].append(int_v4) if vnic_is_primary is True and host_primary_ip4 is None: host_primary_ip4 = int_v4 for ipv6_entry in grab(vnic, "spec.ip.ipV6Config.ipV6Address", fallback=list()): int_v6 = "{}/{}".format(grab(ipv6_entry, "ipAddress"), grab(ipv6_entry, "prefixLength")) if ip_valid_to_add_to_netbox(int_v6, self.permitted_subnets, self.excluded_subnets, vnic_name) is True: vnic_ips[vnic_name].append(int_v6) # set first valid IPv6 address as primary IPv6 # not the best way but maybe we can find more information in "spec.ipRouteSpec" # about default route and we could use that to determine the correct IPv6 address if vnic_is_primary is True and host_primary_ip6 is None: host_primary_ip6 = int_v6 # add host to inventory self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, vnic_data=vnic_data_dict, nic_ips=vnic_ips, p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6) return def add_virtual_machine(self, obj): """ Parse a vCenter VM add to NetBox once all data is gathered. VMs are parsed twice. First only "online" VMs are parsed and added. In the second round also "offline" VMs will be parsed. This helps of VMs are cloned and used for upgrades but then have the same name. First VM is filtered: VM has a cluster and is it permitted was VM with same name and cluster already parsed does the VM pass the vm_include_filter and vm_exclude_filter Then all necessary VM data will be collected. platform, virtual interfaces, virtual cpu/disk/memory interface VLANs, IP addresses Primary IPv4/6 will be determined by interface that provides the default route for this VM Note: IP address information can only be extracted if guest tools are installed and running. Parameters ---------- obj: vim.VirtualMachine virtual machine object to parse """ name = get_string_or_none(grab(obj, "name")) if name is not None and self.strip_vm_domain_name is True: name = name.split(".")[0] # # Filtering # # get VM UUID vm_uuid = grab(obj, "config.uuid") if vm_uuid is None or vm_uuid in self.processed_vm_uuid: return log.debug(f"Parsing vCenter VM: {name}") # get VM power state status = "active" if get_string_or_none(grab(obj, "runtime.powerState")) == "poweredOn" else "offline" # check if vm is template template = grab(obj, "config.template") if bool(self.skip_vm_templates) is True and template is True: log.debug2(f"VM '{name}' is a template. Skipping") return if bool(self.skip_srm_placeholder_vms) is True \ and f"{grab(obj, 'config.managedBy.extensionKey')}".startswith("com.vmware.vcDr"): log.debug2(f"VM '{name}' is a SRM placeholder VM. Skipping") return # ignore offline VMs during first run if self.parsing_vms_the_first_time is True and status == "offline": log.debug2(f"Ignoring {status} VM '{name}' on first run") return # add to processed VMs self.processed_vm_uuid.append(vm_uuid) parent_host = self.get_parent_object_by_class(grab(obj, "runtime.host"), vim.HostSystem) cluster = self.get_parent_object_by_class(parent_host, vim.ClusterComputeResource) if self.set_source_name_as_cluster_group is True: group = self.inventory.get_by_data(NBClusterGroup, data={"name": self.name}) else: group = self.get_parent_object_by_class(cluster, vim.Datacenter) if None in [parent_host, cluster, group]: log.error(f"Requesting host or cluster for Virtual Machine '{name}' failed. Skipping.") return parent_name = grab(parent_host, "name") cluster_name = grab(cluster, "name") cluster_full_name = f"{group.name}/{cluster_name}" nb_cluster_object = self.get_object_from_cache(cluster) # honor strip_host_domain_name if cluster_name is not None and self.strip_host_domain_name is True and \ parent_name.split(".")[0] == cluster_name.split(".")[0]: cluster_name = cluster_name.split(".")[0] cluster_full_name = f"{group.name}/{cluster_name}" # check VM cluster if nb_cluster_object is None: log.debug(f"Virtual machine '{name}' is not part of a permitted cluster. Skipping") return if name in self.processed_vm_names.get(cluster_full_name, list()): log.warning(f"Virtual machine '{name}' for cluster '{cluster_full_name}' already parsed. " "Make sure to use unique VM names. Skipping") return # add vm to processed list if self.processed_vm_names.get(cluster_full_name) is None: self.processed_vm_names[cluster_full_name] = list() self.processed_vm_names[cluster_full_name].append(name) # filter VMs by name if self.passes_filter(name, self.vm_include_filter, self.vm_exclude_filter) is False: return # # Collect data # # check if cluster is a Standalone ESXi site_name = nb_cluster_object.get_site_name() if site_name is None: site_name = self.get_site_name(NBCluster, cluster_full_name) # first check against vm_platform_relation platform = get_string_or_none(grab(obj, "config.guestFullName")) platform = get_string_or_none(grab(obj, "guest.guestFullName", fallback=platform)) if platform is not None: platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) hardware_devices = grab(obj, "config.hardware.device", fallback=list()) disk = int(sum([getattr(comp, "capacityInKB", 0) for comp in hardware_devices if isinstance(comp, vim.vm.device.VirtualDisk) ]) / 1024 / 1024) annotation = None if bool(self.skip_vm_comments) is False: annotation = get_string_or_none(grab(obj, "config.annotation")) # assign vm_tenant_relation tenant_name = self.get_object_relation(name, "vm_tenant_relation") # assign vm_tag_relation vm_tags = self.get_object_relation(name, "vm_tag_relation") # get vCenter tags vm_tags.extend(self.collect_object_tags(obj)) vm_data = { "name": name, "cluster": nb_cluster_object, "status": status, "memory": grab(obj, "config.hardware.memoryMB"), "vcpus": grab(obj, "config.hardware.numCPU"), "disk": disk } if platform is not None: vm_data["platform"] = {"name": platform} if annotation is not None: vm_data["comments"] = annotation if tenant_name is not None: vm_data["tenant"] = {"name": tenant_name} if len(vm_tags) > 0: vm_data["tags"] = vm_tags # add custom fields if present and configured vm_custom_fields = self.get_object_custom_fields(obj) if len(vm_custom_fields) > 0: vm_data["custom_fields"] = vm_custom_fields vm_primary_ip4 = None vm_primary_ip6 = None vm_default_gateway_ip4 = None vm_default_gateway_ip6 = None # check vm routing to determine which is the default interface for each IP version for route in grab(obj, "guest.ipStack.0.ipRouteConfig.ipRoute", fallback=list()): # we found a default route if grab(route, "prefixLength") == 0: try: ip_a = ip_address(grab(route, "network")) except ValueError: continue try: gateway_ip_address = ip_address(grab(route, "gateway.ipAddress")) except ValueError: continue if ip_a.version == 4 and gateway_ip_address is not None: log.debug2(f"Found default IPv4 gateway {gateway_ip_address}") vm_default_gateway_ip4 = gateway_ip_address elif ip_a.version == 6 and gateway_ip_address is not None: log.debug2(f"Found default IPv6 gateway {gateway_ip_address}") vm_default_gateway_ip6 = gateway_ip_address nic_data = dict() nic_ips = dict() # track MAC addresses in order add dummy guest interfaces processed_interface_macs = list() # get VM interfaces for vm_device in hardware_devices: # sample: https://github.com/vmware/pyvmomi-community-samples/blob/master/samples/getvnicinfo.py # not a network interface if not isinstance(vm_device, vim.vm.device.VirtualEthernetCard): continue int_mac = normalize_mac_address(grab(vm_device, "macAddress")) device_class = grab(vm_device, "_wsdlName") log.debug2(f"Parsing device {device_class}: {int_mac}") device_backing = grab(vm_device, "backing") # set defaults int_mtu = None int_mode = None int_network_vlan_ids = None int_network_vlan_id_ranges = None int_network_name = None int_network_private = False processed_interface_macs.append(int_mac) # get info from local vSwitches if isinstance(device_backing, vim.vm.device.VirtualEthernetCard.NetworkBackingInfo): int_network_name = get_string_or_none(grab(device_backing, "deviceName")) int_host_pgroup = grab(self.network_data, f"host_pgroup|{parent_name}|{int_network_name}", separator="|") if int_host_pgroup is not None: int_network_vlan_ids = [int_host_pgroup.get("vlan_id")] int_network_vlan_id_ranges = [str(int_host_pgroup.get("vlan_id"))] int_vswitch_name = int_host_pgroup.get("vswitch") int_vswitch_data = grab(self.network_data, f"vswitch|{parent_name}|{int_vswitch_name}", separator="|") if int_vswitch_data is not None: int_mtu = int_vswitch_data.get("mtu") # get info from distributed port group else: dvs_portgroup_key = grab(device_backing, "port.portgroupKey", fallback="None") int_portgroup_data = grab(self.network_data, f"dpgroup|{dvs_portgroup_key}", separator="|") if int_portgroup_data is not None: int_network_name = grab(int_portgroup_data, "name") int_network_vlan_ids = grab(int_portgroup_data, "vlan_ids") if len(grab(int_portgroup_data, "vlan_id_ranges")) > 0: int_network_vlan_id_ranges = grab(int_portgroup_data, "vlan_id_ranges") else: int_network_vlan_id_ranges = [str(int_network_vlan_ids[0])] int_network_private = grab(int_portgroup_data, "private") int_dvswitch_uuid = grab(device_backing, "port.switchUuid") int_dvswitch_data = grab(self.network_data, f"pswitch|{parent_name}|{int_dvswitch_uuid}", separator="|") if int_dvswitch_data is not None: int_mtu = int_dvswitch_data.get("mtu") int_connected = grab(vm_device, "connectable.connected", fallback=False) int_label = grab(vm_device, "deviceInfo.label", fallback="") int_name = "vNIC {}".format(int_label.split(" ")[-1]) int_full_name = int_name if int_network_name is not None: int_full_name = f"{int_full_name} ({int_network_name})" int_description = f"{int_label} ({device_class})" if int_network_vlan_ids is not None: if len(int_network_vlan_ids) == 1 and int_network_vlan_ids[0] == 4095: vlan_description = "all vlans" int_mode = "tagged-all" else: vlan_description = "vlan ID: %s" % ", ".join(int_network_vlan_id_ranges) if len(int_network_vlan_ids) == 1: int_mode = "access" else: int_mode = "tagged" if int_network_private is True: vlan_description = f"{vlan_description} (private)" int_description = f"{int_description} ({vlan_description})" # find corresponding guest NIC and get IP addresses and connected status for guest_nic in grab(obj, "guest.net", fallback=list()): # get matching guest NIC if int_mac != normalize_mac_address(grab(guest_nic, "macAddress")): continue int_connected = grab(guest_nic, "connected", fallback=int_connected) if nic_ips.get(int_full_name) is None: nic_ips[int_full_name] = list() # grab all valid interface IP addresses for int_ip in grab(guest_nic, "ipConfig.ipAddress", fallback=list()): int_ip_address = f"{int_ip.ipAddress}/{int_ip.prefixLength}" if ip_valid_to_add_to_netbox(int_ip_address, self.permitted_subnets, self.excluded_subnets, int_full_name) is False: continue nic_ips[int_full_name].append(int_ip_address) # check if primary gateways are in the subnet of this IP address # if it matches IP gets chosen as primary IP if vm_default_gateway_ip4 is not None and \ vm_default_gateway_ip4 in ip_interface(int_ip_address).network and \ vm_primary_ip4 is None: vm_primary_ip4 = int_ip_address if vm_default_gateway_ip6 is not None and \ vm_default_gateway_ip6 in ip_interface(int_ip_address).network and \ vm_primary_ip6 is None: vm_primary_ip6 = int_ip_address vm_nic_data = { "name": unquote(int_full_name), "virtual_machine": None, "mac_address": int_mac, "description": unquote(int_description), "enabled": int_connected, } if int_mtu is not None: vm_nic_data["mtu"] = int_mtu if int_mode is not None: vm_nic_data["mode"] = int_mode if int_network_vlan_ids is not None and int_mode != "tagged-all": if len(int_network_vlan_ids) == 1 and int_network_vlan_ids[0] != 0: vm_nic_data["untagged_vlan"] = { "name": unquote(int_network_name), "vid": int_network_vlan_ids[0], "site": { "name": site_name } } else: tagged_vlan_list = list() for int_network_vlan_id in int_network_vlan_ids: if int_network_vlan_id == 0: continue tagged_vlan_list.append({ "name": unquote(f"{int_network_name}-{int_network_vlan_id}"), "vid": int_network_vlan_id, "site": { "name": site_name } }) if len(tagged_vlan_list) > 0: vm_nic_data["tagged_vlans"] = tagged_vlan_list nic_data[int_full_name] = vm_nic_data # find dummy guest NIC interfaces if self.sync_vm_dummy_interfaces is True: for guest_nic in grab(obj, "guest.net", fallback=list()): # get matching guest NIC MAC guest_nic_mac = normalize_mac_address(grab(guest_nic, "macAddress")) # skip interfaces of MAC addresses for already known interfaces if guest_nic_mac is None or guest_nic_mac in processed_interface_macs: continue processed_interface_macs.append(guest_nic_mac) int_full_name = "vNIC Dummy-{}".format("".join(guest_nic_mac.split(":")[-2:])) log.debug2(f"Parsing dummy network device: {guest_nic_mac}") if nic_ips.get(int_full_name) is None: nic_ips[int_full_name] = list() # grab all valid interface IP addresses for int_ip in grab(guest_nic, "ipConfig.ipAddress", fallback=list()): int_ip_address = f"{int_ip.ipAddress}/{int_ip.prefixLength}" if ip_valid_to_add_to_netbox(int_ip_address, self.permitted_subnets, self.excluded_subnets, int_full_name) is False: continue nic_ips[int_full_name].append(int_ip_address) vm_nic_data = { "name": int_full_name, "virtual_machine": None, "mac_address": guest_nic_mac, "enabled": grab(guest_nic, "connected", fallback=False), } if len(nic_ips.get(int_full_name, list())) == 0: log.debug(f"Dummy network interface '{int_full_name}' has no IP addresses assigned. Skipping") continue nic_data[int_full_name] = vm_nic_data # add VM to inventory self.add_device_vm_to_inventory(NBVM, object_data=vm_data, vnic_data=nic_data, nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6) return def update_basic_data(self): """ Returns ------- """ # add source identification tag self.inventory.add_update_object(NBTag, data={ "name": self.source_tag, "description": f"Marks objects synced from vCenter '{self.name}' " f"({self.host_fqdn}) to this NetBox Instance." }) # update virtual site if present this_site_object = self.inventory.get_by_data(NBSite, data={"name": self.site_name}) if this_site_object is not None: this_site_object.update(data={ "name": self.site_name, "comments": f"A default virtual site created to house objects " "that have been synced from this vCenter instance " "and have no predefined site assigned." }) server_role_object = self.inventory.get_by_data(NBDeviceRole, data={"name": "Server"}) if server_role_object is not None: server_role_object.update(data={ "name": "Server", "color": "9e9e9e", "vm_role": True }) # EOF