From 508962c34151378e9bf667b1d71acf71bbf9414d Mon Sep 17 00:00:00 2001 From: "ricardo.bartels@telekom.de" Date: Wed, 13 Oct 2021 01:02:09 +0200 Subject: [PATCH] enhances VLAN assignment to interfaces based on ip address prefixes #40 refs: #40 --- .../sources/check_redfish/import_inventory.py | 25 +- module/sources/common/source_base.py | 499 ++++++++++++------ module/sources/vmware/connection.py | 171 ++---- 3 files changed, 387 insertions(+), 308 deletions(-) diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index da0b74b..f02b7d3 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -810,8 +810,17 @@ class CheckRedfish(SourceBase): # collect ip addresses nic_ips[port_name] = list() - nic_ips[port_name].extend(grab(nic_port, "ipv4_addresses", fallback=list())) - nic_ips[port_name].extend(grab(nic_port, "ipv6_addresses", fallback=list())) + for ipv4_address in grab(nic_port, "ipv4_addresses", fallback=list()): + if ip_valid_to_add_to_netbox(ipv4_address, self.permitted_subnets, port_name) is False: + continue + + nic_ips[port_name].append(ipv4_address) + + for ipv6_address in grab(nic_port, "ipv6_addresses", fallback=list()): + if ip_valid_to_add_to_netbox(ipv6_address, self.permitted_subnets, port_name) is False: + continue + + nic_ips[port_name].append(ipv6_address) data = self.map_object_interfaces_to_current_interfaces(self.device_object, port_data_dict) @@ -830,9 +839,7 @@ class CheckRedfish(SourceBase): del (port_data["mac_address"]) # create or update interface with data - if nic_object is None: - nic_object = self.inventory.add_object(NBInterface, data=port_data, source=self) - else: + if nic_object is not None: if self.overwrite_interface_name is False and port_data.get("name") is not None: del(port_data["name"]) @@ -849,13 +856,7 @@ class CheckRedfish(SourceBase): # update nic object nic_object.update(data=data_to_update, source=self) - # check for interface ips - for nic_ip in nic_ips.get(port_name, list()): - - if ip_valid_to_add_to_netbox(nic_ip, self.permitted_subnets, port_name) is False: - continue - - self.add_ip_address(nic_ip, nic_object, grab(self.device_object, "data.site.data.name")) + self.add_update_interface(nic_object, self.device_object, port_data, nic_ips.get(port_name, list())) def update_manager(self): diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 3bc60f8..cfa7c6d 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -9,7 +9,17 @@ from ipaddress import ip_interface, ip_address, IPv6Address, IPv4Address, IPv6Network, IPv4Network -from module.netbox.inventory import NBDevice, NBVM, NBInterface, NBVMInterface, NBSite, NBPrefix, NBIPAddress, NBVLAN +from module.netbox.inventory import ( + NetBoxObject, + NBDevice, + NBVM, + NBInterface, + NBVMInterface, + NBSite, + NBPrefix, + NBIPAddress, + NBVLAN +) from module.common.logging import get_logger from module.common.misc import grab @@ -171,7 +181,7 @@ class SourceBase: return if not isinstance(ip_to_match, (IPv4Address, IPv6Address)): - raise ValueError("Value of 'ip_to_match' needs to be an IPv4Address or IPv6Address this_object.") + raise ValueError("Value of 'ip_to_match' needs to be an IPv4Address or IPv6Address object.") site_object = None if site_name is not None: @@ -180,6 +190,7 @@ class SourceBase: if site_object is None: log.error(f"Unable to find site '{site_name}' for IP {ip_to_match}. " "Skipping to find Prefix for this IP.") + return current_longest_matching_prefix_length = 0 current_longest_matching_prefix = None @@ -200,207 +211,276 @@ class SourceBase: return current_longest_matching_prefix - def add_ip_address(self, nic_ip, nic_object, site): + def add_update_interface(self, interface_object, device_object, interface_data, interface_ips=None): """ - Try to add an IP address to an interface object. - - Prefix length: - * If the 'nic_ip' does not contain a prefix length then a matching prefix will be looked up. - First a prefix in the same site and the globally. If no existing prefix matches the ip - the ip address can't be added. - * If the 'nic_ip' contains a prefix length then a matching prefix will be looked up the same - way as described above. But even if no existing matching prefix the IP address will be - added to NetBox - - Also some sanity checking will be performed: - * is this ip assigned to another device - * does the matching prefix length match the supplied prefix length + Adds/Updates an interface to/of a NBVM or NBDevice including IP addresses. + Validates/enriches data in following order: + * extract untagged_vlan from data + * try to find tagged_vlan_objects + * add/update interface + * loop over list of IPs and add each IP to the Interface if + * IP is valid + * does not belong to another active interface + * extract prefixes belonging to the IPs + * use VLAN data from prefixes to match/add to the interface Parameters ---------- - nic_ip: str - IP address to add - nic_object: NBInterface, NBVMInterface - The NetBox interface object to add the ip - site: str - The name of the site + interface_object: NBVMInterface, NBInterface, None + object handle of the current interface (if existent, otherwise None) + device_object: NBVM, NBDevice + device object handle this interface belongs to + interface_data: dict + dictionary with interface attributes to add to this interface + interface_ips: list + list of ip addresses which are assigned to this interface Returns ------- - this_ip_object: NBIPAddress - The newly created/updated NetBox IP address object + objects: tuple((NBVMInterface, NBInterface), list) + tuple with interface object that was added/updated and a list of ip address objects which were + added to this interface """ - # get IP and prefix length - try: - if "/" in nic_ip: - ip_object = ip_interface(nic_ip) - else: - ip_object = ip_address(nic_ip) - except ValueError: - log.error(f"IP '{nic_ip}' ({nic_object.get_display_name()}) does not appear " - "to be a valid IP address. Skipping!") - return - - log.debug2(f"Trying to find prefix for IP: {ip_object}") - - possible_ip_vrf = None - possible_ip_tenant = None - - # test for site prefixes first - matching_site_name = site - matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object, matching_site_name) - - # nothing was found then check prefixes with site name - if matching_ip_prefix is None: - matching_site_name = None - matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object) - - # matching prefix found, get data from prefix - if matching_ip_prefix is not None: - - this_prefix = grab(matching_ip_prefix, f"data.{NBPrefix.primary_key}") - if matching_site_name is None: - log.debug2(f"Found IP '{ip_object}' matches global prefix '{this_prefix}'") - else: - log.debug2(f"Found IP '{ip_object}' matches site '{matching_site_name}' prefix " - f"'{this_prefix}'") - - # check if prefix net size and ip address prefix length match - if not isinstance(ip_object, (IPv6Address, IPv4Address)) and \ - this_prefix.prefixlen != ip_object.network.prefixlen: - log.warning(f"IP prefix length of '{ip_object}' ({nic_object.get_display_name()}) " - f"does not match network prefix length '{this_prefix}'!") - - # get prefix data - possible_ip_vrf = grab(matching_ip_prefix, "data.vrf") - prefix_tenant = grab(matching_ip_prefix, "data.tenant") - prefix_vlan = grab(matching_ip_prefix, "data.vlan") - - # get NIC VLAN data - nic_vlan = grab(nic_object, "data.untagged_vlan") - nic_vlan_tenant = None - if nic_vlan is not None: - nic_vlan_tenant = grab(nic_vlan, "data.tenant") - - # check if interface VLAN matches prefix VLAN for IP address - - if isinstance(nic_vlan, NBVLAN) and isinstance(prefix_vlan, NBPrefix) and nic_vlan != prefix_vlan: - log.warning(f"Prefix vlan '{prefix_vlan.get_display_name()}' does not match interface vlan " - f"'{nic_vlan.get_display_name()}' for '{nic_object.get_display_name()}") - - if prefix_tenant is not None: - possible_ip_tenant = prefix_tenant - elif nic_vlan_tenant is not None: - possible_ip_tenant = nic_vlan_tenant + if not isinstance(interface_data, dict): + log.error(f"Attribute 'interface_data' must be a dict() got {type(interface_data)}.") + return None + if type(device_object) == NBVM: + interface_class = NBVMInterface + elif type(device_object) == NBDevice: + interface_class = NBInterface + elif device_object is None: + log.error(f"No device/VM object submitted to attach interface '{grab(interface_data, 'name')}' to.") + return None else: - log_text = f"No matching NetBox prefix for '{ip_object}' found" + log.error(f"Device object for interface '{grab(interface_data, 'name')}' must be a 'NBVM' or 'NBDevice'. " + f"Got {type(device_object)}") + return None - if type(ip_object) in [IPv6Address, IPv4Address]: - log.warning(f"{log_text}. Unable to add IP address to NetBox.") - return None + if interface_object is not None and not isinstance(interface_object, interface_class): + log.error(f"Interface object '{grab(interface_data, 'name')}' must be a '{interface_class.name}'.") + return None + + site_name = grab(device_object, "data.site.data.name") + + # get vlans from interface data and remove it for now from interface data dict + # vlans get added later once we have the prefixes for the IP addresses + untagged_vlan = interface_data.get("untagged_vlan") + if untagged_vlan is not None: + del(interface_data["untagged_vlan"]) + + # check tagged vlans and try to find NetBox VLAN objects based on VLAN id and site + tagged_vlans = list() + for tagged_vlan in interface_data.get("tagged_vlans") or list(): + + if isinstance(tagged_vlan, NBVLAN): + tagged_vlans.append(tagged_vlan) else: - log.debug2(log_text) + tagged_vlans.append(self.get_vlan_object_if_exists(tagged_vlan, site_name)) - if matching_ip_prefix is not None and type(ip_object) in [IPv6Address, IPv4Address]: - this_prefix = grab(matching_ip_prefix, "data.prefix") - if type(this_prefix) in [IPv4Network, IPv6Network]: - ip_object = ip_interface(f"{ip_object}/{this_prefix.prefixlen}") - else: - log.warning(f"{matching_ip_prefix.name} got wrong format. Unable to add IP to NetBox") - return None + if len(tagged_vlans) > 0: + interface_data["tagged_vlans"] = tagged_vlans - # try to find matching IP address object - this_ip_object = None - skip_this_ip = False - for ip in self.inventory.get_all_items(NBIPAddress): + # add object to interface + interface_data[interface_class.secondary_key] = device_object - # check if address matches (without prefix length) - ip_address_string = grab(ip, "data.address", fallback="") + # create or update interface with data + if interface_object is None: + interface_object = self.inventory.add_object(interface_class, data=interface_data, source=self) + else: + interface_object.update(data=interface_data, source=self) - # not a matching address - if not ip_address_string.startswith(f"{ip_object.ip.compressed}/"): + ip_address_objects = list() + matching_ip_prefixes = list() + + # add all interface IPs + for nic_ip in interface_ips or list(): + + # get IP and prefix length + try: + if "/" in nic_ip: + ip_object = ip_interface(nic_ip) + else: + ip_object = ip_address(nic_ip) + except ValueError: + log.error(f"IP '{nic_ip}' ({interface_object.get_display_name()}) does not appear " + "to be a valid IP address. Skipping!") continue - current_nic = grab(ip, "data.assigned_object_id") + log.debug2(f"Trying to find prefix for IP: {ip_object}") - # is it our current ip interface? - if current_nic == nic_object: - this_ip_object = ip - break + possible_ip_vrf = None + possible_ip_tenant = None - # check if IP has the same prefix - # continue if - # * both are in global scope - # * both ara part of the same vrf - if possible_ip_vrf != grab(ip, "data.vrf"): + # test for site prefixes first + matching_site_name = site_name + matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object, matching_site_name) + + # nothing was found then check prefixes without site name + if matching_ip_prefix is None: + matching_site_name = None + matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object) + + # matching prefix found, get data from prefix + if matching_ip_prefix is not None: + + this_prefix = grab(matching_ip_prefix, f"data.{NBPrefix.primary_key}") + if matching_site_name is None: + log.debug2(f"Found IP '{ip_object}' matches global prefix '{this_prefix}'") + else: + log.debug2(f"Found IP '{ip_object}' matches site '{matching_site_name}' prefix " + f"'{this_prefix}'") + + # check if prefix net size and ip address prefix length match + if not isinstance(ip_object, (IPv6Address, IPv4Address)) and \ + this_prefix.prefixlen != ip_object.network.prefixlen: + log.warning(f"IP prefix length of '{ip_object}' ({interface_object.get_display_name()}) " + f"does not match network prefix length '{this_prefix}'!") + + possible_ip_vrf = grab(matching_ip_prefix, "data.vrf") + possible_ip_tenant = grab(matching_ip_prefix, "data.tenant") + + matching_ip_prefixes.append(matching_ip_prefix) + + else: + log_text = f"No matching NetBox prefix for '{ip_object}' found" + + # check if IP address is of type IP interface (includes prefix length) + if type(ip_object) in [IPv6Address, IPv4Address]: + log.warning(f"{log_text}. Unable to add IP address to NetBox.") + continue + else: + log.debug2(log_text) + + # try to add prefix length to IP address if present + if matching_ip_prefix is not None and type(ip_object) in [IPv6Address, IPv4Address]: + this_prefix = grab(matching_ip_prefix, "data.prefix") + if type(this_prefix) in [IPv4Network, IPv6Network]: + ip_object = ip_interface(f"{ip_object}/{this_prefix.prefixlen}") + else: + log.warning(f"{matching_ip_prefix.name} got wrong format. Unable to add IP to NetBox") + continue + + # try to find matching IP address object + this_ip_object = None + skip_this_ip = False + for ip in self.inventory.get_all_items(NBIPAddress): + + # check if address matches (without prefix length) + ip_address_string = grab(ip, "data.address", fallback="") + + # not a matching address + if not ip_address_string.startswith(f"{ip_object.ip.compressed}/"): + continue + + current_nic = grab(ip, "data.assigned_object_id") + + # is it our current ip interface? + if current_nic == interface_object: + this_ip_object = ip + break + + # check if IP has the same prefix + # continue if + # * both are in global scope + # * both are part of the same vrf + if possible_ip_vrf != grab(ip, "data.vrf"): + continue + + # IP address is not assigned to any interface + if not isinstance(current_nic, (NBInterface, NBVMInterface)): + this_ip_object = ip + break + + # get current IP interface status + current_nic_enabled = grab(current_nic, "data.enabled", fallback=True) + this_nic_enabled = grab(interface_object, "data.enabled", fallback=True) + + if current_nic_enabled is True and this_nic_enabled is False: + log.debug(f"Current interface '{current_nic.get_display_name()}' for IP '{ip_object}'" + f" is enabled and this one '{interface_object.get_display_name()}' is disabled. " + f"IP assignment skipped!") + skip_this_ip = True + break + + if current_nic_enabled is False and this_nic_enabled is True: + log.debug(f"Current interface '{current_nic.get_display_name()}' for IP '{ip_object}'" + f" is disabled and this one '{interface_object.get_display_name()}' is enabled. " + f"IP will be assigned to this interface.") + + this_ip_object = ip + + if current_nic_enabled == this_nic_enabled: + state = "enabled" if this_nic_enabled is True else "disabled" + log.warning(f"Current interface '{current_nic.get_display_name()}' for IP " + f"'{ip_object}' and this one '{interface_object.get_display_name()}' are " + f"both {state}. " + f"IP assignment skipped because it is unclear which one is the correct one!") + skip_this_ip = True + break + + if skip_this_ip is True: continue - # IP address is not assigned to any interface - if not isinstance(current_nic, (NBInterface, NBVMInterface)): - this_ip_object = ip + nic_ip_data = { + "address": ip_object.compressed, + "assigned_object_id": interface_object, + } + + if not isinstance(this_ip_object, NBIPAddress): + log.debug(f"No existing {NBIPAddress.name} object found. Creating a new one.") + + if possible_ip_vrf is not None: + nic_ip_data["vrf"] = possible_ip_vrf + if possible_ip_tenant is not None: + nic_ip_data["tenant"] = possible_ip_tenant + + this_ip_object = self.inventory.add_object(NBIPAddress, data=nic_ip_data, source=self) + + # update IP address with additional data if not already present + else: + + log.debug2(f"Found existing NetBox {NBIPAddress.name} object: {this_ip_object.get_display_name()}") + + if grab(this_ip_object, "data.vrf") is None and possible_ip_vrf is not None: + nic_ip_data["vrf"] = possible_ip_vrf + + if grab(this_ip_object, "data.tenant") is None and possible_ip_tenant is not None: + nic_ip_data["tenant"] = possible_ip_tenant + + this_ip_object.update(data=nic_ip_data, source=self) + + ip_address_objects.append(this_ip_object) + + matching_untagged_vlan = None + + for matching_prefix in matching_ip_prefixes: + + prefix_vlan = grab(matching_prefix, "data.vlan") + if prefix_vlan is None: + continue + + if untagged_vlan is None or str(grab(prefix_vlan, "data.vid")) == str(untagged_vlan.get("vid")): + + log.debug(f"Found matching prefix VLAN {prefix_vlan.get_display_name()} for " + f"untagged interface VLAN.") + + matching_untagged_vlan = prefix_vlan break - # get current IP interface status - current_nic_enabled = grab(current_nic, "data.enabled", fallback=True) - this_nic_enabled = grab(nic_object, "data.enabled", fallback=True) + # try to find vlan object if no matching prefix VLAN ws found + if matching_untagged_vlan is None: + matching_untagged_vlan = self.get_vlan_object_if_exists(untagged_vlan, site_name) - if current_nic_enabled is True and this_nic_enabled is False: - log.debug(f"Current interface '{current_nic.get_display_name()}' for IP '{ip_object}'" - f" is enabled and this one '{nic_object.get_display_name()}' is disabled. " - f"IP assignment skipped!") - skip_this_ip = True - break + if matching_untagged_vlan is not None: + vlan_interface_data = dict({"untagged_vlan": matching_untagged_vlan}) + if grab(interface_object, "data.mode") is None: + vlan_interface_data["mode"] = "access" - if current_nic_enabled is False and this_nic_enabled is True: - log.debug(f"Current interface '{current_nic.get_display_name()}' for IP '{ip_object}'" - f" is disabled and this one '{nic_object.get_display_name()}' is enabled. " - f"IP will be assigned to this interface.") + interface_object.update(data=vlan_interface_data, source=self) - this_ip_object = ip - - if current_nic_enabled == this_nic_enabled: - state = "enabled" if this_nic_enabled is True else "disabled" - log.warning(f"Current interface '{current_nic.get_display_name()}' for IP " - f"'{ip_object}' and this one '{nic_object.get_display_name()}' are " - f"both {state}. " - f"IP assignment skipped because it is unclear which one is the correct one!") - skip_this_ip = True - break - - if skip_this_ip is True: - return - - nic_ip_data = { - "address": ip_object.compressed, - "assigned_object_id": nic_object, - } - - if not isinstance(this_ip_object, NBIPAddress): - log.debug(f"No existing {NBIPAddress.name} object found. Creating a new one.") - - if possible_ip_vrf is not None: - nic_ip_data["vrf"] = possible_ip_vrf - if possible_ip_tenant is not None: - nic_ip_data["tenant"] = possible_ip_tenant - - this_ip_object = self.inventory.add_object(NBIPAddress, data=nic_ip_data, source=self) - - # update IP address with additional data if not already present - else: - - log.debug2(f"Found existing NetBox {NBIPAddress.name} object: {this_ip_object.get_display_name()}") - - if grab(this_ip_object, "data.vrf") is None and possible_ip_vrf is not None: - nic_ip_data["vrf"] = possible_ip_vrf - - if grab(this_ip_object, "data.tenant") is None and possible_ip_tenant is not None: - nic_ip_data["tenant"] = possible_ip_tenant - - this_ip_object.update(data=nic_ip_data, source=self) - - return this_ip_object + return interface_object, ip_address_objects @staticmethod def patch_data(object_to_patch, data, overwrite=False): @@ -434,4 +514,73 @@ class SourceBase: return data_to_update + def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): + """ + This function will try to find a matching VLAN object based on 'vlan_data' + Will return matching objects in following order: + * exact match: VLAN id and site match + * global match: VLAN id matches but the VLAN has no site assigned + If nothing matches the input data from 'vlan_data' will be returned + + Parameters + ---------- + vlan_data: dict + dict with NBVLAN data attributes + vlan_site: str + name of site the VLAN could be present + + Returns + ------- + (NBVLAN, dict, None): matching VLAN object, dict or None (content of vlan_data) if no match found + + """ + + if vlan_data is None: + return None + + if not isinstance(vlan_data, dict): + raise ValueError("Value of 'vlan_data' needs to be a dict.") + + # check existing Devices for matches + log.debug2(f"Trying to find a {NBVLAN.name} based on the VLAN id '{vlan_data.get('vid')}'") + + if vlan_data.get("vid") is None: + log.debug("No VLAN id set in vlan_data while trying to find matching VLAN.") + return vlan_data + + if vlan_site is None: + vlan_site = self.inventory.get_by_data(NBSite, data=vlan_data.get("site")) + + return_data = vlan_data + vlan_object_including_site = None + vlan_object_without_site = None + + for vlan in self.inventory.get_all_items(NBVLAN): + + if grab(vlan, "data.vid") != vlan_data.get("vid"): + continue + + if vlan_site is not None and grab(vlan, "data.site") == vlan_site: + vlan_object_including_site = vlan + + if grab(vlan, "data.site") is None: + vlan_object_without_site = vlan + + if isinstance(vlan_object_including_site, NetBoxObject): + return_data = vlan_object_including_site + log.debug2("Found a exact matching %s object: %s" % + (vlan_object_including_site.name, + vlan_object_including_site.get_display_name(including_second_key=True))) + + elif isinstance(vlan_object_without_site, NetBoxObject): + return_data = vlan_object_without_site + log.debug2("Found a global matching %s object: %s" % + (vlan_object_without_site.name, + vlan_object_without_site.get_display_name(including_second_key=True))) + + else: + log.debug2("No matching existing VLAN found for this VLAN id.") + + return return_data + # EOF diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 54345a0..df67152 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -12,6 +12,7 @@ import pprint import re from ipaddress import ip_address, ip_network, ip_interface from socket import gaierror +from urllib.parse import unquote from pyVim.connect import SmartConnectNoSSL, Disconnect from pyVmomi import vim @@ -22,7 +23,6 @@ from module.common.misc import grab, dump, get_string_or_none, plural from module.common.support import normalize_mac_address, ip_valid_to_add_to_netbox from module.netbox.object_classes import ( NetBoxInterfaceType, - NetBoxObject, NBTag, NBManufacturer, NBDeviceType, @@ -659,73 +659,7 @@ class VMWareHandler(SourceBase): f"based on the primary IPv6 '{primary_ip6}'") return device - def get_vlan_object_if_exists(self, vlan_data=None): - """ - This function will try to find a matching VLAN object based on 'vlan_data' - Will return matching objects in following order: - * exact match: VLAN id and site match - * global match: VLAN id matches but the VLAN has no site assigned - If nothing matches the input data from 'vlan_data' will be returned - - Parameters - ---------- - vlan_data: dict - dict with NBVLAN data attributes - - Returns - ------- - (NBVLAN, dict, None): matching VLAN object, dict or None (content of vlan_data) if no match found - - """ - - if vlan_data is None: - return None - - if not isinstance(vlan_data, dict): - raise ValueError("Value of 'vlan_data' needs to be a dict.") - - # check existing Devices for matches - log.debug2(f"Trying to find a {NBVLAN.name} based on the VLAN id '{vlan_data.get('vid')}'") - - if vlan_data.get("vid") is None: - log.debug("No VLAN id set in vlan_data while trying to find matching VLAN.") - return vlan_data - - vlan_site = self.inventory.get_by_data(NBSite, data=vlan_data.get("site")) - - return_data = vlan_data - vlan_object_including_site = None - vlan_object_without_site = None - - for vlan in self.inventory.get_all_items(NBVLAN): - - if grab(vlan, "data.vid") != vlan_data.get("vid"): - continue - - if vlan_site is not None and grab(vlan, "data.site") == vlan_site: - vlan_object_including_site = vlan - - if grab(vlan, "data.site") is None: - vlan_object_without_site = vlan - - if isinstance(vlan_object_including_site, NetBoxObject): - return_data = vlan_object_including_site - log.debug2("Found a exact matching %s object: %s" % - (vlan_object_including_site.name, - vlan_object_including_site.get_display_name(including_second_key=True))) - - elif isinstance(vlan_object_without_site, NetBoxObject): - return_data = vlan_object_without_site - log.debug2("Found a global matching %s object: %s" % - (vlan_object_without_site.name, - vlan_object_without_site.get_display_name(including_second_key=True))) - - else: - log.debug2("No matching existing VLAN found for this VLAN id.") - - return return_data - - def add_device_vm_to_inventory(self, object_type, object_data, site_name, pnic_data=None, vnic_data=None, + def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, vnic_data=None, nic_ips=None, p_ipv4=None, p_ipv6=None): """ Add/update device/VM object in inventory based on gathered data. @@ -775,8 +709,6 @@ class VMWareHandler(SourceBase): NetBoxObject sub class of object to add object_data: dict data of object to add/update - site_name: str - site name this object is part of pnic_data: dict data of physical interfaces of this object, interface name as key vnic_data: dict @@ -884,49 +816,46 @@ class VMWareHandler(SourceBase): # compile all nic data into one dictionary if object_type == NBVM: nic_data = vnic_data - interface_class = NBVMInterface else: nic_data = {**pnic_data, **vnic_data} - interface_class = NBInterface # map interfaces of existing object with discovered interfaces nic_object_dict = self.map_object_interfaces_to_current_interfaces(device_vm_object, nic_data) if object_data.get("status", "") == "active" and (nic_ips is None or len(nic_ips.keys()) == 0): - log.warning(f"No IP addresses for '{object_name}' found!") + log.debug(f"No IP addresses for '{object_name}' found!") + + primary_ipv4_object = None + primary_ipv6_object = None + + if p_ipv4 is not None: + try: + primary_ipv4_object = ip_interface(p_ipv4) + except ValueError: + log.error(f"Primary IPv4 ({p_ipv4}) does not appear to be a valid IP address (needs included suffix).") + + if p_ipv6 is not None: + try: + primary_ipv6_object = ip_interface(p_ipv6) + except ValueError: + log.error(f"Primary IPv6 ({p_ipv6}) does not appear to be a valid IP address (needs included suffix).") for int_name, int_data in nic_data.items(): - # add object to interface - int_data[interface_class.secondary_key] = device_vm_object - - # get current object for this interface if it exists - nic_object = nic_object_dict.get(int_name) - - # create or update interface with data - if nic_object is None: - nic_object = self.inventory.add_object(interface_class, data=int_data, source=self) - else: - nic_object.update(data=int_data, source=self) + # add/update interface with retrieved data + nic_object, ip_address_objects = self.add_update_interface(nic_object_dict.get(int_name), device_vm_object, + int_data, nic_ips.get(int_name, list())) # add all interface IPs - for nic_ip in nic_ips.get(int_name, list()): + for ip_object in ip_address_objects: - # get IP and prefix length - try: - ip_interface_object = ip_interface(nic_ip) - except ValueError: - log.error(f"IP '{nic_ip}' (nic_object.get_display_name()) does not appear " - "to be a valid IP address. Skipping!") - continue - - ip_object = self.add_ip_address(nic_ip, nic_object, site_name) + ip_interface_object = ip_interface(grab(ip_object, "data.address")) if ip_object is None: continue # continue if address is not a primary IP - if nic_ip not in [p_ipv4, p_ipv6]: + if ip_interface_object not in [primary_ipv4_object, primary_ipv6_object]: continue # set/update/remove primary IP addresses @@ -960,7 +889,7 @@ class VMWareHandler(SourceBase): if set_this_primary_ip is True: - log.debug(f"Setting IP '{nic_ip}' as primary IPv{ip_version} for " + log.debug(f"Setting IP '{grab(ip_object, 'data.address')}' as primary IPv{ip_version} for " f"'{device_vm_object.get_display_name()}'") device_vm_object.update(data={f"primary_ip{ip_version}": ip_object}) @@ -1289,7 +1218,7 @@ class VMWareHandler(SourceBase): self.network_data["vswitch"][name] = dict() for vswitch in grab(obj, "config.network.vswitch", fallback=list()): - vswitch_name = grab(vswitch, "name") + vswitch_name = unquote(grab(vswitch, "name")) vswitch_pnics = [str(x) for x in grab(vswitch, "pnic", fallback=list())] @@ -1308,7 +1237,7 @@ class VMWareHandler(SourceBase): for pswitch in grab(obj, "config.network.proxySwitch", fallback=list()): pswitch_uuid = grab(pswitch, "dvsUuid") - pswitch_name = grab(pswitch, "dvsName") + pswitch_name = unquote(grab(pswitch, "dvsName")) pswitch_pnics = [str(x) for x in grab(pswitch, "pnic", fallback=list())] if pswitch_uuid is not None: @@ -1340,7 +1269,7 @@ class VMWareHandler(SourceBase): self.network_data["host_pgroup"][name][pgroup_name] = { "vlan_id": grab(pgroup, "spec.vlanId"), - "vswitch": grab(pgroup, "spec.vswitchName"), + "vswitch": unquote(grab(pgroup, "spec.vswitchName")), "nics": pgroup_nics } @@ -1402,11 +1331,11 @@ class VMWareHandler(SourceBase): }) pnic_data = { - "name": pnic_name, + "name": unquote(pnic_name), "device": None, # will be set once we found the correct device "mac_address": normalize_mac_address(grab(pnic, "mac")), "enabled": bool(grab(pnic, "linkSpeed")), - "description": pnic_description, + "description": unquote(pnic_description), "type": NetBoxInterfaceType(pnic_link_speed).get_this_netbox_type() } @@ -1436,13 +1365,13 @@ class VMWareHandler(SourceBase): if pnic_vlan.get("vid") == 0: continue - tagged_vlan_list.append(self.get_vlan_object_if_exists({ + tagged_vlan_list.append({ "name": pnic_vlan.get("name"), "vid": pnic_vlan.get("vid"), "site": { "name": site_name } - })) + }) if len(tagged_vlan_list) > 0: pnic_data["tagged_vlans"] = tagged_vlan_list @@ -1509,7 +1438,7 @@ class VMWareHandler(SourceBase): # add data vnic_data = { - "name": vnic_name, + "name": unquote(vnic_name), "device": None, # will be set once we found the correct device "mac_address": normalize_mac_address(grab(vnic, "spec.mac")), "enabled": True, # ESXi vmk interface is enabled by default @@ -1521,19 +1450,19 @@ class VMWareHandler(SourceBase): vnic_data["mode"] = vnic_mode if vnic_description is not None: - vnic_data["description"] = vnic_description + vnic_data["description"] = unquote(vnic_description) else: vnic_description = "" if vnic_portgroup_data is not None and vnic_portgroup_vlan_id != 0: - vnic_data["untagged_vlan"] = self.get_vlan_object_if_exists({ - "name": f"ESXi {vnic_portgroup} (ID: {vnic_portgroup_vlan_id}) ({site_name})", + vnic_data["untagged_vlan"] = { + "name": unquote(f"ESXi {vnic_portgroup} (ID: {vnic_portgroup_vlan_id}) ({site_name})"), "vid": vnic_portgroup_vlan_id, "site": { "name": site_name } - }) + } elif vnic_dv_portgroup_data is not None: @@ -1546,13 +1475,13 @@ class VMWareHandler(SourceBase): if vnic_dv_portgroup_data_vlan_id == 0: continue - tagged_vlan_list.append(self.get_vlan_object_if_exists({ - "name": f"{vnic_dv_portgroup_data.get('name')}-{vnic_dv_portgroup_data_vlan_id}", + tagged_vlan_list.append({ + "name": unquote(f"{vnic_dv_portgroup_data.get('name')}-{vnic_dv_portgroup_data_vlan_id}"), "vid": vnic_dv_portgroup_data_vlan_id, "site": { "name": site_name } - })) + }) if len(tagged_vlan_list) > 0: vnic_data["tagged_vlans"] = tagged_vlan_list @@ -1592,7 +1521,7 @@ class VMWareHandler(SourceBase): host_primary_ip6 = int_v6 # add host to inventory - self.add_device_vm_to_inventory(NBDevice, object_data=host_data, site_name=site_name, pnic_data=pnic_data_dict, + self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, vnic_data=vnic_data_dict, nic_ips=vnic_ips, p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6) @@ -1910,10 +1839,10 @@ class VMWareHandler(SourceBase): vm_primary_ip6 = int_ip_address vm_nic_data = { - "name": int_full_name, + "name": unquote(int_full_name), "virtual_machine": None, "mac_address": int_mac, - "description": int_description, + "description": unquote(int_description), "enabled": int_connected, } @@ -1926,13 +1855,13 @@ class VMWareHandler(SourceBase): if len(int_network_vlan_ids) == 1 and int_network_vlan_ids[0] != 0: - vm_nic_data["untagged_vlan"] = self.get_vlan_object_if_exists({ - "name": int_network_name, + vm_nic_data["untagged_vlan"] = { + "name": unquote(int_network_name), "vid": int_network_vlan_ids[0], "site": { "name": site_name } - }) + } else: tagged_vlan_list = list() for int_network_vlan_id in int_network_vlan_ids: @@ -1940,13 +1869,13 @@ class VMWareHandler(SourceBase): if int_network_vlan_id == 0: continue - tagged_vlan_list.append(self.get_vlan_object_if_exists({ - "name": f"{int_network_name}-{int_network_vlan_id}", + tagged_vlan_list.append({ + "name": unquote(f"{int_network_name}-{int_network_vlan_id}"), "vid": int_network_vlan_id, "site": { "name": site_name } - })) + }) if len(tagged_vlan_list) > 0: vm_nic_data["tagged_vlans"] = tagged_vlan_list @@ -1954,7 +1883,7 @@ class VMWareHandler(SourceBase): nic_data[int_full_name] = vm_nic_data # add VM to inventory - self.add_device_vm_to_inventory(NBVM, object_data=vm_data, site_name=site_name, vnic_data=nic_data, + self.add_device_vm_to_inventory(NBVM, object_data=vm_data, vnic_data=nic_data, nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6) return