From 59b1aea44cfbe2e4ccf100142ebc0f9fc165b5e6 Mon Sep 17 00:00:00 2001 From: "ricardo.bartels@telekom.de" Date: Mon, 20 Sep 2021 10:50:27 +0200 Subject: [PATCH] WIP: refactoring source classes #91 --- module/common/support.py | 363 +--------------- .../sources/check_redfish/import_inventory.py | 56 ++- module/sources/common/source_base.py | 388 ++++++++++++++++++ module/sources/vmware/connection.py | 43 +- 4 files changed, 441 insertions(+), 409 deletions(-) create mode 100644 module/sources/common/source_base.py diff --git a/module/common/support.py b/module/common/support.py index d652bba..b050f7f 100644 --- a/module/common/support.py +++ b/module/common/support.py @@ -7,14 +7,12 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . -from ipaddress import ip_interface, ip_address, IPv6Address, IPv4Address, IPv6Network, IPv4Network +from ipaddress import ip_interface, ip_address import asyncio import aiodns from module.common.logging import get_logger -from module.common.misc import grab -from module.netbox.inventory import NBDevice, NBVM, NBInterface, NBVMInterface, NBSite, NBPrefix, NBIPAddress, NBVLAN log = get_logger() @@ -187,363 +185,4 @@ async def reverse_lookup(resolver, ip): return {ip: resolved_name} - -def map_object_interfaces_to_current_interfaces(inventory, device_vm_object, interface_data_dict=None): - """ - Try to match current object interfaces to discovered ones. This will be done - by multiple approaches. Order as following listing whatever matches first will be chosen. - - by simple name: - both interface names match exactly - by MAC address separated by physical and virtual NICs: - MAC address of interfaces match exactly, distinguish between physical and virtual interfaces - by MAC regardless of interface type - MAC address of interfaces match exactly, type of interface does not matter - - If there are interfaces which don't match at all then the unmatched interfaces will be - matched 1:1. Sort both lists (unmatched current interfaces, unmatched new new interfaces) - by name and assign them each other. - - eth0 > vNIC 1 - eth1 > vNIC 2 - ens1 > vNIC 3 - ... > ... - - Parameters - ---------- - inventory: NetBoxInventory - inventory handler - device_vm_object: (NBDevice, NBVM) - object type to look for - interface_data_dict: dict - dictionary with interface data to compare to existing machine - - Returns - ------- - dict: {"$interface_name": associated_interface_object} - if no current current interface was left to match "None" will be returned instead of - a matching interface object - """ - - """ - trying multiple ways to match interfaces - """ - - if not isinstance(device_vm_object, (NBDevice, NBVM)): - raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") - - if not isinstance(interface_data_dict, dict): - raise ValueError(f"Value for 'interface_data_dict' must be a dict, got: {interface_data_dict}") - - log.debug2("Trying to match current object interfaces in NetBox with discovered interfaces") - - current_object_interfaces = { - "virtual": dict(), - "physical": dict() - } - - current_object_interface_names = list() - - return_data = dict() - - # grab current data - for interface in inventory.get_all_interfaces(device_vm_object): - int_mac = grab(interface, "data.mac_address") - int_name = grab(interface, "data.name") - int_type = "virtual" - if "virtual" not in str(grab(interface, "data.type", fallback="virtual")): - int_type = "physical" - - if int_mac is not None: - current_object_interfaces[int_type][int_mac] = interface - current_object_interfaces[int_mac] = interface - - if int_name is not None: - current_object_interfaces[int_name] = interface - current_object_interface_names.append(int_name) - - log.debug2("Found '%d' NICs in Netbox for '%s'" % - (len(current_object_interface_names), device_vm_object.get_display_name())) - - unmatched_interface_names = list() - - for int_name, int_data in interface_data_dict.items(): - - return_data[int_name] = None - - int_mac = grab(int_data, "mac_address", fallback="XX:XX:YY:YY:ZZ:ZZ") - int_type = "virtual" - if "virtual" not in str(grab(int_data, "type", fallback="virtual")): - int_type = "physical" - - # match simply by name - matching_int = None - if int_name in current_object_interface_names: - log.debug2(f"Found 1:1 name match for NIC '{int_name}'") - matching_int = current_object_interfaces.get(int_name) - - # match mac by interface type - elif grab(current_object_interfaces, f"{int_type}.{int_mac}") is not None: - log.debug2(f"Found 1:1 MAC address match for {int_type} NIC '{int_name}'") - matching_int = grab(current_object_interfaces, f"{int_type}.{int_mac}") - - # match mac regardless of interface type - elif current_object_interfaces.get(int_mac) is not None and \ - current_object_interfaces.get(int_mac) not in return_data.values(): - log.debug2(f"Found 1:1 MAC address match for NIC '{int_name}' (ignoring interface type)") - matching_int = current_object_interfaces.get(int_mac) - - if isinstance(matching_int, (NBInterface, NBVMInterface)): - return_data[int_name] = matching_int - # ToDo: - # check why sometimes names are not present anymore and remove fails - if grab(matching_int, "data.name") in current_object_interface_names: - current_object_interface_names.remove(grab(matching_int, "data.name")) - - # no match found, we match the left overs just by #1 -> #1, #2 -> #2, ... - else: - unmatched_interface_names.append(int_name) - - current_object_interface_names.sort() - unmatched_interface_names.sort() - - matching_nics = dict(zip(unmatched_interface_names, current_object_interface_names)) - - for new_int, current_int in matching_nics.items(): - current_int_object = current_object_interfaces.get(current_int) - log.debug2(f"Matching '{new_int}' to NetBox Interface '{current_int_object.get_display_name()}'") - return_data[new_int] = current_int_object - - return return_data - - -def return_longest_matching_prefix_for_ip(inventory=None, ip_to_match=None, site_name=None): - """ - This is a lazy approach to find longest matching prefix to an IP address. - If site_name is set only IP prefixes from that site are matched. - - Parameters - ---------- - inventory: NetBoxInventory - inventory handler - ip_to_match: (IPv4Address, IPv6Address) - IP address to find prefix for - site_name: str - name of the site the prefix needs to be in - - Returns - ------- - (NBPrefix, None): longest matching IP prefix, or None if no matching prefix was found - """ - - if ip_to_match is None or inventory is None: - return - - if not isinstance(ip_to_match, (IPv4Address, IPv6Address)): - raise ValueError("Value of 'ip_to_match' needs to be an IPv4Address or IPv6Address this_object.") - - site_object = None - if site_name is not None: - site_object = inventory.get_by_data(NBSite, data={"name": site_name}) - - if site_object is None: - log.error(f"Unable to find site '{site_name}' for IP {ip_to_match}. " - "Skipping to find Prefix for this IP.") - - current_longest_matching_prefix_length = 0 - current_longest_matching_prefix = None - - for prefix in inventory.get_all_items(NBPrefix): - - if grab(prefix, "data.site") != site_object: - continue - - prefix_network = grab(prefix, f"data.{NBPrefix.primary_key}") - if prefix_network is None: - continue - - if ip_to_match in prefix_network and \ - prefix_network.prefixlen >= current_longest_matching_prefix_length: - - current_longest_matching_prefix_length = prefix_network.prefixlen - current_longest_matching_prefix = prefix - - return current_longest_matching_prefix - - -def add_ip_address(source_handler, nic_ip, nic_object, site): - - # get IP and prefix length - try: - if "/" in nic_ip: - ip_object = ip_interface(nic_ip) - else: - ip_object = ip_address(nic_ip) - except ValueError: - log.error(f"IP '{nic_ip}' ({nic_object.get_display_name()}) does not appear " - "to be a valid IP address. Skipping!") - return - - log.debug2(f"Trying to find prefix for IP: {ip_object}") - - possible_ip_vrf = None - possible_ip_tenant = None - - # test for site prefixes first - matching_site_name = site - matching_ip_prefix = return_longest_matching_prefix_for_ip(source_handler.inventory, - ip_object, - matching_site_name) - - # nothing was found then check prefixes with site name - if matching_ip_prefix is None: - matching_site_name = None - matching_ip_prefix = return_longest_matching_prefix_for_ip(source_handler.inventory, ip_object) - - # matching prefix found, get data from prefix - if matching_ip_prefix is not None: - - this_prefix = grab(matching_ip_prefix, f"data.{NBPrefix.primary_key}") - if matching_site_name is None: - log.debug2(f"Found IP '{ip_object}' matches global prefix '{this_prefix}'") - else: - log.debug2(f"Found IP '{ip_object}' matches site '{matching_site_name}' prefix " - f"'{this_prefix}'") - - # check if prefix net size and ip address prefix length match - if not isinstance(ip_object, (IPv6Address, IPv4Address)) and \ - this_prefix.prefixlen != ip_object.network.prefixlen: - log.warning(f"IP prefix length of '{ip_object}' ({nic_object.get_display_name()}) " - f"does not match network prefix length '{this_prefix}'!") - - # get prefix data - possible_ip_vrf = grab(matching_ip_prefix, "data.vrf") - prefix_tenant = grab(matching_ip_prefix, "data.tenant") - prefix_vlan = grab(matching_ip_prefix, "data.vlan") - - # get NIC VLAN data - nic_vlan = grab(nic_object, "data.untagged_vlan") - nic_vlan_tenant = None - if nic_vlan is not None: - nic_vlan_tenant = grab(nic_vlan, "data.tenant") - - # check if interface VLAN matches prefix VLAN for IP address - - if isinstance(nic_vlan, NBVLAN) and isinstance(prefix_vlan, NBPrefix) and nic_vlan != prefix_vlan: - log.warning(f"Prefix vlan '{prefix_vlan.get_display_name()}' does not match interface vlan " - f"'{nic_vlan.get_display_name()}' for '{nic_object.get_display_name()}") - - if prefix_tenant is not None: - possible_ip_tenant = prefix_tenant - elif nic_vlan_tenant is not None: - possible_ip_tenant = nic_vlan_tenant - - else: - log_text = f"No matching NetBox prefix for '{ip_object}' found" - - if type(ip_object) in [IPv6Address, IPv4Address]: - log.warning(f"{log_text}. Unable to add IP address to NetBox.") - return None - else: - log.debug2(log_text) - - if matching_ip_prefix is not None and type(ip_object) in [IPv6Address, IPv4Address]: - this_prefix = grab(matching_ip_prefix, "data.prefix") - if type(ip_object) in [IPv4Network, IPv6Network]: - ip_object = ip_interface(f"{ip_object}/{this_prefix.prefixlen}") - else: - log.warning(f"{matching_ip_prefix.name} got wrong format. Unable to add IP to NetBox") - return None - - # try to find matching IP address object - this_ip_object = None - skip_this_ip = False - for ip in source_handler.inventory.get_all_items(NBIPAddress): - - # check if address matches (without prefix length) - ip_address_string = grab(ip, "data.address", fallback="") - - # not a matching address - if not ip_address_string.startswith(f"{ip_object.ip.compressed}/"): - continue - - current_nic = grab(ip, "data.assigned_object_id") - - # is it our current ip interface? - if current_nic == nic_object: - this_ip_object = ip - break - - # check if IP has the same prefix - # continue if - # * both are in global scope - # * both ara part of the same vrf - if possible_ip_vrf != grab(ip, "data.vrf"): - continue - - # IP address is not assigned to any interface - if not isinstance(current_nic, (NBInterface, NBVMInterface)): - this_ip_object = ip - break - - # get current IP interface status - current_nic_enabled = grab(current_nic, "data.enabled", fallback=True) - this_nic_enabled = grab(nic_object, "data.enabled", fallback=True) - - if current_nic_enabled is True and this_nic_enabled is False: - log.debug(f"Current interface '{current_nic.get_display_name()}' for IP '{ip_object}'" - f" is enabled and this one '{nic_object.get_display_name()}' is disabled. " - f"IP assignment skipped!") - skip_this_ip = True - break - - if current_nic_enabled is False and this_nic_enabled is True: - log.debug(f"Current interface '{current_nic.get_display_name()}' for IP '{ip_object}'" - f" is disabled and this one '{nic_object.get_display_name()}' is enabled. " - f"IP will be assigned to this interface.") - - this_ip_object = ip - - if current_nic_enabled == this_nic_enabled: - state = "enabled" if this_nic_enabled is True else "disabled" - log.warning(f"Current interface '{current_nic.get_display_name()}' for IP " - f"'{ip_object}' and this one '{nic_object.get_display_name()}' are " - f"both {state}. " - f"IP assignment skipped because it is unclear which one is the correct one!") - skip_this_ip = True - break - - if skip_this_ip is True: - return - - nic_ip_data = { - "address": ip_object.compressed, - "assigned_object_id": nic_object, - } - - if not isinstance(this_ip_object, NBIPAddress): - log.debug(f"No existing {NBIPAddress.name} object found. Creating a new one.") - - if possible_ip_vrf is not None: - nic_ip_data["vrf"] = possible_ip_vrf - if possible_ip_tenant is not None: - nic_ip_data["tenant"] = possible_ip_tenant - - this_ip_object = source_handler.inventory.add_object(NBIPAddress, data=nic_ip_data, source=source_handler) - - # update IP address with additional data if not already present - else: - - log.debug2(f"Found existing NetBox {NBIPAddress.name} object: {this_ip_object.get_display_name()}") - - if grab(this_ip_object, "data.vrf") is None and possible_ip_vrf is not None: - nic_ip_data["vrf"] = possible_ip_vrf - - if grab(this_ip_object, "data.tenant") is None and possible_ip_tenant is not None: - nic_ip_data["tenant"] = possible_ip_tenant - - this_ip_object.update(data=nic_ip_data, source=source_handler) - - return this_ip_object - # EOF diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index a91cc94..7c3ccfc 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -14,14 +14,10 @@ import json from packaging import version +from module.sources.common.source_base import SourceBase from module.common.logging import get_logger from module.common.misc import grab, get_string_or_none -from module.common.support import ( - normalize_mac_address, - ip_valid_to_add_to_netbox, - map_object_interfaces_to_current_interfaces, - add_ip_address -) +from module.common.support import normalize_mac_address, ip_valid_to_add_to_netbox from module.netbox.object_classes import ( NBTag, NBManufacturer, @@ -48,7 +44,7 @@ from module.netbox.inventory import interface_speed_type_mapping log = get_logger() -class CheckRedfish: +class CheckRedfish(SourceBase): minimum_check_redfish_version = "1.2.0" @@ -80,7 +76,9 @@ class CheckRedfish: "permitted_subnets": None, "overwrite_host_name": False, "overwrite_power_supply_name": False, - "overwrite_interface_name": False + "overwrite_power_supply_attributes": True, + "overwrite_interface_name": False, + "overwrite_interface_attributes": True, } init_successful = False @@ -307,8 +305,6 @@ class CheckRedfish: "mark_connected": connected } - if name is not None and self.overwrite_power_supply_name is True: - ps_data["name"] = name if capacity_in_watt is not None: ps_data["maximum_draw"] = capacity_in_watt if firmware is not None: @@ -319,7 +315,11 @@ class CheckRedfish: if ps_object is None: self.inventory.add_object(NBPowerPort, data=ps_data, source=self) else: - ps_object.update(data=ps_data, source=self) + if self.overwrite_power_supply_name is True: + ps_data["name"] = name + + data_to_update = self.patch_data(ps_object, ps_data, self.overwrite_power_supply_attributes) + ps_object.update(data=data_to_update, source=self) ps_index += 1 @@ -695,41 +695,31 @@ class CheckRedfish: nic_ips[port_name].extend(grab(nic_port, "ipv4_addresses", fallback=list())) nic_ips[port_name].extend(grab(nic_port, "ipv6_addresses", fallback=list())) - data = map_object_interfaces_to_current_interfaces(self.inventory, device_object, port_data_dict) + data = self.map_object_interfaces_to_current_interfaces(device_object, port_data_dict) for port_name, port_data in port_data_dict.items(): # get current object for this interface if it exists nic_object = data.get(port_name) - if "inventory_type" in port_data: - del(port_data["inventory_type"]) - if "health" in port_data: - del(port_data["health"]) + # unset "illegal" attributes + for attribute in ["inventory_type", "health"]: + if attribute in port_data: + del(port_data[attribute]) + + # del empty mac address attribute if port_data.get("mac_address") is None: del (port_data["mac_address"]) - if self.overwrite_interface_name is False: - del (port_data["name"]) # create or update interface with data if nic_object is None: nic_object = self.inventory.add_object(NBInterface, data=port_data, source=self) else: - tags = nic_object.get_tags() - if self.source_tag in tags: - tags.remove(self.source_tag) + if self.overwrite_interface_name is False and port_data.get("name") is not None: + del(port_data["name"]) - # no other source for this interface - if len([x for x in tags if x.startswith("Source")]) == 0: - nic_object.update(data=port_data, source=self) - else: - # only append data - data_to_update = dict() - for key, value in port_data.items(): - if str(grab(nic_object, f"data.{key}")) == "": - data_to_update[key] = value - - nic_object.update(data=data_to_update, source=self) + data_to_update = self.patch_data(nic_object, port_data, self.overwrite_interface_attributes) + nic_object.update(data=data_to_update, source=self) # check for interface ips for nic_ip in nic_ips.get(port_name, list()): @@ -737,7 +727,7 @@ class CheckRedfish: if ip_valid_to_add_to_netbox(nic_ip, self.permitted_subnets, port_name) is False: continue - add_ip_address(self, nic_ip, nic_object, grab(device_object, "data.site.data.name")) + self.add_ip_address(nic_ip, nic_object, grab(device_object, "data.site.data.name")) def update_all_items(self, items): diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py new file mode 100644 index 0000000..598825e --- /dev/null +++ b/module/sources/common/source_base.py @@ -0,0 +1,388 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2021 Ricardo Bartels. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +from ipaddress import ip_interface, ip_address, IPv6Address, IPv4Address, IPv6Network, IPv4Network + +from module.netbox.inventory import NBDevice, NBVM, NBInterface, NBVMInterface, NBSite, NBPrefix, NBIPAddress, NBVLAN +from module.common.logging import get_logger +from module.common.misc import grab + +log = get_logger() + + +class SourceBase: + + inventory = None + source_tag = None + + def map_object_interfaces_to_current_interfaces(self, device_vm_object, interface_data_dict=None): + """ + Try to match current object interfaces to discovered ones. This will be done + by multiple approaches. Order as following listing whatever matches first will be chosen. + + by simple name: + both interface names match exactly + by MAC address separated by physical and virtual NICs: + MAC address of interfaces match exactly, distinguish between physical and virtual interfaces + by MAC regardless of interface type + MAC address of interfaces match exactly, type of interface does not matter + + If there are interfaces which don't match at all then the unmatched interfaces will be + matched 1:1. Sort both lists (unmatched current interfaces, unmatched new new interfaces) + by name and assign them each other. + + eth0 > vNIC 1 + eth1 > vNIC 2 + ens1 > vNIC 3 + ... > ... + + Parameters + ---------- + device_vm_object: (NBDevice, NBVM) + object type to look for + interface_data_dict: dict + dictionary with interface data to compare to existing machine + + Returns + ------- + dict: {"$interface_name": associated_interface_object} + if no current current interface was left to match "None" will be returned instead of + a matching interface object + """ + + """ + trying multiple ways to match interfaces + """ + + if not isinstance(device_vm_object, (NBDevice, NBVM)): + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + if not isinstance(interface_data_dict, dict): + raise ValueError(f"Value for 'interface_data_dict' must be a dict, got: {interface_data_dict}") + + log.debug2("Trying to match current object interfaces in NetBox with discovered interfaces") + + current_object_interfaces = { + "virtual": dict(), + "physical": dict() + } + + current_object_interface_names = list() + + return_data = dict() + + # grab current data + for interface in self.inventory.get_all_interfaces(device_vm_object): + int_mac = grab(interface, "data.mac_address") + int_name = grab(interface, "data.name") + int_type = "virtual" + if "virtual" not in str(grab(interface, "data.type", fallback="virtual")): + int_type = "physical" + + if int_mac is not None: + current_object_interfaces[int_type][int_mac] = interface + current_object_interfaces[int_mac] = interface + + if int_name is not None: + current_object_interfaces[int_name] = interface + current_object_interface_names.append(int_name) + + log.debug2("Found '%d' NICs in Netbox for '%s'" % + (len(current_object_interface_names), device_vm_object.get_display_name())) + + unmatched_interface_names = list() + + for int_name, int_data in interface_data_dict.items(): + + return_data[int_name] = None + + int_mac = grab(int_data, "mac_address", fallback="XX:XX:YY:YY:ZZ:ZZ") + int_type = "virtual" + if "virtual" not in str(grab(int_data, "type", fallback="virtual")): + int_type = "physical" + + # match simply by name + matching_int = None + if int_name in current_object_interface_names: + log.debug2(f"Found 1:1 name match for NIC '{int_name}'") + matching_int = current_object_interfaces.get(int_name) + + # match mac by interface type + elif grab(current_object_interfaces, f"{int_type}.{int_mac}") is not None: + log.debug2(f"Found 1:1 MAC address match for {int_type} NIC '{int_name}'") + matching_int = grab(current_object_interfaces, f"{int_type}.{int_mac}") + + # match mac regardless of interface type + elif current_object_interfaces.get(int_mac) is not None and \ + current_object_interfaces.get(int_mac) not in return_data.values(): + log.debug2(f"Found 1:1 MAC address match for NIC '{int_name}' (ignoring interface type)") + matching_int = current_object_interfaces.get(int_mac) + + if isinstance(matching_int, (NBInterface, NBVMInterface)): + return_data[int_name] = matching_int + # ToDo: + # check why sometimes names are not present anymore and remove fails + if grab(matching_int, "data.name") in current_object_interface_names: + current_object_interface_names.remove(grab(matching_int, "data.name")) + + # no match found, we match the left overs just by #1 -> #1, #2 -> #2, ... + else: + unmatched_interface_names.append(int_name) + + current_object_interface_names.sort() + unmatched_interface_names.sort() + + matching_nics = dict(zip(unmatched_interface_names, current_object_interface_names)) + + for new_int, current_int in matching_nics.items(): + current_int_object = current_object_interfaces.get(current_int) + log.debug2(f"Matching '{new_int}' to NetBox Interface '{current_int_object.get_display_name()}'") + return_data[new_int] = current_int_object + + return return_data + + def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None): + """ + This is a lazy approach to find longest matching prefix to an IP address. + If site_name is set only IP prefixes from that site are matched. + + Parameters + ---------- + ip_to_match: (IPv4Address, IPv6Address) + IP address to find prefix for + site_name: str + name of the site the prefix needs to be in + + Returns + ------- + (NBPrefix, None): longest matching IP prefix, or None if no matching prefix was found + """ + + if ip_to_match is None or self.inventory is None: + return + + if not isinstance(ip_to_match, (IPv4Address, IPv6Address)): + raise ValueError("Value of 'ip_to_match' needs to be an IPv4Address or IPv6Address this_object.") + + site_object = None + if site_name is not None: + site_object = self.inventory.get_by_data(NBSite, data={"name": site_name}) + + if site_object is None: + log.error(f"Unable to find site '{site_name}' for IP {ip_to_match}. " + "Skipping to find Prefix for this IP.") + + current_longest_matching_prefix_length = 0 + current_longest_matching_prefix = None + + for prefix in self.inventory.get_all_items(NBPrefix): + + if grab(prefix, "data.site") != site_object: + continue + + prefix_network = grab(prefix, f"data.{NBPrefix.primary_key}") + if prefix_network is None: + continue + + if ip_to_match in prefix_network and \ + prefix_network.prefixlen >= current_longest_matching_prefix_length: + current_longest_matching_prefix_length = prefix_network.prefixlen + current_longest_matching_prefix = prefix + + return current_longest_matching_prefix + + def add_ip_address(self, nic_ip, nic_object, site): + + # get IP and prefix length + try: + if "/" in nic_ip: + ip_object = ip_interface(nic_ip) + else: + ip_object = ip_address(nic_ip) + except ValueError: + log.error(f"IP '{nic_ip}' ({nic_object.get_display_name()}) does not appear " + "to be a valid IP address. Skipping!") + return + + log.debug2(f"Trying to find prefix for IP: {ip_object}") + + possible_ip_vrf = None + possible_ip_tenant = None + + # test for site prefixes first + matching_site_name = site + matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object, matching_site_name) + + # nothing was found then check prefixes with site name + if matching_ip_prefix is None: + matching_site_name = None + matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object) + + # matching prefix found, get data from prefix + if matching_ip_prefix is not None: + + this_prefix = grab(matching_ip_prefix, f"data.{NBPrefix.primary_key}") + if matching_site_name is None: + log.debug2(f"Found IP '{ip_object}' matches global prefix '{this_prefix}'") + else: + log.debug2(f"Found IP '{ip_object}' matches site '{matching_site_name}' prefix " + f"'{this_prefix}'") + + # check if prefix net size and ip address prefix length match + if not isinstance(ip_object, (IPv6Address, IPv4Address)) and \ + this_prefix.prefixlen != ip_object.network.prefixlen: + log.warning(f"IP prefix length of '{ip_object}' ({nic_object.get_display_name()}) " + f"does not match network prefix length '{this_prefix}'!") + + # get prefix data + possible_ip_vrf = grab(matching_ip_prefix, "data.vrf") + prefix_tenant = grab(matching_ip_prefix, "data.tenant") + prefix_vlan = grab(matching_ip_prefix, "data.vlan") + + # get NIC VLAN data + nic_vlan = grab(nic_object, "data.untagged_vlan") + nic_vlan_tenant = None + if nic_vlan is not None: + nic_vlan_tenant = grab(nic_vlan, "data.tenant") + + # check if interface VLAN matches prefix VLAN for IP address + + if isinstance(nic_vlan, NBVLAN) and isinstance(prefix_vlan, NBPrefix) and nic_vlan != prefix_vlan: + log.warning(f"Prefix vlan '{prefix_vlan.get_display_name()}' does not match interface vlan " + f"'{nic_vlan.get_display_name()}' for '{nic_object.get_display_name()}") + + if prefix_tenant is not None: + possible_ip_tenant = prefix_tenant + elif nic_vlan_tenant is not None: + possible_ip_tenant = nic_vlan_tenant + + else: + log_text = f"No matching NetBox prefix for '{ip_object}' found" + + if type(ip_object) in [IPv6Address, IPv4Address]: + log.warning(f"{log_text}. Unable to add IP address to NetBox.") + return None + else: + log.debug2(log_text) + + if matching_ip_prefix is not None and type(ip_object) in [IPv6Address, IPv4Address]: + this_prefix = grab(matching_ip_prefix, "data.prefix") + if type(this_prefix) in [IPv4Network, IPv6Network]: + ip_object = ip_interface(f"{ip_object}/{this_prefix.prefixlen}") + else: + log.warning(f"{matching_ip_prefix.name} got wrong format. Unable to add IP to NetBox") + return None + + # try to find matching IP address object + this_ip_object = None + skip_this_ip = False + for ip in self.inventory.get_all_items(NBIPAddress): + + # check if address matches (without prefix length) + ip_address_string = grab(ip, "data.address", fallback="") + + # not a matching address + if not ip_address_string.startswith(f"{ip_object.ip.compressed}/"): + continue + + current_nic = grab(ip, "data.assigned_object_id") + + # is it our current ip interface? + if current_nic == nic_object: + this_ip_object = ip + break + + # check if IP has the same prefix + # continue if + # * both are in global scope + # * both ara part of the same vrf + if possible_ip_vrf != grab(ip, "data.vrf"): + continue + + # IP address is not assigned to any interface + if not isinstance(current_nic, (NBInterface, NBVMInterface)): + this_ip_object = ip + break + + # get current IP interface status + current_nic_enabled = grab(current_nic, "data.enabled", fallback=True) + this_nic_enabled = grab(nic_object, "data.enabled", fallback=True) + + if current_nic_enabled is True and this_nic_enabled is False: + log.debug(f"Current interface '{current_nic.get_display_name()}' for IP '{ip_object}'" + f" is enabled and this one '{nic_object.get_display_name()}' is disabled. " + f"IP assignment skipped!") + skip_this_ip = True + break + + if current_nic_enabled is False and this_nic_enabled is True: + log.debug(f"Current interface '{current_nic.get_display_name()}' for IP '{ip_object}'" + f" is disabled and this one '{nic_object.get_display_name()}' is enabled. " + f"IP will be assigned to this interface.") + + this_ip_object = ip + + if current_nic_enabled == this_nic_enabled: + state = "enabled" if this_nic_enabled is True else "disabled" + log.warning(f"Current interface '{current_nic.get_display_name()}' for IP " + f"'{ip_object}' and this one '{nic_object.get_display_name()}' are " + f"both {state}. " + f"IP assignment skipped because it is unclear which one is the correct one!") + skip_this_ip = True + break + + if skip_this_ip is True: + return + + nic_ip_data = { + "address": ip_object.compressed, + "assigned_object_id": nic_object, + } + + if not isinstance(this_ip_object, NBIPAddress): + log.debug(f"No existing {NBIPAddress.name} object found. Creating a new one.") + + if possible_ip_vrf is not None: + nic_ip_data["vrf"] = possible_ip_vrf + if possible_ip_tenant is not None: + nic_ip_data["tenant"] = possible_ip_tenant + + this_ip_object = self.inventory.add_object(NBIPAddress, data=nic_ip_data, source=self) + + # update IP address with additional data if not already present + else: + + log.debug2(f"Found existing NetBox {NBIPAddress.name} object: {this_ip_object.get_display_name()}") + + if grab(this_ip_object, "data.vrf") is None and possible_ip_vrf is not None: + nic_ip_data["vrf"] = possible_ip_vrf + + if grab(this_ip_object, "data.tenant") is None and possible_ip_tenant is not None: + nic_ip_data["tenant"] = possible_ip_tenant + + this_ip_object.update(data=nic_ip_data, source=self) + + return this_ip_object + + @staticmethod + def patch_data(object_to_patch, data, overwrite=False): + + if overwrite is True: + return data + + # only append data + data_to_update = dict() + for key, value in data.items(): + current_value = grab(object_to_patch, f"data.{key}") + if current_value is None or str(current_value) == "": + data_to_update[key] = value + + return data_to_update + +# EOF diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 5dd88ee..268b0aa 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -10,29 +10,44 @@ import atexit import pprint import re -from ipaddress import ip_address, ip_network, ip_interface, IPv4Address, IPv6Address +from ipaddress import ip_address, ip_network, ip_interface from socket import gaierror from pyVim.connect import SmartConnectNoSSL, Disconnect from pyVmomi import vim +from module.sources.common.source_base import SourceBase from module.common.logging import get_logger, DEBUG3 from module.common.misc import grab, dump, get_string_or_none, plural -from module.common.support import ( - normalize_mac_address, - ip_valid_to_add_to_netbox, - map_object_interfaces_to_current_interfaces, - return_longest_matching_prefix_for_ip, - add_ip_address +from module.common.support import normalize_mac_address, ip_valid_to_add_to_netbox +from module.netbox.object_classes import ( + NetBoxObject, + NBTag, + NBManufacturer, + NBDeviceType, + NBPlatform, + NBClusterType, + NBClusterGroup, + NBDeviceRole, + NBSite, + NBCluster, + NBDevice, + NBVM, + NBVMInterface, + NBInterface, + NBIPAddress, + NBPrefix, + NBTenant, + NBVRF, + NBVLAN ) -from module.netbox.object_classes import * from module.netbox.inventory import interface_speed_type_mapping log = get_logger() # noinspection PyTypeChecker -class VMWareHandler: +class VMWareHandler(SourceBase): """ Source class to import data from a vCenter instance and add/update NetBox objects based on gathered information """ @@ -81,7 +96,7 @@ class VMWareHandler: "host_tenant_relation": None, "vm_platform_relation": None, "host_role_relation": None, - "vm_role_relation":None, + "vm_role_relation": None, "dns_name_lookup": False, "custom_dns_servers": None, "set_primary_ip": "when-undefined", @@ -398,6 +413,7 @@ class VMWareHandler: except Exception as e: log.error(e) + # noinspection PyArgumentList view_details.get("view_handler")(obj) container_view.Destroy() @@ -689,7 +705,7 @@ class VMWareHandler: if vlan_site is not None and grab(vlan, "data.site") == vlan_site: vlan_object_including_site = vlan - if grab(vlan, "data.site") == None: + if grab(vlan, "data.site") is None: vlan_object_without_site = vlan if isinstance(vlan_object_including_site, NetBoxObject): @@ -874,7 +890,7 @@ class VMWareHandler: interface_class = NBInterface # map interfaces of existing object with discovered interfaces - nic_object_dict = map_object_interfaces_to_current_interfaces(self.inventory, device_vm_object, nic_data) + nic_object_dict = self.map_object_interfaces_to_current_interfaces(device_vm_object, nic_data) if object_data.get("status", "") == "active" and (nic_ips is None or len(nic_ips.keys()) == 0): log.warning(f"No IP addresses for '{object_name}' found!") @@ -904,7 +920,7 @@ class VMWareHandler: "to be a valid IP address. Skipping!") continue - ip_object = add_ip_address(self, nic_ip, nic_object, site_name) + ip_object = self.add_ip_address(nic_ip, nic_object, site_name) if ip_object is None: continue @@ -1223,7 +1239,6 @@ class VMWareHandler: if serial is None: serial = get_string_or_none(identifier_dict.get(serial_num_key)) - # add asset tag if desired and present asset_tag = None