From 29b0296ef0e38c860d8bdeeeccff77079d9e76cf Mon Sep 17 00:00:00 2001 From: "ricardo.bartels@telekom.de" Date: Wed, 20 Nov 2024 22:05:52 +0100 Subject: [PATCH 01/32] fixes typos --- module/common/cli_parser.py | 2 +- module/common/misc.py | 4 +-- module/common/support.py | 4 +-- module/netbox/connection.py | 20 ++++++------- module/netbox/inventory.py | 26 ++++++++-------- module/netbox/object_classes.py | 30 +++++++++---------- module/sources/__init__.py | 4 +-- .../sources/check_redfish/import_inventory.py | 4 +-- module/sources/common/excluded_vlan.py | 1 + module/sources/common/permitted_subnets.py | 2 +- module/sources/common/source_base.py | 14 ++++----- module/sources/vmware/connection.py | 30 +++++++++---------- 12 files changed, 71 insertions(+), 70 deletions(-) diff --git a/module/common/cli_parser.py b/module/common/cli_parser.py index 07d69ac5..824f7fd0 100644 --- a/module/common/cli_parser.py +++ b/module/common/cli_parser.py @@ -23,7 +23,7 @@ def parse_command_line(self_description=None): Parameters ---------- self_description: str - short self description of this program + short self-description of this program Returns ------- diff --git a/module/common/misc.py b/module/common/misc.py index e966a975..1302291e 100644 --- a/module/common/misc.py +++ b/module/common/misc.py @@ -15,7 +15,7 @@ def grab(structure=None, path=None, separator=".", fallback=None): """ get data from a complex object/json structure with a "." separated path information. If a part of a path - is not not present then this function returns the + is not present then this function returns the value of fallback (default: "None"). example structure: @@ -38,7 +38,7 @@ def grab(structure=None, path=None, separator=".", fallback=None): Parameters ---------- structure: dict, list, object - object structure to extract data from + an object structure to extract data from path: str nested path to extract separator: str diff --git a/module/common/support.py b/module/common/support.py index 7fca9ae7..8a3677df 100644 --- a/module/common/support.py +++ b/module/common/support.py @@ -51,9 +51,9 @@ def perform_ptr_lookups(ips, dns_servers=None): Parameters ---------- ips: list - list of IP addresses to look up + a list of IP addresses to look up dns_servers: list - list of DNS servers to use to look up list of IP addresses + a list of DNS servers to use to look up list of IP addresses Returns ------- diff --git a/module/netbox/connection.py b/module/netbox/connection.py index e1b943a6..ccdc2033 100644 --- a/module/netbox/connection.py +++ b/module/netbox/connection.py @@ -217,20 +217,20 @@ def request(self, object_class, req_type="GET", data=None, params=None, nb_id=No Parameters ---------- - object_class: NetBoxObject sub class + object_class: NetBoxObject subclass class definition of the desired NetBox object req_type: str GET, PATCH, PUT, DELETE data: dict - data which shall be send to NetBox + data which shall be sent to NetBox params: dict - dict of URL params which should be passed to NetBox + dictionary of URL params which should be passed to NetBox nb_id: int ID of the NetBox object which will be appended to the requested NetBox URL Returns ------- - (dict, bool, None): of returned NetBox data. If object was requested to be deleted and it was + dict, bool, None: of returned NetBox data. If object was requested to be deleted, and it was successful then True will be returned. None if request failed or was empty """ @@ -373,12 +373,12 @@ def query_current_data(self, netbox_objects_to_query=None): """ Request all current NetBox objects. Use caching whenever possible. Objects must provide "last_updated" attribute to support caching for this object type. - Otherwise it's not possible to query only changed objects since last run. If attribute is + Otherwise, it's not possible to query only changed objects since last run. If attribute is not present all objects will be requested (looking at you *Interfaces) Parameters ---------- - netbox_objects_to_query: list of NetBoxObject sub classes + netbox_objects_to_query: list of NetBoxObject subclasses NetBox items to query """ @@ -585,14 +585,14 @@ def initialize_basic_data(self): def update_object(self, nb_object_sub_class, unset=False, last_run=False): """ - Iterate over all objects of a certain NetBoxObject sub class and add/update them. + Iterate over all objects of a certain NetBoxObject subclass and add/update them. But first update objects which this object class depends on. If some dependencies are unresolvable then these will be removed from the request and re added later to the object to try update object in a third run. Parameters ---------- - nb_object_sub_class: NetBoxObject sub class + nb_object_sub_class: NetBoxObject subclass NetBox objects to update unset: bool True if only unset items should be deleted @@ -660,7 +660,7 @@ def update_object(self, nb_object_sub_class, unset=False, last_run=False): # special case for IP address if isinstance(this_object, NBIPAddress): - # if object is new and and has no id, then we need to remove assigned_object_type from data_to_patch + # if object is new and has no id, then we need to remove assigned_object_type from data_to_patch if "assigned_object_id" in unresolved_dependency_data.keys() and \ "assigned_object_type" in data_to_patch.keys(): del data_to_patch["assigned_object_type"] @@ -674,7 +674,7 @@ def update_object(self, nb_object_sub_class, unset=False, last_run=False): req_type = "POST" action = "Creating new" - # if its not a new object then update it + # if it's not a new object then update it if this_object.is_new is False: nb_id = this_object.nb_id req_type = "PATCH" diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py index ab18d7d4..d07ce98c 100644 --- a/module/netbox/inventory.py +++ b/module/netbox/inventory.py @@ -19,7 +19,7 @@ class NetBoxInventory: """ - Singleton class to manage a inventory of NetBoxObject objects + Singleton class to manage an inventory of NetBoxObject objects """ base_structure = dict() @@ -62,18 +62,18 @@ def get_by_id(self, object_type, nb_id=None): Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to find nb_id: int NetBox ID of object Returns ------- - (NetBoxObject sub class, None): return object instance if object was found, None otherwise + (NetBoxObject subclass, None): return object instance if object was found, None otherwise """ if object_type not in NetBoxObject.__subclasses__(): - raise AttributeError("'%s' object must be a sub class of '%s'." % + raise AttributeError("'%s' object must be a subclass of '%s'." % (object_type.__name__, NetBoxObject.__name__)) if nb_id is None or self.base_structure[object_type.name] is None: @@ -90,18 +90,18 @@ def get_by_data(self, object_type, data=None): Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to find data: dict params of object to match Returns ------- - (NetBoxObject sub class, None): return object instance if object was found, None otherwise + (NetBoxObject subclass, None): return object instance if object was found, None otherwise """ if object_type not in NetBoxObject.__subclasses__(): - raise AttributeError("'%s' object must be a sub class of '%s'." % + raise AttributeError("'%s' object must be a subclass of '%s'." % (object_type.__name__, NetBoxObject.__name__)) if data is None or len(self.get_all_items(object_type)) == 0: @@ -169,7 +169,7 @@ def slug_used(self, object_type: NetBoxObject, slug: str) -> bool: """ if object_type not in NetBoxObject.__subclasses__(): - raise AttributeError("'%s' object must be a sub class of '%s'." % + raise AttributeError("'%s' object must be a subclass of '%s'." % (object_type.__name__, NetBoxObject.__name__)) if "slug" in object_type.data_model.keys(): @@ -185,7 +185,7 @@ def add_object(self, object_type, data=None, read_from_netbox=False, source=None Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to add data: dict Object data to add to the inventory @@ -216,10 +216,10 @@ def add_update_object(self, object_type, data=None, read_from_netbox=False, sour Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to add/update data: dict - data used to create a new object or update a existing object + data used to create a new object or update an existing object read_from_netbox: bool True if data was read directly from NetBox source: object handler of source @@ -265,7 +265,7 @@ def get_all_items(self, object_type): Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to find Returns @@ -274,7 +274,7 @@ def get_all_items(self, object_type): """ if object_type not in NetBoxObject.__subclasses__(): - raise ValueError(f"'{object_type.__name__}' object must be a sub class of '{NetBoxObject.__name__}'.") + raise ValueError(f"'{object_type.__name__}' object must be a subclass of '{NetBoxObject.__name__}'.") return self.base_structure.get(object_type.name, list()) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index dd84862f..043180aa 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -60,7 +60,7 @@ class to handle and determine NetBox interface types "400gbase-x-osfp": 400_000 } - # assign common types for a interface speed value + # assign common types for an interface speed value common_types = { 100: "100base-tx", 1_000: "1000base-t", @@ -99,14 +99,14 @@ def get_netbox_type_list(self) -> list: Returns ------- valid_interface_types: list - list with valid types + a list with valid types """ return list(self.valid_types.keys()) def get_common_type(self) -> str: """ - return a interface type from the common type list + return an interface type from the common type list Returns ------- @@ -160,7 +160,7 @@ def get_speed_human(self) -> str: Returns ------- human_speed: str - human readable string of interface speed + human-readable string of interface speed """ if self.detected_speed == 0: @@ -215,7 +215,7 @@ class NetBoxObject: """ Base class for all NetBox object types. Implements all methods used on a NetBox object. - Sub classes need to have following attributes: + subclasses need to have the following attributes: name: name of the object type (i.e. "virtual machine") api_path: @@ -244,14 +244,14 @@ class NetBoxObject: can be a string with an undefined length bool (class): attribute must be True or False - NetBoxObject sub class: + NetBoxObject subclass: value of this key is a reference to another NetBoxObject of exact defined type list (instance): - value can be one of the predefined values in that list - list of NetBoxObject sub classes: + value can be one of the predefined values in that list. + list of NetBoxObject subclasses: value must be an instance of predefined netBoxObject classes in list - NBObjectList sub class: - value mast be the defined sub class of NBObjectList + NBObjectList subclass: + value mast be the defined subclass of NBObjectList """ @@ -434,7 +434,7 @@ def get_uniq_slug(self, text=None, max_len=50)-> str: new_slug = f"{slug}-{x}" if self.inventory.slug_used(self.__class__, new_slug) is False and len(new_slug) <= max_len: log.info(f"Slug '{slug}' for {self.name} '{text}' has been used. " - f"Assignin slug '{new_slug}'") + f"Assigning slug '{new_slug}'") return new_slug raise ValueError(f"Unable to find uniq slug for {self.name} '{text}'") @@ -750,7 +750,7 @@ def get_display_name(self, data=None, including_second_key=False): def resolve_relations(self): """ - Resolve object relations for this object. Substitute a dict of data with a id with the instantiated + Resolve object relations for this object. Substitute a dict of data with an id with the instantiated reference of this object """ @@ -810,11 +810,11 @@ def resolve_relations(self): def get_dependencies(self): """ - returns a list of NetBoxObject sub classes this object depends on + returns a list of NetBoxObject subclasses this object depends on Returns ------- - list: of NetBoxObject sub classes + list: of NetBoxObject subclasses """ r = [x for x in self.data_model.values() if x in NetBoxObject.__subclasses__()] @@ -1168,7 +1168,7 @@ class NBObjectList(list): Base class of listed NetBox objects. Extends list(). Currently used for tags and untagged VLANs Mandatory attributes: - member_type: NetBoxObject sub class + member_type: NetBoxObject subclass defines the type objects contained in this type of list """ diff --git a/module/sources/__init__.py b/module/sources/__init__.py index dc2b7dcc..cdb1f846 100644 --- a/module/sources/__init__.py +++ b/module/sources/__init__.py @@ -33,7 +33,7 @@ def validate_source(source_class_object=None, state="pre"): source_class_object: Source handler class/object class/object to investigate state: str - pre if validating a class, post if validating a initialized object + pre if validating a class, post if validating an initialized object """ @@ -70,7 +70,7 @@ def validate_source(source_class_object=None, state="pre"): def instantiate_sources(): """ Instantiate a source handler and add necessary attributes. Also - validate source handler pre and post initialization. + validate source handler on pre- and post-initialization. Returns ------- diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index 36cc53b8..84274671 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -895,7 +895,7 @@ def update_all_items(self, items): Parameters ---------- items: list - list of items to update + a list of items to update Returns ------- @@ -974,7 +974,7 @@ def update_item(self, item_data: dict, inventory_object: NBInventoryItem = None) Parameters ---------- item_data: dict - dict with data for item to update + a dict with data for item to update inventory_object: NBInventoryItem, None the NetBox inventory item to update. diff --git a/module/sources/common/excluded_vlan.py b/module/sources/common/excluded_vlan.py index 40c16d3e..99a9626f 100644 --- a/module/sources/common/excluded_vlan.py +++ b/module/sources/common/excluded_vlan.py @@ -46,6 +46,7 @@ def site_matches(self, site_name): return True # string or regex matches + # noinspection PyBroadException try: if ([self.site, site_name]).count(None) == 0 and re.search(f"^{self.site}$", site_name): log.debug2(f"VLAN exclude site name '{site_name}' matches '{self.site}'") diff --git a/module/sources/common/permitted_subnets.py b/module/sources/common/permitted_subnets.py index caffb5b7..36412d38 100644 --- a/module/sources/common/permitted_subnets.py +++ b/module/sources/common/permitted_subnets.py @@ -16,7 +16,7 @@ class PermittedSubnets: """ - initializes and verifies if an IP address is part of an permitted subnet + initializes and verifies if an IP address is part of a permitted subnet """ def __init__(self, config_string: str): diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 86f06b8e..d6e535cb 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -60,7 +60,7 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac MAC address of interfaces match exactly, type of interface does not matter If there are interfaces which don't match at all then the unmatched interfaces will be - matched 1:1. Sort both lists (unmatched current interfaces, unmatched new new interfaces) + matched 1:1. Sort both lists (unmatched current interfaces, unmatched new interfaces) by name and assign them each other. eth0 > vNIC 1 @@ -80,7 +80,7 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac Returns ------- dict: {"$interface_name": associated_interface_object} - if no current current interface was left to match "None" will be returned instead of + if no current interface was left to match "None" will be returned instead of a matching interface object """ @@ -159,7 +159,7 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac if grab(matching_int, "data.name") in current_object_interface_names: current_object_interface_names.remove(grab(matching_int, "data.name")) - # no match found, we match the left overs just by #1 -> #1, #2 -> #2, ... + # no match found, we match the leftovers just by #1 -> #1, #2 -> #2, ... else: unmatched_interface_names.append(int_name) @@ -182,7 +182,7 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None): """ - This is a lazy approach to find longest matching prefix to an IP address. + This is a lazy approach to find the longest matching prefix to an IP address. If site_name is set only IP prefixes from that site are matched. Parameters @@ -254,7 +254,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, interface_data: dict dictionary with interface attributes to add to this interface interface_ips: list - list of ip addresses which are assigned to this interface + a list of ip addresses which are assigned to this interface vmware_object: (vim.HostSystem, vim.VirtualMachine) object to add to list of objects to reevaluate @@ -618,7 +618,7 @@ def patch_data(object_to_patch, data, overwrite=False): Returns ------- data_to_update: dict - dict with data to append/patch + A dict with data to append/patch """ if overwrite is True: @@ -644,7 +644,7 @@ def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): Parameters ---------- vlan_data: dict - dict with NBVLAN data attributes + A dict with NBVLAN data attributes vlan_site: str name of site the VLAN could be present diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index f8f8bfd8..66a00052 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -421,9 +421,9 @@ def passes_filter(name, include_filter, exclude_filter): name: str name of the object to check include_filter: regex object - regex object of include filter + A regex object of include filter exclude_filter: regex object - regex object of exclude filter + A regex object of exclude filter Returns ------- @@ -511,7 +511,7 @@ def get_object_based_on_macs(self, object_type, mac_list=None): object_type: (NBDevice, NBVM) type of NetBox device to find in inventory mac_list: list - list of MAC addresses to compare against NetBox interface objects + a list of MAC addresses to compare against NetBox interface objects Returns ------- @@ -710,7 +710,7 @@ def collect_object_tags(self, obj): Returns ------- tag_list: list - list of NBTag objets retrieved from vCenter for this object + a list of NBTag objets retrieved from vCenter for this object """ if obj is None: @@ -960,7 +960,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v look for longest matching IP Prefix in the same site. If this failed we try to find the longest matching global IP Prefix. - If a IP Prefix was found then we try to get the VRF and VLAN for this prefix. Now we compare + If an IP Prefix was found then we try to get the VRF and VLAN for this prefix. Now we compare if interface VLAN and prefix VLAN match up and warn if they don't. Then we try to add data to the IP address if not already set: @@ -971,7 +971,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v And we also set primary IP4/6 for this object depending on the "set_primary_ip" setting. - If a IP address is set as primary IP for another device then using this IP on another + If an IP address is set as primary IP for another device then using this IP on another device will be rejected by NetBox. Setting "always": @@ -991,7 +991,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v Parameters ---------- object_type: (NBDevice, NBVM) - NetBoxObject sub class of object to add + NetBoxObject subclass of object to add object_data: dict data of object to add/update pnic_data: dict @@ -999,13 +999,13 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v vnic_data: dict data of virtual interfaces of this object, interface name as key nic_ips: dict - dict of ips per interface of this object, interface name as key + a dict of ips per interface of this object, interface name as key p_ipv4: str primary IPv4 as string including netmask/prefix p_ipv6: str primary IPv6 as string including netmask/prefix vmware_object: (vim.HostSystem, vim.VirtualMachine) - vmware object to pass on to 'add_update_interface' method to setup reevaluation + vmware object to pass on to 'add_update_interface' method to set up reevaluation disk_data: list data of discs which belong to a VM @@ -1121,7 +1121,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v object_type == NBVM and disk_data is not None and len(disk_data) > 0: # create pairs of existing and discovered disks. - # currently these disks are only used within the VM model. that's we we use this simple approach and + # currently these disks are only used within the VM model. that's why we use this simple approach and # just rewrite disk as they appear in order. # otherwise we would need to implement a matching function like matching interfaces. disk_zip_list = zip_longest( @@ -1301,7 +1301,7 @@ def add_datacenter(self, obj): Parameters ---------- obj: vim.Datacenter - datacenter object + a datacenter object """ if self.settings.set_source_name_as_cluster_group is True: @@ -2017,8 +2017,8 @@ def add_host(self, obj): vnic_ips[vnic_name].append(int_v6) # set first valid IPv6 address as primary IPv6 - # not the best way but maybe we can find more information in "spec.ipRouteSpec" - # about default route and we could use that to determine the correct IPv6 address + # not the best way, but maybe we can find more information in "spec.ipRouteSpec" + # about default route, and we could use that to determine the correct IPv6 address if vnic_is_primary is True and host_primary_ip6 is None: host_primary_ip6 = int_v6 @@ -2034,10 +2034,10 @@ def add_virtual_machine(self, obj): Parse a vCenter VM add to NetBox once all data is gathered. VMs are parsed twice. First only "online" VMs are parsed and added. In the second - round also "offline" VMs will be parsed. This helps of VMs are cloned and used + round also "offline" VMs will be parsed. This helps if VMs are cloned and used for upgrades but then have the same name. - First VM is filtered: + First VM will be filtered: VM has a cluster and is it permitted was VM with same name and cluster already parsed does the VM pass the vm_include_filter and vm_exclude_filter From 08b244ac56ad69fa9d3b0b89e99ec378ddaec2aa Mon Sep 17 00:00:00 2001 From: "ricardo.bartels@telekom.de" Date: Wed, 20 Nov 2024 22:22:52 +0100 Subject: [PATCH 02/32] mitigates issue of pruning newly used tags #401 --- module/netbox/connection.py | 2 +- module/netbox/object_classes.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/module/netbox/connection.py b/module/netbox/connection.py index ccdc2033..6889398f 100644 --- a/module/netbox/connection.py +++ b/module/netbox/connection.py @@ -928,7 +928,7 @@ def delete_unused_tags(self): if tag_description is None or not tag_description.startswith(self.primary_tag): continue - if tag_tagged_items is None or tag_tagged_items != 0: + if tag_tagged_items is None or tag_tagged_items != 0 or this_tag.used is True: continue log.info(f"Deleting unused tag '{this_tag.get_display_name()}'") diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 043180aa..9e1ac7f1 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -909,6 +909,10 @@ def compile_tags(self, tags, remove=False): tag = self.inventory.add_update_object(NBTag, data={"name": tag_name}) + # mark tag as used in order to not delete newly used tags + if isinstance(tag, NBTag): + tag.used = True + new_tags.append(tag) if tag_name in current_tag_strings and remove is True: @@ -940,7 +944,7 @@ def update_tags(self, tags, remove=False): Parameters ---------- - tags: (str, list, dict, NBTag) + tags: str, list, dict, NBTag tags to parse and add/remove to/from current list of object tags remove: bool True if tags shall be removed, otherwise they will be added @@ -1269,6 +1273,7 @@ def __init__(self, *args, **kwargs): "description": 200, "tagged_items": int } + self.used = False super().__init__(*args, **kwargs) From 13685c5b6018ba032f770c0af9a8264985b16833 Mon Sep 17 00:00:00 2001 From: "ricardo.bartels@telekom.de" Date: Fri, 22 Nov 2024 12:02:20 +0100 Subject: [PATCH 03/32] adds config options vlan_group_relation_by_name and vlan_group_relation_by_id #373 --- module/config/base.py | 4 + module/netbox/__init__.py | 1 + module/netbox/object_classes.py | 22 +++- .../{excluded_vlan.py => handle_vlan.py} | 49 +++++---- module/sources/common/source_base.py | 104 +++++++++++++----- module/sources/vmware/config.py | 73 +++++++++--- module/sources/vmware/connection.py | 1 + settings-example.ini | 9 ++ 8 files changed, 190 insertions(+), 73 deletions(-) rename module/sources/common/{excluded_vlan.py => handle_vlan.py} (63%) diff --git a/module/config/base.py b/module/config/base.py index b65494d5..d6dcfc79 100644 --- a/module/config/base.py +++ b/module/config/base.py @@ -30,6 +30,10 @@ def __eq__(self, other): def __contains__(self, key): return key in self.__dict__ + def __getattr__(self, item): + if item in self: + return getattr(self, item) + return None class ConfigBase: """ diff --git a/module/netbox/__init__.py b/module/netbox/__init__.py index 8904ffe9..a7c37f5d 100644 --- a/module/netbox/__init__.py +++ b/module/netbox/__init__.py @@ -19,6 +19,7 @@ NBVRF, NBVLAN, NBVLANList, + NBVLANGroup, NBPrefix, NBManufacturer, NBDeviceType, diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 9e1ac7f1..b29ae308 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1013,12 +1013,13 @@ def compile_vlans(self, vlans): Parameters ---------- - vlans: list of (dict, NBVLAN) + vlans: list of (dict or NBVLAN) list of VLANs that should be in the returned list Returns ------- - NBVLANList: of parsed VLANs + NBVLANList + of parsed VLANs """ if vlans is None or NBVLANList not in self.data_model.values(): @@ -1361,7 +1362,8 @@ def __init__(self, *args, **kwargs): "site": NBSite, "description": 200, "tenant": NBTenant, - "tags": NBTagList + "tags": NBTagList, + "group": NBVLANGroup } super().__init__(*args, **kwargs) @@ -1402,6 +1404,20 @@ def update(self, data=None, read_from_netbox=False, source=None): super().update(data=data, read_from_netbox=read_from_netbox, source=source) +class NBVLANGroup(NetBoxObject): + name = "VLANGroup" + api_path = "ipam/vlan-groups" + primary_key = "name" + prune = False + + def __init__(self, *args, **kwargs): + self.data_model = { + "name": 100, + "slug": 100, + "description": 200, + "tags": NBTagList + } + super().__init__(*args, **kwargs) class NBVLANList(NBObjectList): member_type = NBVLAN diff --git a/module/sources/common/excluded_vlan.py b/module/sources/common/handle_vlan.py similarity index 63% rename from module/sources/common/excluded_vlan.py rename to module/sources/common/handle_vlan.py index 99a9626f..4c4a5bc7 100644 --- a/module/sources/common/excluded_vlan.py +++ b/module/sources/common/handle_vlan.py @@ -13,19 +13,17 @@ log = get_logger() -class ExcludedVLAN: - """ - initializes and verifies if an VLAN should be excluded from being synced to NetBox - """ +class VLANFilter: - def __init__(self, vlan): + def __init__(self, vlan, filter_type): self._validation_failed = False self.site = None + self.filter_type = filter_type - if vlan is None: + if vlan is None or len(f"{vlan}") == 0: self._validation_failed = True - log.error("submitted VLAN string for VLAN exclusion was 'None'") + log.error(f"submitted VLAN {self.filter_type} string for VLAN was '{"'None'" if vlan is None else "empty" }'") return vlan_split = [x.replace('\\', "") for x in re.split(r'(?= int(self._value.split("-")[1]): - log.error(f"range has to start with the lower id: {self._value}") + log.error(f"VLAN {self.filter_type} range has to start with the lower ID: {self._value}") self._validation_failed = True return @@ -113,22 +117,21 @@ def __init__(self, vlan): for i in self._value.split(',')), [] ) except Exception as e: - log.error(f"unable to extract ids from value '{self._value}': {e}") + log.error(f"unable to extract VLAN IDs from value '{self._value}': {e}") self._validation_failed = True def matches(self, vlan_id, site=None): if self.site_matches(site) is False: + log.debug2(f"VLAN {self.filter_type} site name '{site_name}' matches '{self.site}'") return False try: if int(vlan_id) in self.range: - log.debug2(f"VLAN exclude id '{vlan_id}' matches '{self._value}'") + log.debug2(f"VLAN {self.filter_type} ID '{vlan_id}' matches '{self._value}'") return True except Exception as e: - log.warning(f"Unable to match exclude VLAN id '{vlan_id}' to '{self._value}': {e}") + log.warning(f"Unable to match {self.filter_type} VLAN ID '{vlan_id}' to '{self._value}': {e}") return False return False - - diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index d6e535cb..5f1b431d 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -15,7 +15,7 @@ from module.netbox import * from module.common.logging import get_logger from module.common.misc import grab -from module.sources.common.excluded_vlan import ExcludedVLANName, ExcludedVLANID +from module.sources.common.handle_vlan import FilterVLANByName, FilterVLANByID log = get_logger() @@ -247,27 +247,26 @@ def add_update_interface(self, interface_object, device_object, interface_data, Parameters ---------- - interface_object: NBVMInterface, NBInterface, None + interface_object: NBVMInterface | NBInterface | None object handle of the current interface (if existent, otherwise None) - device_object: NBVM, NBDevice + device_object: NBVM | NBDevice device object handle this interface belongs to interface_data: dict dictionary with interface attributes to add to this interface interface_ips: list a list of ip addresses which are assigned to this interface - vmware_object: (vim.HostSystem, vim.VirtualMachine) + vmware_object: vim.HostSystem | vim.VirtualMachine object to add to list of objects to reevaluate Returns ------- - objects: tuple((NBVMInterface, NBInterface), list) + objects: + tuple of NBVMInterface | NBInterface and list tuple with interface object that was added/updated and a list of ip address objects which were added to this interface """ - ip_tenant_inheritance_order = None - if "ip_tenant_inheritance_order" in self.settings: - ip_tenant_inheritance_order = self.settings.ip_tenant_inheritance_order + ip_tenant_inheritance_order = self.settings.ip_tenant_inheritance_order if not isinstance(interface_data, dict): log.error(f"Attribute 'interface_data' must be a dict() got {type(interface_data)}.") @@ -572,7 +571,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, f"untagged interface VLAN.") if matching_untagged_vlan is not None: - vlan_interface_data["untagged_vlan"] = matching_untagged_vlan + vlan_interface_data["untagged_vlan"] = self.add_vlan_group(matching_untagged_vlan, site_name) if grab(interface_object, "data.mode") is None: vlan_interface_data["mode"] = "access" @@ -591,7 +590,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, matching_tagged_vlan = None if matching_tagged_vlan is not None: - compiled_tagged_vlans.append(matching_tagged_vlan) + compiled_tagged_vlans.append(self.add_vlan_group(matching_tagged_vlan, site_name)) if len(compiled_tagged_vlans) > 0: vlan_interface_data["tagged_vlans"] = compiled_tagged_vlans @@ -633,12 +632,70 @@ def patch_data(object_to_patch, data, overwrite=False): return data_to_update + def add_vlan_group(self, vlan_data, vlan_site) -> NBVLAN | dict: + """ + This function will try to find a matching VLAN group according to the settings. + Name matching will take precedence over ID matching. First match wins. + + If nothing matches the input data from 'vlan_data' will be returned + + Parameters + ---------- + vlan_data: dict | NBVLAN + A dict or NBVLAN object + vlan_site: str | None + name of site for the VLAN + + Returns + ------- + NBVLAN | dict: the input vlan_data enriched with VLAN group if a match was found + + """ + + # get VLAN details + if isinstance(vlan_data, NBVLAN): + vlan_name = grab(vlan_data, "data.name") + vlan_id = grab(vlan_data, "data.vid") + elif isinstance(vlan_data, dict): + vlan_name = vlan_data.get("name") + vlan_id = vlan_data.get("vid") + else: + return vlan_data + + # check existing Devices for matches + log.debug2(f"Trying to find a matching VLAN Group based on the VLAN name '{vlan_name}' and VLAN ID '{vlan_id}'") + + vlan_group = None + for vlan_filter, vlan_group_name in self.settings.vlan_group_relation_by_name or list(): + if vlan_filter.matches(vlan_name, vlan_site): + vlan_group = self.inventory.get_by_data(NBVLANGroup, data={"name": vlan_group_name}) + break + + if vlan_group is None: + for vlan_filter, vlan_group_name in self.settings.vlan_group_relation_by_id or list(): + if vlan_filter.matches(vlan_id, vlan_site): + vlan_group = self.inventory.get_by_data(NBVLANGroup, data={"name": vlan_group_name}) + break + + if vlan_group is not None: + log.debug2(f"Found matching VLAN group '{vlan_group.get_display_name()}'") + if isinstance(vlan_data, NBVLAN): + vlan_data.update(data={"group": vlan_group}) + elif isinstance(vlan_data, dict): + vlan_data["group"] = vlan_group + else: + log.debug2("No matching VLAN group found") + + print(vlan_data) + + return vlan_data + def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): """ This function will try to find a matching VLAN object based on 'vlan_data' Will return matching objects in following order: - * exact match: VLAN id and site match - * global match: VLAN id matches but the VLAN has no site assigned + * exact match: VLAN ID and site match + * global match: VLAN ID matches but the VLAN has no site assigned If nothing matches the input data from 'vlan_data' will be returned Parameters @@ -664,10 +721,10 @@ def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): raise ValueError("Value of 'vlan_data' needs to be a dict.") # check existing Devices for matches - log.debug2(f"Trying to find a {NBVLAN.name} based on the VLAN id '{vlan_data.get('vid')}'") + log.debug2(f"Trying to find a {NBVLAN.name} based on the VLAN ID '{vlan_data.get('vid')}'") if vlan_data.get("vid") is None: - log.debug("No VLAN id set in vlan_data while trying to find matching VLAN.") + log.debug("No VLAN ID set in vlan_data while trying to find matching VLAN.") return vlan_data if vlan_site is None: @@ -703,7 +760,7 @@ def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): vlan_object_without_site.get_display_name(including_second_key=True))) else: - log.debug2("No matching existing VLAN found for this VLAN id.") + log.debug2("No matching existing VLAN found for this VLAN ID.") return return_data @@ -724,17 +781,6 @@ def add_vlan_object_to_netbox(self, vlan_data, site_name=None): """ - # get config data - disable_vlan_sync = False - vlan_sync_exclude_by_name: List[ExcludedVLANName] = list() - vlan_sync_exclude_by_id: List[ExcludedVLANID] = list() - if "disable_vlan_sync" in self.settings: - disable_vlan_sync = self.settings.disable_vlan_sync - if "vlan_sync_exclude_by_name" in self.settings: - vlan_sync_exclude_by_name = self.settings.vlan_sync_exclude_by_name - if "vlan_sync_exclude_by_id" in self.settings: - vlan_sync_exclude_by_id = self.settings.vlan_sync_exclude_by_id - # VLAN is already an existing NetBox VLAN, then it can be reused if isinstance(vlan_data, NetBoxObject): return True @@ -742,7 +788,7 @@ def add_vlan_object_to_netbox(self, vlan_data, site_name=None): if vlan_data is None: return False - if disable_vlan_sync is True: + if self.settings.disable_vlan_sync is True: return False # get VLAN details @@ -757,11 +803,11 @@ def add_vlan_object_to_netbox(self, vlan_data, site_name=None): log.warning(f"Skipping sync of invalid VLAN '{vlan_name}' ID: '{vlan_id}'") return False - for excluded_vlan in vlan_sync_exclude_by_name or list(): + for excluded_vlan in self.settings.vlan_sync_exclude_by_name or list(): if excluded_vlan.matches(vlan_name, site_name): return False - for excluded_vlan in vlan_sync_exclude_by_id or list(): + for excluded_vlan in self.settings.vlan_sync_exclude_by_id or list(): if excluded_vlan.matches(vlan_id, site_name): return False diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 2dc58abf..883c5cce 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -17,7 +17,7 @@ from module.config.group import ConfigOptionGroup from module.sources.common.config import * from module.sources.common.permitted_subnets import PermittedSubnets -from module.sources.common.excluded_vlan import ExcludedVLANID, ExcludedVLANName +from module.sources.common.handle_vlan import FilterVLANByID, FilterVLANByName from module.common.logging import get_logger from module.common.support import normalize_mac_address @@ -321,7 +321,22 @@ def __init__(self): ConfigOption("vlan_sync_exclude_by_id", str, config_example="Frankfurt/25, 1023-1042"), + ConfigOption("vlan_group_relation_by_name", + str, + description="""adds a relation to assign VLAN groups to matching VLANs + by name. Same matching rules as the exclude_by_name option uses are applied. + If name and id relations are defined, the name relation takes precedence. + Fist match wins.""", + config_example="London/Vlan_.* = VLAN Group 1, Tokio/Vlan_.* = VLAN Group 2"), + ConfigOption("vlan_group_relation_by_id", + str, + description="""adds a relation to assign VLAN groups to matching VLANs by ID. + Same matching rules as the exclude_by_id option uses are applied. + Fist match wins. + """, + config_example="1023-1042 = VLAN Group 1, Tokio/2342 = VLAN Group 2") ]), + ConfigOption("track_vm_host", bool, description="""enabling this option will add the ESXi host @@ -430,7 +445,7 @@ def validate_options(self): continue - if "relation" in option.key: + if "relation" in option.key and "vlan_group_relation" not in option.key: relation_data = list() @@ -549,35 +564,57 @@ def validate_options(self): continue - if option.key == "vlan_sync_exclude_by_name": + if option.key in [ "vlan_sync_exclude_by_name", "vlan_sync_exclude_by_id", + "vlan_group_relation_by_name", "vlan_group_relation_by_id" ]: + + if option.key == "vlan_sync_exclude_by_name": + filter_class = FilterVLANByName + filter_type = "exclude" + elif option.key == "vlan_group_relation_by_name": + filter_class = FilterVLANByName + filter_type = "group relation" + elif option.key == "vlan_sync_exclude_by_id": + filter_class = FilterVLANByID + filter_type = "exclude" + elif option.key == "vlan_group_relation_by_id": + filter_class = FilterVLANByID + filter_type = "group relation" + else: + raise ValueError(f"unhandled config option {option.key}") value_list = list() - for excluded_vlan in quoted_split(option.value) or list(): - - excluded_vlan_object = ExcludedVLANName(excluded_vlan) - - if not excluded_vlan_object.is_valid(): - self.set_validation_failed() - continue + for single_option_value in quoted_split(option.value) or list(): - value_list.append(excluded_vlan_object) + relation_name = None + object_name = single_option_value.split("=")[0].strip(' "') - option.set_value(value_list) + if "relation" in option.key: - if option.key == "vlan_sync_exclude_by_id": + if "=" not in single_option_value: + log.error(f"Config option '{option.key}' malformed, got {single_option_value} but " + f"needs key = value relation.") + self.set_validation_failed() + continue - value_list = list() + relation_name = single_option_value.split("=")[1].strip(' "') - for excluded_vlan in quoted_split(option.value) or list(): + if relation_name is not None and len(relation_name) == 0: + log.error(f"Config option '{option.key}' malformed, got '{object_name}' as " + f"object name and relation name was empty.") + self.set_validation_failed() + continue - excluded_vlan_object = ExcludedVLANID(excluded_vlan) + vlan_filter = filter_class(object_name, filter_type) - if not excluded_vlan_object.is_valid(): + if not vlan_filter.is_valid(): self.set_validation_failed() continue - value_list.append(excluded_vlan_object) + if "relation" in option.key: + value_list.append((vlan_filter, relation_name)) + else: + value_list.append(vlan_filter) option.set_value(value_list) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 66a00052..ae34751b 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -71,6 +71,7 @@ class VMWareHandler(SourceBase): NBTenant, NBVRF, NBVLAN, + NBVLANGroup, NBCustomField, NBVirtualDisk ] diff --git a/settings-example.ini b/settings-example.ini index 682bffc9..883a1809 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -334,6 +334,15 @@ password = super-secret ;vlan_sync_exclude_by_name = New York/Storage, Backup, Tokio/DMZ, Madrid/.* ;vlan_sync_exclude_by_id = Frankfurt/25, 1023-1042 +; adds a relation to assign VLAN groups to matching VLANs by name. Same matching rules as +; the exclude_by_name option uses are applied. If name and id relations are defined, the +; name relation takes precedence. Fist match wins. +;vlan_group_relation_by_name = London/Vlan_.* = VLAN Group 1, Tokio/Vlan_.* = VLAN Group 2 + +; adds a relation to assign VLAN groups to matching VLANs by ID. Same matching rules as +; the exclude_by_id option uses are applied. Fist match wins. +;vlan_group_relation_by_id = 1023-1042 = VLAN Group 1, Tokio/2342 = VLAN Group 2 + ; enabling this option will add the ESXi host this VM is running on to the VM details ;track_vm_host = False From 1a0c742c29f6eea718e53ad0f1c75bb6d2c3c579 Mon Sep 17 00:00:00 2001 From: "ricardo.bartels@telekom.de" Date: Wed, 27 Nov 2024 21:37:21 +0100 Subject: [PATCH 04/32] reduce docker file size --- Dockerfile | 17 +++++++++++------ scripts/publi.sh | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e6cb907..f048b553 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,18 +2,23 @@ FROM python:3.11-slim-bookworm AS builder COPY requirements.txt . +ARG VENV=/opt/netbox-sync/venv + # Install dependencies RUN apt-get update && apt-get install -y --no-install-recommends git && \ rm -rf /var/lib/apt/lists/* && \ - python3 -m venv /opt/netbox-sync/venv && \ - /opt/netbox-sync/venv/bin/python3 -m pip install --upgrade pip && \ - /opt/netbox-sync/venv/bin/pip install -r requirements.txt && \ - /opt/netbox-sync/venv/bin/pip install --upgrade git+https://github.com/vmware/vsphere-automation-sdk-python.git + python3 -m venv $VENV && \ + $VENV/bin/python3 -m pip install --upgrade pip && \ + $VENV/bin/pip install -r requirements.txt && \ + $VENV/bin/pip install --upgrade git+https://github.com/vmware/vsphere-automation-sdk-python.git && \ + find $VENV -type d -name "__pycache__" -print0 | xargs -0 -n1 rm -rf FROM python:3.11-slim-bookworm AS netbox-sync +ARG VENV=/opt/netbox-sync/venv + # Copy installed packages -COPY --from=builder /opt/netbox-sync/venv /opt/netbox-sync/venv +COPY --from=builder $VENV $VENV # Add netbox-sync user RUN groupadd --gid 1000 netbox-sync && \ @@ -27,7 +32,7 @@ WORKDIR /app COPY --chown=netbox-sync:netbox-sync . . # Use virtual env packages and allow timezone setup -ENV PATH=/opt/netbox-sync/venv/bin:$PATH +ENV PATH=$VENV/bin:$PATH ENV TZ=Europe/Berlin ENTRYPOINT ["python3", "netbox-sync.py"] diff --git a/scripts/publi.sh b/scripts/publi.sh index 8fcc3484..d5310af1 100755 --- a/scripts/publi.sh +++ b/scripts/publi.sh @@ -33,7 +33,7 @@ unset DOCKER_TLS_VERIFY unset DOCKER_HOST unset DOCKER_CERT_PATH -find module -type d -name "__pycache__" -exec rm -rf {} \; +find module -type d -name "__pycache__" -print0 | xargs -0 -n1 rm -rf docker --config ./docker-tmp login docker --config ./docker-tmp buildx create --use if [[ $FINAL == true ]]; then From 740f1e1fcab7acc5badd0e70faf324be60372217 Mon Sep 17 00:00:00 2001 From: "ricardo.bartels@telekom.de" Date: Thu, 28 Nov 2024 14:29:33 +0100 Subject: [PATCH 05/32] fixes issue with vlan filter #373 --- module/sources/common/handle_vlan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module/sources/common/handle_vlan.py b/module/sources/common/handle_vlan.py index 4c4a5bc7..552b1516 100644 --- a/module/sources/common/handle_vlan.py +++ b/module/sources/common/handle_vlan.py @@ -23,7 +23,7 @@ def __init__(self, vlan, filter_type): if vlan is None or len(f"{vlan}") == 0: self._validation_failed = True - log.error(f"submitted VLAN {self.filter_type} string for VLAN was '{"'None'" if vlan is None else "empty" }'") + log.error(f"submitted VLAN {self.filter_type} string for VLAN was " + "'None'" if vlan is None else "empty") return vlan_split = [x.replace('\\', "") for x in re.split(r'(? Date: Thu, 12 Dec 2024 16:39:18 +0100 Subject: [PATCH 06/32] fixes issue with redfish source and sorting resource with no names --- module/sources/check_redfish/import_inventory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index 84274671..8b481959 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -270,7 +270,7 @@ def update_power_supply(self): if grab(ps, "data.device") == self.device_object: current_ps.append(ps) - current_ps.sort(key=lambda x: grab(x, "data.name")) + current_ps.sort(key=lambda x: grab(x, "data.name") or "") ps_index = 1 ps_items = list() @@ -944,7 +944,7 @@ def update_all_items(self, items): unmatched_inventory_items.append(item) # sort unmatched items by full_name - unmatched_inventory_items.sort(key=lambda x: x.get("full_name")) + unmatched_inventory_items.sort(key=lambda x: x.get("full_name") or "") # iterate over current NetBox inventory items # if name did not match try to assign unmatched items in alphabetical order From 26a228140bd01d97043fd2fb4d02a7e91a975a9c Mon Sep 17 00:00:00 2001 From: "ricardo.bartels@telekom.de" Date: Thu, 12 Dec 2024 16:45:02 +0100 Subject: [PATCH 07/32] adds config option disable collection of ESXi serial #428 --- module/sources/vmware/config.py | 4 ++++ module/sources/vmware/connection.py | 2 +- settings-example.ini | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 883c5cce..8a251593 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -197,6 +197,10 @@ def __init__(self): bool, description="Attempt to collect asset tags from vCenter hosts", default_value=True), + ConfigOption("collect_hardware_serial", + bool, + description="Attempt to collect serials from vCenter hosts", + default_value=True), ConfigOption("dns_name_lookup", bool, description="""Perform a reverse lookup for all collected IP addresses. diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index ae34751b..de072e2b 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1637,7 +1637,7 @@ def add_host(self, obj): serial = None for serial_num_key in ["SerialNumberTag", "ServiceTag", "EnclosureSerialNumberTag"]: - if serial_num_key in identifier_dict.keys(): + if serial_num_key in identifier_dict.keys() and self.settings.collect_hardware_serial is True: log.debug2(f"Found {serial_num_key}: {get_string_or_none(identifier_dict.get(serial_num_key))}") if serial is None: serial = get_string_or_none(identifier_dict.get(serial_num_key)) diff --git a/settings-example.ini b/settings-example.ini index 883a1809..1c51448c 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -241,6 +241,9 @@ password = super-secret ; Attempt to collect asset tags from vCenter hosts ;collect_hardware_asset_tag = True +; Attempt to collect serials from vCenter hosts +;collect_hardware_serial = True + ; Perform a reverse lookup for all collected IP addresses. If a dns name was found it will ; be added to the IP address object in NetBox ;dns_name_lookup = True From 849bd27d4cbb09ebc86fe53d70608011e9481a40 Mon Sep 17 00:00:00 2001 From: Joachim Burket Date: Thu, 23 Jan 2025 13:56:59 +0100 Subject: [PATCH 08/32] add the 'serial' field to NetBox VirtualMachine and set it from VMWare 'instanceUuid' Signed-off-by: Joachim Burket --- module/netbox/object_classes.py | 1 + module/sources/vmware/connection.py | 1 + 2 files changed, 2 insertions(+) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index dd84862f..8cb8ff82 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1624,6 +1624,7 @@ class NBVM(NetBoxObject): def __init__(self, *args, **kwargs): self.data_model = { "name": 64, + "serial": 50, "status": ["offline", "active", "planned", "staged", "failed", "decommissioning"], "cluster": NBCluster, "role": NBDeviceRole, diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index f8f8bfd8..b75d5942 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -2176,6 +2176,7 @@ def add_virtual_machine(self, obj): vm_data = { "name": name, + "serial": vm_uuid, "cluster": nb_cluster_object, "status": status, "memory": vm_memory, From 2963de5141bd498e841dec5d3bc8feda45698fbd Mon Sep 17 00:00:00 2001 From: joachimBurket Date: Thu, 23 Jan 2025 16:14:01 +0100 Subject: [PATCH 09/32] only set the 'serial' field for NetBox version >=4.1.0 Signed-off-by: joachimBurket --- module/sources/vmware/connection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index b75d5942..27c177c8 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -2176,7 +2176,6 @@ def add_virtual_machine(self, obj): vm_data = { "name": name, - "serial": vm_uuid, "cluster": nb_cluster_object, "status": status, "memory": vm_memory, @@ -2197,6 +2196,10 @@ def add_virtual_machine(self, obj): if isinstance(comp, vim.vm.device.VirtualDisk) ]) / 1024 / 1024) + # Add adaptation for the new 'serial' field in NetBox 4.1.0 VM model + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.1.0"): + vm_data["serial"] = vm_uuid + if platform is not None: vm_data["platform"] = {"name": platform} if annotation is not None: From 9400b16eb3703c6e572403afe4ea39b78cb5bb5a Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Fri, 31 Jan 2025 11:14:09 +0100 Subject: [PATCH 10/32] adds support for 'primary_mac_address' in NetBox 4.2 #436 --- module/netbox/__init__.py | 1 + module/netbox/connection.py | 7 +- module/netbox/object_classes.py | 112 +++++++++++++++++++++++++++ module/sources/common/source_base.py | 35 ++++++++- module/sources/vmware/connection.py | 3 +- 5 files changed, 152 insertions(+), 6 deletions(-) diff --git a/module/netbox/__init__.py b/module/netbox/__init__.py index a7c37f5d..a7e44606 100644 --- a/module/netbox/__init__.py +++ b/module/netbox/__init__.py @@ -34,6 +34,7 @@ NBVirtualDisk, NBInterface, NBIPAddress, + NBMACAddress, NBFHRPGroupItem, NBInventoryItem, NBPowerPort diff --git a/module/netbox/connection.py b/module/netbox/connection.py index 6889398f..163aab01 100644 --- a/module/netbox/connection.py +++ b/module/netbox/connection.py @@ -650,7 +650,8 @@ def update_object(self, nb_object_sub_class, unset=False, last_run=False): # resolve dependency issues in last run # primary IP always set in last run if value.get_nb_reference() is None or \ - (key.startswith("primary_ip") and last_run is False): + (key.startswith("primary_ip") and last_run is False) or \ + (key.startswith("primary_mac_address") and last_run is False): unresolved_dependency_data[key] = value else: data_to_patch[key] = value.get_nb_reference() @@ -658,8 +659,8 @@ def update_object(self, nb_object_sub_class, unset=False, last_run=False): else: data_to_patch[key] = value - # special case for IP address - if isinstance(this_object, NBIPAddress): + # special case for IP and MAC addresses + if isinstance(this_object, (NBIPAddress, NBMACAddress)): # if object is new and has no id, then we need to remove assigned_object_type from data_to_patch if "assigned_object_id" in unresolved_dependency_data.keys() and \ "assigned_object_type" in data_to_patch.keys(): diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 2868e44f..319a7a39 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1687,6 +1687,7 @@ def __init__(self, *args, **kwargs): "virtual_machine": NBVM, "enabled": bool, "mac_address": str, + "primary_mac_address": NBMACAddress, "mtu": int, "mode": ["access", "tagged", "tagged-all"], "untagged_vlan": NBVLAN, @@ -1722,6 +1723,7 @@ def __init__(self, *args, **kwargs): "type": NetBoxInterfaceType().get_netbox_type_list(), "enabled": bool, "mac_address": str, + "primary_mac_address": NBMACAddress, "wwn": str, "mgmt_only": bool, "mtu": int, @@ -1913,6 +1915,116 @@ def remove_interface_association(self): if o_type is not None: self.unset_attribute("assigned_object_type") +class NBMACAddress(NetBoxObject): + name = "MAC address" + api_path = "dcim/mac-addresses" + primary_key = "mac_address" + prune = True + min_netbox_version = "4.2" + + def __init__(self, *args, **kwargs): + self.data_model = { + "mac_address": str, + "assigned_object_type": ["dcim.interface", "virtualization.vminterface"], + "assigned_object_id": [NBInterface, NBVMInterface], + "description": 200, + "tags": NBTagList, + } + + # add relation between two attributes + self.data_model_relation = { + "dcim.interface": NBInterface, + "virtualization.vminterface": NBVMInterface, + NBInterface: "dcim.interface", + NBVMInterface: "virtualization.vminterface", + } + super().__init__(*args, **kwargs) + + def resolve_relations(self): + + o_id = self.data.get("assigned_object_id") + o_type = self.data.get("assigned_object_type") + + # this needs special treatment as the object type depends on a second model key + if o_type is not None and o_type not in self.data_model.get("assigned_object_type"): + + log.error(f"Attribute 'assigned_object_type' for '{self.get_display_name()}' invalid: {o_type}") + do_error_exit(f"Error while resolving relations for {self.get_display_name()}") + + if isinstance(o_id, int) and o_type is not None: + self.data["assigned_object_id"] = self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) + + super().resolve_relations() + + + def update(self, data=None, read_from_netbox=False, source=None): + + object_type = data.get("assigned_object_type") + assigned_object = data.get("assigned_object_id") + + # we got an object data structure where we have to find the object + if read_from_netbox is False and assigned_object is not None: + + if not isinstance(assigned_object, NetBoxObject): + + data["assigned_object_id"] = \ + self.inventory.add_update_object(self.data_model_relation.get(object_type), data=assigned_object) + + else: + # noinspection PyTypeChecker + data["assigned_object_type"] = self.data_model_relation.get(type(assigned_object)) + + super().update(data=data, read_from_netbox=read_from_netbox, source=source) + + # we need to tell NetBox which object type this is meant to be + if "assigned_object_id" in self.updated_items: + self.updated_items.append("assigned_object_type") + + # if ip association has been removed we also need to get rid of object type + if "assigned_object_type" in self.updated_items and self.data.get("assigned_object_id") is None \ + and "assigned_object_type" in self.updated_items: + self.updated_items.remove("assigned_object_type") + + def get_interface(self): + o_id = self.data.get("assigned_object_id") + o_type = self.data.get("assigned_object_type") + + if isinstance(o_id, (NBInterface, NBVMInterface)): + return o_id + + if o_type is None or not isinstance(o_id, int): + return + + if o_type not in self.data_model.get("assigned_object_type"): + return + + return self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) + + def get_device_vm(self): + + o_interface = self.get_interface() + + if o_interface is None: + return + + if isinstance(o_interface, NBInterface): + return o_interface.data.get("device") + elif isinstance(o_interface, NBVMInterface): + return o_interface.data.get("virtual_machine") + + def remove_interface_association(self): + o_id = self.data.get("assigned_object_id") + o_type = self.data.get("assigned_object_type") + o_device = self.get_device_vm() + + if grab(o_device, "data.primary_mac_address") is self: + o_device.unset_attribute("primary_mac_address") + + if o_id is not None: + self.unset_attribute("assigned_object_id") + if o_type is not None: + self.unset_attribute("assigned_object_type") + class NBFHRPGroupItem(NetBoxObject): """ diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 5f1b431d..8ad26e18 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -11,6 +11,7 @@ from ipaddress import ip_interface, ip_address, IPv6Address, IPv4Address, IPv6Network, IPv4Network from typing import List +from packaging import version from module.netbox import * from module.common.logging import get_logger @@ -266,6 +267,12 @@ def add_update_interface(self, interface_object, device_object, interface_data, added to this interface """ + # handle change to mac_address object from NetBox 4.2 on + interface_mac_address = None + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + interface_mac_address = interface_data.get("mac_address") + del(interface_data["mac_address"]) + ip_tenant_inheritance_order = self.settings.ip_tenant_inheritance_order if not isinstance(interface_data, dict): @@ -312,6 +319,32 @@ def add_update_interface(self, interface_object, device_object, interface_data, else: interface_object.update(data=interface_data, source=self) + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0") and \ + interface_mac_address is not None : + + primary_mac_address_object = grab(interface_object, "data.primary_mac_address") + + if primary_mac_address_object is None or grab(primary_mac_address_object, "data.mac_address") != interface_mac_address: + + primary_mac_address_object = None + for mac_address_object in self.inventory.get_all_items(NBMACAddress): + if grab(mac_address_object, "data.mac_address") == interface_mac_address and grab(mac_address_object, "data.assigned_object_id") is None: + primary_mac_address_object = mac_address_object + break + + primary_mac_address_data = { + "mac_address": interface_mac_address, + "assigned_object_id": interface_object, + "assigned_object_type": interface_class + } + + if primary_mac_address_object is None: + primary_mac_address_object = self.inventory.add_object(NBMACAddress, data=primary_mac_address_data, source=self) + else: + primary_mac_address_object.update(data=primary_mac_address_data, source=self) + + interface_object.update(data={"primary_mac_address": primary_mac_address_object}, source=self) + ip_address_objects = list() matching_ip_prefixes = list() # add all interface IPs @@ -686,8 +719,6 @@ def add_vlan_group(self, vlan_data, vlan_site) -> NBVLAN | dict: else: log.debug2("No matching VLAN group found") - print(vlan_data) - return vlan_data def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 094d2d51..66eae7c9 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -73,7 +73,8 @@ class VMWareHandler(SourceBase): NBVLAN, NBVLANGroup, NBCustomField, - NBVirtualDisk + NBVirtualDisk, + NBMACAddress ] source_type = "vmware" From 80ac276b7beaa4fd18d2b5437abab9c3037dc75b Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 19 Feb 2025 22:45:02 +0100 Subject: [PATCH 11/32] adds support for site scope on cluster object in NetBox 4.2.0 #436 --- module/netbox/object_classes.py | 28 +++++++++++++++++++++++++++- module/sources/common/source_base.py | 3 ++- module/sources/vmware/connection.py | 9 +++++++-- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 319a7a39..0fc45c54 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1167,6 +1167,17 @@ def get_site_name(self, data=None): if isinstance(this_site, dict): return this_site.get("name") + this_scope_type = this_data_set.get("scope_type") + this_site = this_data_set.get("scope_id") + + if this_scope_type == "dcim.site" and this_site is not None: + + if isinstance(this_site, NetBoxObject): + return this_site.get_display_name() + + if isinstance(this_site, dict): + return this_site.get("name") + class NBObjectList(list): """ @@ -1589,11 +1600,26 @@ def __init__(self, *args, **kwargs): "type": NBClusterType, "tenant": NBTenant, "group": NBClusterGroup, - "site": NBSite, + "scope_type": ["dcim.site", "dcim.sitegroup", "dcim.location", "dcim.region"], + "scope_id": NBSite, "tags": NBTagList } super().__init__(*args, **kwargs) + def update(self, data=None, read_from_netbox=False, source=None): + + # Add adaption for change in NetBox 4.2.0 Device model + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + if data.get("site") is not None: + data["scope_id"] = data.get("site") + data["scope_type"] = "dcim.site" + del data["site"] + + if data.get("scope_id") is not None: + data["scope_type"] = "dcim.site" + + super().update(data=data, read_from_netbox=read_from_netbox, source=source) + class NBDevice(NetBoxObject): name = "device" diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 8ad26e18..cdd046c0 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -281,7 +281,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, if type(device_object) == NBVM: interface_class = NBVMInterface - site_name = grab(device_object, "data.cluster.data.site.data.name") + interface_cluster = grab(device_object, "data.cluster") + site_name = interface_cluster.get_site_name() elif type(device_object) == NBDevice: interface_class = NBInterface site_name = grab(device_object, "data.site.data.name") diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 66eae7c9..82133f59 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1378,10 +1378,15 @@ def add_cluster(self, obj): data = { "name": name, "type": {"name": "VMware ESXi"}, - "group": group, - "site": {"name": site_name} + "group": group } + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + data["scope_id"] = {"name": site_name} + data["scope_type"] = "dcim.site" + else: + data["site"] = {"name": site_name} + tenant_name = self.get_object_relation(full_cluster_name, "cluster_tenant_relation") if tenant_name is not None: data["tenant"] = {"name": tenant_name} From 13311abce924290fd46cd8f987b92f9fc1135b38 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 19 Feb 2025 23:10:51 +0100 Subject: [PATCH 12/32] adds vm guest prettyName in order to extract correct VM platform #429 --- module/sources/vmware/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 82133f59..8749d3b6 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -2156,6 +2156,7 @@ def add_virtual_machine(self, obj): # first check against vm_platform_relation platform = get_string_or_none(grab(obj, "config.guestFullName")) platform = get_string_or_none(grab(obj, "guest.guestFullName", fallback=platform)) + platform = get_string_or_none(grab(obj, "guest.guestDetailedData.prettyName", fallback=platform)) if platform is not None: platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) From f88c5f73edb12cf8e186752737ec9b13caad3ce9 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 19 Feb 2025 23:30:29 +0100 Subject: [PATCH 13/32] adds possibility to match VM by serial for NetNox >= 4.1 instances --- module/sources/vmware/connection.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 8749d3b6..f49d119a 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1064,6 +1064,11 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v device_vm_object = self.inventory.get_by_data(object_type, data={"asset_tag": object_data.get("asset_tag")}) + # look for VMs with same serial + if object_type == NBVM and device_vm_object is None and object_data.get("serial") is not None: + log.debug2(f"No match found. Trying to find {object_type.name} based on serial number") + device_vm_object = self.inventory.get_by_data(object_type, data={"serial": object_data.get("serial")}) + if device_vm_object is not None: log.debug2("Found a matching %s object: %s" % (object_type.name, device_vm_object.get_display_name(including_second_key=True))) From 04cb13c8e0a945873e9f7a82dc76dff531750c3d Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Thu, 20 Feb 2025 17:43:35 +0100 Subject: [PATCH 14/32] adds custom field for host power status and combines two others #379 #440 --- module/netbox/object_classes.py | 3 +- .../sources/check_redfish/import_inventory.py | 60 ++++++++++++++++--- module/sources/common/source_base.py | 4 +- module/sources/vmware/connection.py | 8 +-- 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 0fc45c54..f5639fbc 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -630,7 +630,8 @@ def update(self, data=None, read_from_netbox=False, source=None): if self.data_model.get(key) == NBCustomField: if current_value is None: current_value = dict() - new_value_str = str({**current_value, **new_value}) + new_value = {**current_value, **new_value} + new_value_str = str(new_value) elif isinstance(new_value, (NetBoxObject, NBObjectList)): new_value_str = str(new_value.get_display_name()) else: diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index 8b481959..a8c3aa20 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -214,11 +214,6 @@ def update_device(self): log.error(f"No system data found for '{self.device_object.get_display_name()}' in inventory file.") return - # get status - status = "offline" - if get_string_or_none(grab(system, "power_state")) == "On": - status = "active" - serial = get_string_or_none(grab(system, "serial")) name = get_string_or_none(grab(system, "host_name")) manufacturer = get_string_or_none(grab(system, "manufacturer")) @@ -230,9 +225,9 @@ def update_device(self): "name": manufacturer }, }, - "status": status, "custom_fields": { - "health": get_string_or_none(grab(system, "health_status")) + "health": get_string_or_none(grab(system, "health_status")), + "power_state": get_string_or_none(grab(system, "power_state")) } } @@ -399,6 +394,7 @@ def update_fan(self): def update_memory(self): items = list() + memory_size_total = 0 for memory in grab(self.inventory_file_content, "inventory.memory", fallback=list()): if grab(memory, "operation_status") in ["NotPresent", "Absent"]: @@ -416,6 +412,8 @@ def update_memory(self): if size_in_mb == 0 or (health_status is None and grab(memory, "operation_status") != "GoodInUse"): continue + memory_size_total += size_in_mb + name_details = list() if dimm_type is not None: name_details.append(f"{dimm_type}") @@ -448,9 +446,21 @@ def update_memory(self): self.update_all_items(items) + if memory_size_total > 0: + memory_size_total = memory_size_total / 1024 + memory_size_unit = "GB" + if memory_size_total >= 1024: + memory_size_total = memory_size_total / 1024 + memory_size_unit = "TB" + + custom_fields_data = {"custom_fields": {"host_memory": f"{memory_size_total} {memory_size_unit}"}} + self.device_object.update(data=custom_fields_data, source=self) + def update_proc(self): items = list() + num_cores = 0 + cpu_name = "" for processor in grab(self.inventory_file_content, "inventory.processor", fallback=list()): if grab(processor, "operation_status") in ["NotPresent", "Absent"]: @@ -465,6 +475,7 @@ def update_proc(self): health_status = get_string_or_none(grab(processor, "health_status")) name = f"{socket} ({model})" + cpu_name = model if current_speed is not None: current_speed = f"{current_speed / 1000}GHz" @@ -477,6 +488,7 @@ def update_proc(self): description.append(f"{instruction_set}") if cores is not None: description.append(f"Cores: {cores}") + num_cores += int(cores) if threads is not None: description.append(f"Threads: {threads}") @@ -493,6 +505,10 @@ def update_proc(self): self.update_all_items(items) + if num_cores > 0: + custom_fields_data = {"custom_fields": {"host_cpu_cores": f"{num_cores} {cpu_name}"}} + self.device_object.update(data=custom_fields_data, source=self) + def update_physical_drive(self): items = list() @@ -1037,6 +1053,36 @@ def add_necessary_base_objects(self): "description": f"Marks objects synced from check_redfish inventory '{self.name}' to this NetBox Instance." }) + self.add_update_custom_field({ + "name": "host_cpu_cores", + "label": "Physical CPU Cores", + "object_types": [ + "dcim.device" + ], + "type": "text", + "description": f"Reported Host CPU cores" + }) + + self.add_update_custom_field({ + "name": "host_memory", + "label": "Memory", + "object_types": [ + "dcim.device" + ], + "type": "text", + "description": f"Reported size of Memory" + }) + + self.add_update_custom_field({ + "name": "power_state", + "label": "Power State", + "object_types": [ + "dcim.device" + ], + "type": "text", + "description": "Device power state" + }) + # add Firmware self.add_update_custom_field({ "name": "firmware", diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index cdd046c0..56a8331d 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -269,7 +269,9 @@ def add_update_interface(self, interface_object, device_object, interface_data, # handle change to mac_address object from NetBox 4.2 on interface_mac_address = None - if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0") and \ + interface_data.get("mac_address") is not None: + interface_mac_address = interface_data.get("mac_address") del(interface_data["mac_address"]) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index f49d119a..cae51b93 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -797,22 +797,22 @@ def get_object_custom_fields(self, obj): if num_cpu_cores is not None: custom_field = self.add_update_custom_field({ - "name": "vcsa_host_cpu_cores", + "name": "host_cpu_cores", "label": "Physical CPU Cores", "object_types": [object_type], "type": "text", - "description": f"vCenter '{self.name}' reported Host CPU cores" + "description": f"Reported Host CPU cores" }) return_custom_fields[grab(custom_field, "data.name")] = f"{num_cpu_cores} {cpu_model}" if isinstance(memory_size, int): custom_field = self.add_update_custom_field({ - "name": "vcsa_host_memory", + "name": "host_memory", "label": "Memory", "object_types": [object_type], "type": "text", - "description": f"vCenter '{self.name}' reported Memory" + "description": f"Reported size of Memory" }) memory_size = round(memory_size / 1024 ** 3) From c12cb81430462af572ded0c597fcff2a92c45ddc Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Fri, 21 Feb 2025 21:38:55 +0100 Subject: [PATCH 15/32] improves the vlan group relation settings #373 --- module/netbox/__init__.py | 1 + module/netbox/object_classes.py | 92 +++++++++++++++++- module/sources/common/source_base.py | 140 ++++++++++++++++++++------- module/sources/vmware/connection.py | 1 + 4 files changed, 194 insertions(+), 40 deletions(-) diff --git a/module/netbox/__init__.py b/module/netbox/__init__.py index a7e44606..ef9cbe81 100644 --- a/module/netbox/__init__.py +++ b/module/netbox/__init__.py @@ -16,6 +16,7 @@ NBTagList, NBTenant, NBSite, + NBSiteGroup, NBVRF, NBVLAN, NBVLANList, diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index f5639fbc..86f02fd2 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1338,7 +1338,25 @@ def __init__(self, *args, **kwargs): "slug": 100, "comments": str, "tenant": NBTenant, - "tags": NBTagList + "tags": NBTagList, + "group": NBSiteGroup + } + super().__init__(*args, **kwargs) + +class NBSiteGroup(NetBoxObject): + """ + This object is currently not used directly in any class. + It is used to handle scopes properly. + """ + name = "site group" + api_path = "dcim/site-groups" + primary_key = "name" + prune = False + + def __init__(self, *args, **kwargs): + self.data_model = { + "name": 100, + "slug": 100, } super().__init__(*args, **kwargs) @@ -1404,7 +1422,15 @@ def get_display_name(self, data=None, including_second_key=False): site_name = self.get_site_name(this_data_set) if site_name is not None: - my_name = f"{vlan_id} ({site_name})" + my_name = f"{vlan_id} (site: {site_name})" + + this_group = this_data_set.get("group") + if this_group is not None: + vlan_id = this_data_set.get(self.primary_key) + group_name = grab(this_group, "data.name") + + if group_name is not None: + my_name = f"{vlan_id} (group: {group_name})" return my_name @@ -1427,10 +1453,70 @@ def __init__(self, *args, **kwargs): "name": 100, "slug": 100, "description": 200, - "tags": NBTagList + "tags": NBTagList, + "scope_type": ["dcim.site", "dcim.sitegroup", "virtualization.cluster", "virtualization.clustergroup"], + "scope_id": [NBSite, NBSiteGroup, NBCluster, NBClusterGroup], } + # add relation between two attributes + self.data_model_relation = { + "dcim.site": NBSite, + "dcim.sitegroup": NBSiteGroup, + "virtualization.cluster": NBCluster, + "virtualization.clustergroup": NBClusterGroup, + NBSite: "dcim.site", + NBSiteGroup: "dcim.sitegroup", + NBCluster: "virtualization.cluster", + NBClusterGroup: "virtualization.clustergroup" + } + super().__init__(*args, **kwargs) + def resolve_relations(self): + + o_id = self.data.get("scope_id") + o_type = self.data.get("scope_type") + + if isinstance(o_id, int) and o_type is not None and self.data_model_relation.get(o_type) is not None: + self.data["scope_id"] = self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) + elif not isinstance(o_id, NetBoxObject): + log.debug(f"{self.name} '{self.data.get('name')}' scope type '{o_type}' for '{grab(self, 'data.scope.name')}' is currently not supported") + self.data["scope_id"] = "" + + super().resolve_relations() + + def matches_site_cluster(self, site=None, cluster=None) -> bool: + """ + tries to figure out if this vlan group matches a certain site or cluster + + Parameters + ---------- + site: NBSite + the site object to match to + cluster: NBCluster + the cluster object to match to + + Returns + ------- + bool: True if matches one of the params + + """ + if isinstance(site, NBSite): + if isinstance(self.data["scope_id"], NBSite) and self.data["scope_id"] == site: + return True + if (isinstance(self.data["scope_id"], NBSiteGroup) and + self.data["scope_id"] == grab(site, "data.group")): + return True + + if isinstance(cluster, NBCluster): + if isinstance(self.data["scope_id"], NBCluster) and self.data["scope_id"] == cluster: + return True + if (isinstance(self.data["scope_id"], NBClusterGroup) and + self.data["scope_id"] == grab(cluster, "data.group")): + return True + + return False + + class NBVLANList(NBObjectList): member_type = NBVLAN diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 56a8331d..4763a1b4 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -281,10 +281,12 @@ def add_update_interface(self, interface_object, device_object, interface_data, log.error(f"Attribute 'interface_data' must be a dict() got {type(interface_data)}.") return None + device_object_cluster = grab(device_object, "data.cluster") + device_object_site = grab(device_object, "data.site") + if type(device_object) == NBVM: interface_class = NBVMInterface - interface_cluster = grab(device_object, "data.cluster") - site_name = interface_cluster.get_site_name() + site_name = device_object_cluster.get_site_name() elif type(device_object) == NBDevice: interface_class = NBInterface site_name = grab(device_object, "data.site.data.name") @@ -596,7 +598,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, vlan_interface_data = dict() if untagged_vlan is not None or (untagged_vlan is None and len(tagged_vlans) == 0): if matching_untagged_vlan is None and untagged_vlan is not None: - matching_untagged_vlan = self.get_vlan_object_if_exists(untagged_vlan, site_name) + matching_untagged_vlan = self.get_vlan_object_if_exists(untagged_vlan, device_object_site, + device_object_cluster) # don't sync newly discovered VLANs to NetBox if self.add_vlan_object_to_netbox(matching_untagged_vlan, site_name) is False: @@ -607,7 +610,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, f"untagged interface VLAN.") if matching_untagged_vlan is not None: - vlan_interface_data["untagged_vlan"] = self.add_vlan_group(matching_untagged_vlan, site_name) + vlan_interface_data["untagged_vlan"] = self.add_vlan_group(matching_untagged_vlan, site_name, + device_object_cluster) if grab(interface_object, "data.mode") is None: vlan_interface_data["mode"] = "access" @@ -619,14 +623,16 @@ def add_update_interface(self, interface_object, device_object, interface_data, log.debug2(f"Found matching prefix VLAN {matching_tagged_vlan.get_display_name()} for " f"tagged interface VLAN.") else: - matching_tagged_vlan = self.get_vlan_object_if_exists(tagged_vlan, site_name) + matching_tagged_vlan = self.get_vlan_object_if_exists(tagged_vlan, device_object_site, + device_object_cluster) # don't sync newly discovered VLANs to NetBox if self.add_vlan_object_to_netbox(matching_tagged_vlan, site_name) is False: matching_tagged_vlan = None if matching_tagged_vlan is not None: - compiled_tagged_vlans.append(self.add_vlan_group(matching_tagged_vlan, site_name)) + compiled_tagged_vlans.append(self.add_vlan_group(matching_tagged_vlan, site_name, + device_object_cluster)) if len(compiled_tagged_vlans) > 0: vlan_interface_data["tagged_vlans"] = compiled_tagged_vlans @@ -668,19 +674,20 @@ def patch_data(object_to_patch, data, overwrite=False): return data_to_update - def add_vlan_group(self, vlan_data, vlan_site) -> NBVLAN | dict: + def add_vlan_group(self, vlan_data, vlan_site, vlan_cluster) -> NBVLAN | dict: """ This function will try to find a matching VLAN group according to the settings. Name matching will take precedence over ID matching. First match wins. - If nothing matches the input data from 'vlan_data' will be returned + If nothing matches the input data the submitted 'vlan_data' will be returned Parameters ---------- vlan_data: dict | NBVLAN A dict or NBVLAN object - vlan_site: str | None + vlan_site: NBSite | str | None name of site for the VLAN + vlan_cluster: NBCluster | str | None Returns ------- @@ -690,45 +697,79 @@ def add_vlan_group(self, vlan_data, vlan_site) -> NBVLAN | dict: # get VLAN details if isinstance(vlan_data, NBVLAN): + if vlan_data.is_new is False: + return vlan_data + vlan_name = grab(vlan_data, "data.name") vlan_id = grab(vlan_data, "data.vid") + # vlan already has a group attached + if grab(vlan_data, "data.group") is not None: + return vlan_data + elif isinstance(vlan_data, dict): vlan_name = vlan_data.get("name") vlan_id = vlan_data.get("vid") else: return vlan_data - # check existing Devices for matches - log.debug2(f"Trying to find a matching VLAN Group based on the VLAN name '{vlan_name}' and VLAN ID '{vlan_id}'") + if isinstance(vlan_site, str): + vlan_site = self.inventory.get_by_data(NBSite, data={"name": vlan_site}) + + if isinstance(vlan_cluster, str): + vlan_cluster = self.inventory.get_by_data(NBCluster, data={"name": vlan_cluster}) + + log_text = f"Trying to find a matching VLAN Group based on the VLAN name '{vlan_name}'" + if vlan_site is not None: + log_text += f", site '{vlan_site.get_display_name()}'" + if vlan_cluster is not None: + log_text += f", cluster '{vlan_cluster.get_display_name()}'" + log_text += f" and VLAN ID '{vlan_id}'" + log.debug(log_text) vlan_group = None for vlan_filter, vlan_group_name in self.settings.vlan_group_relation_by_name or list(): if vlan_filter.matches(vlan_name, vlan_site): - vlan_group = self.inventory.get_by_data(NBVLANGroup, data={"name": vlan_group_name}) - break + for inventory_vlan_group in self.inventory.get_all_items(NBVLANGroup): + + if grab(inventory_vlan_group, "data.name") != vlan_group_name: + continue + + if inventory_vlan_group.matches_site_cluster(vlan_site, vlan_cluster): + vlan_group = inventory_vlan_group + break if vlan_group is None: for vlan_filter, vlan_group_name in self.settings.vlan_group_relation_by_id or list(): if vlan_filter.matches(vlan_id, vlan_site): - vlan_group = self.inventory.get_by_data(NBVLANGroup, data={"name": vlan_group_name}) - break + for inventory_vlan_group in self.inventory.get_all_items(NBVLANGroup): + + if grab(inventory_vlan_group, "data.name") != vlan_group_name: + continue + + if inventory_vlan_group.matches_site_cluster(vlan_site, vlan_cluster): + vlan_group = inventory_vlan_group + break if vlan_group is not None: log.debug2(f"Found matching VLAN group '{vlan_group.get_display_name()}'") if isinstance(vlan_data, NBVLAN): vlan_data.update(data={"group": vlan_group}) + if vlan_data.data.get("site") is not None: + vlan_data.unset_attribute("site") elif isinstance(vlan_data, dict): vlan_data["group"] = vlan_group + if vlan_data.get("site") is not None: + del(vlan_data["site"]) else: log.debug2("No matching VLAN group found") return vlan_data - def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): + def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None, vlan_cluster=None): """ This function will try to find a matching VLAN object based on 'vlan_data' Will return matching objects in following order: - * exact match: VLAN ID and site match + * exact match: VLAN ID and site or VLAN Group which matches site or cluster * global match: VLAN ID matches but the VLAN has no site assigned If nothing matches the input data from 'vlan_data' will be returned @@ -736,8 +777,10 @@ def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): ---------- vlan_data: dict A dict with NBVLAN data attributes - vlan_site: str - name of site the VLAN could be present + vlan_site: NBSite + site object + vlan_cluster: NBCluster + cluster object Returns ------- @@ -766,32 +809,55 @@ def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): elif isinstance(vlan_site, str): vlan_site = self.inventory.get_by_data(NBSite, data={"name": vlan_site}) + if isinstance(vlan_cluster, str): + vlan_cluster = self.inventory.get_by_data(NBCluster, data={"name": vlan_cluster}) + return_data = vlan_data - vlan_object_including_site = None - vlan_object_without_site = None + vlan_object_by_site = None + vlan_object_by_group = None + vlan_object_global = None for vlan in self.inventory.get_all_items(NBVLAN): if grab(vlan, "data.vid") != vlan_data.get("vid"): continue + # try finding matching VLAN by site if vlan_site is not None and grab(vlan, "data.site") == vlan_site: - vlan_object_including_site = vlan - - if grab(vlan, "data.site") is None: - vlan_object_without_site = vlan - - if isinstance(vlan_object_including_site, NetBoxObject): - return_data = vlan_object_including_site - log.debug2("Found a exact matching %s object: %s" % - (vlan_object_including_site.name, - vlan_object_including_site.get_display_name(including_second_key=True))) - - elif isinstance(vlan_object_without_site, NetBoxObject): - return_data = vlan_object_without_site - log.debug2("Found a global matching %s object: %s" % - (vlan_object_without_site.name, - vlan_object_without_site.get_display_name(including_second_key=True))) + vlan_object_by_site = vlan + break + + # try find matching VLAN by group + if grab(vlan, "data.group") is not None: + vlan_group = grab(vlan, "data.group") + if vlan_group.matches_site_cluster(vlan_site, vlan_cluster): + vlan_object_by_group = vlan + break + + if grab(vlan, "data.site") is None and grab(vlan, "data.group") is None: + vlan_object_global = vlan + + if isinstance(vlan_object_by_site, NetBoxObject): + return_data = vlan_object_by_site + log.debug2(f"Found a {return_data.name} object which matches the site '{vlan_site.get_display_name()}': %s" + % vlan_object_by_site.get_display_name(including_second_key=True)) + + elif isinstance(vlan_object_by_group, NetBoxObject): + return_data = vlan_object_by_group + vlan_group_object = grab(vlan_object_by_group, "data.group") + vlan_group_object_scope_object = grab(vlan_object_by_group, "data.scope_id") + scope_details = "" + if vlan_group_object_scope_object is not None: + scope_details = (f" ({vlan_group_object_scope_object.name} " + f"{vlan_group_object_scope_object.get_display_name()})") + log.debug2(f"Found a {return_data.name} object which matches the {vlan_group_object.name} " + f"'{vlan_group_object.get_display_name()}'{scope_details}: %s" % + vlan_object_by_group.get_display_name(including_second_key=True)) + + elif isinstance(vlan_object_global, NetBoxObject): + return_data = vlan_object_global + log.debug2(f"Found a global matching {return_data.name} object: %s" % + vlan_object_global.get_display_name(including_second_key=True)) else: log.debug2("No matching existing VLAN found for this VLAN ID.") diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index cae51b93..417e3e9f 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -61,6 +61,7 @@ class VMWareHandler(SourceBase): NBClusterGroup, NBDeviceRole, NBSite, + NBSiteGroup, NBCluster, NBDevice, NBVM, From c783da62ac256bb9c7e5c047a80cd00f0afa4b9b Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Fri, 21 Feb 2025 22:30:09 +0100 Subject: [PATCH 16/32] fixes issue with tagging of VLANs if VLAN sync is disabled #392 --- module/netbox/inventory.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py index d07ce98c..47944e12 100644 --- a/module/netbox/inventory.py +++ b/module/netbox/inventory.py @@ -335,6 +335,11 @@ def tag_all_the_things(self, netbox_handler): # if object was found in source if this_object.source is not None: + # skip tagging of VLANs if vlan sync is disabled + if object_type == NBVLAN and \ + grab(this_object.source, "settings.disable_vlan_sync", fallback=False) is True: + continue + this_object.add_tags([netbox_handler.primary_tag, this_object.source.source_tag]) # if object was orphaned remove tag again From 1117bf1b29eb45b6c80c6170f14a86394ce79d5b Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Fri, 21 Feb 2025 23:17:48 +0100 Subject: [PATCH 17/32] fixes issue with mac address wrongly tagged orphaned --- module/netbox/object_classes.py | 1 - .../sources/check_redfish/import_inventory.py | 7 +++++++ module/sources/common/source_base.py | 17 ++++++++--------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 86f02fd2..e5e548f4 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1453,7 +1453,6 @@ def __init__(self, *args, **kwargs): "name": 100, "slug": 100, "description": 200, - "tags": NBTagList, "scope_type": ["dcim.site", "dcim.sitegroup", "virtualization.cluster", "virtualization.clustergroup"], "scope_id": [NBSite, NBSiteGroup, NBCluster, NBClusterGroup], } diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index a8c3aa20..eff99455 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -41,14 +41,17 @@ class CheckRedfish(SourceBase): NBClusterGroup, NBDeviceRole, NBSite, + NBSiteGroup, NBCluster, NBDevice, NBInterface, + NBMACAddress, NBIPAddress, NBPrefix, NBTenant, NBVRF, NBVLAN, + NBVLANGroup, NBPowerPort, NBInventoryItem, NBCustomField @@ -858,6 +861,7 @@ def update_network_interface(self): this_link_type = port_data.get("type") mgmt_only = port_data.get("mgmt_only") + mac_address = port_data.get("mac_address") data_to_update = self.patch_data(nic_object, port_data, self.settings.overwrite_interface_attributes) # always overwrite nic type if discovered @@ -866,6 +870,9 @@ def update_network_interface(self): data_to_update["mgmt_only"] = mgmt_only + if mac_address is not None: + data_to_update["mac_address"] = mac_address + port_data = data_to_update self.add_update_interface(nic_object, self.device_object, port_data, nic_ips.get(port_name, list())) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 4763a1b4..ec368d72 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -10,13 +10,11 @@ import re from ipaddress import ip_interface, ip_address, IPv6Address, IPv4Address, IPv6Network, IPv4Network -from typing import List from packaging import version from module.netbox import * from module.common.logging import get_logger from module.common.misc import grab -from module.sources.common.handle_vlan import FilterVLANByName, FilterVLANByID log = get_logger() @@ -271,7 +269,6 @@ def add_update_interface(self, interface_object, device_object, interface_data, interface_mac_address = None if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0") and \ interface_data.get("mac_address") is not None: - interface_mac_address = interface_data.get("mac_address") del(interface_data["mac_address"]) @@ -329,6 +326,12 @@ def add_update_interface(self, interface_object, device_object, interface_data, primary_mac_address_object = grab(interface_object, "data.primary_mac_address") + primary_mac_address_data = { + "mac_address": interface_mac_address, + "assigned_object_id": interface_object, + "assigned_object_type": interface_class + } + if primary_mac_address_object is None or grab(primary_mac_address_object, "data.mac_address") != interface_mac_address: primary_mac_address_object = None @@ -337,16 +340,12 @@ def add_update_interface(self, interface_object, device_object, interface_data, primary_mac_address_object = mac_address_object break - primary_mac_address_data = { - "mac_address": interface_mac_address, - "assigned_object_id": interface_object, - "assigned_object_type": interface_class - } - if primary_mac_address_object is None: primary_mac_address_object = self.inventory.add_object(NBMACAddress, data=primary_mac_address_data, source=self) else: primary_mac_address_object.update(data=primary_mac_address_data, source=self) + else: + primary_mac_address_object.update(data=primary_mac_address_data, source=self) interface_object.update(data={"primary_mac_address": primary_mac_address_object}, source=self) From e5619c33c9728b80be20a69f8f86cac58909eca5 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Sat, 22 Feb 2025 10:21:00 +0100 Subject: [PATCH 18/32] better mitigates issue with assigning VLAN groups #373 --- module/sources/common/source_base.py | 15 +++++++++++++-- module/sources/vmware/config.py | 7 +++++-- settings-example.ini | 6 ++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index ec368d72..3eecb1c9 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -701,6 +701,7 @@ def add_vlan_group(self, vlan_data, vlan_site, vlan_cluster) -> NBVLAN | dict: vlan_name = grab(vlan_data, "data.name") vlan_id = grab(vlan_data, "data.vid") + vlan_current_site = grab(vlan_data, "data.site") # vlan already has a group attached if grab(vlan_data, "data.group") is not None: return vlan_data @@ -708,6 +709,7 @@ def add_vlan_group(self, vlan_data, vlan_site, vlan_cluster) -> NBVLAN | dict: elif isinstance(vlan_data, dict): vlan_name = vlan_data.get("name") vlan_id = vlan_data.get("vid") + vlan_current_site = vlan_data.get("site") else: return vlan_data @@ -717,6 +719,9 @@ def add_vlan_group(self, vlan_data, vlan_site, vlan_cluster) -> NBVLAN | dict: if isinstance(vlan_cluster, str): vlan_cluster = self.inventory.get_by_data(NBCluster, data={"name": vlan_cluster}) + if isinstance(vlan_current_site, dict): + vlan_current_site = self.inventory.get_by_data(NBSite, data=vlan_current_site) + log_text = f"Trying to find a matching VLAN Group based on the VLAN name '{vlan_name}'" if vlan_site is not None: log_text += f", site '{vlan_site.get_display_name()}'" @@ -751,13 +756,19 @@ def add_vlan_group(self, vlan_data, vlan_site, vlan_cluster) -> NBVLAN | dict: if vlan_group is not None: log.debug2(f"Found matching VLAN group '{vlan_group.get_display_name()}'") + """ + If a VLAN group has been found we also need to check if the vlan site and the scope of the VLAN group are + matching. If the VLAN group has a different scope then site, we need to remove the site from the VLAN. + + Mitigation for: https://github.com/netbox-community/netbox/issues/18706 + """ if isinstance(vlan_data, NBVLAN): vlan_data.update(data={"group": vlan_group}) - if vlan_data.data.get("site") is not None: + if vlan_current_site is not vlan_group.data.get("scope_id"): vlan_data.unset_attribute("site") elif isinstance(vlan_data, dict): vlan_data["group"] = vlan_group - if vlan_data.get("site") is not None: + if vlan_current_site is not vlan_group.data.get("scope_id"): del(vlan_data["site"]) else: log.debug2("No matching VLAN group found") diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 8a251593..e73a8dc2 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -330,13 +330,16 @@ def __init__(self): description="""adds a relation to assign VLAN groups to matching VLANs by name. Same matching rules as the exclude_by_name option uses are applied. If name and id relations are defined, the name relation takes precedence. - Fist match wins.""", + Fist match wins. Only newly discovered VLANs which are not present in + NetBox will be assigned a VLAN group. + """, config_example="London/Vlan_.* = VLAN Group 1, Tokio/Vlan_.* = VLAN Group 2"), ConfigOption("vlan_group_relation_by_id", str, description="""adds a relation to assign VLAN groups to matching VLANs by ID. Same matching rules as the exclude_by_id option uses are applied. - Fist match wins. + Fist match wins. Only newly discovered VLANs which are not present in + NetBox will be assigned a VLAN group. """, config_example="1023-1042 = VLAN Group 1, Tokio/2342 = VLAN Group 2") ]), diff --git a/settings-example.ini b/settings-example.ini index 1c51448c..7b507a48 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -339,11 +339,13 @@ password = super-secret ; adds a relation to assign VLAN groups to matching VLANs by name. Same matching rules as ; the exclude_by_name option uses are applied. If name and id relations are defined, the -; name relation takes precedence. Fist match wins. +; name relation takes precedence. Fist match wins. Only newly discovered VLANs which are +; not present in NetBox will be assigned a VLAN group. ;vlan_group_relation_by_name = London/Vlan_.* = VLAN Group 1, Tokio/Vlan_.* = VLAN Group 2 ; adds a relation to assign VLAN groups to matching VLANs by ID. Same matching rules as -; the exclude_by_id option uses are applied. Fist match wins. +; the exclude_by_id option uses are applied. Fist match wins. Only newly discovered VLANs +; which are not present in NetBox will be assigned a VLAN group. ;vlan_group_relation_by_id = 1023-1042 = VLAN Group 1, Tokio/2342 = VLAN Group 2 ; enabling this option will add the ESXi host this VM is running on to the VM details From ff4957023a2b64d702348bf788976e889e66a52e Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Mon, 24 Feb 2025 08:00:13 +0100 Subject: [PATCH 19/32] adds check if permitted subnet contains prefix length #441 --- module/sources/common/permitted_subnets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/module/sources/common/permitted_subnets.py b/module/sources/common/permitted_subnets.py index 36412d38..47217c10 100644 --- a/module/sources/common/permitted_subnets.py +++ b/module/sources/common/permitted_subnets.py @@ -41,6 +41,10 @@ def __init__(self, config_string: str): excluded = True subnet = subnet[1:].strip() + if "/" not in subnet: + log.error(f"permitted subnet '{subnet}' is missing the prefix length (i.e.: {subnet}/24)") + self._validation_failed = True + try: if excluded is True: self.excluded_subnets.append(ip_network(subnet)) From 6274052d5afc8f8ce632fab7ec8e1946c44871a3 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Mon, 24 Feb 2025 21:46:14 +0100 Subject: [PATCH 20/32] skip IP address handling if no VMWare tools are installed or if VM is switched off #326 --- module/sources/common/source_base.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 3eecb1c9..c64b9f1c 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -349,11 +349,22 @@ def add_update_interface(self, interface_object, device_object, interface_data, interface_object.update(data={"primary_mac_address": primary_mac_address_object}, source=self) + # skip handling of IPs for VMs with not installed/running guest tools + skip_ip_handling = False + if ("vim.VirtualMachine" in str(type(vmware_object)) and + grab(vmware_object,'guest.toolsStatus') != "toolsOk"): + guest_tool_satus = str(grab(vmware_object,'guest.toolsStatus')).replace("tools", "") + log.debug(f"VM '{device_object.name}' guest tool status is '{guest_tool_satus}', skipping IP handling") + skip_ip_handling = True + ip_address_objects = list() matching_ip_prefixes = list() # add all interface IPs for nic_ip in interface_ips or list(): + if skip_ip_handling is True: + continue + # get IP and prefix length try: if "/" in nic_ip: @@ -559,6 +570,10 @@ def add_update_interface(self, interface_object, device_object, interface_data, ip_address_objects.append(this_ip_object) for current_ip in interface_object.get_ip_addresses(): + + if skip_ip_handling is True: + continue + if current_ip not in ip_address_objects: log.info(f"{current_ip.name} is no longer assigned to {interface_object.get_display_name()} and " f"therefore removed from this interface") From e798b30df5bd8a30f818a7c05e18f06ca3fa6714 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Mon, 24 Feb 2025 22:34:58 +0100 Subject: [PATCH 21/32] fixes issue with reassigned IP addresses of subinterfaces within VM #302 --- module/sources/common/source_base.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index c64b9f1c..c39ac08e 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -351,8 +351,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, # skip handling of IPs for VMs with not installed/running guest tools skip_ip_handling = False - if ("vim.VirtualMachine" in str(type(vmware_object)) and - grab(vmware_object,'guest.toolsStatus') != "toolsOk"): + if type(device_object) == NBVM and grab(vmware_object,'guest.toolsStatus') != "toolsOk": guest_tool_satus = str(grab(vmware_object,'guest.toolsStatus')).replace("tools", "") log.debug(f"VM '{device_object.name}' guest tool status is '{guest_tool_satus}', skipping IP handling") skip_ip_handling = True @@ -530,6 +529,20 @@ def add_update_interface(self, interface_object, device_object, interface_data, "assigned_object_id": interface_object, } + # skip reassignment if IP is assigned to sub interface of a VM + if type(device_object) == NBVM and grab(ip.get_interface(), "data.parent") is not None: + current_ip_nic = ip.get_interface() + current_ip_nic_parent = grab(current_ip_nic, "data.parent") + if isinstance(current_ip_nic_parent, dict): + current_ip_nic_parent = self.inventory.get_by_id(NBVMInterface, + nb_id=current_ip_nic_parent.get("id")) + + if current_ip_nic_parent == interface_object: + log.debug(f"{ip.name} '{ip.get_display_name()}' is assigned to sub interface " + f"'{current_ip_nic.get_display_name()}' of '{interface_object.get_display_name()}'. " + f"Not changing assignment") + nic_ip_data["assigned_object_id"] = current_ip_nic + # grab tenant from device/vm if prefix didn't provide a tenant ip_tenant = None if isinstance(ip_tenant_inheritance_order, list) and "disabled" not in ip_tenant_inheritance_order: From 758e864d143b2fff611a44e451aa04b22002e280 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Mon, 24 Feb 2025 23:03:47 +0100 Subject: [PATCH 22/32] ignore reassignment and deletion if IP has role anycast #344 --- module/netbox/object_classes.py | 1 + module/sources/common/source_base.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index e5e548f4..397fa916 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1902,6 +1902,7 @@ def __init__(self, *args, **kwargs): "assigned_object_type": ["dcim.interface", "virtualization.vminterface", "ipam.fhrpgroup"], "assigned_object_id": [NBInterface, NBVMInterface, NBFHRPGroupItem], "description": 200, + "role": ["loopback", "secondary", "anycast", "vip", "vrrp", "hsrp", "glbp", "carp"], "dns_name": 255, "tags": NBTagList, "tenant": NBTenant, diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index c39ac08e..396c0e82 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -499,6 +499,12 @@ def add_update_interface(self, interface_object, device_object, interface_data, this_ip_object = ip + if grab(ip, "data.role.value") == "anycast": + log.debug(f"{ip.name} '{ip.get_display_name()}' is an Anycast address and " + f"can be assigned to multiple interfaces at the same time.") + skip_this_ip = True + break + if current_nic_enabled == this_nic_enabled: this_log_handler = log.warning @@ -587,6 +593,11 @@ def add_update_interface(self, interface_object, device_object, interface_data, if skip_ip_handling is True: continue + if grab(current_ip, "data.role.value") == "anycast": + log.debug2(f"{ip.name} '{ip.get_display_name()}' is an Anycast address and will " + f"NOT be deleted from interface") + continue + if current_ip not in ip_address_objects: log.info(f"{current_ip.name} is no longer assigned to {interface_object.get_display_name()} and " f"therefore removed from this interface") From eebede0ccf2f184fd552389f9ee165259dfa6de3 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Tue, 25 Feb 2025 08:11:02 +0100 Subject: [PATCH 23/32] fixes implementation for guest tools prettyName #429 --- module/sources/vmware/connection.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 417e3e9f..734a2300 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -29,7 +29,7 @@ from module.sources.common.source_base import SourceBase from module.sources.vmware.config import VMWareConfig from module.common.logging import get_logger, DEBUG3 -from module.common.misc import grab, dump, get_string_or_none, plural +from module.common.misc import grab, dump, get_string_or_none, plural, quoted_split from module.common.support import normalize_mac_address from module.netbox.inventory import NetBoxInventory from module.netbox import * @@ -2162,7 +2162,14 @@ def add_virtual_machine(self, obj): # first check against vm_platform_relation platform = get_string_or_none(grab(obj, "config.guestFullName")) platform = get_string_or_none(grab(obj, "guest.guestFullName", fallback=platform)) - platform = get_string_or_none(grab(obj, "guest.guestDetailedData.prettyName", fallback=platform)) + + # extract prettyName from extraConfig exposed by guest tools + extra_config = [x.value for x in grab(obj, "config.extraConfig", fallback=[]) + if x.key == "guestOS.detailed.data"] + if len(extra_config) > 0: + pretty_name = [x for x in quoted_split(extra_config[0].replace("' ", "', ")) if x.startswith("prettyName")] + if len(pretty_name) > 0: + platform = pretty_name[0].replace("prettyName='","") if platform is not None: platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) From c47e4ae0f308f271f85c4e77a786127af0720c8f Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 26 Feb 2025 07:01:06 +0100 Subject: [PATCH 24/32] bumps version to 1.8.0 --- LICENSE.txt | 2 +- README.md | 2 +- module/__init__.py | 6 +++--- module/common/__init__.py | 2 +- module/common/cli_parser.py | 2 +- module/common/config.py | 2 +- module/common/logging.py | 2 +- module/common/misc.py | 2 +- module/common/support.py | 2 +- module/config/__init__.py | 2 +- module/config/base.py | 2 +- module/config/file_output.py | 2 +- module/config/files.py | 2 +- module/config/formatter.py | 2 +- module/config/group.py | 2 +- module/config/option.py | 2 +- module/config/parser.py | 2 +- module/netbox/__init__.py | 2 +- module/netbox/config.py | 2 +- module/netbox/connection.py | 2 +- module/netbox/inventory.py | 2 +- module/netbox/manufacturer_mapping.py | 2 +- module/netbox/object_classes.py | 2 +- module/sources/__init__.py | 2 +- module/sources/check_redfish/config.py | 2 +- module/sources/check_redfish/import_inventory.py | 2 +- module/sources/common/config.py | 2 +- module/sources/common/handle_vlan.py | 2 +- module/sources/common/permitted_subnets.py | 2 +- module/sources/common/source_base.py | 2 +- module/sources/vmware/config.py | 2 +- module/sources/vmware/connection.py | 2 +- netbox-sync.py | 2 +- scripts/publi.sh | 2 +- scripts/set_version.sh | 4 ++-- settings-example.ini | 2 +- 36 files changed, 39 insertions(+), 39 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 44aae69a..cefeaf2f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 - 2021 Ricardo Bartels +Copyright (c) 2020 - 2025 Ricardo Bartels Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 173b9d0c..b09612da 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ usage: netbox-sync.py [-h] [-c settings.ini [settings.ini ...]] [-g] Sync objects from various sources to NetBox -Version: 1.7.0 (2024-10-21) +Version: 1.8.0 (2025-02-26) Project URL: https://github.com/bb-ricardo/netbox-sync options: diff --git a/module/__init__.py b/module/__init__.py index b283262e..ee32b85e 100644 --- a/module/__init__.py +++ b/module/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -7,8 +7,8 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . -__version__ = "1.7.0" -__version_date__ = "2024-10-21" +__version__ = "1.8.0" +__version_date__ = "2025-02-26" __author__ = "Ricardo Bartels " __description__ = "NetBox Sync" __license__ = "MIT" diff --git a/module/common/__init__.py b/module/common/__init__.py index 376af17d..4b85b91e 100644 --- a/module/common/__init__.py +++ b/module/common/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/common/cli_parser.py b/module/common/cli_parser.py index 824f7fd0..3d890cf2 100644 --- a/module/common/cli_parser.py +++ b/module/common/cli_parser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/common/config.py b/module/common/config.py index 28ed919f..bb8dd06d 100644 --- a/module/common/config.py +++ b/module/common/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/common/logging.py b/module/common/logging.py index 020f38a2..6bc76500 100644 --- a/module/common/logging.py +++ b/module/common/logging.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/common/misc.py b/module/common/misc.py index 1302291e..77038dc3 100644 --- a/module/common/misc.py +++ b/module/common/misc.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/common/support.py b/module/common/support.py index 8a3677df..e800da10 100644 --- a/module/common/support.py +++ b/module/common/support.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/__init__.py b/module/config/__init__.py index 6aadb483..02d7fef5 100644 --- a/module/config/__init__.py +++ b/module/config/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/base.py b/module/config/base.py index d6dcfc79..d9876c8c 100644 --- a/module/config/base.py +++ b/module/config/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/file_output.py b/module/config/file_output.py index 00870044..3b2f6e5c 100644 --- a/module/config/file_output.py +++ b/module/config/file_output.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/files.py b/module/config/files.py index 93ad830c..2b734cc1 100644 --- a/module/config/files.py +++ b/module/config/files.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/formatter.py b/module/config/formatter.py index b832df8f..1fef402b 100644 --- a/module/config/formatter.py +++ b/module/config/formatter.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/group.py b/module/config/group.py index ce30a0ac..8880147b 100644 --- a/module/config/group.py +++ b/module/config/group.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/option.py b/module/config/option.py index 8ee3882f..e792269b 100644 --- a/module/config/option.py +++ b/module/config/option.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/parser.py b/module/config/parser.py index 2c01c3ed..d46256db 100644 --- a/module/config/parser.py +++ b/module/config/parser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/__init__.py b/module/netbox/__init__.py index ef9cbe81..6891c375 100644 --- a/module/netbox/__init__.py +++ b/module/netbox/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/config.py b/module/netbox/config.py index c82aa3d9..8f943c06 100644 --- a/module/netbox/config.py +++ b/module/netbox/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/connection.py b/module/netbox/connection.py index 163aab01..5c86d26f 100644 --- a/module/netbox/connection.py +++ b/module/netbox/connection.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py index 47944e12..2daa39b6 100644 --- a/module/netbox/inventory.py +++ b/module/netbox/inventory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/manufacturer_mapping.py b/module/netbox/manufacturer_mapping.py index b3554eb9..d3aa2f74 100644 --- a/module/netbox/manufacturer_mapping.py +++ b/module/netbox/manufacturer_mapping.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 397fa916..a8c3e94a 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/__init__.py b/module/sources/__init__.py index cdb1f846..4be4120f 100644 --- a/module/sources/__init__.py +++ b/module/sources/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/check_redfish/config.py b/module/sources/check_redfish/config.py index 4d729318..2c21fff9 100644 --- a/module/sources/check_redfish/config.py +++ b/module/sources/check_redfish/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index eff99455..7a893baa 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/common/config.py b/module/sources/common/config.py index c2417eb7..5fa7b3f2 100644 --- a/module/sources/common/config.py +++ b/module/sources/common/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/common/handle_vlan.py b/module/sources/common/handle_vlan.py index 552b1516..18bf986c 100644 --- a/module/sources/common/handle_vlan.py +++ b/module/sources/common/handle_vlan.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/common/permitted_subnets.py b/module/sources/common/permitted_subnets.py index 47217c10..a4c88e87 100644 --- a/module/sources/common/permitted_subnets.py +++ b/module/sources/common/permitted_subnets.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 396c0e82..89793cff 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index e73a8dc2..9fc618b0 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 734a2300..10d1b17d 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/netbox-sync.py b/netbox-sync.py index 7fdd436a..e499838b 100755 --- a/netbox-sync.py +++ b/netbox-sync.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/scripts/publi.sh b/scripts/publi.sh index d5310af1..2a585aca 100755 --- a/scripts/publi.sh +++ b/scripts/publi.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/scripts/set_version.sh b/scripts/set_version.sh index 92e65fc6..f947e0eb 100755 --- a/scripts/set_version.sh +++ b/scripts/set_version.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -12,7 +12,7 @@ EXAMPLE_CONFIG_FILE="settings-example.ini" VERSION_DATA_FILE="module/__init__.py" README_FILE="README.md" VERSION_TO_SET="$1" -COPYRIGHT_PATTERN="# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved." +COPYRIGHT_PATTERN="# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved." BASE_PATH="$(realpath "$(dirname "${0}")/..")" # shellcheck disable=SC2181 diff --git a/settings-example.ini b/settings-example.ini index 7b507a48..a730dd23 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -1,5 +1,5 @@ ;;; Welcome to the NetBox Sync configuration file. -;;; Version: 1.7.0 (2024-10-21) +;;; Version: 1.8.0 (2025-02-26) ;;; Project URL: https://github.com/bb-ricardo/netbox-sync ; The values in this file override the default values used by the system if a config From d7ca5d36a2481e9149fa6cc6ec075df0360e5762 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 26 Feb 2025 08:05:30 +0100 Subject: [PATCH 25/32] adds config option 'vm_exclude_by_tag_filter' #393 --- module/sources/vmware/config.py | 17 ++++++++++++++++- module/sources/vmware/connection.py | 11 ++++++++++- settings-example.ini | 5 +++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 9fc618b0..50b62d7a 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -108,6 +108,15 @@ def __init__(self): str, description="simply include/exclude VMs"), ConfigOption("vm_include_filter", str) ]), + ConfigOption("vm_exclude_by_tag_filter", + str, + description="""defines a comma separated list of vCenter tags which (if assigned to a VM) + will exclude this VM from being synced to NetBox. The config option 'vm_tag_source' + determines which tags are collected for VMs. + """, + config_example="tag-a, tag-b" + ), + ConfigOptionGroup(title="relations", options=[ ConfigOption("cluster_site_relation", @@ -439,7 +448,7 @@ def validate_options(self): if option.value is None: continue - if "filter" in option.key: + if "filter" in option.key and "vm_exclude_by_tag_filter" not in option.key: re_compiled = None try: @@ -452,6 +461,12 @@ def validate_options(self): continue + if option.key == "vm_exclude_by_tag_filter": + + option.set_value(quoted_split(option.value)) + + continue + if "relation" in option.key and "vlan_group_relation" not in option.key: relation_data = list() diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 10d1b17d..267e8fc4 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -2187,7 +2187,16 @@ def add_virtual_machine(self, obj): vm_tags = self.get_object_relation(name, "vm_tag_relation") # get vCenter tags - vm_tags.extend(self.collect_object_tags(obj)) + vcenter_tags = self.collect_object_tags(obj) + + # check if VM tag excludes VM from being synced to NetBox + for sync_exclude_tag in self.settings.vm_exclude_by_tag_filter or list(): + if sync_exclude_tag in vcenter_tags: + log.debug(f"Virtual machine vCenter tag '{sync_exclude_tag}' in matches 'vm_exclude_by_tag_filter'. " + f"Skipping") + return + + vm_tags.extend(vcenter_tags) # vm memory depending on setting vm_memory = grab(obj, "config.hardware.memoryMB", fallback=0) diff --git a/settings-example.ini b/settings-example.ini index a730dd23..b2880086 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -172,6 +172,11 @@ password = super-secret ;vm_exclude_filter = ;vm_include_filter = +; defines a comma separated list of vCenter tags which (if assigned to a VM) will exclude +; this VM from being synced to NetBox. The config option 'vm_tag_source' determines which +; tags are collected for VMs. +;vm_exclude_by_tag_filter = tag-a, tag-b + ; relations options ; This option defines which vCenter cluster is part of a NetBox site. From 8f9fa015a77699d46e962821e15d100805e9a973 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 26 Feb 2025 09:45:46 +0100 Subject: [PATCH 26/32] fixes issue with finding ip prefixx in NetBox 4.2 and reassignes VRFs to IPs #397 --- module/netbox/object_classes.py | 69 ++++++++++++++++++++++++---- module/sources/common/source_base.py | 24 ++++------ 2 files changed, 70 insertions(+), 23 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index a8c3e94a..2d76020e 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1442,6 +1442,7 @@ def update(self, data=None, read_from_netbox=False, source=None): super().update(data=data, read_from_netbox=read_from_netbox, source=source) + class NBVLANGroup(NetBoxObject): name = "VLANGroup" api_path = "ipam/vlan-groups" @@ -1477,8 +1478,9 @@ def resolve_relations(self): if isinstance(o_id, int) and o_type is not None and self.data_model_relation.get(o_type) is not None: self.data["scope_id"] = self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) - elif not isinstance(o_id, NetBoxObject): - log.debug(f"{self.name} '{self.data.get('name')}' scope type '{o_type}' for '{grab(self, 'data.scope.name')}' is currently not supported") + elif o_id is not None and not isinstance(o_id, NetBoxObject): + log.debug(f"{self.name} '{self.data.get('name')}' scope type '{o_type}' for " + f"'{grab(self, 'data.scope.name')}' is currently not supported") self.data["scope_id"] = "" super().resolve_relations() @@ -1500,16 +1502,16 @@ def matches_site_cluster(self, site=None, cluster=None) -> bool: """ if isinstance(site, NBSite): - if isinstance(self.data["scope_id"], NBSite) and self.data["scope_id"] == site: + if isinstance(self.data.get("scope_id"), NBSite) and self.data.get("scope_id") == site: return True - if (isinstance(self.data["scope_id"], NBSiteGroup) and + if (isinstance(self.data.get("scope_id"), NBSiteGroup) and self.data["scope_id"] == grab(site, "data.group")): return True if isinstance(cluster, NBCluster): - if isinstance(self.data["scope_id"], NBCluster) and self.data["scope_id"] == cluster: + if isinstance(self.data.get("scope_id"), NBCluster) and self.data.get("scope_id") == cluster: return True - if (isinstance(self.data["scope_id"], NBClusterGroup) and + if (isinstance(self.data.get("scope_id"), NBClusterGroup) and self.data["scope_id"] == grab(cluster, "data.group")): return True @@ -1545,19 +1547,29 @@ def __init__(self, *args, **kwargs): self.data_model = { "prefix": [IPv4Network, IPv6Network], "site": NBSite, + "scope_type": ["dcim.site", "dcim.sitegroup"], + "scope_id": [NBSite, NBSiteGroup], "tenant": NBTenant, "vlan": NBVLAN, "vrf": NBVRF, "description": 200, "tags": NBTagList } + # add relation between two attributes + self.data_model_relation = { + "dcim.site": NBSite, + "dcim.sitegroup": NBSiteGroup, + NBSite: "dcim.site", + NBSiteGroup: "dcim.sitegroup", + } + super().__init__(*args, **kwargs) def update(self, data=None, read_from_netbox=False, source=None): # prefixes are parsed into ip_networks data_prefix = data.get(self.primary_key) - if data_prefix is not None and not isinstance(data_prefix, (IPv4Network, IPv6Network)): + if not isinstance(data_prefix, (IPv4Network, IPv6Network)) and data_prefix is not None: try: data[self.primary_key] = ip_network(data_prefix) except ValueError as e: @@ -1569,6 +1581,47 @@ def update(self, data=None, read_from_netbox=False, source=None): if read_from_netbox is False: raise ValueError(f"Adding {self.name} by this program is currently not implemented.") + def resolve_relations(self): + + o_id = self.data.get("scope_id") + o_type = self.data.get("scope_type") + + if isinstance(o_id, int) and o_type is not None and self.data_model_relation.get(o_type) is not None: + self.data["scope_id"] = self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) + elif o_id is not None and not isinstance(o_id, NetBoxObject): + log.debug(f"{self.name} '{self.data.get('name')}' scope type '{o_type}' for " + f"'{grab(self, 'data.scope.name')}' is currently not supported") + self.data["scope_id"] = "" + + super().resolve_relations() + + def matches_site(self, site=None) -> bool: + """ + tries to figure out if this prefix matches a certain site or site group + + Parameters + ---------- + site: NBSite + the site object to match to + + Returns + ------- + bool: True if matches one of the params + + """ + if isinstance(site, NBSite): + if isinstance(self.data.get("scope_id"), NBSite) and self.data.get("scope_id") == site: + return True + if (isinstance(self.data.get("scope_id"), NBSiteGroup) and + self.data["scope_id"] == grab(site, "data.group")): + return True + + # compatible for NetBox versions < 4.2.0 + if self.data.get("site") == site: + return True + + return False + class NBManufacturer(NetBoxObject): name = "manufacturer" @@ -1686,7 +1739,7 @@ def __init__(self, *args, **kwargs): "type": NBClusterType, "tenant": NBTenant, "group": NBClusterGroup, - "scope_type": ["dcim.site", "dcim.sitegroup", "dcim.location", "dcim.region"], + "scope_type": ["dcim.site", "dcim.sitegroup"], "scope_id": NBSite, "tags": NBTagList } diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 89793cff..8c0dcd0f 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -216,7 +216,7 @@ def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None for prefix in self.inventory.get_all_items(NBPrefix): - if grab(prefix, "data.site") != site_object: + if not prefix.matches_site(site_object): continue prefix_network = grab(prefix, f"data.{NBPrefix.primary_key}") @@ -286,7 +286,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, site_name = device_object_cluster.get_site_name() elif type(device_object) == NBDevice: interface_class = NBInterface - site_name = grab(device_object, "data.site.data.name") + site_name = device_object.get_site_name() elif device_object is None: log.error(f"No device/VM object submitted to attach interface '{grab(interface_data, 'name')}' to.") return None @@ -415,7 +415,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, # check if IP address is of type IP interface (includes prefix length) if type(ip_object) in [IPv6Address, IPv4Address]: - log.warning(f"{log_text}. Unable to add IP address to NetBox.") + log.warning(f"{log_text}. Unable to add IP address to NetBox") continue else: log.debug2(log_text) @@ -426,7 +426,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, if type(this_prefix) in [IPv4Network, IPv6Network]: ip_object = ip_interface(f"{ip_object}/{this_prefix.prefixlen}") else: - log.warning(f"{matching_ip_prefix.name} got wrong format. Unable to add IP to NetBox") + log.warning(f"{matching_ip_prefix.name} got wrong format. Unable to add IP address to NetBox") continue # try to find matching IP address object @@ -563,14 +563,14 @@ def add_update_interface(self, interface_object, device_object, interface_data, ip_tenant = prefix_tenant break + if possible_ip_vrf is not None: + nic_ip_data["vrf"] = possible_ip_vrf + if ip_tenant is not None: + nic_ip_data["tenant"] = ip_tenant + if not isinstance(this_ip_object, NBIPAddress): log.debug(f"No existing {NBIPAddress.name} object found. Creating a new one.") - if possible_ip_vrf is not None: - nic_ip_data["vrf"] = possible_ip_vrf - if ip_tenant is not None: - nic_ip_data["tenant"] = ip_tenant - this_ip_object = self.inventory.add_object(NBIPAddress, data=nic_ip_data, source=self) # update IP address with additional data if not already present @@ -578,12 +578,6 @@ def add_update_interface(self, interface_object, device_object, interface_data, log.debug2(f"Found existing NetBox {NBIPAddress.name} object: {this_ip_object.get_display_name()}") - if grab(this_ip_object, "data.vrf") is None and possible_ip_vrf is not None: - nic_ip_data["vrf"] = possible_ip_vrf - - if grab(this_ip_object, "data.tenant") is None and ip_tenant is not None: - nic_ip_data["tenant"] = ip_tenant - this_ip_object.update(data=nic_ip_data, source=self) ip_address_objects.append(this_ip_object) From 64f1c3a664e50e1a07cbcebe85ce43765b592310 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Thu, 27 Feb 2025 22:08:29 +0100 Subject: [PATCH 27/32] fixes issue with detection of running guest tools #326 #443 --- module/sources/common/source_base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 8c0dcd0f..6d2c71d0 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -351,9 +351,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, # skip handling of IPs for VMs with not installed/running guest tools skip_ip_handling = False - if type(device_object) == NBVM and grab(vmware_object,'guest.toolsStatus') != "toolsOk": - guest_tool_satus = str(grab(vmware_object,'guest.toolsStatus')).replace("tools", "") - log.debug(f"VM '{device_object.name}' guest tool status is '{guest_tool_satus}', skipping IP handling") + if type(device_object) == NBVM and grab(vmware_object,'guest.toolsRunningStatus') != "guestToolsRunning": + log.debug(f"VM '{device_object.name}' guest tool status is 'NotRunning', skipping IP handling") skip_ip_handling = True ip_address_objects = list() From c6d617cdc6c69a890aafa9ea60c85e595e474dc5 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Fri, 28 Feb 2025 01:10:06 +0100 Subject: [PATCH 28/32] improves handling of object scopes --- module/netbox/object_classes.py | 360 +++++++++++++++++---------- module/sources/common/source_base.py | 26 +- 2 files changed, 245 insertions(+), 141 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 2d76020e..52da05d5 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -13,7 +13,7 @@ # noinspection PyUnresolvedReferences from packaging import version -from module.common.misc import grab, do_error_exit +from module.common.misc import grab from module.common.logging import get_logger from module.netbox.manufacturer_mapping import sanitize_manufacturer_name @@ -211,27 +211,75 @@ def get_this_netbox_type(self) -> str: return self.get_common_type() +class NetBoxMappings: + """ + Adds singleton to map between NetBox object class and "object_type" attribute of that class + this is used for objects scopes and IP address and MAC address objects + """ + + mapping = dict() + + def __new__(cls): + it = cls.__dict__.get("__it__") + if it is not None: + return it + cls.__it__ = it = object.__new__(cls) + it.init() + return it + + def init(self): + + for classDefinition in NetBoxObject.__subclasses__(): + + if classDefinition.object_type is not None: + self.mapping[classDefinition] = classDefinition.object_type + self.mapping[classDefinition.object_type] = classDefinition + + def get(self, needle): + if isinstance(needle, NetBoxObject): + return self.mapping.get(type(needle)) + else: + return self.mapping.get(needle) + + def scopes_object_types(self, scopes_list) -> list: + result = list() + + if not isinstance(scopes_list, list): + raise ValueError("value for 'scopes_list' must be a list") + + for scope in scopes_list: + result.append(self.get(scope)) + + return result + + class NetBoxObject: """ Base class for all NetBox object types. Implements all methods used on a NetBox object. subclasses need to have the following attributes: - name: + name: string name of the object type (i.e. "virtual machine") - api_path: + api_path: string NetBox api path of object type (i.e: "virtualization/virtual-machines") - primary_key: + object_type: string + NetBox object type (i.e: "virtualization.virtualmachine") to handle scopes for this NetBox object + primary_key: string name of the data model key which represents the primary key of this object besides id (i.e: "name") - data_model: + data_model: string dict of permitted data keys and possible values (see description below) + prune: bool + defines if this object type will be pruned by netbox-sync optional attributes - secondary_key: + secondary_key: string name of the data model key which represents the secondary key of this object besides id - enforce_secondary_key: - bool if secondary key of an object shall be added to name when get_display_name() method is called - min_netbox_version: + enforce_secondary_key: bool + if secondary key of an object shall be added to name when get_display_name() method is called + min_netbox_version: string defines since which NetBox version this object is available + read_only: bool + defines if this is a read only object class and can't be changed within netbox-sync The data_model attribute needs to be a dict describing the data model in NetBox. Key must be string. @@ -259,9 +307,11 @@ class NetBoxObject: api_path = "" primary_key = "" data_model = {} + object_type = None min_netbox_version = "0.0" + read_only = False # _mandatory_attrs must be set at subclasses - _mandatory_attrs = ("name", "api_path", "primary_key", "data_model") + _mandatory_attrs = ("name", "api_path", "primary_key", "data_model", "object_type") # just skip this object if a mandatory attribute is missing skip_object_if_mandatory_attr_is_missing = False @@ -270,7 +320,7 @@ class NetBoxObject: inventory = None def __init__(self, data=None, read_from_netbox=False, inventory=None, source=None): - if not all(getattr(self, attr) for attr in self._mandatory_attrs): + if not all(getattr(self, attr) for attr in self._mandatory_attrs) or hasattr(self, "prune") is False: raise ValueError( f"FATAL: not all mandatory attributes {self._mandatory_attrs} " f"are set in {self.__class__.__name__}." @@ -315,7 +365,7 @@ def to_dict(self): continue if callable(value) is True: continue - if key in ["inventory", "default_attributes", "data_model_relation"]: + if key in ["inventory", "default_attributes", "data_model_relation", "mapping", "scopes"]: continue if key == "source": value = getattr(value, "name", None) @@ -488,6 +538,9 @@ def update(self, data=None, read_from_netbox=False, source=None): return + if self.read_only is True: + raise ValueError(f"Adding {self.name} by this program is currently not implemented.") + self.set_source(source) display_name = self.get_display_name(data) @@ -809,6 +862,24 @@ def resolve_relations(self): log.error(f"Problems resolving relation '{key}' for object '{self.get_display_name()}' and " f"value '{data_value}'") + def resolve_scoped_relations(self, id_attr, type_attr): + + o_id = self.data.get(id_attr) + o_type = self.data.get(type_attr) + + if hasattr(self, "mapping") is True: + mapping = getattr(self, "mapping") + else: + return + + if isinstance(o_id, int) and o_type is not None and mapping.get(o_type) is not None: + self.data[id_attr] = self.inventory.get_by_id(mapping.get(o_type), nb_id=o_id) + elif o_id is not None and not isinstance(o_id, NetBoxObject): + o_id_name = grab(self, f"data.{id_attr}.name") + log.debug(f"{self.name} '{self.data.get('name')}' {type_attr} '{o_type}' for " + f"'{o_id_name}' is currently not supported") + self.data[type_attr] = "" + def get_dependencies(self): """ returns a list of NetBoxObject subclasses this object depends on @@ -1197,20 +1268,22 @@ def get_display_name(self): class NBCustomField(NetBoxObject): name = "custom field" api_path = "extras/custom-fields" + object_type = "extras.customfield" primary_key = "name" prune = False - # used by this software - valid_object_types = [ - "dcim.device", - "dcim.interface", - "dcim.inventoryitem", - "dcim.powerport", - "virtualization.clustergroup", - "virtualization.vminterface", - "virtualization.virtualmachine" - ] def __init__(self, *args, **kwargs): + # used by this software + self.valid_object_types = [ + NBDevice.object_type, + NBInterface.object_type, + NBInventoryItem.object_type, + NBPowerPort.object_type, + NBClusterGroup.object_type, + NBVMInterface.object_type, + NBVM.object_type + ] + self.data_model = { "object_types": list, # field name (object_types) for NetBox < 4.0.0 @@ -1275,6 +1348,7 @@ def update(self, data=None, read_from_netbox=False, source=None): class NBTag(NetBoxObject): name = "tag" api_path = "extras/tags" + object_type = "extras.tag" primary_key = "name" prune = False @@ -1312,6 +1386,7 @@ def get_nb_reference(self): class NBTenant(NetBoxObject): name = "tenant" api_path = "tenancy/tenants" + object_type = "tenancy.tenant" primary_key = "name" prune = False @@ -1326,9 +1401,45 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) +# class NBLocation(NetBoxObject): +# name = "location" +# api_path = "dcim/locations" +# object_type = "dcim.location" +# primary_key = "name" +# prune = False +# read_only = True +# +# def __init__(self, *args, **kwargs): +# self.data_model = { +# "name": 100, +# "slug": 100, +# "site": NBSite, +# "tags": NBTagList +# } +# super().__init__(*args, **kwargs) +# +# +# class NBRegion(NetBoxObject): +# name = "region" +# api_path = "dcim/regions" +# object_type = "dcim.region" +# primary_key = "name" +# prune = False +# read_only = True +# +# def __init__(self, *args, **kwargs): +# self.data_model = { +# "name": 100, +# "slug": 100, +# "tags": NBTagList +# } +# super().__init__(*args, **kwargs) + + class NBSite(NetBoxObject): name = "site" api_path = "dcim/sites" + object_type = "dcim.site" primary_key = "name" prune = False @@ -1344,14 +1455,12 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) class NBSiteGroup(NetBoxObject): - """ - This object is currently not used directly in any class. - It is used to handle scopes properly. - """ name = "site group" api_path = "dcim/site-groups" + object_type = "dcim.sitegroup" primary_key = "name" prune = False + read_only = True def __init__(self, *args, **kwargs): self.data_model = { @@ -1364,8 +1473,10 @@ def __init__(self, *args, **kwargs): class NBVRF(NetBoxObject): name = "VRF" api_path = "ipam/vrfs" + object_type = "ipam.vrf" primary_key = "name" prune = False + read_only = True def __init__(self, *args, **kwargs): self.data_model = { @@ -1380,6 +1491,7 @@ def __init__(self, *args, **kwargs): class NBVLAN(NetBoxObject): name = "VLAN" api_path = "ipam/vlans" + object_type = "ipam.vlan" primary_key = "vid" secondary_key = "name" enforce_secondary_key = True @@ -1446,43 +1558,29 @@ def update(self, data=None, read_from_netbox=False, source=None): class NBVLANGroup(NetBoxObject): name = "VLANGroup" api_path = "ipam/vlan-groups" + object_type = "ipam.vlangroup" primary_key = "name" prune = False + read_only = True def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBSite, NBSiteGroup, NBCluster, NBClusterGroup + ] self.data_model = { "name": 100, "slug": 100, "description": 200, - "scope_type": ["dcim.site", "dcim.sitegroup", "virtualization.cluster", "virtualization.clustergroup"], - "scope_id": [NBSite, NBSiteGroup, NBCluster, NBClusterGroup], - } - # add relation between two attributes - self.data_model_relation = { - "dcim.site": NBSite, - "dcim.sitegroup": NBSiteGroup, - "virtualization.cluster": NBCluster, - "virtualization.clustergroup": NBClusterGroup, - NBSite: "dcim.site", - NBSiteGroup: "dcim.sitegroup", - NBCluster: "virtualization.cluster", - NBClusterGroup: "virtualization.clustergroup" + "scope_type": self.mapping.scopes_object_types(self.scopes), + "scope_id": self.scopes, } super().__init__(*args, **kwargs) def resolve_relations(self): - o_id = self.data.get("scope_id") - o_type = self.data.get("scope_type") - - if isinstance(o_id, int) and o_type is not None and self.data_model_relation.get(o_type) is not None: - self.data["scope_id"] = self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) - elif o_id is not None and not isinstance(o_id, NetBoxObject): - log.debug(f"{self.name} '{self.data.get('name')}' scope type '{o_type}' for " - f"'{grab(self, 'data.scope.name')}' is currently not supported") - self.data["scope_id"] = "" - + self.resolve_scoped_relations("scope_id", "scope_type") super().resolve_relations() def matches_site_cluster(self, site=None, cluster=None) -> bool: @@ -1540,28 +1638,27 @@ def get_nb_reference(self): class NBPrefix(NetBoxObject): name = "IP prefix" api_path = "ipam/prefixes" + object_type = "ipam.prefix" primary_key = "prefix" + read_only = True prune = False def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBSite, NBSiteGroup + ] self.data_model = { "prefix": [IPv4Network, IPv6Network], "site": NBSite, - "scope_type": ["dcim.site", "dcim.sitegroup"], - "scope_id": [NBSite, NBSiteGroup], + "scope_type": self.mapping.scopes_object_types(self.scopes), + "scope_id": self.scopes, "tenant": NBTenant, "vlan": NBVLAN, "vrf": NBVRF, "description": 200, "tags": NBTagList } - # add relation between two attributes - self.data_model_relation = { - "dcim.site": NBSite, - "dcim.sitegroup": NBSiteGroup, - NBSite: "dcim.site", - NBSiteGroup: "dcim.sitegroup", - } super().__init__(*args, **kwargs) @@ -1578,21 +1675,10 @@ def update(self, data=None, read_from_netbox=False, source=None): super().update(data=data, read_from_netbox=read_from_netbox, source=source) - if read_from_netbox is False: - raise ValueError(f"Adding {self.name} by this program is currently not implemented.") def resolve_relations(self): - o_id = self.data.get("scope_id") - o_type = self.data.get("scope_type") - - if isinstance(o_id, int) and o_type is not None and self.data_model_relation.get(o_type) is not None: - self.data["scope_id"] = self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) - elif o_id is not None and not isinstance(o_id, NetBoxObject): - log.debug(f"{self.name} '{self.data.get('name')}' scope type '{o_type}' for " - f"'{grab(self, 'data.scope.name')}' is currently not supported") - self.data["scope_id"] = "" - + self.resolve_scoped_relations("scope_id", "scope_type") super().resolve_relations() def matches_site(self, site=None) -> bool: @@ -1622,10 +1708,34 @@ def matches_site(self, site=None) -> bool: return False + def get_scope_display_name(self): + + if self.data.get("scope_id") is not None: + if isinstance(self.data.get("scope_id"), NetBoxObject): + scope_object = self.data.get("scope_id") + return f"{scope_object.name} '{scope_object.get_display_name()}'" + if isinstance(self.data.get("scope_id"), dict) and self.data.get("scope_type") is not None: + + if isinstance(self.data.get("scope_type"), str): + scope_class = self.mapping.get(self.data.get("scope_type")) + else: + scope_class = self.data.get("scope_type") + + if scope_class is not None: + return f"{scope_class.name} '{grab(self, 'data.scope_id.data.name')}'" + + if self.data.get("site") is not None: + if isinstance(self.data.get("site"), NetBoxObject): + site_name = self.data.get("site").get_display_name() + return f"site '{site_name}'" + elif isinstance(self.data.get("site"), dict): + return f"site '{grab(self, 'data.scope_id.data.name')}'" + class NBManufacturer(NetBoxObject): name = "manufacturer" api_path = "dcim/manufacturers" + object_type = "dcim.manufacturer" primary_key = "name" prune = False @@ -1642,6 +1752,7 @@ def __init__(self, *args, **kwargs): class NBDeviceType(NetBoxObject): name = "device type" api_path = "dcim/device-types" + object_type = "dcim.devicetype" primary_key = "model" prune = False @@ -1659,6 +1770,7 @@ def __init__(self, *args, **kwargs): class NBPlatform(NetBoxObject): name = "platform" api_path = "dcim/platforms" + object_type = "dcim.platform" primary_key = "name" prune = False @@ -1676,6 +1788,7 @@ def __init__(self, *args, **kwargs): class NBClusterType(NetBoxObject): name = "cluster type" api_path = "virtualization/cluster-types" + object_type = "virtualization.clustertype" primary_key = "name" prune = False @@ -1692,6 +1805,7 @@ def __init__(self, *args, **kwargs): class NBClusterGroup(NetBoxObject): name = "cluster group" api_path = "virtualization/cluster-groups" + object_type = "virtualization.clustergroup" primary_key = "name" prune = False @@ -1709,6 +1823,7 @@ def __init__(self, *args, **kwargs): class NBDeviceRole(NetBoxObject): name = "device role" api_path = "dcim/device-roles" + object_type = "dcim.devicerole" primary_key = "name" prune = False @@ -1727,19 +1842,25 @@ def __init__(self, *args, **kwargs): class NBCluster(NetBoxObject): name = "cluster" api_path = "virtualization/clusters" + object_type = "virtualization.cluster" primary_key = "name" secondary_key = "site" prune = False # include_secondary_key_if_present = True def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBSite, NBSiteGroup + ] self.data_model = { "name": 100, "comments": str, "type": NBClusterType, "tenant": NBTenant, "group": NBClusterGroup, - "scope_type": ["dcim.site", "dcim.sitegroup"], + "scope_type": self.mapping.scopes_object_types(self.scopes), + # currently only site is supported as a scope "scope_id": NBSite, "tags": NBTagList } @@ -1759,10 +1880,16 @@ def update(self, data=None, read_from_netbox=False, source=None): super().update(data=data, read_from_netbox=read_from_netbox, source=source) + def resolve_relations(self): + + self.resolve_scoped_relations("scope_id", "scope_type") + super().resolve_relations() + class NBDevice(NetBoxObject): name = "device" api_path = "dcim/devices" + object_type = "dcim.device" primary_key = "name" secondary_key = "site" prune = True @@ -1803,6 +1930,7 @@ def update(self, data=None, read_from_netbox=False, source=None): class NBVM(NetBoxObject): name = "virtual machine" api_path = "virtualization/virtual-machines" + object_type = "virtualization.virtualmachine" primary_key = "name" secondary_key = "cluster" prune = True @@ -1841,6 +1969,7 @@ def get_virtual_disks(self): class NBVMInterface(NetBoxObject): name = "virtual machine interface" api_path = "virtualization/interfaces" + object_type = "virtualization.vminterface" primary_key = "name" secondary_key = "virtual_machine" enforce_secondary_key = True @@ -1875,6 +2004,7 @@ def get_ip_addresses(self): class NBInterface(NetBoxObject): name = "interface" api_path = "dcim/interfaces" + object_type = "dcim.interface" primary_key = "name" secondary_key = "device" enforce_secondary_key = True @@ -1898,7 +2028,7 @@ def __init__(self, *args, **kwargs): "untagged_vlan": NBVLAN, "tagged_vlans": NBVLANList, "description": 200, - "connection_status": bool, + "mark_connected": bool, "tags": NBTagList, "parent": object } @@ -1927,16 +2057,18 @@ def update(self, data=None, read_from_netbox=False, source=None): class NBVirtualDisk(NetBoxObject): name = "Virtual Disk" api_path = "virtualization/virtual-disks" + object_type = "virtualization.virtualdisk" primary_key = "name" secondary_key = "virtual_machine" min_netbox_version = "3.7" + prune = True def __init__(self, *args, **kwargs): self.data_model = { "name": 64, "virtual_machine": NBVM, "description": 200, - "size": int, # in GB + "size": int, # in MB "tags": NBTagList } super().__init__(*args, **kwargs) @@ -1945,15 +2077,20 @@ def __init__(self, *args, **kwargs): class NBIPAddress(NetBoxObject): name = "IP address" api_path = "ipam/ip-addresses" + object_type = "ipam.ipaddress" primary_key = "address" is_primary = False prune = True def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBInterface, NBVMInterface, NBFHRPGroupItem + ] self.data_model = { "address": str, - "assigned_object_type": ["dcim.interface", "virtualization.vminterface", "ipam.fhrpgroup"], - "assigned_object_id": [NBInterface, NBVMInterface, NBFHRPGroupItem], + "assigned_object_type": self.mapping.scopes_object_types(self.scopes), + "assigned_object_id": self.scopes, "description": 200, "role": ["loopback", "secondary", "anycast", "vip", "vrrp", "hsrp", "glbp", "carp"], "dns_name": 255, @@ -1961,31 +2098,11 @@ def __init__(self, *args, **kwargs): "tenant": NBTenant, "vrf": NBVRF } - # add relation between two attributes - self.data_model_relation = { - "dcim.interface": NBInterface, - "virtualization.vminterface": NBVMInterface, - "ipam.fhrpgroup": NBFHRPGroupItem, - NBInterface: "dcim.interface", - NBVMInterface: "virtualization.vminterface", - NBFHRPGroupItem: "ipam.fhrpgroup" - } super().__init__(*args, **kwargs) def resolve_relations(self): - o_id = self.data.get("assigned_object_id") - o_type = self.data.get("assigned_object_type") - - # this needs special treatment as the object type depends on a second model key - if o_type is not None and o_type not in self.data_model.get("assigned_object_type"): - - log.error(f"Attribute 'assigned_object_type' for '{self.get_display_name()}' invalid: {o_type}") - do_error_exit(f"Error while resolving relations for {self.get_display_name()}") - - if isinstance(o_id, int) and o_type is not None: - self.data["assigned_object_id"] = self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) - + self.resolve_scoped_relations("assigned_object_id", "assigned_object_type") super().resolve_relations() def update(self, data=None, read_from_netbox=False, source=None): @@ -2011,11 +2128,11 @@ def update(self, data=None, read_from_netbox=False, source=None): if not isinstance(assigned_object, NetBoxObject): data["assigned_object_id"] = \ - self.inventory.add_update_object(self.data_model_relation.get(object_type), data=assigned_object) + self.inventory.add_update_object(self.mapping.get(object_type), data=assigned_object) else: # noinspection PyTypeChecker - data["assigned_object_type"] = self.data_model_relation.get(type(assigned_object)) + data["assigned_object_type"] = self.mapping.get(assigned_object) super().update(data=data, read_from_netbox=read_from_netbox, source=source) @@ -2052,7 +2169,7 @@ def get_interface(self): if o_type not in self.data_model.get("assigned_object_type"): return - return self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) + return self.inventory.get_by_id(self.mapping.get(o_type), nb_id=o_id) def get_device_vm(self): @@ -2084,45 +2201,30 @@ def remove_interface_association(self): class NBMACAddress(NetBoxObject): name = "MAC address" api_path = "dcim/mac-addresses" + object_type = "ipam.macaddress" primary_key = "mac_address" prune = True min_netbox_version = "4.2" def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBInterface, NBVMInterface + ] self.data_model = { "mac_address": str, - "assigned_object_type": ["dcim.interface", "virtualization.vminterface"], - "assigned_object_id": [NBInterface, NBVMInterface], + "assigned_object_type": self.mapping.scopes_object_types(self.scopes), + "assigned_object_id": self.scopes, "description": 200, "tags": NBTagList, } - - # add relation between two attributes - self.data_model_relation = { - "dcim.interface": NBInterface, - "virtualization.vminterface": NBVMInterface, - NBInterface: "dcim.interface", - NBVMInterface: "virtualization.vminterface", - } super().__init__(*args, **kwargs) def resolve_relations(self): - o_id = self.data.get("assigned_object_id") - o_type = self.data.get("assigned_object_type") - - # this needs special treatment as the object type depends on a second model key - if o_type is not None and o_type not in self.data_model.get("assigned_object_type"): - - log.error(f"Attribute 'assigned_object_type' for '{self.get_display_name()}' invalid: {o_type}") - do_error_exit(f"Error while resolving relations for {self.get_display_name()}") - - if isinstance(o_id, int) and o_type is not None: - self.data["assigned_object_id"] = self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) - + self.resolve_scoped_relations("assigned_object_id", "assigned_object_type") super().resolve_relations() - def update(self, data=None, read_from_netbox=False, source=None): object_type = data.get("assigned_object_type") @@ -2134,11 +2236,11 @@ def update(self, data=None, read_from_netbox=False, source=None): if not isinstance(assigned_object, NetBoxObject): data["assigned_object_id"] = \ - self.inventory.add_update_object(self.data_model_relation.get(object_type), data=assigned_object) + self.inventory.add_update_object(self.mapping.get(object_type), data=assigned_object) else: # noinspection PyTypeChecker - data["assigned_object_type"] = self.data_model_relation.get(type(assigned_object)) + data["assigned_object_type"] = self.mapping.get(assigned_object) super().update(data=data, read_from_netbox=read_from_netbox, source=source) @@ -2164,7 +2266,7 @@ def get_interface(self): if o_type not in self.data_model.get("assigned_object_type"): return - return self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) + return self.inventory.get_by_id(self.mapping.get(o_type), nb_id=o_id) def get_device_vm(self): @@ -2193,14 +2295,12 @@ def remove_interface_association(self): class NBFHRPGroupItem(NetBoxObject): - """ - This object is currently not used directly in any class. - It is used to handle IP address object properly. - """ name = "FHRP group" api_path = "ipam/fhrp-groups" + object_type = "ipam.fhrpgroup" primary_key = "group_id" prune = False + read_only = True def __init__(self, *args, **kwargs): self.data_model = { @@ -2216,6 +2316,7 @@ def __init__(self, *args, **kwargs): class NBInventoryItem(NetBoxObject): name = "inventory item" api_path = "dcim/inventory-items" + object_type = "dcim.inventoryitem" primary_key = "name" secondary_key = "device" prune = True @@ -2240,6 +2341,7 @@ def __init__(self, *args, **kwargs): class NBPowerPort(NetBoxObject): name = "power port" api_path = "dcim/power-ports" + object_type = "dcim.powerport" primary_key = "name" secondary_key = "device" prune = True diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 6d2c71d0..9830ab11 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -179,7 +179,7 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac return return_data - def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None): + def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None) -> NBPrefix|None: """ This is a lazy approach to find the longest matching prefix to an IP address. If site_name is set only IP prefixes from that site are matched. @@ -332,11 +332,13 @@ def add_update_interface(self, interface_object, device_object, interface_data, "assigned_object_type": interface_class } - if primary_mac_address_object is None or grab(primary_mac_address_object, "data.mac_address") != interface_mac_address: + if (primary_mac_address_object is None or + grab(primary_mac_address_object, "data.mac_address") != interface_mac_address): primary_mac_address_object = None for mac_address_object in self.inventory.get_all_items(NBMACAddress): - if grab(mac_address_object, "data.mac_address") == interface_mac_address and grab(mac_address_object, "data.assigned_object_id") is None: + if (grab(mac_address_object, "data.mac_address") == interface_mac_address and + grab(mac_address_object, "data.assigned_object_id") is None): primary_mac_address_object = mac_address_object break @@ -380,22 +382,21 @@ def add_update_interface(self, interface_object, device_object, interface_data, prefix_tenant = None # test for site prefixes first - matching_site_name = site_name - matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object, matching_site_name) + matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object, site_name) # nothing was found then check prefixes without site name if matching_ip_prefix is None: - matching_site_name = None matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object) # matching prefix found, get data from prefix if matching_ip_prefix is not None: this_prefix = grab(matching_ip_prefix, f"data.{NBPrefix.primary_key}") - if matching_site_name is None: + prefix_scope = matching_ip_prefix.get_scope_display_name() + if prefix_scope is None: log.debug2(f"Found IP '{ip_object}' matches global prefix '{this_prefix}'") else: - log.debug2(f"Found IP '{ip_object}' matches site '{matching_site_name}' prefix " + log.debug2(f"Found IP '{ip_object}' matches {prefix_scope} prefix " f"'{this_prefix}'") # check if prefix net size and ip address prefix length match @@ -535,15 +536,16 @@ def add_update_interface(self, interface_object, device_object, interface_data, } # skip reassignment if IP is assigned to sub interface of a VM - if type(device_object) == NBVM and grab(ip.get_interface(), "data.parent") is not None: - current_ip_nic = ip.get_interface() + if (type(device_object) == NBVM and this_ip_object is not None and + grab(this_ip_object.get_interface(), "data.parent") is not None): + current_ip_nic = this_ip_object.get_interface() current_ip_nic_parent = grab(current_ip_nic, "data.parent") if isinstance(current_ip_nic_parent, dict): current_ip_nic_parent = self.inventory.get_by_id(NBVMInterface, nb_id=current_ip_nic_parent.get("id")) if current_ip_nic_parent == interface_object: - log.debug(f"{ip.name} '{ip.get_display_name()}' is assigned to sub interface " + log.debug(f"{this_ip_object.name} '{this_ip_object.get_display_name()}' is assigned to sub interface " f"'{current_ip_nic.get_display_name()}' of '{interface_object.get_display_name()}'. " f"Not changing assignment") nic_ip_data["assigned_object_id"] = current_ip_nic @@ -587,7 +589,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, continue if grab(current_ip, "data.role.value") == "anycast": - log.debug2(f"{ip.name} '{ip.get_display_name()}' is an Anycast address and will " + log.debug2(f"{current_ip.name} '{current_ip.get_display_name()}' is an Anycast address and will " f"NOT be deleted from interface") continue From 4a610952f575fbbbcaa4bfa507cdc37a64bf683b Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Mon, 3 Mar 2025 21:35:19 +0100 Subject: [PATCH 29/32] adds CDP info to ESXi host nic description if available #350 --- module/sources/vmware/connection.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 267e8fc4..9ab89744 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1768,6 +1768,14 @@ def add_host(self, obj): # now iterate over all physical interfaces and collect data pnic_data_dict = dict() + pnic_hints = dict() + # noinspection PyBroadException + try: + for hint in obj.configManager.networkSystem.QueryNetworkHint(""): + pnic_hints[hint.device] = hint + except Exception: + pass + for pnic in grab(obj, "config.network.pnic", fallback=list()): pnic_name = grab(pnic, "device") @@ -1820,7 +1828,6 @@ def add_host(self, obj): # check vlans on this pnic pnic_vlans = list() - for pg_name, pg_data in self.network_data["host_pgroup"][name].items(): if pnic_name in pg_data.get("nics", list()): @@ -1831,6 +1838,15 @@ def add_host(self, obj): pnic_mac_address = normalize_mac_address(grab(pnic, "mac")) + if pnic_hints.get(pnic_name) is not None: + pnic_switch_port = grab(pnic_hints.get(pnic_name), 'connectedSwitchPort') + if pnic_switch_port is not None: + pnic_sp_sys_name = grab(pnic_switch_port, 'systemName') + if pnic_sp_sys_name is None: + pnic_sp_sys_name = grab(pnic_switch_port, 'devId') + if pnic_sp_sys_name is not None: + pnic_description += f" (conn: {pnic_sp_sys_name} - {grab(pnic_switch_port, 'portId')})" + if self.settings.host_nic_exclude_by_mac_list is not None and \ pnic_mac_address in self.settings.host_nic_exclude_by_mac_list: log.debug2(f"Host NIC with MAC '{pnic_mac_address}' excluded from sync. Skipping") From ae5246f6e7e2744eb2af87cc215036c9cdf0f578 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Tue, 4 Mar 2025 00:17:56 +0100 Subject: [PATCH 30/32] fixes issue with properly assigned MAC addresses --- module/netbox/object_classes.py | 23 ++++++++++++++++++++-- module/sources/common/source_base.py | 29 ++++++++++++++++++---------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 52da05d5..96df5fd9 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1454,6 +1454,7 @@ def __init__(self, *args, **kwargs): } super().__init__(*args, **kwargs) + class NBSiteGroup(NetBoxObject): name = "site group" api_path = "dcim/site-groups" @@ -1995,11 +1996,20 @@ def get_ip_addresses(self): result_list = list() for ip_object in self.inventory.get_all_items(NBIPAddress): - if grab(ip_object, "data.assigned_object_id") == self: + if ip_object.data.get("assigned_object_id") == self: result_list.append(ip_object) return result_list + def get_mac_addresses(self): + + result_list = list() + for mac_object in self.inventory.get_all_items(NBMACAddress): + if mac_object.data.get("assigned_object_id") == self: + result_list.append(mac_object) + + return result_list + class NBInterface(NetBoxObject): name = "interface" @@ -2038,11 +2048,20 @@ def get_ip_addresses(self): result_list = list() for ip_object in self.inventory.get_all_items(NBIPAddress): - if grab(ip_object, "data.assigned_object_id") == self: + if ip_object.data.get("assigned_object_id") == self: result_list.append(ip_object) return result_list + def get_mac_addresses(self): + + result_list = list() + for mac_object in self.inventory.get_all_items(NBMACAddress): + if mac_object.data.get("assigned_object_id") == self: + result_list.append(mac_object) + + return result_list + def update(self, data=None, read_from_netbox=False, source=None): # remove definition of interface type if a parent interface is set as it only supports virtual types diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 9830ab11..fc81a160 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -322,9 +322,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, interface_object.update(data=interface_data, source=self) if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0") and \ - interface_mac_address is not None : - - primary_mac_address_object = grab(interface_object, "data.primary_mac_address") + interface_mac_address is not None: primary_mac_address_data = { "mac_address": interface_mac_address, @@ -332,20 +330,31 @@ def add_update_interface(self, interface_object, device_object, interface_data, "assigned_object_type": interface_class } - if (primary_mac_address_object is None or - grab(primary_mac_address_object, "data.mac_address") != interface_mac_address): + primary_mac_address_object = None + # check for associated MAC addresses on existing interface + if interface_object.is_new is False: + current_primary_mac_address_object = grab(interface_object, "data.primary_mac_address") + if grab(current_primary_mac_address_object, "data.mac_address") == interface_mac_address: + primary_mac_address_object = current_primary_mac_address_object + for mac_address_object in interface_object.get_mac_addresses(): + if (primary_mac_address_object is None and + grab(mac_address_object, "data.mac_address") == interface_mac_address): + primary_mac_address_object = mac_address_object + if mac_address_object is not primary_mac_address_object: + mac_address_object.remove_interface_association() - primary_mac_address_object = None + # if a new interface or not matching assigned MAC address, try to find an existing unassigned mac address + if primary_mac_address_object is None: for mac_address_object in self.inventory.get_all_items(NBMACAddress): if (grab(mac_address_object, "data.mac_address") == interface_mac_address and grab(mac_address_object, "data.assigned_object_id") is None): primary_mac_address_object = mac_address_object break - if primary_mac_address_object is None: - primary_mac_address_object = self.inventory.add_object(NBMACAddress, data=primary_mac_address_data, source=self) - else: - primary_mac_address_object.update(data=primary_mac_address_data, source=self) + # of no existing mac address could be found, create a new one + if primary_mac_address_object is None: + primary_mac_address_object = self.inventory.add_object(NBMACAddress, data=primary_mac_address_data, + source=self) else: primary_mac_address_object.update(data=primary_mac_address_data, source=self) From 341e6ecc1bb2217fa970a1879372e48a2f6b3139 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Fri, 7 Mar 2025 20:22:49 +0100 Subject: [PATCH 31/32] bump version to 1.8.0 and current date --- README.md | 2 +- module/__init__.py | 2 +- settings-example.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b09612da..0430a672 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ usage: netbox-sync.py [-h] [-c settings.ini [settings.ini ...]] [-g] Sync objects from various sources to NetBox -Version: 1.8.0 (2025-02-26) +Version: 1.8.0 (2025-03-07) Project URL: https://github.com/bb-ricardo/netbox-sync options: diff --git a/module/__init__.py b/module/__init__.py index ee32b85e..32323f94 100644 --- a/module/__init__.py +++ b/module/__init__.py @@ -8,7 +8,7 @@ # repository or visit: . __version__ = "1.8.0" -__version_date__ = "2025-02-26" +__version_date__ = "2025-03-07" __author__ = "Ricardo Bartels " __description__ = "NetBox Sync" __license__ = "MIT" diff --git a/settings-example.ini b/settings-example.ini index b2880086..14027f2d 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -1,5 +1,5 @@ ;;; Welcome to the NetBox Sync configuration file. -;;; Version: 1.8.0 (2025-02-26) +;;; Version: 1.8.0 (2025-03-07) ;;; Project URL: https://github.com/bb-ricardo/netbox-sync ; The values in this file override the default values used by the system if a config From 2e4c0f316361c497d2c77bc5f6661df26acc77d3 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Fri, 7 Mar 2025 21:15:03 +0100 Subject: [PATCH 32/32] adds note about issue with VLAN group --- docs/source_vmware.md | 4 ++-- module/sources/vmware/config.py | 4 +++- settings-example.ini | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/source_vmware.md b/docs/source_vmware.md index bff3fa7a..11f1d03e 100644 --- a/docs/source_vmware.md +++ b/docs/source_vmware.md @@ -28,7 +28,7 @@ IP addresses and interfaces will be added to NetBox. Try to find object first based on the object data, interface MAC addresses and primary IPs. 1. try to find by name and cluster/site 2. try to find by mac addresses interfaces -3. try to find by serial number (1st) or asset tag (2nd) (ESXi host) +3. try to find by serial number (1st) or asset tag (2nd) 4. try to find by primary IP ### Finding hosts and VMs from discovered data @@ -49,7 +49,7 @@ If two or more NetBox object (device, vm) with matching MACs were found, compare NetBox object (device, vm) with the highest amount of matching interfaces. If the ratio of matching interface MAC addresses exceeds 2.0 then the top matching NetBox object (device, vm) is chosen as desired object. -If the ratio is below 2.0 then None will be chosen. The probability is to low that +If the ratio is below 2.0 then None will be chosen. The probability is too low that this one is the correct one. #### 3. Try to find a NetBox object based on the primary IP (v4 or v6) address diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 50b62d7a..7e448a28 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -340,7 +340,9 @@ def __init__(self): by name. Same matching rules as the exclude_by_name option uses are applied. If name and id relations are defined, the name relation takes precedence. Fist match wins. Only newly discovered VLANs which are not present in - NetBox will be assigned a VLAN group. + NetBox will be assigned a VLAN group. Supported scopes for a VLAN group + are "site", "site-group", "cluster" and "cluster-group". Scopes are buggy + in NetBox https://github.com/netbox-community/netbox/issues/18706 """, config_example="London/Vlan_.* = VLAN Group 1, Tokio/Vlan_.* = VLAN Group 2"), ConfigOption("vlan_group_relation_by_id", diff --git a/settings-example.ini b/settings-example.ini index 14027f2d..432d4c2c 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -345,7 +345,9 @@ password = super-secret ; adds a relation to assign VLAN groups to matching VLANs by name. Same matching rules as ; the exclude_by_name option uses are applied. If name and id relations are defined, the ; name relation takes precedence. Fist match wins. Only newly discovered VLANs which are -; not present in NetBox will be assigned a VLAN group. +; not present in NetBox will be assigned a VLAN group. Supported scopes for a VLAN group +; are "site", "site-group", "cluster" and "cluster-group". Scopes are buggy in NetBox +; https://github.com/netbox-community/netbox/issues/18706 ;vlan_group_relation_by_name = London/Vlan_.* = VLAN Group 1, Tokio/Vlan_.* = VLAN Group 2 ; adds a relation to assign VLAN groups to matching VLANs by ID. Same matching rules as