diff --git a/Dockerfile b/Dockerfile index 9e6cb907..f048b553 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,18 +2,23 @@ FROM python:3.11-slim-bookworm AS builder COPY requirements.txt . +ARG VENV=/opt/netbox-sync/venv + # Install dependencies RUN apt-get update && apt-get install -y --no-install-recommends git && \ rm -rf /var/lib/apt/lists/* && \ - python3 -m venv /opt/netbox-sync/venv && \ - /opt/netbox-sync/venv/bin/python3 -m pip install --upgrade pip && \ - /opt/netbox-sync/venv/bin/pip install -r requirements.txt && \ - /opt/netbox-sync/venv/bin/pip install --upgrade git+https://github.com/vmware/vsphere-automation-sdk-python.git + python3 -m venv $VENV && \ + $VENV/bin/python3 -m pip install --upgrade pip && \ + $VENV/bin/pip install -r requirements.txt && \ + $VENV/bin/pip install --upgrade git+https://github.com/vmware/vsphere-automation-sdk-python.git && \ + find $VENV -type d -name "__pycache__" -print0 | xargs -0 -n1 rm -rf FROM python:3.11-slim-bookworm AS netbox-sync +ARG VENV=/opt/netbox-sync/venv + # Copy installed packages -COPY --from=builder /opt/netbox-sync/venv /opt/netbox-sync/venv +COPY --from=builder $VENV $VENV # Add netbox-sync user RUN groupadd --gid 1000 netbox-sync && \ @@ -27,7 +32,7 @@ WORKDIR /app COPY --chown=netbox-sync:netbox-sync . . # Use virtual env packages and allow timezone setup -ENV PATH=/opt/netbox-sync/venv/bin:$PATH +ENV PATH=$VENV/bin:$PATH ENV TZ=Europe/Berlin ENTRYPOINT ["python3", "netbox-sync.py"] diff --git a/LICENSE.txt b/LICENSE.txt index 44aae69a..cefeaf2f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 - 2021 Ricardo Bartels +Copyright (c) 2020 - 2025 Ricardo Bartels Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 173b9d0c..0430a672 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ usage: netbox-sync.py [-h] [-c settings.ini [settings.ini ...]] [-g] Sync objects from various sources to NetBox -Version: 1.7.0 (2024-10-21) +Version: 1.8.0 (2025-03-07) Project URL: https://github.com/bb-ricardo/netbox-sync options: diff --git a/docs/source_vmware.md b/docs/source_vmware.md index bff3fa7a..11f1d03e 100644 --- a/docs/source_vmware.md +++ b/docs/source_vmware.md @@ -28,7 +28,7 @@ IP addresses and interfaces will be added to NetBox. Try to find object first based on the object data, interface MAC addresses and primary IPs. 1. try to find by name and cluster/site 2. try to find by mac addresses interfaces -3. try to find by serial number (1st) or asset tag (2nd) (ESXi host) +3. try to find by serial number (1st) or asset tag (2nd) 4. try to find by primary IP ### Finding hosts and VMs from discovered data @@ -49,7 +49,7 @@ If two or more NetBox object (device, vm) with matching MACs were found, compare NetBox object (device, vm) with the highest amount of matching interfaces. If the ratio of matching interface MAC addresses exceeds 2.0 then the top matching NetBox object (device, vm) is chosen as desired object. -If the ratio is below 2.0 then None will be chosen. The probability is to low that +If the ratio is below 2.0 then None will be chosen. The probability is too low that this one is the correct one. #### 3. Try to find a NetBox object based on the primary IP (v4 or v6) address diff --git a/module/__init__.py b/module/__init__.py index b283262e..32323f94 100644 --- a/module/__init__.py +++ b/module/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -7,8 +7,8 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . -__version__ = "1.7.0" -__version_date__ = "2024-10-21" +__version__ = "1.8.0" +__version_date__ = "2025-03-07" __author__ = "Ricardo Bartels " __description__ = "NetBox Sync" __license__ = "MIT" diff --git a/module/common/__init__.py b/module/common/__init__.py index 376af17d..4b85b91e 100644 --- a/module/common/__init__.py +++ b/module/common/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/common/cli_parser.py b/module/common/cli_parser.py index 07d69ac5..3d890cf2 100644 --- a/module/common/cli_parser.py +++ b/module/common/cli_parser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -23,7 +23,7 @@ def parse_command_line(self_description=None): Parameters ---------- self_description: str - short self description of this program + short self-description of this program Returns ------- diff --git a/module/common/config.py b/module/common/config.py index 28ed919f..bb8dd06d 100644 --- a/module/common/config.py +++ b/module/common/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/common/logging.py b/module/common/logging.py index 020f38a2..6bc76500 100644 --- a/module/common/logging.py +++ b/module/common/logging.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/common/misc.py b/module/common/misc.py index e966a975..77038dc3 100644 --- a/module/common/misc.py +++ b/module/common/misc.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -15,7 +15,7 @@ def grab(structure=None, path=None, separator=".", fallback=None): """ get data from a complex object/json structure with a "." separated path information. If a part of a path - is not not present then this function returns the + is not present then this function returns the value of fallback (default: "None"). example structure: @@ -38,7 +38,7 @@ def grab(structure=None, path=None, separator=".", fallback=None): Parameters ---------- structure: dict, list, object - object structure to extract data from + an object structure to extract data from path: str nested path to extract separator: str diff --git a/module/common/support.py b/module/common/support.py index 7fca9ae7..e800da10 100644 --- a/module/common/support.py +++ b/module/common/support.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -51,9 +51,9 @@ def perform_ptr_lookups(ips, dns_servers=None): Parameters ---------- ips: list - list of IP addresses to look up + a list of IP addresses to look up dns_servers: list - list of DNS servers to use to look up list of IP addresses + a list of DNS servers to use to look up list of IP addresses Returns ------- diff --git a/module/config/__init__.py b/module/config/__init__.py index 6aadb483..02d7fef5 100644 --- a/module/config/__init__.py +++ b/module/config/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/base.py b/module/config/base.py index b65494d5..d9876c8c 100644 --- a/module/config/base.py +++ b/module/config/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -30,6 +30,10 @@ def __eq__(self, other): def __contains__(self, key): return key in self.__dict__ + def __getattr__(self, item): + if item in self: + return getattr(self, item) + return None class ConfigBase: """ diff --git a/module/config/file_output.py b/module/config/file_output.py index 00870044..3b2f6e5c 100644 --- a/module/config/file_output.py +++ b/module/config/file_output.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/files.py b/module/config/files.py index 93ad830c..2b734cc1 100644 --- a/module/config/files.py +++ b/module/config/files.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/formatter.py b/module/config/formatter.py index b832df8f..1fef402b 100644 --- a/module/config/formatter.py +++ b/module/config/formatter.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/group.py b/module/config/group.py index ce30a0ac..8880147b 100644 --- a/module/config/group.py +++ b/module/config/group.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/option.py b/module/config/option.py index 8ee3882f..e792269b 100644 --- a/module/config/option.py +++ b/module/config/option.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/config/parser.py b/module/config/parser.py index 2c01c3ed..d46256db 100644 --- a/module/config/parser.py +++ b/module/config/parser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/__init__.py b/module/netbox/__init__.py index 8904ffe9..6891c375 100644 --- a/module/netbox/__init__.py +++ b/module/netbox/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -16,9 +16,11 @@ NBTagList, NBTenant, NBSite, + NBSiteGroup, NBVRF, NBVLAN, NBVLANList, + NBVLANGroup, NBPrefix, NBManufacturer, NBDeviceType, @@ -33,6 +35,7 @@ NBVirtualDisk, NBInterface, NBIPAddress, + NBMACAddress, NBFHRPGroupItem, NBInventoryItem, NBPowerPort diff --git a/module/netbox/config.py b/module/netbox/config.py index c82aa3d9..8f943c06 100644 --- a/module/netbox/config.py +++ b/module/netbox/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/connection.py b/module/netbox/connection.py index e1b943a6..5c86d26f 100644 --- a/module/netbox/connection.py +++ b/module/netbox/connection.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -217,20 +217,20 @@ def request(self, object_class, req_type="GET", data=None, params=None, nb_id=No Parameters ---------- - object_class: NetBoxObject sub class + object_class: NetBoxObject subclass class definition of the desired NetBox object req_type: str GET, PATCH, PUT, DELETE data: dict - data which shall be send to NetBox + data which shall be sent to NetBox params: dict - dict of URL params which should be passed to NetBox + dictionary of URL params which should be passed to NetBox nb_id: int ID of the NetBox object which will be appended to the requested NetBox URL Returns ------- - (dict, bool, None): of returned NetBox data. If object was requested to be deleted and it was + dict, bool, None: of returned NetBox data. If object was requested to be deleted, and it was successful then True will be returned. None if request failed or was empty """ @@ -373,12 +373,12 @@ def query_current_data(self, netbox_objects_to_query=None): """ Request all current NetBox objects. Use caching whenever possible. Objects must provide "last_updated" attribute to support caching for this object type. - Otherwise it's not possible to query only changed objects since last run. If attribute is + Otherwise, it's not possible to query only changed objects since last run. If attribute is not present all objects will be requested (looking at you *Interfaces) Parameters ---------- - netbox_objects_to_query: list of NetBoxObject sub classes + netbox_objects_to_query: list of NetBoxObject subclasses NetBox items to query """ @@ -585,14 +585,14 @@ def initialize_basic_data(self): def update_object(self, nb_object_sub_class, unset=False, last_run=False): """ - Iterate over all objects of a certain NetBoxObject sub class and add/update them. + Iterate over all objects of a certain NetBoxObject subclass and add/update them. But first update objects which this object class depends on. If some dependencies are unresolvable then these will be removed from the request and re added later to the object to try update object in a third run. Parameters ---------- - nb_object_sub_class: NetBoxObject sub class + nb_object_sub_class: NetBoxObject subclass NetBox objects to update unset: bool True if only unset items should be deleted @@ -650,7 +650,8 @@ def update_object(self, nb_object_sub_class, unset=False, last_run=False): # resolve dependency issues in last run # primary IP always set in last run if value.get_nb_reference() is None or \ - (key.startswith("primary_ip") and last_run is False): + (key.startswith("primary_ip") and last_run is False) or \ + (key.startswith("primary_mac_address") and last_run is False): unresolved_dependency_data[key] = value else: data_to_patch[key] = value.get_nb_reference() @@ -658,9 +659,9 @@ def update_object(self, nb_object_sub_class, unset=False, last_run=False): else: data_to_patch[key] = value - # special case for IP address - if isinstance(this_object, NBIPAddress): - # if object is new and and has no id, then we need to remove assigned_object_type from data_to_patch + # special case for IP and MAC addresses + if isinstance(this_object, (NBIPAddress, NBMACAddress)): + # if object is new and has no id, then we need to remove assigned_object_type from data_to_patch if "assigned_object_id" in unresolved_dependency_data.keys() and \ "assigned_object_type" in data_to_patch.keys(): del data_to_patch["assigned_object_type"] @@ -674,7 +675,7 @@ def update_object(self, nb_object_sub_class, unset=False, last_run=False): req_type = "POST" action = "Creating new" - # if its not a new object then update it + # if it's not a new object then update it if this_object.is_new is False: nb_id = this_object.nb_id req_type = "PATCH" @@ -928,7 +929,7 @@ def delete_unused_tags(self): if tag_description is None or not tag_description.startswith(self.primary_tag): continue - if tag_tagged_items is None or tag_tagged_items != 0: + if tag_tagged_items is None or tag_tagged_items != 0 or this_tag.used is True: continue log.info(f"Deleting unused tag '{this_tag.get_display_name()}'") diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py index ab18d7d4..2daa39b6 100644 --- a/module/netbox/inventory.py +++ b/module/netbox/inventory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -19,7 +19,7 @@ class NetBoxInventory: """ - Singleton class to manage a inventory of NetBoxObject objects + Singleton class to manage an inventory of NetBoxObject objects """ base_structure = dict() @@ -62,18 +62,18 @@ def get_by_id(self, object_type, nb_id=None): Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to find nb_id: int NetBox ID of object Returns ------- - (NetBoxObject sub class, None): return object instance if object was found, None otherwise + (NetBoxObject subclass, None): return object instance if object was found, None otherwise """ if object_type not in NetBoxObject.__subclasses__(): - raise AttributeError("'%s' object must be a sub class of '%s'." % + raise AttributeError("'%s' object must be a subclass of '%s'." % (object_type.__name__, NetBoxObject.__name__)) if nb_id is None or self.base_structure[object_type.name] is None: @@ -90,18 +90,18 @@ def get_by_data(self, object_type, data=None): Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to find data: dict params of object to match Returns ------- - (NetBoxObject sub class, None): return object instance if object was found, None otherwise + (NetBoxObject subclass, None): return object instance if object was found, None otherwise """ if object_type not in NetBoxObject.__subclasses__(): - raise AttributeError("'%s' object must be a sub class of '%s'." % + raise AttributeError("'%s' object must be a subclass of '%s'." % (object_type.__name__, NetBoxObject.__name__)) if data is None or len(self.get_all_items(object_type)) == 0: @@ -169,7 +169,7 @@ def slug_used(self, object_type: NetBoxObject, slug: str) -> bool: """ if object_type not in NetBoxObject.__subclasses__(): - raise AttributeError("'%s' object must be a sub class of '%s'." % + raise AttributeError("'%s' object must be a subclass of '%s'." % (object_type.__name__, NetBoxObject.__name__)) if "slug" in object_type.data_model.keys(): @@ -185,7 +185,7 @@ def add_object(self, object_type, data=None, read_from_netbox=False, source=None Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to add data: dict Object data to add to the inventory @@ -216,10 +216,10 @@ def add_update_object(self, object_type, data=None, read_from_netbox=False, sour Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to add/update data: dict - data used to create a new object or update a existing object + data used to create a new object or update an existing object read_from_netbox: bool True if data was read directly from NetBox source: object handler of source @@ -265,7 +265,7 @@ def get_all_items(self, object_type): Parameters ---------- - object_type: NetBoxObject sub class + object_type: NetBoxObject subclass object type to find Returns @@ -274,7 +274,7 @@ def get_all_items(self, object_type): """ if object_type not in NetBoxObject.__subclasses__(): - raise ValueError(f"'{object_type.__name__}' object must be a sub class of '{NetBoxObject.__name__}'.") + raise ValueError(f"'{object_type.__name__}' object must be a subclass of '{NetBoxObject.__name__}'.") return self.base_structure.get(object_type.name, list()) @@ -335,6 +335,11 @@ def tag_all_the_things(self, netbox_handler): # if object was found in source if this_object.source is not None: + # skip tagging of VLANs if vlan sync is disabled + if object_type == NBVLAN and \ + grab(this_object.source, "settings.disable_vlan_sync", fallback=False) is True: + continue + this_object.add_tags([netbox_handler.primary_tag, this_object.source.source_tag]) # if object was orphaned remove tag again diff --git a/module/netbox/manufacturer_mapping.py b/module/netbox/manufacturer_mapping.py index b3554eb9..d3aa2f74 100644 --- a/module/netbox/manufacturer_mapping.py +++ b/module/netbox/manufacturer_mapping.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index dd84862f..96df5fd9 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -13,7 +13,7 @@ # noinspection PyUnresolvedReferences from packaging import version -from module.common.misc import grab, do_error_exit +from module.common.misc import grab from module.common.logging import get_logger from module.netbox.manufacturer_mapping import sanitize_manufacturer_name @@ -60,7 +60,7 @@ class to handle and determine NetBox interface types "400gbase-x-osfp": 400_000 } - # assign common types for a interface speed value + # assign common types for an interface speed value common_types = { 100: "100base-tx", 1_000: "1000base-t", @@ -99,14 +99,14 @@ def get_netbox_type_list(self) -> list: Returns ------- valid_interface_types: list - list with valid types + a list with valid types """ return list(self.valid_types.keys()) def get_common_type(self) -> str: """ - return a interface type from the common type list + return an interface type from the common type list Returns ------- @@ -160,7 +160,7 @@ def get_speed_human(self) -> str: Returns ------- human_speed: str - human readable string of interface speed + human-readable string of interface speed """ if self.detected_speed == 0: @@ -211,27 +211,75 @@ def get_this_netbox_type(self) -> str: return self.get_common_type() +class NetBoxMappings: + """ + Adds singleton to map between NetBox object class and "object_type" attribute of that class + this is used for objects scopes and IP address and MAC address objects + """ + + mapping = dict() + + def __new__(cls): + it = cls.__dict__.get("__it__") + if it is not None: + return it + cls.__it__ = it = object.__new__(cls) + it.init() + return it + + def init(self): + + for classDefinition in NetBoxObject.__subclasses__(): + + if classDefinition.object_type is not None: + self.mapping[classDefinition] = classDefinition.object_type + self.mapping[classDefinition.object_type] = classDefinition + + def get(self, needle): + if isinstance(needle, NetBoxObject): + return self.mapping.get(type(needle)) + else: + return self.mapping.get(needle) + + def scopes_object_types(self, scopes_list) -> list: + result = list() + + if not isinstance(scopes_list, list): + raise ValueError("value for 'scopes_list' must be a list") + + for scope in scopes_list: + result.append(self.get(scope)) + + return result + + class NetBoxObject: """ Base class for all NetBox object types. Implements all methods used on a NetBox object. - Sub classes need to have following attributes: - name: + subclasses need to have the following attributes: + name: string name of the object type (i.e. "virtual machine") - api_path: + api_path: string NetBox api path of object type (i.e: "virtualization/virtual-machines") - primary_key: + object_type: string + NetBox object type (i.e: "virtualization.virtualmachine") to handle scopes for this NetBox object + primary_key: string name of the data model key which represents the primary key of this object besides id (i.e: "name") - data_model: + data_model: string dict of permitted data keys and possible values (see description below) + prune: bool + defines if this object type will be pruned by netbox-sync optional attributes - secondary_key: + secondary_key: string name of the data model key which represents the secondary key of this object besides id - enforce_secondary_key: - bool if secondary key of an object shall be added to name when get_display_name() method is called - min_netbox_version: + enforce_secondary_key: bool + if secondary key of an object shall be added to name when get_display_name() method is called + min_netbox_version: string defines since which NetBox version this object is available + read_only: bool + defines if this is a read only object class and can't be changed within netbox-sync The data_model attribute needs to be a dict describing the data model in NetBox. Key must be string. @@ -244,14 +292,14 @@ class NetBoxObject: can be a string with an undefined length bool (class): attribute must be True or False - NetBoxObject sub class: + NetBoxObject subclass: value of this key is a reference to another NetBoxObject of exact defined type list (instance): - value can be one of the predefined values in that list - list of NetBoxObject sub classes: + value can be one of the predefined values in that list. + list of NetBoxObject subclasses: value must be an instance of predefined netBoxObject classes in list - NBObjectList sub class: - value mast be the defined sub class of NBObjectList + NBObjectList subclass: + value mast be the defined subclass of NBObjectList """ @@ -259,9 +307,11 @@ class NetBoxObject: api_path = "" primary_key = "" data_model = {} + object_type = None min_netbox_version = "0.0" + read_only = False # _mandatory_attrs must be set at subclasses - _mandatory_attrs = ("name", "api_path", "primary_key", "data_model") + _mandatory_attrs = ("name", "api_path", "primary_key", "data_model", "object_type") # just skip this object if a mandatory attribute is missing skip_object_if_mandatory_attr_is_missing = False @@ -270,7 +320,7 @@ class NetBoxObject: inventory = None def __init__(self, data=None, read_from_netbox=False, inventory=None, source=None): - if not all(getattr(self, attr) for attr in self._mandatory_attrs): + if not all(getattr(self, attr) for attr in self._mandatory_attrs) or hasattr(self, "prune") is False: raise ValueError( f"FATAL: not all mandatory attributes {self._mandatory_attrs} " f"are set in {self.__class__.__name__}." @@ -315,7 +365,7 @@ def to_dict(self): continue if callable(value) is True: continue - if key in ["inventory", "default_attributes", "data_model_relation"]: + if key in ["inventory", "default_attributes", "data_model_relation", "mapping", "scopes"]: continue if key == "source": value = getattr(value, "name", None) @@ -434,7 +484,7 @@ def get_uniq_slug(self, text=None, max_len=50)-> str: new_slug = f"{slug}-{x}" if self.inventory.slug_used(self.__class__, new_slug) is False and len(new_slug) <= max_len: log.info(f"Slug '{slug}' for {self.name} '{text}' has been used. " - f"Assignin slug '{new_slug}'") + f"Assigning slug '{new_slug}'") return new_slug raise ValueError(f"Unable to find uniq slug for {self.name} '{text}'") @@ -488,6 +538,9 @@ def update(self, data=None, read_from_netbox=False, source=None): return + if self.read_only is True: + raise ValueError(f"Adding {self.name} by this program is currently not implemented.") + self.set_source(source) display_name = self.get_display_name(data) @@ -630,7 +683,8 @@ def update(self, data=None, read_from_netbox=False, source=None): if self.data_model.get(key) == NBCustomField: if current_value is None: current_value = dict() - new_value_str = str({**current_value, **new_value}) + new_value = {**current_value, **new_value} + new_value_str = str(new_value) elif isinstance(new_value, (NetBoxObject, NBObjectList)): new_value_str = str(new_value.get_display_name()) else: @@ -750,7 +804,7 @@ def get_display_name(self, data=None, including_second_key=False): def resolve_relations(self): """ - Resolve object relations for this object. Substitute a dict of data with a id with the instantiated + Resolve object relations for this object. Substitute a dict of data with an id with the instantiated reference of this object """ @@ -808,13 +862,31 @@ def resolve_relations(self): log.error(f"Problems resolving relation '{key}' for object '{self.get_display_name()}' and " f"value '{data_value}'") + def resolve_scoped_relations(self, id_attr, type_attr): + + o_id = self.data.get(id_attr) + o_type = self.data.get(type_attr) + + if hasattr(self, "mapping") is True: + mapping = getattr(self, "mapping") + else: + return + + if isinstance(o_id, int) and o_type is not None and mapping.get(o_type) is not None: + self.data[id_attr] = self.inventory.get_by_id(mapping.get(o_type), nb_id=o_id) + elif o_id is not None and not isinstance(o_id, NetBoxObject): + o_id_name = grab(self, f"data.{id_attr}.name") + log.debug(f"{self.name} '{self.data.get('name')}' {type_attr} '{o_type}' for " + f"'{o_id_name}' is currently not supported") + self.data[type_attr] = "" + def get_dependencies(self): """ - returns a list of NetBoxObject sub classes this object depends on + returns a list of NetBoxObject subclasses this object depends on Returns ------- - list: of NetBoxObject sub classes + list: of NetBoxObject subclasses """ r = [x for x in self.data_model.values() if x in NetBoxObject.__subclasses__()] @@ -909,6 +981,10 @@ def compile_tags(self, tags, remove=False): tag = self.inventory.add_update_object(NBTag, data={"name": tag_name}) + # mark tag as used in order to not delete newly used tags + if isinstance(tag, NBTag): + tag.used = True + new_tags.append(tag) if tag_name in current_tag_strings and remove is True: @@ -940,7 +1016,7 @@ def update_tags(self, tags, remove=False): Parameters ---------- - tags: (str, list, dict, NBTag) + tags: str, list, dict, NBTag tags to parse and add/remove to/from current list of object tags remove: bool True if tags shall be removed, otherwise they will be added @@ -1009,12 +1085,13 @@ def compile_vlans(self, vlans): Parameters ---------- - vlans: list of (dict, NBVLAN) + vlans: list of (dict or NBVLAN) list of VLANs that should be in the returned list Returns ------- - NBVLANList: of parsed VLANs + NBVLANList + of parsed VLANs """ if vlans is None or NBVLANList not in self.data_model.values(): @@ -1162,13 +1239,24 @@ def get_site_name(self, data=None): if isinstance(this_site, dict): return this_site.get("name") + this_scope_type = this_data_set.get("scope_type") + this_site = this_data_set.get("scope_id") + + if this_scope_type == "dcim.site" and this_site is not None: + + if isinstance(this_site, NetBoxObject): + return this_site.get_display_name() + + if isinstance(this_site, dict): + return this_site.get("name") + class NBObjectList(list): """ Base class of listed NetBox objects. Extends list(). Currently used for tags and untagged VLANs Mandatory attributes: - member_type: NetBoxObject sub class + member_type: NetBoxObject subclass defines the type objects contained in this type of list """ @@ -1180,20 +1268,22 @@ def get_display_name(self): class NBCustomField(NetBoxObject): name = "custom field" api_path = "extras/custom-fields" + object_type = "extras.customfield" primary_key = "name" prune = False - # used by this software - valid_object_types = [ - "dcim.device", - "dcim.interface", - "dcim.inventoryitem", - "dcim.powerport", - "virtualization.clustergroup", - "virtualization.vminterface", - "virtualization.virtualmachine" - ] def __init__(self, *args, **kwargs): + # used by this software + self.valid_object_types = [ + NBDevice.object_type, + NBInterface.object_type, + NBInventoryItem.object_type, + NBPowerPort.object_type, + NBClusterGroup.object_type, + NBVMInterface.object_type, + NBVM.object_type + ] + self.data_model = { "object_types": list, # field name (object_types) for NetBox < 4.0.0 @@ -1258,6 +1348,7 @@ def update(self, data=None, read_from_netbox=False, source=None): class NBTag(NetBoxObject): name = "tag" api_path = "extras/tags" + object_type = "extras.tag" primary_key = "name" prune = False @@ -1269,6 +1360,7 @@ def __init__(self, *args, **kwargs): "description": 200, "tagged_items": int } + self.used = False super().__init__(*args, **kwargs) @@ -1294,6 +1386,7 @@ def get_nb_reference(self): class NBTenant(NetBoxObject): name = "tenant" api_path = "tenancy/tenants" + object_type = "tenancy.tenant" primary_key = "name" prune = False @@ -1308,9 +1401,45 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) +# class NBLocation(NetBoxObject): +# name = "location" +# api_path = "dcim/locations" +# object_type = "dcim.location" +# primary_key = "name" +# prune = False +# read_only = True +# +# def __init__(self, *args, **kwargs): +# self.data_model = { +# "name": 100, +# "slug": 100, +# "site": NBSite, +# "tags": NBTagList +# } +# super().__init__(*args, **kwargs) +# +# +# class NBRegion(NetBoxObject): +# name = "region" +# api_path = "dcim/regions" +# object_type = "dcim.region" +# primary_key = "name" +# prune = False +# read_only = True +# +# def __init__(self, *args, **kwargs): +# self.data_model = { +# "name": 100, +# "slug": 100, +# "tags": NBTagList +# } +# super().__init__(*args, **kwargs) + + class NBSite(NetBoxObject): name = "site" api_path = "dcim/sites" + object_type = "dcim.site" primary_key = "name" prune = False @@ -1320,7 +1449,24 @@ def __init__(self, *args, **kwargs): "slug": 100, "comments": str, "tenant": NBTenant, - "tags": NBTagList + "tags": NBTagList, + "group": NBSiteGroup + } + super().__init__(*args, **kwargs) + + +class NBSiteGroup(NetBoxObject): + name = "site group" + api_path = "dcim/site-groups" + object_type = "dcim.sitegroup" + primary_key = "name" + prune = False + read_only = True + + def __init__(self, *args, **kwargs): + self.data_model = { + "name": 100, + "slug": 100, } super().__init__(*args, **kwargs) @@ -1328,8 +1474,10 @@ def __init__(self, *args, **kwargs): class NBVRF(NetBoxObject): name = "VRF" api_path = "ipam/vrfs" + object_type = "ipam.vrf" primary_key = "name" prune = False + read_only = True def __init__(self, *args, **kwargs): self.data_model = { @@ -1344,6 +1492,7 @@ def __init__(self, *args, **kwargs): class NBVLAN(NetBoxObject): name = "VLAN" api_path = "ipam/vlans" + object_type = "ipam.vlan" primary_key = "vid" secondary_key = "name" enforce_secondary_key = True @@ -1356,7 +1505,8 @@ def __init__(self, *args, **kwargs): "site": NBSite, "description": 200, "tenant": NBTenant, - "tags": NBTagList + "tags": NBTagList, + "group": NBVLANGroup } super().__init__(*args, **kwargs) @@ -1385,7 +1535,15 @@ def get_display_name(self, data=None, including_second_key=False): site_name = self.get_site_name(this_data_set) if site_name is not None: - my_name = f"{vlan_id} ({site_name})" + my_name = f"{vlan_id} (site: {site_name})" + + this_group = this_data_set.get("group") + if this_group is not None: + vlan_id = this_data_set.get(self.primary_key) + group_name = grab(this_group, "data.name") + + if group_name is not None: + my_name = f"{vlan_id} (group: {group_name})" return my_name @@ -1398,6 +1556,67 @@ def update(self, data=None, read_from_netbox=False, source=None): super().update(data=data, read_from_netbox=read_from_netbox, source=source) +class NBVLANGroup(NetBoxObject): + name = "VLANGroup" + api_path = "ipam/vlan-groups" + object_type = "ipam.vlangroup" + primary_key = "name" + prune = False + read_only = True + + def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBSite, NBSiteGroup, NBCluster, NBClusterGroup + ] + self.data_model = { + "name": 100, + "slug": 100, + "description": 200, + "scope_type": self.mapping.scopes_object_types(self.scopes), + "scope_id": self.scopes, + } + + super().__init__(*args, **kwargs) + + def resolve_relations(self): + + self.resolve_scoped_relations("scope_id", "scope_type") + super().resolve_relations() + + def matches_site_cluster(self, site=None, cluster=None) -> bool: + """ + tries to figure out if this vlan group matches a certain site or cluster + + Parameters + ---------- + site: NBSite + the site object to match to + cluster: NBCluster + the cluster object to match to + + Returns + ------- + bool: True if matches one of the params + + """ + if isinstance(site, NBSite): + if isinstance(self.data.get("scope_id"), NBSite) and self.data.get("scope_id") == site: + return True + if (isinstance(self.data.get("scope_id"), NBSiteGroup) and + self.data["scope_id"] == grab(site, "data.group")): + return True + + if isinstance(cluster, NBCluster): + if isinstance(self.data.get("scope_id"), NBCluster) and self.data.get("scope_id") == cluster: + return True + if (isinstance(self.data.get("scope_id"), NBClusterGroup) and + self.data["scope_id"] == grab(cluster, "data.group")): + return True + + return False + + class NBVLANList(NBObjectList): member_type = NBVLAN @@ -1420,26 +1639,35 @@ def get_nb_reference(self): class NBPrefix(NetBoxObject): name = "IP prefix" api_path = "ipam/prefixes" + object_type = "ipam.prefix" primary_key = "prefix" + read_only = True prune = False def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBSite, NBSiteGroup + ] self.data_model = { "prefix": [IPv4Network, IPv6Network], "site": NBSite, + "scope_type": self.mapping.scopes_object_types(self.scopes), + "scope_id": self.scopes, "tenant": NBTenant, "vlan": NBVLAN, "vrf": NBVRF, "description": 200, "tags": NBTagList } + super().__init__(*args, **kwargs) def update(self, data=None, read_from_netbox=False, source=None): # prefixes are parsed into ip_networks data_prefix = data.get(self.primary_key) - if data_prefix is not None and not isinstance(data_prefix, (IPv4Network, IPv6Network)): + if not isinstance(data_prefix, (IPv4Network, IPv6Network)) and data_prefix is not None: try: data[self.primary_key] = ip_network(data_prefix) except ValueError as e: @@ -1448,13 +1676,67 @@ def update(self, data=None, read_from_netbox=False, source=None): super().update(data=data, read_from_netbox=read_from_netbox, source=source) - if read_from_netbox is False: - raise ValueError(f"Adding {self.name} by this program is currently not implemented.") + + def resolve_relations(self): + + self.resolve_scoped_relations("scope_id", "scope_type") + super().resolve_relations() + + def matches_site(self, site=None) -> bool: + """ + tries to figure out if this prefix matches a certain site or site group + + Parameters + ---------- + site: NBSite + the site object to match to + + Returns + ------- + bool: True if matches one of the params + + """ + if isinstance(site, NBSite): + if isinstance(self.data.get("scope_id"), NBSite) and self.data.get("scope_id") == site: + return True + if (isinstance(self.data.get("scope_id"), NBSiteGroup) and + self.data["scope_id"] == grab(site, "data.group")): + return True + + # compatible for NetBox versions < 4.2.0 + if self.data.get("site") == site: + return True + + return False + + def get_scope_display_name(self): + + if self.data.get("scope_id") is not None: + if isinstance(self.data.get("scope_id"), NetBoxObject): + scope_object = self.data.get("scope_id") + return f"{scope_object.name} '{scope_object.get_display_name()}'" + if isinstance(self.data.get("scope_id"), dict) and self.data.get("scope_type") is not None: + + if isinstance(self.data.get("scope_type"), str): + scope_class = self.mapping.get(self.data.get("scope_type")) + else: + scope_class = self.data.get("scope_type") + + if scope_class is not None: + return f"{scope_class.name} '{grab(self, 'data.scope_id.data.name')}'" + + if self.data.get("site") is not None: + if isinstance(self.data.get("site"), NetBoxObject): + site_name = self.data.get("site").get_display_name() + return f"site '{site_name}'" + elif isinstance(self.data.get("site"), dict): + return f"site '{grab(self, 'data.scope_id.data.name')}'" class NBManufacturer(NetBoxObject): name = "manufacturer" api_path = "dcim/manufacturers" + object_type = "dcim.manufacturer" primary_key = "name" prune = False @@ -1471,6 +1753,7 @@ def __init__(self, *args, **kwargs): class NBDeviceType(NetBoxObject): name = "device type" api_path = "dcim/device-types" + object_type = "dcim.devicetype" primary_key = "model" prune = False @@ -1488,6 +1771,7 @@ def __init__(self, *args, **kwargs): class NBPlatform(NetBoxObject): name = "platform" api_path = "dcim/platforms" + object_type = "dcim.platform" primary_key = "name" prune = False @@ -1505,6 +1789,7 @@ def __init__(self, *args, **kwargs): class NBClusterType(NetBoxObject): name = "cluster type" api_path = "virtualization/cluster-types" + object_type = "virtualization.clustertype" primary_key = "name" prune = False @@ -1521,6 +1806,7 @@ def __init__(self, *args, **kwargs): class NBClusterGroup(NetBoxObject): name = "cluster group" api_path = "virtualization/cluster-groups" + object_type = "virtualization.clustergroup" primary_key = "name" prune = False @@ -1538,6 +1824,7 @@ def __init__(self, *args, **kwargs): class NBDeviceRole(NetBoxObject): name = "device role" api_path = "dcim/device-roles" + object_type = "dcim.devicerole" primary_key = "name" prune = False @@ -1556,27 +1843,54 @@ def __init__(self, *args, **kwargs): class NBCluster(NetBoxObject): name = "cluster" api_path = "virtualization/clusters" + object_type = "virtualization.cluster" primary_key = "name" secondary_key = "site" prune = False # include_secondary_key_if_present = True def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBSite, NBSiteGroup + ] self.data_model = { "name": 100, "comments": str, "type": NBClusterType, "tenant": NBTenant, "group": NBClusterGroup, - "site": NBSite, + "scope_type": self.mapping.scopes_object_types(self.scopes), + # currently only site is supported as a scope + "scope_id": NBSite, "tags": NBTagList } super().__init__(*args, **kwargs) + def update(self, data=None, read_from_netbox=False, source=None): + + # Add adaption for change in NetBox 4.2.0 Device model + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + if data.get("site") is not None: + data["scope_id"] = data.get("site") + data["scope_type"] = "dcim.site" + del data["site"] + + if data.get("scope_id") is not None: + data["scope_type"] = "dcim.site" + + super().update(data=data, read_from_netbox=read_from_netbox, source=source) + + def resolve_relations(self): + + self.resolve_scoped_relations("scope_id", "scope_type") + super().resolve_relations() + class NBDevice(NetBoxObject): name = "device" api_path = "dcim/devices" + object_type = "dcim.device" primary_key = "name" secondary_key = "site" prune = True @@ -1617,6 +1931,7 @@ def update(self, data=None, read_from_netbox=False, source=None): class NBVM(NetBoxObject): name = "virtual machine" api_path = "virtualization/virtual-machines" + object_type = "virtualization.virtualmachine" primary_key = "name" secondary_key = "cluster" prune = True @@ -1624,6 +1939,7 @@ class NBVM(NetBoxObject): def __init__(self, *args, **kwargs): self.data_model = { "name": 64, + "serial": 50, "status": ["offline", "active", "planned", "staged", "failed", "decommissioning"], "cluster": NBCluster, "role": NBDeviceRole, @@ -1654,6 +1970,7 @@ def get_virtual_disks(self): class NBVMInterface(NetBoxObject): name = "virtual machine interface" api_path = "virtualization/interfaces" + object_type = "virtualization.vminterface" primary_key = "name" secondary_key = "virtual_machine" enforce_secondary_key = True @@ -1665,6 +1982,7 @@ def __init__(self, *args, **kwargs): "virtual_machine": NBVM, "enabled": bool, "mac_address": str, + "primary_mac_address": NBMACAddress, "mtu": int, "mode": ["access", "tagged", "tagged-all"], "untagged_vlan": NBVLAN, @@ -1678,15 +1996,25 @@ def get_ip_addresses(self): result_list = list() for ip_object in self.inventory.get_all_items(NBIPAddress): - if grab(ip_object, "data.assigned_object_id") == self: + if ip_object.data.get("assigned_object_id") == self: result_list.append(ip_object) return result_list + def get_mac_addresses(self): + + result_list = list() + for mac_object in self.inventory.get_all_items(NBMACAddress): + if mac_object.data.get("assigned_object_id") == self: + result_list.append(mac_object) + + return result_list + class NBInterface(NetBoxObject): name = "interface" api_path = "dcim/interfaces" + object_type = "dcim.interface" primary_key = "name" secondary_key = "device" enforce_secondary_key = True @@ -1700,6 +2028,7 @@ def __init__(self, *args, **kwargs): "type": NetBoxInterfaceType().get_netbox_type_list(), "enabled": bool, "mac_address": str, + "primary_mac_address": NBMACAddress, "wwn": str, "mgmt_only": bool, "mtu": int, @@ -1709,7 +2038,7 @@ def __init__(self, *args, **kwargs): "untagged_vlan": NBVLAN, "tagged_vlans": NBVLANList, "description": 200, - "connection_status": bool, + "mark_connected": bool, "tags": NBTagList, "parent": object } @@ -1719,11 +2048,20 @@ def get_ip_addresses(self): result_list = list() for ip_object in self.inventory.get_all_items(NBIPAddress): - if grab(ip_object, "data.assigned_object_id") == self: + if ip_object.data.get("assigned_object_id") == self: result_list.append(ip_object) return result_list + def get_mac_addresses(self): + + result_list = list() + for mac_object in self.inventory.get_all_items(NBMACAddress): + if mac_object.data.get("assigned_object_id") == self: + result_list.append(mac_object) + + return result_list + def update(self, data=None, read_from_netbox=False, source=None): # remove definition of interface type if a parent interface is set as it only supports virtual types @@ -1738,16 +2076,18 @@ def update(self, data=None, read_from_netbox=False, source=None): class NBVirtualDisk(NetBoxObject): name = "Virtual Disk" api_path = "virtualization/virtual-disks" + object_type = "virtualization.virtualdisk" primary_key = "name" secondary_key = "virtual_machine" min_netbox_version = "3.7" + prune = True def __init__(self, *args, **kwargs): self.data_model = { "name": 64, "virtual_machine": NBVM, "description": 200, - "size": int, # in GB + "size": int, # in MB "tags": NBTagList } super().__init__(*args, **kwargs) @@ -1756,46 +2096,32 @@ def __init__(self, *args, **kwargs): class NBIPAddress(NetBoxObject): name = "IP address" api_path = "ipam/ip-addresses" + object_type = "ipam.ipaddress" primary_key = "address" is_primary = False prune = True def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBInterface, NBVMInterface, NBFHRPGroupItem + ] self.data_model = { "address": str, - "assigned_object_type": ["dcim.interface", "virtualization.vminterface", "ipam.fhrpgroup"], - "assigned_object_id": [NBInterface, NBVMInterface, NBFHRPGroupItem], + "assigned_object_type": self.mapping.scopes_object_types(self.scopes), + "assigned_object_id": self.scopes, "description": 200, + "role": ["loopback", "secondary", "anycast", "vip", "vrrp", "hsrp", "glbp", "carp"], "dns_name": 255, "tags": NBTagList, "tenant": NBTenant, "vrf": NBVRF } - # add relation between two attributes - self.data_model_relation = { - "dcim.interface": NBInterface, - "virtualization.vminterface": NBVMInterface, - "ipam.fhrpgroup": NBFHRPGroupItem, - NBInterface: "dcim.interface", - NBVMInterface: "virtualization.vminterface", - NBFHRPGroupItem: "ipam.fhrpgroup" - } super().__init__(*args, **kwargs) def resolve_relations(self): - o_id = self.data.get("assigned_object_id") - o_type = self.data.get("assigned_object_type") - - # this needs special treatment as the object type depends on a second model key - if o_type is not None and o_type not in self.data_model.get("assigned_object_type"): - - log.error(f"Attribute 'assigned_object_type' for '{self.get_display_name()}' invalid: {o_type}") - do_error_exit(f"Error while resolving relations for {self.get_display_name()}") - - if isinstance(o_id, int) and o_type is not None: - self.data["assigned_object_id"] = self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) - + self.resolve_scoped_relations("assigned_object_id", "assigned_object_type") super().resolve_relations() def update(self, data=None, read_from_netbox=False, source=None): @@ -1821,11 +2147,11 @@ def update(self, data=None, read_from_netbox=False, source=None): if not isinstance(assigned_object, NetBoxObject): data["assigned_object_id"] = \ - self.inventory.add_update_object(self.data_model_relation.get(object_type), data=assigned_object) + self.inventory.add_update_object(self.mapping.get(object_type), data=assigned_object) else: # noinspection PyTypeChecker - data["assigned_object_type"] = self.data_model_relation.get(type(assigned_object)) + data["assigned_object_type"] = self.mapping.get(assigned_object) super().update(data=data, read_from_netbox=read_from_netbox, source=source) @@ -1862,7 +2188,7 @@ def get_interface(self): if o_type not in self.data_model.get("assigned_object_type"): return - return self.inventory.get_by_id(self.data_model_relation.get(o_type), nb_id=o_id) + return self.inventory.get_by_id(self.mapping.get(o_type), nb_id=o_id) def get_device_vm(self): @@ -1891,16 +2217,109 @@ def remove_interface_association(self): if o_type is not None: self.unset_attribute("assigned_object_type") +class NBMACAddress(NetBoxObject): + name = "MAC address" + api_path = "dcim/mac-addresses" + object_type = "ipam.macaddress" + primary_key = "mac_address" + prune = True + min_netbox_version = "4.2" + + def __init__(self, *args, **kwargs): + self.mapping = NetBoxMappings() + self.scopes = [ + NBInterface, NBVMInterface + ] + self.data_model = { + "mac_address": str, + "assigned_object_type": self.mapping.scopes_object_types(self.scopes), + "assigned_object_id": self.scopes, + "description": 200, + "tags": NBTagList, + } + super().__init__(*args, **kwargs) + + def resolve_relations(self): + + self.resolve_scoped_relations("assigned_object_id", "assigned_object_type") + super().resolve_relations() + + def update(self, data=None, read_from_netbox=False, source=None): + + object_type = data.get("assigned_object_type") + assigned_object = data.get("assigned_object_id") + + # we got an object data structure where we have to find the object + if read_from_netbox is False and assigned_object is not None: + + if not isinstance(assigned_object, NetBoxObject): + + data["assigned_object_id"] = \ + self.inventory.add_update_object(self.mapping.get(object_type), data=assigned_object) + + else: + # noinspection PyTypeChecker + data["assigned_object_type"] = self.mapping.get(assigned_object) + + super().update(data=data, read_from_netbox=read_from_netbox, source=source) + + # we need to tell NetBox which object type this is meant to be + if "assigned_object_id" in self.updated_items: + self.updated_items.append("assigned_object_type") + + # if ip association has been removed we also need to get rid of object type + if "assigned_object_type" in self.updated_items and self.data.get("assigned_object_id") is None \ + and "assigned_object_type" in self.updated_items: + self.updated_items.remove("assigned_object_type") + + def get_interface(self): + o_id = self.data.get("assigned_object_id") + o_type = self.data.get("assigned_object_type") + + if isinstance(o_id, (NBInterface, NBVMInterface)): + return o_id + + if o_type is None or not isinstance(o_id, int): + return + + if o_type not in self.data_model.get("assigned_object_type"): + return + + return self.inventory.get_by_id(self.mapping.get(o_type), nb_id=o_id) + + def get_device_vm(self): + + o_interface = self.get_interface() + + if o_interface is None: + return + + if isinstance(o_interface, NBInterface): + return o_interface.data.get("device") + elif isinstance(o_interface, NBVMInterface): + return o_interface.data.get("virtual_machine") + + def remove_interface_association(self): + o_id = self.data.get("assigned_object_id") + o_type = self.data.get("assigned_object_type") + o_device = self.get_device_vm() + + if grab(o_device, "data.primary_mac_address") is self: + o_device.unset_attribute("primary_mac_address") + + if o_id is not None: + self.unset_attribute("assigned_object_id") + if o_type is not None: + self.unset_attribute("assigned_object_type") + class NBFHRPGroupItem(NetBoxObject): - """ - This object is currently not used directly in any class. - It is used to handle IP address object properly. - """ name = "FHRP group" api_path = "ipam/fhrp-groups" + object_type = "ipam.fhrpgroup" primary_key = "group_id" prune = False + read_only = True def __init__(self, *args, **kwargs): self.data_model = { @@ -1916,6 +2335,7 @@ def __init__(self, *args, **kwargs): class NBInventoryItem(NetBoxObject): name = "inventory item" api_path = "dcim/inventory-items" + object_type = "dcim.inventoryitem" primary_key = "name" secondary_key = "device" prune = True @@ -1940,6 +2360,7 @@ def __init__(self, *args, **kwargs): class NBPowerPort(NetBoxObject): name = "power port" api_path = "dcim/power-ports" + object_type = "dcim.powerport" primary_key = "name" secondary_key = "device" prune = True diff --git a/module/sources/__init__.py b/module/sources/__init__.py index dc2b7dcc..4be4120f 100644 --- a/module/sources/__init__.py +++ b/module/sources/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -33,7 +33,7 @@ def validate_source(source_class_object=None, state="pre"): source_class_object: Source handler class/object class/object to investigate state: str - pre if validating a class, post if validating a initialized object + pre if validating a class, post if validating an initialized object """ @@ -70,7 +70,7 @@ def validate_source(source_class_object=None, state="pre"): def instantiate_sources(): """ Instantiate a source handler and add necessary attributes. Also - validate source handler pre and post initialization. + validate source handler on pre- and post-initialization. Returns ------- diff --git a/module/sources/check_redfish/config.py b/module/sources/check_redfish/config.py index 4d729318..2c21fff9 100644 --- a/module/sources/check_redfish/config.py +++ b/module/sources/check_redfish/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index 36cc53b8..7a893baa 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -41,14 +41,17 @@ class CheckRedfish(SourceBase): NBClusterGroup, NBDeviceRole, NBSite, + NBSiteGroup, NBCluster, NBDevice, NBInterface, + NBMACAddress, NBIPAddress, NBPrefix, NBTenant, NBVRF, NBVLAN, + NBVLANGroup, NBPowerPort, NBInventoryItem, NBCustomField @@ -214,11 +217,6 @@ def update_device(self): log.error(f"No system data found for '{self.device_object.get_display_name()}' in inventory file.") return - # get status - status = "offline" - if get_string_or_none(grab(system, "power_state")) == "On": - status = "active" - serial = get_string_or_none(grab(system, "serial")) name = get_string_or_none(grab(system, "host_name")) manufacturer = get_string_or_none(grab(system, "manufacturer")) @@ -230,9 +228,9 @@ def update_device(self): "name": manufacturer }, }, - "status": status, "custom_fields": { - "health": get_string_or_none(grab(system, "health_status")) + "health": get_string_or_none(grab(system, "health_status")), + "power_state": get_string_or_none(grab(system, "power_state")) } } @@ -270,7 +268,7 @@ def update_power_supply(self): if grab(ps, "data.device") == self.device_object: current_ps.append(ps) - current_ps.sort(key=lambda x: grab(x, "data.name")) + current_ps.sort(key=lambda x: grab(x, "data.name") or "") ps_index = 1 ps_items = list() @@ -399,6 +397,7 @@ def update_fan(self): def update_memory(self): items = list() + memory_size_total = 0 for memory in grab(self.inventory_file_content, "inventory.memory", fallback=list()): if grab(memory, "operation_status") in ["NotPresent", "Absent"]: @@ -416,6 +415,8 @@ def update_memory(self): if size_in_mb == 0 or (health_status is None and grab(memory, "operation_status") != "GoodInUse"): continue + memory_size_total += size_in_mb + name_details = list() if dimm_type is not None: name_details.append(f"{dimm_type}") @@ -448,9 +449,21 @@ def update_memory(self): self.update_all_items(items) + if memory_size_total > 0: + memory_size_total = memory_size_total / 1024 + memory_size_unit = "GB" + if memory_size_total >= 1024: + memory_size_total = memory_size_total / 1024 + memory_size_unit = "TB" + + custom_fields_data = {"custom_fields": {"host_memory": f"{memory_size_total} {memory_size_unit}"}} + self.device_object.update(data=custom_fields_data, source=self) + def update_proc(self): items = list() + num_cores = 0 + cpu_name = "" for processor in grab(self.inventory_file_content, "inventory.processor", fallback=list()): if grab(processor, "operation_status") in ["NotPresent", "Absent"]: @@ -465,6 +478,7 @@ def update_proc(self): health_status = get_string_or_none(grab(processor, "health_status")) name = f"{socket} ({model})" + cpu_name = model if current_speed is not None: current_speed = f"{current_speed / 1000}GHz" @@ -477,6 +491,7 @@ def update_proc(self): description.append(f"{instruction_set}") if cores is not None: description.append(f"Cores: {cores}") + num_cores += int(cores) if threads is not None: description.append(f"Threads: {threads}") @@ -493,6 +508,10 @@ def update_proc(self): self.update_all_items(items) + if num_cores > 0: + custom_fields_data = {"custom_fields": {"host_cpu_cores": f"{num_cores} {cpu_name}"}} + self.device_object.update(data=custom_fields_data, source=self) + def update_physical_drive(self): items = list() @@ -842,6 +861,7 @@ def update_network_interface(self): this_link_type = port_data.get("type") mgmt_only = port_data.get("mgmt_only") + mac_address = port_data.get("mac_address") data_to_update = self.patch_data(nic_object, port_data, self.settings.overwrite_interface_attributes) # always overwrite nic type if discovered @@ -850,6 +870,9 @@ def update_network_interface(self): data_to_update["mgmt_only"] = mgmt_only + if mac_address is not None: + data_to_update["mac_address"] = mac_address + port_data = data_to_update self.add_update_interface(nic_object, self.device_object, port_data, nic_ips.get(port_name, list())) @@ -895,7 +918,7 @@ def update_all_items(self, items): Parameters ---------- items: list - list of items to update + a list of items to update Returns ------- @@ -944,7 +967,7 @@ def update_all_items(self, items): unmatched_inventory_items.append(item) # sort unmatched items by full_name - unmatched_inventory_items.sort(key=lambda x: x.get("full_name")) + unmatched_inventory_items.sort(key=lambda x: x.get("full_name") or "") # iterate over current NetBox inventory items # if name did not match try to assign unmatched items in alphabetical order @@ -974,7 +997,7 @@ def update_item(self, item_data: dict, inventory_object: NBInventoryItem = None) Parameters ---------- item_data: dict - dict with data for item to update + a dict with data for item to update inventory_object: NBInventoryItem, None the NetBox inventory item to update. @@ -1037,6 +1060,36 @@ def add_necessary_base_objects(self): "description": f"Marks objects synced from check_redfish inventory '{self.name}' to this NetBox Instance." }) + self.add_update_custom_field({ + "name": "host_cpu_cores", + "label": "Physical CPU Cores", + "object_types": [ + "dcim.device" + ], + "type": "text", + "description": f"Reported Host CPU cores" + }) + + self.add_update_custom_field({ + "name": "host_memory", + "label": "Memory", + "object_types": [ + "dcim.device" + ], + "type": "text", + "description": f"Reported size of Memory" + }) + + self.add_update_custom_field({ + "name": "power_state", + "label": "Power State", + "object_types": [ + "dcim.device" + ], + "type": "text", + "description": "Device power state" + }) + # add Firmware self.add_update_custom_field({ "name": "firmware", diff --git a/module/sources/common/config.py b/module/sources/common/config.py index c2417eb7..5fa7b3f2 100644 --- a/module/sources/common/config.py +++ b/module/sources/common/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/common/excluded_vlan.py b/module/sources/common/handle_vlan.py similarity index 60% rename from module/sources/common/excluded_vlan.py rename to module/sources/common/handle_vlan.py index 40c16d3e..18bf986c 100644 --- a/module/sources/common/excluded_vlan.py +++ b/module/sources/common/handle_vlan.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -13,19 +13,17 @@ log = get_logger() -class ExcludedVLAN: - """ - initializes and verifies if an VLAN should be excluded from being synced to NetBox - """ +class VLANFilter: - def __init__(self, vlan): + def __init__(self, vlan, filter_type): self._validation_failed = False self.site = None + self.filter_type = filter_type - if vlan is None: + if vlan is None or len(f"{vlan}") == 0: self._validation_failed = True - log.error("submitted VLAN string for VLAN exclusion was 'None'") + log.error(f"submitted VLAN {self.filter_type} string for VLAN was " + "'None'" if vlan is None else "empty") return vlan_split = [x.replace('\\', "") for x in re.split(r'(?= int(self._value.split("-")[1]): - log.error(f"range has to start with the lower id: {self._value}") + log.error(f"VLAN {self.filter_type} range has to start with the lower ID: {self._value}") self._validation_failed = True return @@ -112,22 +117,21 @@ def __init__(self, vlan): for i in self._value.split(',')), [] ) except Exception as e: - log.error(f"unable to extract ids from value '{self._value}': {e}") + log.error(f"unable to extract VLAN IDs from value '{self._value}': {e}") self._validation_failed = True def matches(self, vlan_id, site=None): if self.site_matches(site) is False: + log.debug2(f"VLAN {self.filter_type} site name '{site_name}' matches '{self.site}'") return False try: if int(vlan_id) in self.range: - log.debug2(f"VLAN exclude id '{vlan_id}' matches '{self._value}'") + log.debug2(f"VLAN {self.filter_type} ID '{vlan_id}' matches '{self._value}'") return True except Exception as e: - log.warning(f"Unable to match exclude VLAN id '{vlan_id}' to '{self._value}': {e}") + log.warning(f"Unable to match {self.filter_type} VLAN ID '{vlan_id}' to '{self._value}': {e}") return False return False - - diff --git a/module/sources/common/permitted_subnets.py b/module/sources/common/permitted_subnets.py index caffb5b7..a4c88e87 100644 --- a/module/sources/common/permitted_subnets.py +++ b/module/sources/common/permitted_subnets.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -16,7 +16,7 @@ class PermittedSubnets: """ - initializes and verifies if an IP address is part of an permitted subnet + initializes and verifies if an IP address is part of a permitted subnet """ def __init__(self, config_string: str): @@ -41,6 +41,10 @@ def __init__(self, config_string: str): excluded = True subnet = subnet[1:].strip() + if "/" not in subnet: + log.error(f"permitted subnet '{subnet}' is missing the prefix length (i.e.: {subnet}/24)") + self._validation_failed = True + try: if excluded is True: self.excluded_subnets.append(ip_network(subnet)) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 86f06b8e..fc81a160 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -10,12 +10,11 @@ import re from ipaddress import ip_interface, ip_address, IPv6Address, IPv4Address, IPv6Network, IPv4Network -from typing import List +from packaging import version from module.netbox import * from module.common.logging import get_logger from module.common.misc import grab -from module.sources.common.excluded_vlan import ExcludedVLANName, ExcludedVLANID log = get_logger() @@ -60,7 +59,7 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac MAC address of interfaces match exactly, type of interface does not matter If there are interfaces which don't match at all then the unmatched interfaces will be - matched 1:1. Sort both lists (unmatched current interfaces, unmatched new new interfaces) + matched 1:1. Sort both lists (unmatched current interfaces, unmatched new interfaces) by name and assign them each other. eth0 > vNIC 1 @@ -80,7 +79,7 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac Returns ------- dict: {"$interface_name": associated_interface_object} - if no current current interface was left to match "None" will be returned instead of + if no current interface was left to match "None" will be returned instead of a matching interface object """ @@ -159,7 +158,7 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac if grab(matching_int, "data.name") in current_object_interface_names: current_object_interface_names.remove(grab(matching_int, "data.name")) - # no match found, we match the left overs just by #1 -> #1, #2 -> #2, ... + # no match found, we match the leftovers just by #1 -> #1, #2 -> #2, ... else: unmatched_interface_names.append(int_name) @@ -180,9 +179,9 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac return return_data - def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None): + def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None) -> NBPrefix|None: """ - This is a lazy approach to find longest matching prefix to an IP address. + This is a lazy approach to find the longest matching prefix to an IP address. If site_name is set only IP prefixes from that site are matched. Parameters @@ -217,7 +216,7 @@ def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None for prefix in self.inventory.get_all_items(NBPrefix): - if grab(prefix, "data.site") != site_object: + if not prefix.matches_site(site_object): continue prefix_network = grab(prefix, f"data.{NBPrefix.primary_key}") @@ -247,38 +246,47 @@ def add_update_interface(self, interface_object, device_object, interface_data, Parameters ---------- - interface_object: NBVMInterface, NBInterface, None + interface_object: NBVMInterface | NBInterface | None object handle of the current interface (if existent, otherwise None) - device_object: NBVM, NBDevice + device_object: NBVM | NBDevice device object handle this interface belongs to interface_data: dict dictionary with interface attributes to add to this interface interface_ips: list - list of ip addresses which are assigned to this interface - vmware_object: (vim.HostSystem, vim.VirtualMachine) + a list of ip addresses which are assigned to this interface + vmware_object: vim.HostSystem | vim.VirtualMachine object to add to list of objects to reevaluate Returns ------- - objects: tuple((NBVMInterface, NBInterface), list) + objects: + tuple of NBVMInterface | NBInterface and list tuple with interface object that was added/updated and a list of ip address objects which were added to this interface """ - ip_tenant_inheritance_order = None - if "ip_tenant_inheritance_order" in self.settings: - ip_tenant_inheritance_order = self.settings.ip_tenant_inheritance_order + # handle change to mac_address object from NetBox 4.2 on + interface_mac_address = None + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0") and \ + interface_data.get("mac_address") is not None: + interface_mac_address = interface_data.get("mac_address") + del(interface_data["mac_address"]) + + ip_tenant_inheritance_order = self.settings.ip_tenant_inheritance_order if not isinstance(interface_data, dict): log.error(f"Attribute 'interface_data' must be a dict() got {type(interface_data)}.") return None + device_object_cluster = grab(device_object, "data.cluster") + device_object_site = grab(device_object, "data.site") + if type(device_object) == NBVM: interface_class = NBVMInterface - site_name = grab(device_object, "data.cluster.data.site.data.name") + site_name = device_object_cluster.get_site_name() elif type(device_object) == NBDevice: interface_class = NBInterface - site_name = grab(device_object, "data.site.data.name") + site_name = device_object.get_site_name() elif device_object is None: log.error(f"No device/VM object submitted to attach interface '{grab(interface_data, 'name')}' to.") return None @@ -313,11 +321,59 @@ def add_update_interface(self, interface_object, device_object, interface_data, else: interface_object.update(data=interface_data, source=self) + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0") and \ + interface_mac_address is not None: + + primary_mac_address_data = { + "mac_address": interface_mac_address, + "assigned_object_id": interface_object, + "assigned_object_type": interface_class + } + + primary_mac_address_object = None + # check for associated MAC addresses on existing interface + if interface_object.is_new is False: + current_primary_mac_address_object = grab(interface_object, "data.primary_mac_address") + if grab(current_primary_mac_address_object, "data.mac_address") == interface_mac_address: + primary_mac_address_object = current_primary_mac_address_object + for mac_address_object in interface_object.get_mac_addresses(): + if (primary_mac_address_object is None and + grab(mac_address_object, "data.mac_address") == interface_mac_address): + primary_mac_address_object = mac_address_object + if mac_address_object is not primary_mac_address_object: + mac_address_object.remove_interface_association() + + # if a new interface or not matching assigned MAC address, try to find an existing unassigned mac address + if primary_mac_address_object is None: + for mac_address_object in self.inventory.get_all_items(NBMACAddress): + if (grab(mac_address_object, "data.mac_address") == interface_mac_address and + grab(mac_address_object, "data.assigned_object_id") is None): + primary_mac_address_object = mac_address_object + break + + # of no existing mac address could be found, create a new one + if primary_mac_address_object is None: + primary_mac_address_object = self.inventory.add_object(NBMACAddress, data=primary_mac_address_data, + source=self) + else: + primary_mac_address_object.update(data=primary_mac_address_data, source=self) + + interface_object.update(data={"primary_mac_address": primary_mac_address_object}, source=self) + + # skip handling of IPs for VMs with not installed/running guest tools + skip_ip_handling = False + if type(device_object) == NBVM and grab(vmware_object,'guest.toolsRunningStatus') != "guestToolsRunning": + log.debug(f"VM '{device_object.name}' guest tool status is 'NotRunning', skipping IP handling") + skip_ip_handling = True + ip_address_objects = list() matching_ip_prefixes = list() # add all interface IPs for nic_ip in interface_ips or list(): + if skip_ip_handling is True: + continue + # get IP and prefix length try: if "/" in nic_ip: @@ -335,22 +391,21 @@ def add_update_interface(self, interface_object, device_object, interface_data, prefix_tenant = None # test for site prefixes first - matching_site_name = site_name - matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object, matching_site_name) + matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object, site_name) # nothing was found then check prefixes without site name if matching_ip_prefix is None: - matching_site_name = None matching_ip_prefix = self.return_longest_matching_prefix_for_ip(ip_object) # matching prefix found, get data from prefix if matching_ip_prefix is not None: this_prefix = grab(matching_ip_prefix, f"data.{NBPrefix.primary_key}") - if matching_site_name is None: + prefix_scope = matching_ip_prefix.get_scope_display_name() + if prefix_scope is None: log.debug2(f"Found IP '{ip_object}' matches global prefix '{this_prefix}'") else: - log.debug2(f"Found IP '{ip_object}' matches site '{matching_site_name}' prefix " + log.debug2(f"Found IP '{ip_object}' matches {prefix_scope} prefix " f"'{this_prefix}'") # check if prefix net size and ip address prefix length match @@ -369,7 +424,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, # check if IP address is of type IP interface (includes prefix length) if type(ip_object) in [IPv6Address, IPv4Address]: - log.warning(f"{log_text}. Unable to add IP address to NetBox.") + log.warning(f"{log_text}. Unable to add IP address to NetBox") continue else: log.debug2(log_text) @@ -380,7 +435,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, if type(this_prefix) in [IPv4Network, IPv6Network]: ip_object = ip_interface(f"{ip_object}/{this_prefix.prefixlen}") else: - log.warning(f"{matching_ip_prefix.name} got wrong format. Unable to add IP to NetBox") + log.warning(f"{matching_ip_prefix.name} got wrong format. Unable to add IP address to NetBox") continue # try to find matching IP address object @@ -453,6 +508,12 @@ def add_update_interface(self, interface_object, device_object, interface_data, this_ip_object = ip + if grab(ip, "data.role.value") == "anycast": + log.debug(f"{ip.name} '{ip.get_display_name()}' is an Anycast address and " + f"can be assigned to multiple interfaces at the same time.") + skip_this_ip = True + break + if current_nic_enabled == this_nic_enabled: this_log_handler = log.warning @@ -483,6 +544,21 @@ def add_update_interface(self, interface_object, device_object, interface_data, "assigned_object_id": interface_object, } + # skip reassignment if IP is assigned to sub interface of a VM + if (type(device_object) == NBVM and this_ip_object is not None and + grab(this_ip_object.get_interface(), "data.parent") is not None): + current_ip_nic = this_ip_object.get_interface() + current_ip_nic_parent = grab(current_ip_nic, "data.parent") + if isinstance(current_ip_nic_parent, dict): + current_ip_nic_parent = self.inventory.get_by_id(NBVMInterface, + nb_id=current_ip_nic_parent.get("id")) + + if current_ip_nic_parent == interface_object: + log.debug(f"{this_ip_object.name} '{this_ip_object.get_display_name()}' is assigned to sub interface " + f"'{current_ip_nic.get_display_name()}' of '{interface_object.get_display_name()}'. " + f"Not changing assignment") + nic_ip_data["assigned_object_id"] = current_ip_nic + # grab tenant from device/vm if prefix didn't provide a tenant ip_tenant = None if isinstance(ip_tenant_inheritance_order, list) and "disabled" not in ip_tenant_inheritance_order: @@ -497,14 +573,14 @@ def add_update_interface(self, interface_object, device_object, interface_data, ip_tenant = prefix_tenant break + if possible_ip_vrf is not None: + nic_ip_data["vrf"] = possible_ip_vrf + if ip_tenant is not None: + nic_ip_data["tenant"] = ip_tenant + if not isinstance(this_ip_object, NBIPAddress): log.debug(f"No existing {NBIPAddress.name} object found. Creating a new one.") - if possible_ip_vrf is not None: - nic_ip_data["vrf"] = possible_ip_vrf - if ip_tenant is not None: - nic_ip_data["tenant"] = ip_tenant - this_ip_object = self.inventory.add_object(NBIPAddress, data=nic_ip_data, source=self) # update IP address with additional data if not already present @@ -512,17 +588,20 @@ def add_update_interface(self, interface_object, device_object, interface_data, log.debug2(f"Found existing NetBox {NBIPAddress.name} object: {this_ip_object.get_display_name()}") - if grab(this_ip_object, "data.vrf") is None and possible_ip_vrf is not None: - nic_ip_data["vrf"] = possible_ip_vrf - - if grab(this_ip_object, "data.tenant") is None and ip_tenant is not None: - nic_ip_data["tenant"] = ip_tenant - this_ip_object.update(data=nic_ip_data, source=self) ip_address_objects.append(this_ip_object) for current_ip in interface_object.get_ip_addresses(): + + if skip_ip_handling is True: + continue + + if grab(current_ip, "data.role.value") == "anycast": + log.debug2(f"{current_ip.name} '{current_ip.get_display_name()}' is an Anycast address and will " + f"NOT be deleted from interface") + continue + if current_ip not in ip_address_objects: log.info(f"{current_ip.name} is no longer assigned to {interface_object.get_display_name()} and " f"therefore removed from this interface") @@ -561,7 +640,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, vlan_interface_data = dict() if untagged_vlan is not None or (untagged_vlan is None and len(tagged_vlans) == 0): if matching_untagged_vlan is None and untagged_vlan is not None: - matching_untagged_vlan = self.get_vlan_object_if_exists(untagged_vlan, site_name) + matching_untagged_vlan = self.get_vlan_object_if_exists(untagged_vlan, device_object_site, + device_object_cluster) # don't sync newly discovered VLANs to NetBox if self.add_vlan_object_to_netbox(matching_untagged_vlan, site_name) is False: @@ -572,7 +652,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, f"untagged interface VLAN.") if matching_untagged_vlan is not None: - vlan_interface_data["untagged_vlan"] = matching_untagged_vlan + vlan_interface_data["untagged_vlan"] = self.add_vlan_group(matching_untagged_vlan, site_name, + device_object_cluster) if grab(interface_object, "data.mode") is None: vlan_interface_data["mode"] = "access" @@ -584,14 +665,16 @@ def add_update_interface(self, interface_object, device_object, interface_data, log.debug2(f"Found matching prefix VLAN {matching_tagged_vlan.get_display_name()} for " f"tagged interface VLAN.") else: - matching_tagged_vlan = self.get_vlan_object_if_exists(tagged_vlan, site_name) + matching_tagged_vlan = self.get_vlan_object_if_exists(tagged_vlan, device_object_site, + device_object_cluster) # don't sync newly discovered VLANs to NetBox if self.add_vlan_object_to_netbox(matching_tagged_vlan, site_name) is False: matching_tagged_vlan = None if matching_tagged_vlan is not None: - compiled_tagged_vlans.append(matching_tagged_vlan) + compiled_tagged_vlans.append(self.add_vlan_group(matching_tagged_vlan, site_name, + device_object_cluster)) if len(compiled_tagged_vlans) > 0: vlan_interface_data["tagged_vlans"] = compiled_tagged_vlans @@ -618,7 +701,7 @@ def patch_data(object_to_patch, data, overwrite=False): Returns ------- data_to_update: dict - dict with data to append/patch + A dict with data to append/patch """ if overwrite is True: @@ -633,20 +716,124 @@ def patch_data(object_to_patch, data, overwrite=False): return data_to_update - def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): + def add_vlan_group(self, vlan_data, vlan_site, vlan_cluster) -> NBVLAN | dict: + """ + This function will try to find a matching VLAN group according to the settings. + Name matching will take precedence over ID matching. First match wins. + + If nothing matches the input data the submitted 'vlan_data' will be returned + + Parameters + ---------- + vlan_data: dict | NBVLAN + A dict or NBVLAN object + vlan_site: NBSite | str | None + name of site for the VLAN + vlan_cluster: NBCluster | str | None + + Returns + ------- + NBVLAN | dict: the input vlan_data enriched with VLAN group if a match was found + + """ + + # get VLAN details + if isinstance(vlan_data, NBVLAN): + if vlan_data.is_new is False: + return vlan_data + + vlan_name = grab(vlan_data, "data.name") + vlan_id = grab(vlan_data, "data.vid") + vlan_current_site = grab(vlan_data, "data.site") + # vlan already has a group attached + if grab(vlan_data, "data.group") is not None: + return vlan_data + + elif isinstance(vlan_data, dict): + vlan_name = vlan_data.get("name") + vlan_id = vlan_data.get("vid") + vlan_current_site = vlan_data.get("site") + else: + return vlan_data + + if isinstance(vlan_site, str): + vlan_site = self.inventory.get_by_data(NBSite, data={"name": vlan_site}) + + if isinstance(vlan_cluster, str): + vlan_cluster = self.inventory.get_by_data(NBCluster, data={"name": vlan_cluster}) + + if isinstance(vlan_current_site, dict): + vlan_current_site = self.inventory.get_by_data(NBSite, data=vlan_current_site) + + log_text = f"Trying to find a matching VLAN Group based on the VLAN name '{vlan_name}'" + if vlan_site is not None: + log_text += f", site '{vlan_site.get_display_name()}'" + if vlan_cluster is not None: + log_text += f", cluster '{vlan_cluster.get_display_name()}'" + log_text += f" and VLAN ID '{vlan_id}'" + log.debug(log_text) + + vlan_group = None + for vlan_filter, vlan_group_name in self.settings.vlan_group_relation_by_name or list(): + if vlan_filter.matches(vlan_name, vlan_site): + for inventory_vlan_group in self.inventory.get_all_items(NBVLANGroup): + + if grab(inventory_vlan_group, "data.name") != vlan_group_name: + continue + + if inventory_vlan_group.matches_site_cluster(vlan_site, vlan_cluster): + vlan_group = inventory_vlan_group + break + + if vlan_group is None: + for vlan_filter, vlan_group_name in self.settings.vlan_group_relation_by_id or list(): + if vlan_filter.matches(vlan_id, vlan_site): + for inventory_vlan_group in self.inventory.get_all_items(NBVLANGroup): + + if grab(inventory_vlan_group, "data.name") != vlan_group_name: + continue + + if inventory_vlan_group.matches_site_cluster(vlan_site, vlan_cluster): + vlan_group = inventory_vlan_group + break + + if vlan_group is not None: + log.debug2(f"Found matching VLAN group '{vlan_group.get_display_name()}'") + """ + If a VLAN group has been found we also need to check if the vlan site and the scope of the VLAN group are + matching. If the VLAN group has a different scope then site, we need to remove the site from the VLAN. + + Mitigation for: https://github.com/netbox-community/netbox/issues/18706 + """ + if isinstance(vlan_data, NBVLAN): + vlan_data.update(data={"group": vlan_group}) + if vlan_current_site is not vlan_group.data.get("scope_id"): + vlan_data.unset_attribute("site") + elif isinstance(vlan_data, dict): + vlan_data["group"] = vlan_group + if vlan_current_site is not vlan_group.data.get("scope_id"): + del(vlan_data["site"]) + else: + log.debug2("No matching VLAN group found") + + return vlan_data + + def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None, vlan_cluster=None): """ This function will try to find a matching VLAN object based on 'vlan_data' Will return matching objects in following order: - * exact match: VLAN id and site match - * global match: VLAN id matches but the VLAN has no site assigned + * exact match: VLAN ID and site or VLAN Group which matches site or cluster + * global match: VLAN ID matches but the VLAN has no site assigned If nothing matches the input data from 'vlan_data' will be returned Parameters ---------- vlan_data: dict - dict with NBVLAN data attributes - vlan_site: str - name of site the VLAN could be present + A dict with NBVLAN data attributes + vlan_site: NBSite + site object + vlan_cluster: NBCluster + cluster object Returns ------- @@ -664,10 +851,10 @@ def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): raise ValueError("Value of 'vlan_data' needs to be a dict.") # check existing Devices for matches - log.debug2(f"Trying to find a {NBVLAN.name} based on the VLAN id '{vlan_data.get('vid')}'") + log.debug2(f"Trying to find a {NBVLAN.name} based on the VLAN ID '{vlan_data.get('vid')}'") if vlan_data.get("vid") is None: - log.debug("No VLAN id set in vlan_data while trying to find matching VLAN.") + log.debug("No VLAN ID set in vlan_data while trying to find matching VLAN.") return vlan_data if vlan_site is None: @@ -675,35 +862,58 @@ def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None): elif isinstance(vlan_site, str): vlan_site = self.inventory.get_by_data(NBSite, data={"name": vlan_site}) + if isinstance(vlan_cluster, str): + vlan_cluster = self.inventory.get_by_data(NBCluster, data={"name": vlan_cluster}) + return_data = vlan_data - vlan_object_including_site = None - vlan_object_without_site = None + vlan_object_by_site = None + vlan_object_by_group = None + vlan_object_global = None for vlan in self.inventory.get_all_items(NBVLAN): if grab(vlan, "data.vid") != vlan_data.get("vid"): continue + # try finding matching VLAN by site if vlan_site is not None and grab(vlan, "data.site") == vlan_site: - vlan_object_including_site = vlan - - if grab(vlan, "data.site") is None: - vlan_object_without_site = vlan - - if isinstance(vlan_object_including_site, NetBoxObject): - return_data = vlan_object_including_site - log.debug2("Found a exact matching %s object: %s" % - (vlan_object_including_site.name, - vlan_object_including_site.get_display_name(including_second_key=True))) + vlan_object_by_site = vlan + break + + # try find matching VLAN by group + if grab(vlan, "data.group") is not None: + vlan_group = grab(vlan, "data.group") + if vlan_group.matches_site_cluster(vlan_site, vlan_cluster): + vlan_object_by_group = vlan + break - elif isinstance(vlan_object_without_site, NetBoxObject): - return_data = vlan_object_without_site - log.debug2("Found a global matching %s object: %s" % - (vlan_object_without_site.name, - vlan_object_without_site.get_display_name(including_second_key=True))) + if grab(vlan, "data.site") is None and grab(vlan, "data.group") is None: + vlan_object_global = vlan + + if isinstance(vlan_object_by_site, NetBoxObject): + return_data = vlan_object_by_site + log.debug2(f"Found a {return_data.name} object which matches the site '{vlan_site.get_display_name()}': %s" + % vlan_object_by_site.get_display_name(including_second_key=True)) + + elif isinstance(vlan_object_by_group, NetBoxObject): + return_data = vlan_object_by_group + vlan_group_object = grab(vlan_object_by_group, "data.group") + vlan_group_object_scope_object = grab(vlan_object_by_group, "data.scope_id") + scope_details = "" + if vlan_group_object_scope_object is not None: + scope_details = (f" ({vlan_group_object_scope_object.name} " + f"{vlan_group_object_scope_object.get_display_name()})") + log.debug2(f"Found a {return_data.name} object which matches the {vlan_group_object.name} " + f"'{vlan_group_object.get_display_name()}'{scope_details}: %s" % + vlan_object_by_group.get_display_name(including_second_key=True)) + + elif isinstance(vlan_object_global, NetBoxObject): + return_data = vlan_object_global + log.debug2(f"Found a global matching {return_data.name} object: %s" % + vlan_object_global.get_display_name(including_second_key=True)) else: - log.debug2("No matching existing VLAN found for this VLAN id.") + log.debug2("No matching existing VLAN found for this VLAN ID.") return return_data @@ -724,17 +934,6 @@ def add_vlan_object_to_netbox(self, vlan_data, site_name=None): """ - # get config data - disable_vlan_sync = False - vlan_sync_exclude_by_name: List[ExcludedVLANName] = list() - vlan_sync_exclude_by_id: List[ExcludedVLANID] = list() - if "disable_vlan_sync" in self.settings: - disable_vlan_sync = self.settings.disable_vlan_sync - if "vlan_sync_exclude_by_name" in self.settings: - vlan_sync_exclude_by_name = self.settings.vlan_sync_exclude_by_name - if "vlan_sync_exclude_by_id" in self.settings: - vlan_sync_exclude_by_id = self.settings.vlan_sync_exclude_by_id - # VLAN is already an existing NetBox VLAN, then it can be reused if isinstance(vlan_data, NetBoxObject): return True @@ -742,7 +941,7 @@ def add_vlan_object_to_netbox(self, vlan_data, site_name=None): if vlan_data is None: return False - if disable_vlan_sync is True: + if self.settings.disable_vlan_sync is True: return False # get VLAN details @@ -757,11 +956,11 @@ def add_vlan_object_to_netbox(self, vlan_data, site_name=None): log.warning(f"Skipping sync of invalid VLAN '{vlan_name}' ID: '{vlan_id}'") return False - for excluded_vlan in vlan_sync_exclude_by_name or list(): + for excluded_vlan in self.settings.vlan_sync_exclude_by_name or list(): if excluded_vlan.matches(vlan_name, site_name): return False - for excluded_vlan in vlan_sync_exclude_by_id or list(): + for excluded_vlan in self.settings.vlan_sync_exclude_by_id or list(): if excluded_vlan.matches(vlan_id, site_name): return False diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 2dc58abf..7e448a28 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -17,7 +17,7 @@ from module.config.group import ConfigOptionGroup from module.sources.common.config import * from module.sources.common.permitted_subnets import PermittedSubnets -from module.sources.common.excluded_vlan import ExcludedVLANID, ExcludedVLANName +from module.sources.common.handle_vlan import FilterVLANByID, FilterVLANByName from module.common.logging import get_logger from module.common.support import normalize_mac_address @@ -108,6 +108,15 @@ def __init__(self): str, description="simply include/exclude VMs"), ConfigOption("vm_include_filter", str) ]), + ConfigOption("vm_exclude_by_tag_filter", + str, + description="""defines a comma separated list of vCenter tags which (if assigned to a VM) + will exclude this VM from being synced to NetBox. The config option 'vm_tag_source' + determines which tags are collected for VMs. + """, + config_example="tag-a, tag-b" + ), + ConfigOptionGroup(title="relations", options=[ ConfigOption("cluster_site_relation", @@ -197,6 +206,10 @@ def __init__(self): bool, description="Attempt to collect asset tags from vCenter hosts", default_value=True), + ConfigOption("collect_hardware_serial", + bool, + description="Attempt to collect serials from vCenter hosts", + default_value=True), ConfigOption("dns_name_lookup", bool, description="""Perform a reverse lookup for all collected IP addresses. @@ -321,7 +334,27 @@ def __init__(self): ConfigOption("vlan_sync_exclude_by_id", str, config_example="Frankfurt/25, 1023-1042"), + ConfigOption("vlan_group_relation_by_name", + str, + description="""adds a relation to assign VLAN groups to matching VLANs + by name. Same matching rules as the exclude_by_name option uses are applied. + If name and id relations are defined, the name relation takes precedence. + Fist match wins. Only newly discovered VLANs which are not present in + NetBox will be assigned a VLAN group. Supported scopes for a VLAN group + are "site", "site-group", "cluster" and "cluster-group". Scopes are buggy + in NetBox https://github.com/netbox-community/netbox/issues/18706 + """, + config_example="London/Vlan_.* = VLAN Group 1, Tokio/Vlan_.* = VLAN Group 2"), + ConfigOption("vlan_group_relation_by_id", + str, + description="""adds a relation to assign VLAN groups to matching VLANs by ID. + Same matching rules as the exclude_by_id option uses are applied. + Fist match wins. Only newly discovered VLANs which are not present in + NetBox will be assigned a VLAN group. + """, + config_example="1023-1042 = VLAN Group 1, Tokio/2342 = VLAN Group 2") ]), + ConfigOption("track_vm_host", bool, description="""enabling this option will add the ESXi host @@ -417,7 +450,7 @@ def validate_options(self): if option.value is None: continue - if "filter" in option.key: + if "filter" in option.key and "vm_exclude_by_tag_filter" not in option.key: re_compiled = None try: @@ -430,7 +463,13 @@ def validate_options(self): continue - if "relation" in option.key: + if option.key == "vm_exclude_by_tag_filter": + + option.set_value(quoted_split(option.value)) + + continue + + if "relation" in option.key and "vlan_group_relation" not in option.key: relation_data = list() @@ -549,35 +588,57 @@ def validate_options(self): continue - if option.key == "vlan_sync_exclude_by_name": + if option.key in [ "vlan_sync_exclude_by_name", "vlan_sync_exclude_by_id", + "vlan_group_relation_by_name", "vlan_group_relation_by_id" ]: + + if option.key == "vlan_sync_exclude_by_name": + filter_class = FilterVLANByName + filter_type = "exclude" + elif option.key == "vlan_group_relation_by_name": + filter_class = FilterVLANByName + filter_type = "group relation" + elif option.key == "vlan_sync_exclude_by_id": + filter_class = FilterVLANByID + filter_type = "exclude" + elif option.key == "vlan_group_relation_by_id": + filter_class = FilterVLANByID + filter_type = "group relation" + else: + raise ValueError(f"unhandled config option {option.key}") value_list = list() - for excluded_vlan in quoted_split(option.value) or list(): - - excluded_vlan_object = ExcludedVLANName(excluded_vlan) - - if not excluded_vlan_object.is_valid(): - self.set_validation_failed() - continue + for single_option_value in quoted_split(option.value) or list(): - value_list.append(excluded_vlan_object) + relation_name = None + object_name = single_option_value.split("=")[0].strip(' "') - option.set_value(value_list) + if "relation" in option.key: - if option.key == "vlan_sync_exclude_by_id": + if "=" not in single_option_value: + log.error(f"Config option '{option.key}' malformed, got {single_option_value} but " + f"needs key = value relation.") + self.set_validation_failed() + continue - value_list = list() + relation_name = single_option_value.split("=")[1].strip(' "') - for excluded_vlan in quoted_split(option.value) or list(): + if relation_name is not None and len(relation_name) == 0: + log.error(f"Config option '{option.key}' malformed, got '{object_name}' as " + f"object name and relation name was empty.") + self.set_validation_failed() + continue - excluded_vlan_object = ExcludedVLANID(excluded_vlan) + vlan_filter = filter_class(object_name, filter_type) - if not excluded_vlan_object.is_valid(): + if not vlan_filter.is_valid(): self.set_validation_failed() continue - value_list.append(excluded_vlan_object) + if "relation" in option.key: + value_list.append((vlan_filter, relation_name)) + else: + value_list.append(vlan_filter) option.set_value(value_list) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index f8f8bfd8..9ab89744 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -29,7 +29,7 @@ from module.sources.common.source_base import SourceBase from module.sources.vmware.config import VMWareConfig from module.common.logging import get_logger, DEBUG3 -from module.common.misc import grab, dump, get_string_or_none, plural +from module.common.misc import grab, dump, get_string_or_none, plural, quoted_split from module.common.support import normalize_mac_address from module.netbox.inventory import NetBoxInventory from module.netbox import * @@ -61,6 +61,7 @@ class VMWareHandler(SourceBase): NBClusterGroup, NBDeviceRole, NBSite, + NBSiteGroup, NBCluster, NBDevice, NBVM, @@ -71,8 +72,10 @@ class VMWareHandler(SourceBase): NBTenant, NBVRF, NBVLAN, + NBVLANGroup, NBCustomField, - NBVirtualDisk + NBVirtualDisk, + NBMACAddress ] source_type = "vmware" @@ -421,9 +424,9 @@ def passes_filter(name, include_filter, exclude_filter): name: str name of the object to check include_filter: regex object - regex object of include filter + A regex object of include filter exclude_filter: regex object - regex object of exclude filter + A regex object of exclude filter Returns ------- @@ -511,7 +514,7 @@ def get_object_based_on_macs(self, object_type, mac_list=None): object_type: (NBDevice, NBVM) type of NetBox device to find in inventory mac_list: list - list of MAC addresses to compare against NetBox interface objects + a list of MAC addresses to compare against NetBox interface objects Returns ------- @@ -710,7 +713,7 @@ def collect_object_tags(self, obj): Returns ------- tag_list: list - list of NBTag objets retrieved from vCenter for this object + a list of NBTag objets retrieved from vCenter for this object """ if obj is None: @@ -795,22 +798,22 @@ def get_object_custom_fields(self, obj): if num_cpu_cores is not None: custom_field = self.add_update_custom_field({ - "name": "vcsa_host_cpu_cores", + "name": "host_cpu_cores", "label": "Physical CPU Cores", "object_types": [object_type], "type": "text", - "description": f"vCenter '{self.name}' reported Host CPU cores" + "description": f"Reported Host CPU cores" }) return_custom_fields[grab(custom_field, "data.name")] = f"{num_cpu_cores} {cpu_model}" if isinstance(memory_size, int): custom_field = self.add_update_custom_field({ - "name": "vcsa_host_memory", + "name": "host_memory", "label": "Memory", "object_types": [object_type], "type": "text", - "description": f"vCenter '{self.name}' reported Memory" + "description": f"Reported size of Memory" }) memory_size = round(memory_size / 1024 ** 3) @@ -960,7 +963,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v look for longest matching IP Prefix in the same site. If this failed we try to find the longest matching global IP Prefix. - If a IP Prefix was found then we try to get the VRF and VLAN for this prefix. Now we compare + If an IP Prefix was found then we try to get the VRF and VLAN for this prefix. Now we compare if interface VLAN and prefix VLAN match up and warn if they don't. Then we try to add data to the IP address if not already set: @@ -971,7 +974,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v And we also set primary IP4/6 for this object depending on the "set_primary_ip" setting. - If a IP address is set as primary IP for another device then using this IP on another + If an IP address is set as primary IP for another device then using this IP on another device will be rejected by NetBox. Setting "always": @@ -991,7 +994,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v Parameters ---------- object_type: (NBDevice, NBVM) - NetBoxObject sub class of object to add + NetBoxObject subclass of object to add object_data: dict data of object to add/update pnic_data: dict @@ -999,13 +1002,13 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v vnic_data: dict data of virtual interfaces of this object, interface name as key nic_ips: dict - dict of ips per interface of this object, interface name as key + a dict of ips per interface of this object, interface name as key p_ipv4: str primary IPv4 as string including netmask/prefix p_ipv6: str primary IPv6 as string including netmask/prefix vmware_object: (vim.HostSystem, vim.VirtualMachine) - vmware object to pass on to 'add_update_interface' method to setup reevaluation + vmware object to pass on to 'add_update_interface' method to set up reevaluation disk_data: list data of discs which belong to a VM @@ -1062,6 +1065,11 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v device_vm_object = self.inventory.get_by_data(object_type, data={"asset_tag": object_data.get("asset_tag")}) + # look for VMs with same serial + if object_type == NBVM and device_vm_object is None and object_data.get("serial") is not None: + log.debug2(f"No match found. Trying to find {object_type.name} based on serial number") + device_vm_object = self.inventory.get_by_data(object_type, data={"serial": object_data.get("serial")}) + if device_vm_object is not None: log.debug2("Found a matching %s object: %s" % (object_type.name, device_vm_object.get_display_name(including_second_key=True))) @@ -1121,7 +1129,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v object_type == NBVM and disk_data is not None and len(disk_data) > 0: # create pairs of existing and discovered disks. - # currently these disks are only used within the VM model. that's we we use this simple approach and + # currently these disks are only used within the VM model. that's why we use this simple approach and # just rewrite disk as they appear in order. # otherwise we would need to implement a matching function like matching interfaces. disk_zip_list = zip_longest( @@ -1301,7 +1309,7 @@ def add_datacenter(self, obj): Parameters ---------- obj: vim.Datacenter - datacenter object + a datacenter object """ if self.settings.set_source_name_as_cluster_group is True: @@ -1376,10 +1384,15 @@ def add_cluster(self, obj): data = { "name": name, "type": {"name": "VMware ESXi"}, - "group": group, - "site": {"name": site_name} + "group": group } + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + data["scope_id"] = {"name": site_name} + data["scope_type"] = "dcim.site" + else: + data["site"] = {"name": site_name} + tenant_name = self.get_object_relation(full_cluster_name, "cluster_tenant_relation") if tenant_name is not None: data["tenant"] = {"name": tenant_name} @@ -1636,7 +1649,7 @@ def add_host(self, obj): serial = None for serial_num_key in ["SerialNumberTag", "ServiceTag", "EnclosureSerialNumberTag"]: - if serial_num_key in identifier_dict.keys(): + if serial_num_key in identifier_dict.keys() and self.settings.collect_hardware_serial is True: log.debug2(f"Found {serial_num_key}: {get_string_or_none(identifier_dict.get(serial_num_key))}") if serial is None: serial = get_string_or_none(identifier_dict.get(serial_num_key)) @@ -1755,6 +1768,14 @@ def add_host(self, obj): # now iterate over all physical interfaces and collect data pnic_data_dict = dict() + pnic_hints = dict() + # noinspection PyBroadException + try: + for hint in obj.configManager.networkSystem.QueryNetworkHint(""): + pnic_hints[hint.device] = hint + except Exception: + pass + for pnic in grab(obj, "config.network.pnic", fallback=list()): pnic_name = grab(pnic, "device") @@ -1807,7 +1828,6 @@ def add_host(self, obj): # check vlans on this pnic pnic_vlans = list() - for pg_name, pg_data in self.network_data["host_pgroup"][name].items(): if pnic_name in pg_data.get("nics", list()): @@ -1818,6 +1838,15 @@ def add_host(self, obj): pnic_mac_address = normalize_mac_address(grab(pnic, "mac")) + if pnic_hints.get(pnic_name) is not None: + pnic_switch_port = grab(pnic_hints.get(pnic_name), 'connectedSwitchPort') + if pnic_switch_port is not None: + pnic_sp_sys_name = grab(pnic_switch_port, 'systemName') + if pnic_sp_sys_name is None: + pnic_sp_sys_name = grab(pnic_switch_port, 'devId') + if pnic_sp_sys_name is not None: + pnic_description += f" (conn: {pnic_sp_sys_name} - {grab(pnic_switch_port, 'portId')})" + if self.settings.host_nic_exclude_by_mac_list is not None and \ pnic_mac_address in self.settings.host_nic_exclude_by_mac_list: log.debug2(f"Host NIC with MAC '{pnic_mac_address}' excluded from sync. Skipping") @@ -2017,8 +2046,8 @@ def add_host(self, obj): vnic_ips[vnic_name].append(int_v6) # set first valid IPv6 address as primary IPv6 - # not the best way but maybe we can find more information in "spec.ipRouteSpec" - # about default route and we could use that to determine the correct IPv6 address + # not the best way, but maybe we can find more information in "spec.ipRouteSpec" + # about default route, and we could use that to determine the correct IPv6 address if vnic_is_primary is True and host_primary_ip6 is None: host_primary_ip6 = int_v6 @@ -2034,10 +2063,10 @@ def add_virtual_machine(self, obj): Parse a vCenter VM add to NetBox once all data is gathered. VMs are parsed twice. First only "online" VMs are parsed and added. In the second - round also "offline" VMs will be parsed. This helps of VMs are cloned and used + round also "offline" VMs will be parsed. This helps if VMs are cloned and used for upgrades but then have the same name. - First VM is filtered: + First VM will be filtered: VM has a cluster and is it permitted was VM with same name and cluster already parsed does the VM pass the vm_include_filter and vm_exclude_filter @@ -2150,6 +2179,14 @@ def add_virtual_machine(self, obj): platform = get_string_or_none(grab(obj, "config.guestFullName")) platform = get_string_or_none(grab(obj, "guest.guestFullName", fallback=platform)) + # extract prettyName from extraConfig exposed by guest tools + extra_config = [x.value for x in grab(obj, "config.extraConfig", fallback=[]) + if x.key == "guestOS.detailed.data"] + if len(extra_config) > 0: + pretty_name = [x for x in quoted_split(extra_config[0].replace("' ", "', ")) if x.startswith("prettyName")] + if len(pretty_name) > 0: + platform = pretty_name[0].replace("prettyName='","") + if platform is not None: platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) @@ -2166,7 +2203,16 @@ def add_virtual_machine(self, obj): vm_tags = self.get_object_relation(name, "vm_tag_relation") # get vCenter tags - vm_tags.extend(self.collect_object_tags(obj)) + vcenter_tags = self.collect_object_tags(obj) + + # check if VM tag excludes VM from being synced to NetBox + for sync_exclude_tag in self.settings.vm_exclude_by_tag_filter or list(): + if sync_exclude_tag in vcenter_tags: + log.debug(f"Virtual machine vCenter tag '{sync_exclude_tag}' in matches 'vm_exclude_by_tag_filter'. " + f"Skipping") + return + + vm_tags.extend(vcenter_tags) # vm memory depending on setting vm_memory = grab(obj, "config.hardware.memoryMB", fallback=0) @@ -2196,6 +2242,10 @@ def add_virtual_machine(self, obj): if isinstance(comp, vim.vm.device.VirtualDisk) ]) / 1024 / 1024) + # Add adaptation for the new 'serial' field in NetBox 4.1.0 VM model + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.1.0"): + vm_data["serial"] = vm_uuid + if platform is not None: vm_data["platform"] = {"name": platform} if annotation is not None: diff --git a/netbox-sync.py b/netbox-sync.py index 7fdd436a..e499838b 100755 --- a/netbox-sync.py +++ b/netbox-sync.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # diff --git a/scripts/publi.sh b/scripts/publi.sh index 8fcc3484..2a585aca 100755 --- a/scripts/publi.sh +++ b/scripts/publi.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -33,7 +33,7 @@ unset DOCKER_TLS_VERIFY unset DOCKER_HOST unset DOCKER_CERT_PATH -find module -type d -name "__pycache__" -exec rm -rf {} \; +find module -type d -name "__pycache__" -print0 | xargs -0 -n1 rm -rf docker --config ./docker-tmp login docker --config ./docker-tmp buildx create --use if [[ $FINAL == true ]]; then diff --git a/scripts/set_version.sh b/scripts/set_version.sh index 92e65fc6..f947e0eb 100755 --- a/scripts/set_version.sh +++ b/scripts/set_version.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -12,7 +12,7 @@ EXAMPLE_CONFIG_FILE="settings-example.ini" VERSION_DATA_FILE="module/__init__.py" README_FILE="README.md" VERSION_TO_SET="$1" -COPYRIGHT_PATTERN="# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved." +COPYRIGHT_PATTERN="# Copyright (c) 2020 - 2025 Ricardo Bartels. All rights reserved." BASE_PATH="$(realpath "$(dirname "${0}")/..")" # shellcheck disable=SC2181 diff --git a/settings-example.ini b/settings-example.ini index 682bffc9..432d4c2c 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -1,5 +1,5 @@ ;;; Welcome to the NetBox Sync configuration file. -;;; Version: 1.7.0 (2024-10-21) +;;; Version: 1.8.0 (2025-03-07) ;;; Project URL: https://github.com/bb-ricardo/netbox-sync ; The values in this file override the default values used by the system if a config @@ -172,6 +172,11 @@ password = super-secret ;vm_exclude_filter = ;vm_include_filter = +; defines a comma separated list of vCenter tags which (if assigned to a VM) will exclude +; this VM from being synced to NetBox. The config option 'vm_tag_source' determines which +; tags are collected for VMs. +;vm_exclude_by_tag_filter = tag-a, tag-b + ; relations options ; This option defines which vCenter cluster is part of a NetBox site. @@ -241,6 +246,9 @@ password = super-secret ; Attempt to collect asset tags from vCenter hosts ;collect_hardware_asset_tag = True +; Attempt to collect serials from vCenter hosts +;collect_hardware_serial = True + ; Perform a reverse lookup for all collected IP addresses. If a dns name was found it will ; be added to the IP address object in NetBox ;dns_name_lookup = True @@ -334,6 +342,19 @@ password = super-secret ;vlan_sync_exclude_by_name = New York/Storage, Backup, Tokio/DMZ, Madrid/.* ;vlan_sync_exclude_by_id = Frankfurt/25, 1023-1042 +; adds a relation to assign VLAN groups to matching VLANs by name. Same matching rules as +; the exclude_by_name option uses are applied. If name and id relations are defined, the +; name relation takes precedence. Fist match wins. Only newly discovered VLANs which are +; not present in NetBox will be assigned a VLAN group. Supported scopes for a VLAN group +; are "site", "site-group", "cluster" and "cluster-group". Scopes are buggy in NetBox +; https://github.com/netbox-community/netbox/issues/18706 +;vlan_group_relation_by_name = London/Vlan_.* = VLAN Group 1, Tokio/Vlan_.* = VLAN Group 2 + +; adds a relation to assign VLAN groups to matching VLANs by ID. Same matching rules as +; the exclude_by_id option uses are applied. Fist match wins. Only newly discovered VLANs +; which are not present in NetBox will be assigned a VLAN group. +;vlan_group_relation_by_id = 1023-1042 = VLAN Group 1, Tokio/2342 = VLAN Group 2 + ; enabling this option will add the ESXi host this VM is running on to the VM details ;track_vm_host = False