diff --git a/README.md b/README.md index 1aa22f34..6023ca84 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ This is a tool to sync data from different sources to a NetBox instance. Available source types: * VMware vCenter Server * AWS EC2, EBS, and network interfaces, with opt-in S3, Lambda, and EKS inventory (see [AWS source](docs/source_aws.md)) +* oVirt / Red Hat Virtualization (see [oVirt source](docs/source_ovirt.md)) * [bb-ricardo/check_redfish](https://github.com/bb-Ricardo/check_redfish) inventory files **IMPORTANT: READ INSTRUCTIONS CAREFULLY BEFORE RUNNING THIS PROGRAM** @@ -43,6 +44,9 @@ This ensures stale objects are removed from NetBox keeping an accurate current s * Optional boto3 SDK, AWS read-only credentials, and automatic or explicit region discovery * Optional AWS Organizations discovery with collector permissions and a trusted inventory role in each member account * Existing VPC-specific NetBox VRFs for IP import; see [AWS setup](docs/source_aws.md) +#### Source: oVirt (if used) +* oVirt / Red Hat Virtualization Engine v4 API +* Optional SDK and native libraries: see [installation and configuration](docs/source_ovirt.md) # Installing * here we assume we install in ```/opt``` diff --git a/docs/source_ovirt.md b/docs/source_ovirt.md new file mode 100644 index 00000000..4ebde754 --- /dev/null +++ b/docs/source_ovirt.md @@ -0,0 +1,64 @@ +# Source: Ovirt + +## Installation + +The SDK is optional: installations using other sources do not need it. Selecting +an oVirt source without the SDK logs an error and skips that source. + +On Debian/Ubuntu, install the SDK's build dependencies before installing Python +packages: + +```sh +sudo apt-get install gcc python3-dev libxml2-dev libcurl4-openssl-dev +pip install -r requirements-ovirt.txt +``` + +With uv, use `uv sync --extra ovirt` after installing the native libraries. +The SDK contains a native extension; a compiler and Python development headers +matching your Python version are needed when no wheel is available. The runtime +needs libxml2 and libcurl. + +For Docker, enable the optional source in both build stages: + +```sh +docker build --build-arg INSTALL_OVIRT=true -t netbox-sync:ovirt . +``` + +## Configuration + +```ini +[source/production-ovirt] +type = ovirt +url = https://engine.example.com/ovirt-engine/api +username = inventory@internal +password = replace-me +validate_tls_certs = True +ca_file = /etc/pki/ovirt-engine/ca.pem +permitted_subnets = 10.0.0.0/8, fd00::/8 +cluster_site_relation = DC/Production = Main Site +``` + +Use an account with read access to datacenters, clusters, hosts, VMs, disks, +interfaces, and reported devices. The existing source options remain supported, +including name filters, site/tenant/role/tag relations, domain stripping, +`set_vm_name_to_uuid`, `skip_vm_comments`, `skip_vm_platform`, and +`host_custom_object_attributes` / `vm_custom_object_attributes` (dot-separated +SDK attribute paths). Generate a full example with `python netbox-sync.py -g`. + +Guest IP discovery requires the guest agent to report network devices. VMs +without agent data still receive their configured interfaces and disks. As with +other sources, bare guest IPs need a matching prefix in NetBox to determine their +prefix length. Addresses without a matching prefix are not imported; netbox-sync +does not guess a host-length prefix. Only addresses allowed by `permitted_subnets` +are imported. +`ip_tenant_inheritance_order`, `preserve_primary_ips`, `skip_fhrp_group_ips`, and +interface exclusion filters use the current shared network-sync behavior. + +NetBox 4.2+ cluster sites use scoped relations and interfaces use MAC address +objects. NetBox 3.7+ disks are separate virtual disk objects, with the size units +appropriate to the NetBox version. Older versions retain aggregate VM disk size. +Missing CPU or disk size data is not replaced with fabricated values. + +Begin with a dry run (`python netbox-sync.py -n`) and inspect the log before +allowing writes. Unit tests use a fake SDK and the real in-memory NetBox +inventory; they do not replace validation against your live oVirt Engine. diff --git a/module/config/file_output.py b/module/config/file_output.py index 9acf4e5a..b7191f86 100644 --- a/module/config/file_output.py +++ b/module/config/file_output.py @@ -17,6 +17,7 @@ from module.sources.vmware.config import VMWareConfig from module.sources.check_redfish.config import CheckRedfishConfig from module.sources.hetzner.config import HetznerConfig +from module.sources.ovirt.config import OVirtConfig from module.sources.aws.config import AWSConfig from module.common.logging import get_logger from module.config import default_config_file_path, source_config_section_name @@ -37,7 +38,8 @@ class ConfigFileOutput(DescriptionFormatterMixin): VMWareConfig, CheckRedfishConfig, HetznerConfig, - AWSConfig + AWSConfig, + OVirtConfig ] header = f"Welcome to the {__description__} configuration file." diff --git a/module/sources/__init__.py b/module/sources/__init__.py index 5cdc0f1b..fe3e461b 100644 --- a/module/sources/__init__.py +++ b/module/sources/__init__.py @@ -10,6 +10,7 @@ # define all available sources here from module.sources.vmware.connection import VMWareHandler from module.sources.hetzner.connection import HetznerHandler +from module.sources.ovirt.connection import OVirtHandler from module.sources.aws.connection import AWSHandler from module.sources.check_redfish.import_inventory import CheckRedfish @@ -20,7 +21,7 @@ from module.config import source_config_section_name # list of valid sources -valid_sources = [VMWareHandler, CheckRedfish, HetznerHandler, AWSHandler] +valid_sources = [VMWareHandler, CheckRedfish, HetznerHandler, AWSHandler, OVirtHandler] def validate_source(source_class_object=None, state="pre"): diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 7538e5de..cc6f3bdc 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -50,6 +50,14 @@ def implements(cls, source_type): def finish(self): pass + def object_synced_by_other_source(self, nb_object): + """Check the current source or stored tags for ownership by another configured source.""" + if nb_object.source is not None: + return nb_object.source is not self + + other_source_tags = [x.source_tag for x in self.inventory.source_list if x is not self] + return len(set(nb_object.get_tags()).intersection(other_source_tags)) > 0 + def ip_is_primary_ip_of_object(self, ip_object, device_vm_object) -> bool: """ Check if a NBIPAddress object is currently set as primary IPv4 or IPv6 @@ -510,7 +518,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, # skip handling of IPs for VMs with not installed/running guest tools skip_ip_handling = False - if type(device_object) == NBVM and grab(vmware_object,'guest.toolsRunningStatus') != "guestToolsRunning": + if type(device_object) == NBVM and vmware_object is not None and \ + grab(vmware_object, 'guest.toolsRunningStatus') != "guestToolsRunning": log.debug(f"VM '{device_object.name}' guest tool status is 'NotRunning', skipping IP handling") skip_ip_handling = True diff --git a/module/sources/ovirt/config.py b/module/sources/ovirt/config.py new file mode 100644 index 00000000..36456e16 --- /dev/null +++ b/module/sources/ovirt/config.py @@ -0,0 +1,377 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +import re +from ipaddress import ip_address + +from module.common.misc import quoted_split +from module.config import source_config_section_name +from module.config.base import ConfigBase +from module.config.option import ConfigOption +from module.config.group import ConfigOptionGroup +from module.sources.common.config import * +from module.sources.common.permitted_subnets import PermittedSubnets +from module.common.logging import get_logger +from module.common.support import normalize_mac_address + +log = get_logger() + + +class OVirtConfig(ConfigBase): + + section_name = source_config_section_name + source_name = None + source_name_example = "my-ovirt-example" + + def __init__(self): + self.options = [ + ConfigOption(**config_option_enabled_definition), + + ConfigOption(**{**config_option_type_definition, "config_example": "ovirt"}), + + ConfigOption("url", + str, + description="host name / IP address of the oVirt API", + config_example="https://engine40.example.com/ovirt-engine/api", + mandatory=True), + + ConfigOption("username", + str, + description="username to use to log into oVirt", + config_example="ovirtuser", + mandatory=True), + + ConfigOption("password", + str, + description="password to use to log into oVirt", + config_example="supersecret", + sensitive=True, + mandatory=True), + + ConfigOption("ca_file", + str, + description="path to the CA file for oVirt", + config_example="ca.pem"), + + ConfigOption("validate_tls_certs", + bool, + description="""Enforces TLS certificate validation. + If oVirt uses a valid TLS certificate then this option should be set + to 'true' to ensure a secure connection.""", + default_value=False), + + ConfigOption(**config_option_permitted_subnets_definition), + ConfigOption(**config_option_ip_tenant_inheritance_order_definition), + ConfigOption("preserve_primary_ips", bool, default_value=False, + description="Keep primary IPs when no longer reported by the guest agent."), + ConfigOption("skip_fhrp_group_ips", bool, default_value=False, + description="Do not reassign IPs belonging to an FHRP group."), + ConfigOption("disable_vlan_sync", bool, default_value=False, + description="Do not create newly discovered VLANs."), + + ConfigOptionGroup(title="filter", + description="""filters can be used to include/exclude certain objects from importing + into NetBox. Include filters are checked first and exclude filters after. + An object name has to pass both filters to be synced to NetBox. + If a filter is unset it will be ignored. Filters are all treated as regex expressions! + If more then one expression should match, a '|' needs to be used + """, + config_example="""Example: (exclude all VMs with "replica" in their name + and all VMs starting with "backup"): vm_exclude_filter = .*replica.*|^backup.*""", + options=[ + ConfigOption("cluster_exclude_filter", + str, + description="""If a cluster is excluded from sync then ALL VMs and HOSTS + inside the cluster will be ignored! a cluster can be specified + as "Cluster-name" or "Datacenter-name/Cluster-name" if + multiple clusters have the same name"""), + ConfigOption("cluster_include_filter", str), + ConfigOption("host_exclude_filter", + str, + description="""This will only include/exclude the host, + not the VM if Host is part of a multi host cluster"""), + ConfigOption("host_include_filter", str), + ConfigOption("vm_exclude_filter", + str, description="simply include/exclude VMs"), + ConfigOption("vm_include_filter", str), + ConfigOption("host_interface_exclude_filter", str), + ConfigOption("vm_interface_exclude_filter", str) + ]), + ConfigOptionGroup(title="relations", + options=[ + ConfigOption("cluster_site_relation", + str, + description="""\ + This option defines which oVirt cluster is part of a NetBox site. + This is done with a comma separated key = value list. + key: defines the cluster name as regex + value: defines the NetBox site name (use quotes if name contains commas) + This is a quite important config setting as IP addresses, prefixes, VLANs + and VRFs are site dependent. In order to assign the correct prefix to an IP + address it is important to pick the correct site. + A VM always depends on the cluster site relation + a cluster can be specified as "Cluster-name" or + "Datacenter-name/Cluster-name" if multiple clusters have the same name + """, + config_example="Cluster_NYC = New York, Cluster_FFM.* = Frankfurt, Datacenter_TOKIO/.* = Tokio"), + ConfigOption("host_site_relation", + str, + description="""Same as cluster site but on host level. + If unset it will fall back to cluster_site_relation""", + config_example="nyc02.* = New York, ffm01.* = Frankfurt"), + ConfigOption("cluster_tenant_relation", + str, + description="""\ + This option defines which cluster/host/VM belongs to which tenant. + This is done with a comma separated key = value list. + key: defines a hosts/VM name as regex + value: defines the NetBox tenant name (use quotes if name contains commas) + a cluster can be specified as "Cluster-name" or + "Datacenter-name/Cluster-name" if multiple clusters have the same name + """, + config_example="Cluster_NYC.* = Customer A"), + ConfigOption("host_tenant_relation", str, config_example="esxi300.* = Infrastructure"), + ConfigOption("vm_tenant_relation", str, config_example="grafana.* = Infrastructure"), + ConfigOption("vm_platform_relation", + str, + description="""\ + This option defines custom platforms if the VMWare created platforms are not suitable. + Pretty much a mapping of VMWare platform name to your own platform name. + This is done with a comma separated key = value list. + key: defines a VMWare returned platform name + value: defines the desired NetBox platform name""", + config_example="centos-7.* = centos7, microsoft-windows-server-2016.* = Windows2016"), + ConfigOption("host_role_relation", + str, + description="""\ + Define the NetBox device role used for hosts. The default is + set to "Server". This is done with a comma separated key = value list. + key: defines host(s) name as regex + value: defines the NetBox role name (use quotes if name contains commas) + """, + default_value=".* = Server"), + ConfigOption("vm_role_relation", + str, + description="""\ + Define the NetBox device role used for VMs. This is done with a + comma separated key = value list, same as 'host_role_relation'. + key: defines VM(s) name as regex + value: defines the NetBox role name (use quotes if name contains commas) + """, + config_example=".* = Server"), + ConfigOption("cluster_tag_relation", + str, + description="""\ + Define NetBox tags which are assigned to a cluster, host or VM. This is + done with a comma separated key = value list. + key: defines a hosts/VM name as regex + value: defines the NetBox tag (use quotes if name contains commas) + a cluster can be specified as "Cluster-name" or + "Datacenter-name/Cluster-name" if multiple clusters have the same name""", + config_example="Cluster_NYC.* = Infrastructure"), + ConfigOption("host_tag_relation", str, config_example="esxi300.* = Infrastructure"), + ConfigOption("vm_tag_relation", str, config_example="grafana.* = Infrastructure") + ]), + ConfigOption("dns_name_lookup", + bool, + description="""Perform a reverse lookup for all collected IP addresses. + If a dns name was found it will be added to the IP address object in NetBox + """, + default_value=True), + ConfigOption("custom_dns_servers", + str, + description="use custom DNS server to do the reverse lookups", + config_example="192.168.1.11, 192.168.1.12"), + ConfigOption("set_primary_ip", + str, + description="""\ + define how the primary IPs should be set + possible values: + + always: will remove primary IP from the object where this address is + currently set as primary and moves it to new object + + when-undefined: + only sets primary IP if undefined, will cause ERRORs if same IP is + assigned more then once to different hosts and IP is set as the + objects primary IP + + never: don't set any primary IPs, will cause the same ERRORs + as "when-undefined" + """, + default_value="when-undefined"), + ConfigOption("skip_vm_comments", + bool, + description="do not set notes to the UUID or name of the VM", + default_value=False), + ConfigOption("skip_vm_platform", + bool, + description="do not sync the VM operating system to the NetBox platform", + default_value=False), + ConfigOption("strip_host_domain_name", + bool, + description="strip domain part from host name before syncing device to NetBox", + default_value=False), + ConfigOption("strip_vm_domain_name", + bool, + description="strip domain part from VM name before syncing VM to NetBox", + default_value=False), + ConfigOptionGroup(title="custom object attributes", + description="""\ + add arbitrary host/vm object attributes as custom fields to NetBox. + multiple attributes can be defined comma separated. + to get a list of available attributes use '-l DEBUG3' as cli param (CAREFUL: output might be long) + and here 'https://gist.github.com/bb-Ricardo/538768487bdac4efafabe56e005cb4ef' can be seen how to + access these attributes + """, + options=[ + ConfigOption("host_custom_object_attributes", + str, + config_example="uuid"), + ConfigOption("vm_custom_object_attributes", + str, + config_example="uuid") + ]), + ConfigOption("set_source_name_as_cluster_group", + bool, + description="""this will set the sources name as cluster group name instead of the datacenter. + This works if the oVirt CP has ONLY ONE datacenter configured. + Otherwise it will rename all datacenters to the source name!""", + default_value=False), + ConfigOption("set_vm_name_to_uuid", + bool, + description="Set the name in Netbox to the VM UUID instead of name", + default_value=False), + + # removed settings + ConfigOption("netbox_host_device_role", + str, + deprecation_message="You need to switch to 'host_role_relation'.", + removed=True), + ConfigOption("netbox_vm_device_role", + str, + deprecation_message="You need to switch to 'vm_role_relation'.", + removed=True), + ConfigOption("sync_tags", + bool, + deprecation_message="You need to switch to 'host_tag_relation', " + + "'vm_tag_relation' or 'cluster_tag_relation'", + removed=True), + ConfigOption("sync_parent_tags", + bool, + deprecation_message="You need to switch to 'host_tag_relation', " + + "'vm_tag_relation' or 'cluster_tag_relation'", + removed=True) + ] + + super().__init__() + + def validate_options(self): + + for option in self.options: + + if option.value is None: + continue + + if "filter" in option.key: + + re_compiled = None + try: + re_compiled = re.compile(option.value) + except re.error as e: + log.error(f"Problem parsing regular expression for '{self.source_name}.{option.key}': {e}") + self.set_validation_failed() + + option.set_value(re_compiled) + + continue + + if "relation" in option.key: + + relation_data = list() + + relation_type = option.key.split("_")[1] + + for relation in quoted_split(option.value): + + object_name, separator, relation_name = relation.partition("=") + object_name = object_name.strip(' "') + relation_name = relation_name.strip(' "') + + if not separator or len(object_name) == 0 or len(relation_name) == 0: + log.error(f"Config option '{relation}' malformed got '{object_name}' for " + f"object name and '{relation_name}' for {relation_type} name.") + self.set_validation_failed() + continue + + try: + re_compiled = re.compile(object_name) + except re.error as e: + log.error(f"Problem parsing regular expression '{object_name}' for '{relation}': {e}") + self.set_validation_failed() + continue + + relation_data.append({ + "object_regex": re_compiled, + "assigned_name": relation_name + }) + + option.set_value(relation_data) + + continue + + if "custom_object_attributes" in option.key: + + option.set_value(quoted_split(option.value)) + + continue + + if option.key == "ip_tenant_inheritance_order": + order = quoted_split(option.value) + if len(order) > 2 or any(item not in ("device", "prefix", "disabled") for item in order): + log.error(f"Invalid IP tenant inheritance order '{option.value}' for '{self.source_name}'.") + self.set_validation_failed() + option.set_value(order) + continue + + if option.key == "set_primary_ip": + if option.value not in ["always", "when-undefined", "never"]: + log.error(f"Primary IP option '{option.key}' value '{option.value}' invalid.") + self.set_validation_failed() + + if option.key == "custom_dns_servers": + + dns_name_lookup = self.get_option_by_name("dns_name_lookup") + + if not isinstance(dns_name_lookup, ConfigOption) or dns_name_lookup.value is False: + continue + + custom_dns_servers = quoted_split(option.value) + + tested_custom_dns_servers = list() + for custom_dns_server in custom_dns_servers: + try: + tested_custom_dns_servers.append(str(ip_address(custom_dns_server))) + except ValueError: + log.error(f"Config option 'custom_dns_servers' value '{custom_dns_server}' " + f"does not appear to be an IP address.") + self.set_validation_failed() + + option.set_value(tested_custom_dns_servers) + + continue + + permitted_subnets_option = self.get_option_by_name("permitted_subnets") + + if permitted_subnets_option is not None: + permitted_subnets = PermittedSubnets(permitted_subnets_option.value) + if permitted_subnets.validation_failed is True: + self.set_validation_failed() + + permitted_subnets_option.set_value(permitted_subnets) diff --git a/module/sources/ovirt/connection.py b/module/sources/ovirt/connection.py new file mode 100644 index 00000000..67b64240 --- /dev/null +++ b/module/sources/ovirt/connection.py @@ -0,0 +1,1340 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2021 Ricardo Bartels. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +import pprint +import socket +from ipaddress import ip_address, ip_interface + +from packaging import version +try: + import ovirtsdk4 as sdk + import ovirtsdk4.types as types +except ModuleNotFoundError as exc: + if exc.name != "ovirtsdk4": + raise + sdk = None + types = None + +from module.sources.common.source_base import SourceBase +from module.sources.ovirt.config import OVirtConfig +from module.common.logging import get_logger, DEBUG3 +from module.common.misc import grab, dump, get_string_or_none +from module.common.support import normalize_mac_address +from module.netbox.inventory import NetBoxInventory +from module.netbox import * + +log = get_logger() + + +# noinspection PyTypeChecker +class OVirtHandler(SourceBase): + """ + Source class to import data from a oVirt instance and add/update NetBox objects based on gathered information + """ + + dependent_netbox_objects = [ + NBTag, + NBManufacturer, + NBDeviceType, + NBPlatform, + NBClusterType, + NBClusterGroup, + NBDeviceRole, + NBSite, + NBCluster, + NBDevice, + NBVM, + NBVMInterface, + NBInterface, + NBIPAddress, + NBPrefix, + NBTenant, + NBVRF, + NBVLAN, + NBCustomField, + NBMACAddress, + NBVirtualDisk + ] + + source_type = "ovirt" + + # internal vars + session = None + tag_session = None + + site_name = None + + def __init__(self, name=None): + + if name is None: + raise ValueError(f"Invalid value for attribute 'name': '{name}'.") + + self.inventory = NetBoxInventory() + self.name = name + + # parse settings + settings_handler = OVirtConfig() + settings_handler.source_name = self.name + self.settings = settings_handler.parse() + + self.set_source_tag() + self.site_name = f"oVirt: {name}" + + if self.settings.enabled is False: + log.info(f"Source '{name}' is currently disabled. Skipping") + return + + self.create_ovirt_session() + + if self.session is None: + log.info(f"Source '{name}' is currently unavailable. Skipping") + return + + self.init_successful = True + self.permitted_clusters = dict() + self.processed_host_names = dict() + self.processed_vm_names = dict() + self.processed_vm_uuid = list() + self.object_cache = dict() + self.parsing_vms_the_first_time = True + self.disk_cache = dict() + + def create_ovirt_session(self): + """ + Initialize session with oVirt + + Returns + ------- + bool: if initialization was successful or not + """ + + if self.session is not None: + return True + + if sdk is None: + log.error("The oVirt SDK is missing. Install dependencies from requirements-ovirt.txt.") + return False + + log.debug(f"Starting oVirt connection to '{self.settings.url}'") + + session = None + try: + session = sdk.Connection( + url=self.settings.url, + username=self.settings.username, + password=self.settings.password, + ca_file=self.settings.ca_file, + insecure=(not self.settings.validate_tls_certs), + debug=False, + log=log, + ) + session.test(raise_exception=True) + except Exception as e: + if session is not None: + try: + session.close() + except Exception as close_error: + log.warning(f"Unable to close failed oVirt session: {close_error}") + log.error(f"Unable to connect to oVirt instance '{self.settings.url}'. {e}") + return False + + self.session = session + log.info(f"Successfully connected to oVirt '{self.settings.url}'") + + return True + + def finish(self): + if self.session is not None: + self.session.close() + + def apply(self): + """ + Main source handler method. This method is called for each source from "main" program + to retrieve data from it source and apply it to the NetBox inventory. + + Every update of new/existing objects fot this source has to happen here. + """ + + log.info(f"Query data from oVirt: '{self.settings.url}'") + + """ + Mapping of object type keywords to view types and handlers + + iterate over all VMs twice. + + To handle VMs with the same name in a cluster we first + iterate over all VMs and look only at the active ones + and sync these first. + Then we iterate a second time to catch the rest. + + This has been implemented to support migration scenarios + where you create the same machines with a different setup + like a new version or something. This way NetBox will be + updated primarily with the actual active VM data. + """ + + system_service = self.session.system_service() + + disks = system_service.disks_service().list() + for disk in disks: + self.disk_cache[disk.id] = disk + + datacenters = system_service.data_centers_service().list() + for datacenter in datacenters: + self.add_datacenter(datacenter) + + clusters = system_service.clusters_service().list() + for cluster in clusters: + self.add_cluster(cluster) + + hosts = system_service.hosts_service().list(follow="nics") + for host in hosts: + self.add_host(host) + + vms = system_service.vms_service().list(follow="nics,disk_attachments,reported_devices") + for vm in sorted(vms, key=lambda item: item.status != types.VmStatus.UP): + self.add_virtual_machine(vm) + + self.update_basic_data() + + @staticmethod + def passes_filter(name, include_filter, exclude_filter): + """ + checks if object name passes a defined object filter. + + Parameters + ---------- + name: str + name of the object to check + include_filter: regex object + regex object of include filter + exclude_filter: regex object + regex object of exclude filter + + Returns + ------- + bool: True if all filter passed, otherwise False + """ + + # first includes + if include_filter is not None and not include_filter.match(name): + log.debug(f"Object '{name}' did not match include filter '{include_filter.pattern}'. Skipping") + return False + + # second excludes + if exclude_filter is not None and exclude_filter.match(name): + log.debug(f"Object '{name}' matched exclude filter '{exclude_filter.pattern}'. Skipping") + return False + + return True + + def get_site_name(self, object_type, object_name, cluster_name=""): + """ + Return a site name for a NBCluster or NBDevice depending on config options + host_site_relation and cluster_site_relation + + Parameters + ---------- + object_type: (NBCluster, NBDevice) + object type to check site relation for + object_name: str + object name to check site relation for + cluster_name: str + cluster name of NBDevice to check for site name + + Returns + ------- + str: site name if a relation was found + """ + + if object_type not in [NBCluster, NBDevice]: + raise ValueError(f"Object must be a '{NBCluster.name}' or '{NBDevice.name}'.") + + log.debug2(f"Trying to find site name for {object_type.name} '{object_name}'") + + # check if site was provided in config + relation_name = "host_site_relation" if object_type == NBDevice else "cluster_site_relation" + + site_name = self.get_object_relation(object_name, relation_name) + + if object_type == NBDevice and site_name is None: + site_name = self.get_site_name(NBCluster, cluster_name) + if site_name is not None: + log.debug2(f"Found a matching cluster site for {object_name}, using site '{site_name}'") + + # set default site name + if site_name is None: + site_name = self.site_name + log.debug(f"No site relation for '{object_name}' found, using default site '{site_name}'") + + return site_name + + def get_object_based_on_macs(self, object_type, mac_list=None): + """ + Try to find a NetBox object based on list of MAC addresses. + + Iterate over all interfaces of this object type and compare MAC address with list of desired MAC + addresses. If match was found store related machine object and count every correct match. + + If exactly one machine with matching interfaces was found then this one will be returned. + + If two or more machines with matching MACs are found compare the two machines with + the highest amount of matching interfaces. If the ration of matching interfaces + exceeds 2.0 then the top matching machine is chosen as desired object. + + If the ration is below 2.0 then None will be returned. The probability is too low that + this one is the correct one. + + None will also be returned if no machine was found at all. + + Parameters + ---------- + object_type: (NBDevice, NBVM) + type of NetBox device to find in inventory + mac_list: list + list of MAC addresses to compare against NetBox interface objects + + Returns + ------- + (NBDevice, NBVM, None): object instance of found device, otherwise None + """ + + object_to_return = None + + if object_type not in [NBDevice, NBVM]: + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + if mac_list is None or not isinstance(mac_list, list) or len(mac_list) == 0: + return + + interface_typ = NBInterface if object_type == NBDevice else NBVMInterface + + objects_with_matching_macs = dict() + matching_object = None + + for interface in self.inventory.get_all_items(interface_typ): + + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + mac = grab(grab(interface, "data.primary_mac_address"), "data.mac_address") + else: + mac = grab(interface, "data.mac_address") + if mac in mac_list: + + matching_object = grab(interface, f"data.{interface.secondary_key}") + if not isinstance(matching_object, (NBDevice, NBVM)): + continue + + log.debug2("Found matching MAC '%s' on %s '%s'" % + (mac, object_type.name, + matching_object.get_display_name(including_second_key=True))) + + if objects_with_matching_macs.get(matching_object) is None: + objects_with_matching_macs[matching_object] = 1 + else: + objects_with_matching_macs[matching_object] += 1 + + # try to find object based on amount of matching MAC addresses + num_devices_witch_matching_macs = len(objects_with_matching_macs.keys()) + + if num_devices_witch_matching_macs == 1 and isinstance(matching_object, (NBDevice, NBVM)): + + log.debug2("Found one %s '%s' based on MAC addresses and using it" % + (object_type.name, matching_object.get_display_name(including_second_key=True))) + + object_to_return = list(objects_with_matching_macs.keys())[0] + + elif num_devices_witch_matching_macs > 1: + + log.debug2(f"Found {num_devices_witch_matching_macs} {object_type.name}s with matching MAC addresses") + + # now select the two top matches + first_choice, second_choice = \ + sorted(objects_with_matching_macs, key=objects_with_matching_macs.get, reverse=True)[0:2] + + first_choice_matches = objects_with_matching_macs.get(first_choice) + second_choice_matches = objects_with_matching_macs.get(second_choice) + + log.debug2(f"The top candidate {first_choice.get_display_name()} with {first_choice_matches} matches") + log.debug2(f"The second candidate {second_choice.get_display_name()} with {second_choice_matches} matches") + + # get ratio between + matching_ration = first_choice_matches / second_choice_matches + + # only pick the first one if the ration exceeds 2 + if matching_ration >= 2.0: + log.debug2(f"The matching ratio of {matching_ration} is high enough " + f"to select {first_choice.get_display_name()} as desired {object_type.name}") + object_to_return = first_choice + else: + log.debug2("Both candidates have a similar amount of " + "matching interface MAC addresses. Using NONE of them!") + + return object_to_return + + def get_object_based_on_primary_ip(self, object_type, primary_ip4=None, primary_ip6=None): + """ + Try to find a NBDevice or NBVM based on the primary IP address. If an exact + match was found the device/vm object will be returned immediately without + checking of the other primary IP address (if defined). + + Parameters + ---------- + object_type: (NBDevice, NBVM) + object type to look for + primary_ip4: str + primary IPv4 address of object to find + primary_ip6: str + primary IPv6 address of object to find + + Returns + ------- + + """ + + def _matches_device_primary_ip(device_primary_ip, ip_needle): + + ip = None + if device_primary_ip is not None and ip_needle is not None: + if isinstance(device_primary_ip, dict): + ip = grab(device_primary_ip, "address") + + elif isinstance(device_primary_ip, NBIPAddress): + ip = grab(device_primary_ip, "data.address") + + elif isinstance(device_primary_ip, int): + ip = self.inventory.get_by_id(NBIPAddress, nb_id=device_primary_ip) + ip = grab(ip, "data.address") + + if ip is not None and ip.split("/")[0] == ip_needle: + return True + + return False + + if object_type not in [NBDevice, NBVM]: + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + if primary_ip4 is None and primary_ip6 is None: + return + + if primary_ip4 is not None: + primary_ip4 = str(primary_ip4).split("/")[0] + + if primary_ip6 is not None: + primary_ip6 = str(primary_ip6).split("/")[0] + + for device in self.inventory.get_all_items(object_type): + + if _matches_device_primary_ip(grab(device, "data.primary_ip4"), primary_ip4) is True: + log.debug2(f"Found existing host '{device.get_display_name()}' " + f"based on the primary IPv4 '{primary_ip4}'") + return device + + if _matches_device_primary_ip(grab(device, "data.primary_ip6"), primary_ip6) is True: + log.debug2(f"Found existing host '{device.get_display_name()}' " + f"based on the primary IPv6 '{primary_ip6}'") + return device + + def get_object_relation(self, name, relation, fallback=None): + """ + + Parameters + ---------- + name: str + name of the object to find a relation for + relation: str + name of the config variable relation (i.e: vm_tag_relation) + fallback: str + fallback string if no relation matched + + Returns + ------- + data: str, list, None + string of matching relation or list of matching tags + """ + + resolved_list = list() + for single_relation in grab(self.settings, relation, fallback=list()): + object_regex = single_relation.get("object_regex") + match_found = False + if object_regex.match(name): + resolved_name = single_relation.get("assigned_name") + log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) for {name}") + resolved_list.append(resolved_name) + match_found = True + + # special cluster condition + if match_found is False and grab(f"{relation}".split("_"), "0") == "cluster": + + stripped_name = "/".join(name.split("/")[1:]) + if object_regex.match(stripped_name): + + resolved_name = single_relation.get("assigned_name") + log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) " + f"for {stripped_name}") + resolved_list.append(resolved_name) + + if grab(f"{relation}".split("_"), "1") == "tag": + return resolved_list + + else: + resolved_name = fallback + if len(resolved_list) >= 1: + resolved_name = resolved_list[0] + if len(resolved_list) > 1: + log.debug(f"Found {len(resolved_list)} matches for {name} in {relation}." + f" Using first on: {resolved_name}") + + return resolved_name + + def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, vnic_data=None, + nic_ips=None, p_ipv4=None, p_ipv6=None, ovirt_object=None, + keep_undiscovered_ips=False): + """ + Add/update device/VM object in inventory based on gathered data. + + Try to find object first based on the object data, interface MAC addresses and primary IPs. + 1. try to find by name and cluster/site + 2. try to find by mac addresses interfaces + 3. try to find by primary IP + + IP addresses for each interface are added here as well. First they will be checked and added + if all checks pass. For each IP address a matching IP prefix will be searched for. First we + look for longest matching IP Prefix in the same site. If this failed we try to find the longest + matching global IP Prefix. + + If a IP Prefix was found then we try to get the VRF and VLAN for this prefix. Now we compare + if interface VLAN and prefix VLAN match up and warn if they don't. Then we try to add data to + the IP address if not already set: + + add prefix VRF if VRF for this IP is undefined + add tenant if tenant for this IP is undefined + 1. try prefix tenant + 2. if prefix tenant is undefined try VLAN tenant + + And we also set primary IP4/6 for this object depending on the "set_primary_ip" setting. + + If a IP address is set as primary IP for another device then using this IP on another + device will be rejected by NetBox. + + Setting "always": + check all NBDevice and NBVM objects if this IP address is set as primary IP to any + other object then this one. If we found another object, then we unset the primary_ip* + for the found object and assign it to this object. + + This setting will also reset the primary IP if it has been changed in NetBox + + Setting "when-undefined": + Will set the primary IP for this object if primary_ip4/6 is undefined. Will cause a + NetBox error if IP has been assigned to a different object as well + + Setting "never": + Well, the attribute primary_ip4/6 will never be touched/changed. + + Parameters + ---------- + object_type: (NBDevice, NBVM) + NetBoxObject sub class of object to add + object_data: dict + data of object to add/update + pnic_data: dict + data of physical interfaces of this object, interface name as key + vnic_data: dict + data of virtual interfaces of this object, interface name as key + nic_ips: dict + dict of ips per interface of this object, interface name as key + p_ipv4: str + primary IPv4 as string including netmask/prefix + p_ipv6: str + primary IPv6 as string including netmask/prefix + + """ + + if object_type not in [NBDevice, NBVM]: + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + pnic_data = pnic_data or {} + vnic_data = vnic_data or {} + nic_ips = nic_ips or {} + + if log.level == DEBUG3: + + log.debug3("function: add_device_vm_to_inventory") + log.debug3(f"Object type {object_type}") + pprint.pprint(object_data) + pprint.pprint(pnic_data) + pprint.pprint(vnic_data) + pprint.pprint(nic_ips) + pprint.pprint(p_ipv4) + pprint.pprint(p_ipv6) + + # check existing Devices for matches + log.debug2(f"Trying to find a {object_type.name} based on the collected name, cluster, IP and MAC addresses") + + device_vm_object = self.inventory.get_by_data(object_type, data=object_data) + + if device_vm_object is not None: + log.debug2("Found a exact matching %s object: %s" % + (object_type.name, device_vm_object.get_display_name(including_second_key=True))) + + # keep searching if no exact match was found + else: + + log.debug2(f"No exact match found. Trying to find {object_type.name} based on MAC addresses") + + # on VMs vnic data is used, on physical devices pnic data is used + mac_source_data = vnic_data if object_type == NBVM else pnic_data + + nic_macs = [x.get("mac_address") for x in mac_source_data.values() if x.get("mac_address") is not None] + + device_vm_object = self.get_object_based_on_macs(object_type, nic_macs) + + if device_vm_object is not None: + log.debug2("Found a matching %s object: %s" % + (object_type.name, device_vm_object.get_display_name(including_second_key=True))) + + # keep looking for devices with the same primary IP + else: + + log.debug2(f"No match found. Trying to find {object_type.name} based on primary IP addresses") + + device_vm_object = self.get_object_based_on_primary_ip(object_type, p_ipv4, p_ipv6) + + if device_vm_object is None: + object_name = object_data.get(object_type.primary_key) + log.debug(f"No existing {object_type.name} object for {object_name}. Creating a new {object_type.name}.") + device_vm_object = self.inventory.add_object(object_type, data=object_data, source=self) + else: + device_vm_object.update(data=object_data, source=self) + + # add object to cache + self.add_object_to_cache(ovirt_object, device_vm_object) + + # update role according to config settings + object_name = object_data.get(object_type.primary_key) + role_name = self.get_object_relation(object_name, + "host_role_relation" if object_type == NBDevice else "vm_role_relation") + + if object_type == NBDevice: + if role_name is None: + role_name = "Server" + device_vm_object.update(data={"device_role": {"name": role_name}}) + if object_type == NBVM and role_name is not None: + device_vm_object.update(data={"role": {"name": role_name}}) + + # compile all nic data into one dictionary + if object_type == NBVM: + nic_data = vnic_data + interface_exclude_filter = self.settings.vm_interface_exclude_filter + else: + nic_data = {**pnic_data, **vnic_data} + interface_exclude_filter = self.settings.host_interface_exclude_filter + + if interface_exclude_filter is not None: + for interface in self.inventory.get_all_interfaces(device_vm_object): + interface_name = grab(interface, "data.name") + if interface_name is not None and interface_exclude_filter.match(interface_name) and \ + self.source_tag in interface.get_tags(): + interface.set_source(self) + + for interface_name in list(nic_data): + if interface_exclude_filter.match(interface_name): + log.debug(f"Discovered interface '{interface_name}' matches interface_exclude_filter. " + "Excluding it from sync") + del nic_data[interface_name] + nic_ips.pop(interface_name, None) + + # map interfaces of existing object with discovered interfaces + nic_object_dict = self.map_object_interfaces_to_current_interfaces(device_vm_object, nic_data) + + if object_data.get("status", "") == "active" and (nic_ips is None or len(nic_ips.keys()) == 0): + log.debug(f"No IP addresses for '{object_name}' found!") + + primary_ipv4_object = None + primary_ipv6_object = None + + if p_ipv4 is not None: + try: + primary_ipv4_object = ip_interface(p_ipv4) + except ValueError: + log.error(f"Primary IPv4 ({p_ipv4}) does not appear to be a valid IP address (needs included suffix).") + + if p_ipv6 is not None: + try: + primary_ipv6_object = ip_interface(p_ipv6) + except ValueError: + log.error(f"Primary IPv6 ({p_ipv6}) does not appear to be a valid IP address (needs included suffix).") + + for int_name, int_data in nic_data.items(): + + # add/update interface with retrieved data + nic_object, ip_address_objects = self.add_update_interface(nic_object_dict.get(int_name), device_vm_object, + int_data, nic_ips.get(int_name, list()), + keep_undiscovered_ips=keep_undiscovered_ips) + + # add all interface IPs + for ip_object in ip_address_objects: + + if ip_object is None: + continue + + ip_interface_object = ip_interface(grab(ip_object, "data.address")) + + # continue if address is not a primary IP + if ip_interface_object not in [primary_ipv4_object, primary_ipv6_object]: + continue + + # set/update/remove primary IP addresses + set_this_primary_ip = False + ip_version = ip_interface_object.ip.version + if self.settings.set_primary_ip == "always": + + for object_type in [NBDevice, NBVM]: + + # new IPs don't need to be removed from other devices/VMs + if ip_object.is_new is True: + break + + for devices_vms in self.inventory.get_all_items(object_type): + + # device has no primary IP of this version + this_primary_ip = grab(devices_vms, f"data.primary_ip{ip_version}") + + # we found this exact object + if devices_vms == device_vm_object: + continue + + # device has the same object assigned + if this_primary_ip == ip_object: + devices_vms.unset_attribute(f"primary_ip{ip_version}") + + set_this_primary_ip = True + + elif self.settings.set_primary_ip != "never" and \ + grab(device_vm_object, f"data.primary_ip{ip_version}") is None: + set_this_primary_ip = True + + if set_this_primary_ip is True: + + log.debug(f"Setting IP '{grab(ip_object, 'data.address')}' as primary IPv{ip_version} for " + f"'{device_vm_object.get_display_name()}'") + device_vm_object.update(data={f"primary_ip{ip_version}": ip_object}) + + return device_vm_object + + def add_object_to_cache(self, vm_object, netbox_object): + + if None in [vm_object, netbox_object]: + return + + vm_class_name = vm_object.__class__.__name__ + vm_object_id = vm_object.id + + if self.object_cache.get(vm_class_name) is None: + self.object_cache[vm_class_name] = dict() + + self.object_cache[vm_class_name][vm_object_id] = netbox_object + + def get_object_from_cache(self, vm_object): + + if vm_object is None: + return + + vm_class_name = vm_object.__class__.__name__ + vm_object_id = vm_object.id + + if self.object_cache.get(vm_class_name) is None: + return + + return self.object_cache[vm_class_name].get(vm_object_id) + + def add_datacenter(self, obj): + """ + Add a oVirt datacenter as a NBClusterGroup to NetBox + + Parameters + ---------- + obj: types.DataCenter + datacenter object + + """ + if self.settings.set_source_name_as_cluster_group is True: + name = self.name + else: + name = get_string_or_none(grab(obj, "name")) + + if name is None: + return + + log.debug(f"Parsing oVirt datacenter: {name}") + + object_data = {"name": name} + + if self.settings.set_source_name_as_cluster_group is True: + label = "Datacenter Name" + custom_field = self.add_update_custom_field({ + "name": f"ovirt_{label}", + "label": label, + "object_types": ["virtualization.clustergroup"], + "type": "text", + "description": f"oVirt '{self.name}' synced custom attribute '{label}'" + }) + + object_data["custom_fields"] = { + grab(custom_field, "data.name"): get_string_or_none(grab(obj, "name")) + } + + self.add_object_to_cache(obj, self.inventory.add_update_object(NBClusterGroup, data=object_data, source=self)) + + def add_cluster(self, obj): + """ + Add a oVirt Cluster as a NBCluster to NetBox. Cluster name is checked against + cluster_include_filter and cluster_exclude_filter config setting. Also adds + cluster and site_name to "self.permitted_clusters" so hosts and VMs can be + checked if they are part of a permitted cluster. + + Parameters + ---------- + obj: types.Cluster + cluster to add + """ + + name = get_string_or_none(grab(obj, "name")) + if self.settings.set_source_name_as_cluster_group is True: + group = self.inventory.get_by_data(NBClusterGroup, data={"name": self.name}) + else: + group = self.get_object_from_cache(obj.data_center) + + if name is None or group is None: + return + + group_name = grab(group, "data.name") + full_cluster_name = f"{group_name}/{name}" + + log.debug(f"Parsing oVirt Cluster: {full_cluster_name}") + + # check for full name and then for cluster name only + include = self.settings.cluster_include_filter + exclude = self.settings.cluster_exclude_filter + if (include is not None and not any(include.match(value) for value in (full_cluster_name, name))) or \ + (exclude is not None and any(exclude.match(value) for value in (full_cluster_name, name))): + return + + site_name = self.get_site_name(NBCluster, full_cluster_name) + + data = { + "name": name, + "type": {"name": "oVirt"}, + "group": group + } + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + data["scope_type"] = "dcim.site" + data["scope_id"] = self.inventory.add_update_object(NBSite, data={"name": site_name}) + else: + data["site"] = {"name": site_name} + + tenant_name = self.get_object_relation(full_cluster_name, "cluster_tenant_relation") + if tenant_name is not None: + data["tenant"] = {"name": tenant_name} + + cluster_tags = self.get_object_relation(full_cluster_name, "cluster_tag_relation") + if len(cluster_tags) > 0: + data["tags"] = cluster_tags + + # try to find cluster including cluster group + log.debug2("Trying to find a matching existing cluster") + cluster_object = None + fallback_cluster_object = None + for cluster_candidate in self.inventory.get_all_items(NBCluster): + if grab(cluster_candidate, "data.name") != name: + continue + if cluster_candidate.get_site_name() not in (None, site_name) and \ + self.object_synced_by_other_source(cluster_candidate): + continue + + # try to find a cluster with matching site + if cluster_candidate.get_site_name() == site_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'site' are matching") + break + + if grab(cluster_candidate, "data.group") is not None and \ + grab(cluster_candidate, "data.group.data.name") == group_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'cluster group' are matching") + break + + if grab(cluster_candidate, "data.tenant") is not None and \ + tenant_name is not None and \ + grab(cluster_candidate, "data.tenant.data.name") == tenant_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'tenant' are matching") + break + + # if only the name matches and there are multiple cluster with the same name we choose the first + # cluster returned from netbox. This needs to be done to not ignore possible matches in one of + # the next iterations + if fallback_cluster_object is None: + fallback_cluster_object = cluster_candidate + + if cluster_object is None and fallback_cluster_object is not None: + log.debug2(f"Found an existing cluster where 'name' " + f"matches (NetBox id: {fallback_cluster_object.get_nb_reference()})") + cluster_object = fallback_cluster_object + + if cluster_object is not None: + cluster_object.update(data=data, source=self) + else: + cluster_object = self.inventory.add_update_object(NBCluster, data=data, source=self) + + self.add_object_to_cache(obj, cluster_object) + + def add_host(self, obj): + """ + Parse a oVirt host to NetBox once all data is gathered. + + First host is filtered: + host has a cluster and is it permitted + was host with same name and site already parsed + does the host pass the host_include_filter and host_exclude_filter + + Then all necessary host data will be collected. + + Primary IPv4/6 will be determined by 'address' value + Other IP's are listed in the 'network attachments' + + Parameters + ---------- + obj: types.Host + host object to parse + """ + + name = get_string_or_none(grab(obj, "name")) + + if name is not None and self.settings.strip_host_domain_name is True: + name = name.split(".")[0] + + # parse data + log.debug(f"Parsing oVirt host: {name}") + + # + # Filtering + # + + # manage site and cluster + cluster_object = obj.cluster + + if cluster_object is None: + log.error(f"Requesting cluster for host '{name}' failed. Skipping.") + return + + if log.level == DEBUG3: + try: + log.info("Cluster data") + dump(cluster_object) + except Exception as e: + log.error(e) + + # get cluster object + nb_cluster_object = self.get_object_from_cache(cluster_object) + + if nb_cluster_object is None: + log.debug(f"Host '{name}' is not part of a permitted cluster. Skipping") + return + + cluster_name = get_string_or_none(grab(nb_cluster_object, "data.name")) + + # get a site for this host + if self.settings.set_source_name_as_cluster_group is True: + group = self.inventory.get_by_data(NBClusterGroup, data={"name": self.name}) + else: + group = grab(nb_cluster_object, "data.group") + group_name = grab(group, "data.name") + site_name = self.get_site_name(NBDevice, name, f"{group_name}/{cluster_name}") + + if name in self.processed_host_names.get(site_name, list()): + log.warning(f"Host '{name}' for site '{site_name}' already parsed. " + "Make sure to use unique host names. Skipping") + return + + # add host to processed list + if self.processed_host_names.get(site_name) is None: + self.processed_host_names[site_name] = list() + + self.processed_host_names[site_name].append(name) + + # filter hosts by name + if self.passes_filter(name, self.settings.host_include_filter, self.settings.host_exclude_filter) is False: + return + + # + # Collecting data + # + + # collect all necessary data + manufacturer = get_string_or_none(grab(obj, "hardware_information.manufacturer")) + model = get_string_or_none(grab(obj, "hardware_information.product_name")) + platform = " ".join(str(value) for value in ( + grab(obj, "os.type"), grab(obj, "os.version.full_version")) if value is not None) or None + + # if the device vendor/model cannot be retrieved (due to problem on the host), + # set a dummy value so the host still gets synced + if manufacturer is None: + manufacturer = "Generic Vendor" + if model is None: + model = "Generic Model" + + # get status + status = "offline" + if obj.status == types.HostStatus.UP: + status = "active" + + # try to find serial + serial = get_string_or_none(grab(obj, "hardware_information.uuid")) + + # add asset tag if desired and present + asset_tag = get_string_or_none(grab(obj, "hardware_information.serial_number")) + + # get host_tenant_relation + tenant_name = self.get_object_relation(name, "host_tenant_relation") + + # get host_tag_relation + host_tags = self.get_object_relation(name, "host_tag_relation") + + # prepare host data model + host_data = { + "name": name, + "device_type": { + "model": model, + "manufacturer": { + "name": manufacturer + } + }, + "site": {"name": site_name}, + "cluster": nb_cluster_object, + "status": status + } + + # add data if present + if serial is not None: + host_data["serial"] = serial + if asset_tag is not None: + host_data["asset_tag"] = asset_tag + if platform is not None: + host_data["platform"] = {"name": platform} + if tenant_name is not None: + host_data["tenant"] = {"name": tenant_name} + if len(host_tags) > 0: + host_data["tags"] = host_tags + self.add_custom_attributes(obj, host_data, NBDevice) + + # Get all NICs information + pnic_data_dict = dict() + vnic_data_dict = dict() + vnic_ips = dict() + + host_primary_ip4 = None + host_primary_ip6 = None + host_address = get_string_or_none(grab(obj, "address")) + host_addresses = set() + if host_address is not None: + try: + host_addresses.add(ip_address(host_address)) + except ValueError: + try: + host_addresses.update( + ip_address(result[4][0]) + for result in socket.getaddrinfo(host_address, None, type=socket.SOCK_STREAM) + ) + except (OSError, ValueError) as e: + log.warning(f"Unable to resolve oVirt host address '{host_address}': {e}") + + for nic in obj.nics or []: + nic_type = "other" + if nic.speed: + nic_type = NetBoxInterfaceType(nic.speed / 1000000).get_this_netbox_type() + + mac_address = None + if nic.mac: + mac_address = normalize_mac_address(grab(nic, "mac.address")) + if nic.base_interface is None and nic.bonding is None: + # Physical Interface + pnic_data = { + "name": nic.name, + "device": None, + "enabled": (nic.status == types.NicStatus.UP), + "type": nic_type, + "mtu": nic.mtu, + "mac_address": mac_address + } + pnic_data_dict[nic.name] = pnic_data + elif nic.bonding is not None: + # Bond + vnic_data = { + "name": nic.name, + "device": None, + "enabled": (nic.status == types.NicStatus.UP), + "type": "virtual", + "mtu": nic.mtu, + "mac_address": mac_address + } + vnic_data_dict[nic.name] = vnic_data + else: + # Bridge + vnic_data = { + "name": nic.name, + "device": None, + "enabled": (nic.status == types.NicStatus.UP), + "type": "virtual", + "mtu": nic.mtu, + "mac_address": mac_address + } + if nic.vlan is not None: + vnic_data["tagged_vlans"] = [{ + "name": f"VLAN-{nic.vlan.id}", + "vid": nic.vlan.id, + "site": { + "name": site_name + }}] + vnic_data_dict[nic.name] = vnic_data + # IP Info + if nic.ip is not None and nic.ip.address is not None and nic.ip.netmask is not None: + int_v4 = "{}/{}".format(nic.ip.address, nic.ip.netmask) + vnic_name = nic.name + if vnic_ips.get(vnic_name) is None: + vnic_ips[vnic_name] = list() + + if self.settings.permitted_subnets.permitted(int_v4, interface_name=vnic_name) is True: + vnic_ips[vnic_name].append(int_v4) + + if host_primary_ip4 is None and ip_address(nic.ip.address) in host_addresses: + host_primary_ip4 = int_v4 + + if nic.ipv6 is not None and nic.ipv6.address is not None and nic.ipv6.netmask is not None: + int_v6 = "{}/{}".format(nic.ipv6.address, nic.ipv6.netmask) + vnic_name = nic.name + if vnic_ips.get(vnic_name) is None: + vnic_ips[vnic_name] = list() + + if self.settings.permitted_subnets.permitted(int_v6, interface_name=vnic_name) is True: + vnic_ips[vnic_name].append(int_v6) + if host_primary_ip6 is None and ip_address(nic.ipv6.address) in host_addresses: + host_primary_ip6 = int_v6 + + # add host to inventory + self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, + vnic_data=vnic_data_dict, nic_ips=vnic_ips, + p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6, ovirt_object=obj) + + return + + def add_virtual_machine(self, obj): + """Sync a VM, including stopped VMs without host or guest-agent data.""" + name = get_string_or_none(obj.name) + if name is None: + log.error(f"VM '{obj.id}' has no name. Skipping.") + return + if name is not None and self.settings.strip_vm_domain_name is True: + name = name.split(".")[0] + + vm_uuid = obj.id + if self.settings.set_vm_name_to_uuid: + name = vm_uuid + log.debug(f"Parsing oVirt VM: {name}") + status = "active" if obj.status == types.VmStatus.UP else "offline" + nb_cluster_object = self.get_object_from_cache(obj.cluster) + if nb_cluster_object is None: + log.debug(f"Virtual machine '{name}' is not part of a permitted cluster. Skipping") + return + group_name = grab(nb_cluster_object, "data.group.data.name") + cluster_name = grab(nb_cluster_object, "data.name") + cluster_full_name = f"{group_name}/{cluster_name}" + if name in self.processed_vm_names.get(cluster_full_name, []) and \ + vm_uuid not in self.processed_vm_uuid: + log.warning(f"Virtual machine '{name}' for cluster '{cluster_full_name}' already parsed. " + "Make sure to use unique VM names. Skipping") + return + + if self.passes_filter(name, self.settings.vm_include_filter, self.settings.vm_exclude_filter) is False: + return + self.processed_vm_names.setdefault(cluster_full_name, []).append(name) + self.processed_vm_uuid.append(vm_uuid) + + site_name = nb_cluster_object.get_site_name() + api_version = version.parse(self.inventory.netbox_api_version) + tenant_name = self.get_object_relation(name, "vm_tenant_relation") + vm_tags = self.get_object_relation(name, "vm_tag_relation") + vm_data = { + "name": name, + "cluster": nb_cluster_object, + "status": status, + "memory": int(obj.memory / 1024**2) if obj.memory is not None else None, + } + cores = grab(obj, "cpu.topology.cores") + sockets = grab(obj, "cpu.topology.sockets") + if cores is not None and sockets is not None: + vm_data["vcpus"] = cores * sockets + if api_version >= version.parse("3.3.0"): + vm_data["site"] = {"name": site_name} + if api_version >= version.parse("4.1.0"): + vm_data["serial"] = vm_uuid + host_object = self.get_object_from_cache(obj.host) + if host_object is not None: + vm_data["device"] = host_object + if self.settings.skip_vm_platform is False: + platform = " ".join(str(value) for value in ( + grab(obj, "guest_operating_system.distribution"), + grab(obj, "guest_operating_system.version.full_version")) if value).strip() or None + if platform is not None: + platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) + vm_data["platform"] = {"name": platform} + if self.settings.skip_vm_comments is False and not self.settings.set_vm_name_to_uuid: + vm_data["comments"] = vm_uuid + if tenant_name is not None: + vm_data["tenant"] = {"name": tenant_name} + if len(vm_tags) > 0: + vm_data["tags"] = vm_tags + + self.add_custom_attributes(obj, vm_data, NBVM) + if self.settings.set_vm_name_to_uuid: + custom_field = self.add_update_custom_field({ + "name": "ovirt_vm_name", + "label": "name", + "object_types": ["virtualization.virtualmachine"], + "type": "text", + "description": f"oVirt '{self.name}' synced object attribute 'name'", + }) + vm_data.setdefault("custom_fields", {})[custom_field.data["name"]] = obj.name + + disk_data = [] + total_disk = 0 + disk_sizes_complete = True + for attachment in obj.disk_attachments or []: + disk = self.disk_cache.get(attachment.disk.id) + size = grab(disk, "provisioned_size") + if size is None: + log.warning(f"VM '{name}' disk '{attachment.disk.id}' has no provisioned size. Skipping disk.") + disk_sizes_complete = False + continue + total_disk += size + unit = 1024**2 if api_version >= version.parse("4.1.0") else 1024**3 + disk_data.append({ + "name": grab(disk, "alias") or grab(disk, "name") or attachment.disk.id, + "size": max(1, int(size / unit)), + }) + if api_version < version.parse("3.7.0") and disk_sizes_complete: + vm_data["disk"] = int(total_disk / 1024**3) + + vm_primary_ip4 = None + vm_primary_ip6 = None + vm_nic_dict = {} + nic_ips = {} + nic_names_by_mac = {} + for nic in obj.nics or []: + mac = normalize_mac_address(grab(nic, "mac.address")) + vm_nic_dict[nic.name] = { + "name": nic.name, "mac_address": mac, "enabled": nic.plugged is not False, + } + if mac is not None: + nic_names_by_mac[mac] = nic.name + + for reporteddevice in obj.reported_devices or []: + if reporteddevice.type != types.ReportedDeviceType.NETWORK: + continue + nic = reporteddevice.name + mac = normalize_mac_address(grab(reporteddevice, "mac.address")) + hardware_name = nic_names_by_mac.get(mac) + if hardware_name is not None and hardware_name != nic: + vm_nic_dict.pop(hardware_name, None) + vm_nic_dict[nic] = { + "name": nic, "mac_address": mac, "description": nic, "enabled": True, + } + for ip in reporteddevice.ips or []: + try: + address = ip_address(ip.address) + except ValueError: + log.warning(f"VM '{name}' reported invalid address '{ip.address}'. Skipping address.") + continue + matched_prefix = self.return_longest_matching_prefix_for_ip(address, site_name) + if matched_prefix is None: + matched_prefix = self.return_longest_matching_prefix_for_ip(address) + ip_addr = str(address) + if matched_prefix is not None: + this_prefix = grab(matched_prefix, "data.prefix") + ip_addr = f"{ip_addr}/{this_prefix.prefixlen}" + if self.settings.permitted_subnets.permitted(ip_addr, interface_name=nic) is True: + nic_ips.setdefault(nic, []).append(ip_addr) + if address.version == 4 and vm_primary_ip4 is None: + vm_primary_ip4 = ip_addr + if address.version == 6 and vm_primary_ip6 is None: + vm_primary_ip6 = ip_addr + else: + log.debug(f"Virtual machine '{name}' address '{ip_addr}' is not valid to add. Skipping") + + nbvm = self.add_device_vm_to_inventory( + NBVM, object_data=vm_data, vnic_data=vm_nic_dict, nic_ips=nic_ips, + p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6, ovirt_object=obj, + keep_undiscovered_ips=not bool(obj.reported_devices)) + if api_version >= version.parse("3.7.0"): + for data in disk_data: + data["virtual_machine"] = nbvm + self.inventory.add_update_object(NBVirtualDisk, data=data, source=self) + + def add_custom_attributes(self, obj, data, object_type): + attributes = self.settings.host_custom_object_attributes if object_type == NBDevice \ + else self.settings.vm_custom_object_attributes + for attribute in attributes or []: + value = grab(obj, attribute) + if value is None: + log.debug(f"oVirt object '{obj.id}' has no custom attribute '{attribute}'.") + continue + field = self.add_update_custom_field({ + "name": f"ovirt_{attribute}", + "label": attribute, + "object_types": [object_type.object_type], + "type": "text", + "description": f"oVirt '{self.name}' synced object attribute '{attribute}'", + }) + data.setdefault("custom_fields", {})[field.data["name"]] = str(value) + + def update_basic_data(self): + """ + + Returns + ------- + + """ + + # add source identification tag + self.inventory.add_update_object(NBTag, data={ + "name": self.source_tag, + "description": f"Marks objects synced from oVirt '{self.name}' " + f"({self.settings.url}) to this NetBox Instance." + }) + + # update virtual site if present + this_site_object = self.inventory.get_by_data(NBSite, data={"name": self.site_name}) + + if this_site_object is not None: + this_site_object.update(data={ + "name": self.site_name, + "comments": "A default virtual site created to house objects " + "that have been synced from this oVirt instance " + "and have no predefined site assigned." + }) + + server_role_object = self.inventory.get_by_data(NBDeviceRole, data={"name": "Server"}) + + if server_role_object is not None: + role_data = {"name": "Server", "vm_role": True} + if server_role_object.is_new is True: + role_data["color"] = "9e9e9e" + + server_role_object.update(data=role_data) + +# EOF diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index cde3c534..86547f4f 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -2425,29 +2425,6 @@ def add_datacenter(self, obj): self.add_object_to_cache(obj, self.inventory.add_update_object(NBClusterGroup, data=object_data, source=self)) - def object_synced_by_other_source(self, nb_object): - """ - Check if a NetBox object is maintained by a different configured source. During - this run that is the source which touched the object, for objects read from NetBox - the source tags decide. - - Parameters - ---------- - nb_object: NetBoxObject - object to check - - Returns - ------- - bool: True if another configured source synced this object - """ - - if nb_object.source is not None: - return nb_object.source is not self - - other_source_tags = [x.source_tag for x in self.inventory.source_list if x is not self] - - return len(set(nb_object.get_tags()).intersection(other_source_tags)) > 0 - def add_cluster(self, obj): """ Add a vCenter cluster as a NBCluster to NetBox. Cluster name is checked against diff --git a/pyproject.toml b/pyproject.toml index 0cf28b08..d691929e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,9 @@ dependencies = [ aws = [ "boto3>=1.40,<2", ] +ovirt = [ + "ovirt-engine-sdk-python==4.6.3", +] test = [ "pytest>=9.1.1", ] diff --git a/requirements-ovirt.txt b/requirements-ovirt.txt new file mode 100644 index 00000000..2a582b34 --- /dev/null +++ b/requirements-ovirt.txt @@ -0,0 +1,2 @@ +-r requirements.txt +ovirt-engine-sdk-python==4.6.3 diff --git a/settings-example.ini b/settings-example.ini index bf8ab5af..994467df 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -748,4 +748,38 @@ regions = auto ;organization_account_ids = 111122223333 ; Leave organization_external_id unset unless the member trust policy requires it. +[source/my-ovirt-example] + +; Optional SDK: pip install -r requirements-ovirt.txt +; Full options can be generated with -g; see docs/source_ovirt.md. +;enabled = True +;type = ovirt +url = +username = +password = +;validate_tls_certs = True +;ca_file = /etc/pki/ovirt-engine/ca.pem +;permitted_subnets = 10.0.0.0/8, fd00::/8 +;cluster_site_relation = DC/Production = Main Site +;cluster_include_filter = +;cluster_exclude_filter = +;host_include_filter = +;host_exclude_filter = +;vm_include_filter = +;vm_exclude_filter = +;host_interface_exclude_filter = +;vm_interface_exclude_filter = +;ip_tenant_inheritance_order = device, prefix +;set_primary_ip = when-undefined +;preserve_primary_ips = False +;skip_fhrp_group_ips = False +;skip_vm_comments = False +;skip_vm_platform = False +;set_vm_name_to_uuid = False +;set_source_name_as_cluster_group = False +;strip_host_domain_name = False +;strip_vm_domain_name = False +;host_custom_object_attributes = id +;vm_custom_object_attributes = id + ;EOF diff --git a/tests/test_ovirt_source.py b/tests/test_ovirt_source.py new file mode 100644 index 00000000..3dc49940 --- /dev/null +++ b/tests/test_ovirt_source.py @@ -0,0 +1,349 @@ +import os +import socket +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from module.config.file_output import ConfigFileOutput +from module.config.parser import ConfigParser +from module.netbox.connection import NetBoxHandler +from module.netbox import ( + NBCluster, NBDevice, NBInterface, NBIPAddress, NBMACAddress, NBPrefix, NBSite, + NBVirtualDisk, NBVM, NBVMInterface, +) +from module.sources import instantiate_sources, valid_sources +from module.sources.ovirt import connection +from module.sources.ovirt.config import OVirtConfig + + +def obj(**kwargs): + return SimpleNamespace(**kwargs) + + +@pytest.fixture(autouse=True) +def windows_config_paths(monkeypatch): + if os.name == "nt": + monkeypatch.setattr(ConfigParser, "get_config_file_path", staticmethod(os.path.abspath)) + + +@pytest.fixture +def ovirt_source(inventory, load_config, monkeypatch): + def make(extra="", api_version="4.3.0"): + inventory.netbox_api_version = api_version + load_config(f""" +[source/test] +type = ovirt +url = https://engine.example.com/ovirt-engine/api +username = admin@internal +password = test-only +permitted_subnets = 10.0.0.0/8, 2001:db8::/32 +dns_name_lookup = False +{extra} +""") + monkeypatch.setattr(connection, "types", obj( + HostStatus=obj(UP="up"), NicStatus=obj(UP="up"), + VmStatus=obj(UP="up"), ReportedDeviceType=obj(NETWORK="network"), + IpVersion=obj(V4="v4", V6="v6"), + )) + session = Mock() + monkeypatch.setattr(connection, "sdk", obj(Connection=Mock(return_value=session))) + sources = instantiate_sources() + assert len(sources) == 1 + source = sources[0] + assert source.init_successful + return source + return make + + +def populate(source, status="up", reported=True, disks=True, host_address="10.0.0.1"): + datacenter = obj(id="dc", name="DC") + cluster = obj(id="cluster", name="Production", data_center=datacenter) + source.add_datacenter(datacenter) + source.add_cluster(cluster) + nic = obj( + name="eth0", mac=obj(address="52:54:00:00:00:01"), speed=1000000000, + base_interface=None, bonding=None, status="up", mtu=1500, vlan=None, + ip=obj(address="10.0.0.1", netmask="255.255.255.0"), ipv6=None, + ) + host = obj( + id="host", name="host.example.com", cluster=obj(id="cluster"), + hardware_information=obj( + manufacturer="Example", product_name="Server", uuid="host-uuid", + serial_number="asset", + ), + os=obj(type="Linux", version=obj(full_version="9")), status="up", + nics=[nic], address=host_address, + ) + source.add_host(host) + vm = obj( + id="vm-uuid", name="vm.example.com", status=status, + host=obj(id="host") if status == "up" else None, cluster=obj(id="cluster"), + memory=2 * 1024**3, cpu=obj(topology=obj(cores=2, sockets=2, threads=1)), + os=obj(type="linux"), + guest_operating_system=obj(distribution="Debian", version=obj(full_version="12")), + disk_attachments=[obj(disk=obj(id="disk"))] if disks else [], + nics=[obj(name="nic1", mac=obj(address="52:54:00:00:00:02"), plugged=True)], + reported_devices=[ + obj(name="ens3", type="network", mac=obj(address="52:54:00:00:00:02"), + ips=[obj(address="10.0.0.2", version="v4"), + obj(address="2001:db8::2", version="v6")]), + ] if reported else [], + ) + source.disk_cache["disk"] = obj( + id="disk", name="root", alias="root", provisioned_size=10 * 1024**3, + actual_size=5 * 1024**3, + ) + return vm, host + + +def test_source_and_config_are_registered(): + assert connection.OVirtHandler in valid_sources + assert OVirtConfig in ConfigFileOutput.source_config_list + assert NBMACAddress in connection.OVirtHandler.dependent_netbox_objects + assert NBVirtualDisk in connection.OVirtHandler.dependent_netbox_objects + + +@pytest.mark.parametrize("api_version", ["3.6.0", "3.7.0", "4.1.0", "4.3.0"]) +def test_host_vm_network_and_disk_sync(ovirt_source, inventory, api_version): + source = ovirt_source(api_version=api_version) + inventory.add_object(NBPrefix, data={"id": 1, "prefix": "10.0.0.0/24"}, read_from_netbox=True) + inventory.add_object(NBPrefix, data={"id": 2, "prefix": "2001:db8::/64"}, read_from_netbox=True) + vm, _ = populate(source) + source.add_virtual_machine(vm) + nbvm, = inventory.get_all_items(NBVM) + host, = inventory.get_all_items(NBDevice) + cluster, = inventory.get_all_items(NBCluster) + assert nbvm.data["vcpus"] == 4 + assert nbvm.data["memory"] == 2048 + assert nbvm.data["platform"].data["name"] == "Debian 12" + assert nbvm.data["device"] is host + assert cluster.get_site_name() == source.site_name + assert nbvm.data["primary_ip4"].data["address"] == "10.0.0.2/24" + assert nbvm.data["primary_ip6"].data["address"] == "2001:db8::2/64" + assert host.data["primary_ip4"].data["address"] == "10.0.0.1/24" + assert inventory.get_all_items(NBInterface)[0].data["type"] == "1000base-t" + interface, = inventory.get_all_items(NBVMInterface) + assert interface.data["name"] == "ens3" + assert len(inventory.get_all_items(NBIPAddress)) == 3 + if api_version >= "4.2": + assert cluster.data["scope_type"] == "dcim.site" + assert isinstance(cluster.data["scope_id"], NBSite) + assert len(inventory.get_all_items(NBMACAddress)) == 2 + assert interface.data["primary_mac_address"].data["mac_address"] == "52:54:00:00:00:02" + if api_version < "3.7": + assert nbvm.data["disk"] == 10 + else: + disk, = inventory.get_all_items(NBVirtualDisk) + assert disk.data["virtual_machine"] is nbvm + assert disk.data["size"] == (10 if api_version < "4.1" else 10240) + source.add_virtual_machine(vm) + assert len(inventory.get_all_items(NBVM)) == 1 + assert len(inventory.get_all_items(NBVMInterface)) == 1 + + +def test_host_primary_ip_matches_fqdn(ovirt_source, inventory, monkeypatch): + source = ovirt_source() + inventory.add_object(NBPrefix, data={"id": 1, "prefix": "10.0.0.0/24"}, read_from_netbox=True) + monkeypatch.setattr(connection.socket, "getaddrinfo", Mock(return_value=[ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("10.0.0.1", 0)), + ])) + populate(source, host_address="host.example.com") + host, = inventory.get_all_items(NBDevice) + assert host.data["primary_ip4"].data["address"] == "10.0.0.1/24" + + +def test_offline_vm_without_agent_or_cpu_is_synced(ovirt_source, inventory): + source = ovirt_source() + vm, _ = populate(source, status="down", reported=False) + vm.cpu = None + vm.guest_operating_system = None + source.add_virtual_machine(vm) + nbvm, = inventory.get_all_items(NBVM) + assert nbvm.data["status"] == "offline" + assert "vcpus" not in nbvm.data + interface, = inventory.get_all_items(NBVMInterface) + assert interface.data["name"] == "nic1" + + +def test_excluded_vm_interface_is_not_created(ovirt_source, inventory): + source = ovirt_source("vm_interface_exclude_filter = ^nic1$") + vm, _ = populate(source) + vm.reported_devices[0].name = "nic1" + mac_count = len(inventory.get_all_items(NBMACAddress)) + + source.add_virtual_machine(vm) + + assert inventory.get_all_items(NBVMInterface) == [] + assert len(inventory.get_all_items(NBMACAddress)) == mac_count + + +def test_excluded_existing_vm_interface_remains_claimed(ovirt_source, inventory): + source = ovirt_source("vm_interface_exclude_filter = ^nic1$") + vm, _ = populate(source) + vm.reported_devices[0].name = "nic1" + cluster, = inventory.get_all_items(NBCluster) + existing_vm = inventory.add_object( + NBVM, data={"id": 1, "name": vm.name, "cluster": cluster}, read_from_netbox=True) + existing_interface = inventory.add_object( + NBVMInterface, + data={"id": 2, "name": "nic1", "virtual_machine": existing_vm, + "mac_address": "52:54:00:00:00:02"}, + read_from_netbox=True, + ) + existing_interface.add_tags([NetBoxHandler.primary_tag, source.source_tag]) + + source.add_virtual_machine(vm) + + assert existing_interface.source is source + assert inventory.get_all_items(NBVMInterface) == [existing_interface] + + +@pytest.mark.parametrize("guest_os", [ + None, + obj(distribution="", version=obj(full_version="")), +], ids=["missing", "empty"]) +def test_offline_vm_keeps_last_guest_platform(ovirt_source, inventory, guest_os): + source = ovirt_source() + vm, _ = populate(source) + source.add_virtual_machine(vm) + nbvm, = inventory.get_all_items(NBVM) + platform = nbvm.data["platform"] + + vm.status = "down" + vm.guest_operating_system = guest_os + source.add_virtual_machine(vm) + + assert nbvm.data["platform"] is platform + + +def test_unknown_disk_size_is_not_invented(ovirt_source, inventory, caplog): + source = ovirt_source() + vm, _ = populate(source) + source.disk_cache["disk"].provisioned_size = None + source.add_virtual_machine(vm) + assert len(inventory.get_all_items(NBVM)) == 1 + assert inventory.get_all_items(NBVirtualDisk) == [] + assert "provisioned size" in caplog.text + + +def test_vm_rename_matches_existing_primary_mac(ovirt_source, inventory): + source = ovirt_source() + vm, _ = populate(source) + source.add_virtual_machine(vm) + original, = inventory.get_all_items(NBVM) + vm.name = "renamed-vm" + source.add_virtual_machine(vm) + assert inventory.get_all_items(NBVM) == [original] + assert original.data["name"] == "renamed-vm" + + +def test_lost_guest_agent_preserves_existing_ip(ovirt_source, inventory): + source = ovirt_source() + inventory.add_object(NBPrefix, data={"id": 1, "prefix": "10.0.0.0/24"}, read_from_netbox=True) + vm, _ = populate(source) + source.add_virtual_machine(vm) + nbvm, = inventory.get_all_items(NBVM) + primary = nbvm.data["primary_ip4"] + vm.reported_devices = [] + vm.status = "down" + source.add_virtual_machine(vm) + assert nbvm.data["primary_ip4"] is primary + assert primary.data["assigned_object_id"].data["virtual_machine"] is nbvm + + +def test_cluster_filter_accepts_qualified_name(ovirt_source, inventory): + source = ovirt_source("cluster_include_filter = DC/Production") + vm, _ = populate(source) + source.add_virtual_machine(vm) + assert len(inventory.get_all_items(NBVM)) == 1 + + +def test_different_sources_do_not_move_each_others_clusters(ovirt_source, inventory): + source = ovirt_source() + vm, _ = populate(source) + original, = inventory.get_all_items(NBCluster) + other = connection.OVirtHandler.__new__(connection.OVirtHandler) + other.name = "other" + other.set_source_tag() + other.settings = source.settings + other.site_name = "Other site" + other.inventory = inventory + other.object_cache = {} + inventory.add_source(other) + datacenter = obj(id="dc-other", name="DC") + other.add_datacenter(datacenter) + other.add_cluster(obj(id="other-cluster", name="Production", data_center=datacenter)) + assert len(inventory.get_all_items(NBCluster)) == 2 + assert original.get_site_name() == source.site_name + + +def test_uuid_name_custom_attributes_and_skip_platform(ovirt_source, inventory): + source = ovirt_source(""" +set_vm_name_to_uuid = True +vm_custom_object_attributes = id +host_custom_object_attributes = id +skip_vm_platform = True +""") + vm, _ = populate(source) + source.add_virtual_machine(vm) + nbvm, = inventory.get_all_items(NBVM) + host, = inventory.get_all_items(NBDevice) + assert nbvm.data["name"] == vm.id + assert "platform" not in nbvm.data + assert nbvm.data["custom_fields"]["ovirt_vm_name"] == vm.name + assert nbvm.data["custom_fields"]["ovirt_id"] == vm.id + assert host.data["custom_fields"]["ovirt_id"] == "host" + + +def test_missing_sdk_is_reported_without_breaking_other_sources( + inventory, load_config, monkeypatch, caplog): + load_config(""" +[source/test] +type = ovirt +url = https://engine.example.com/ovirt-engine/api +username = admin@internal +password = test-only +""") + monkeypatch.setattr(connection, "sdk", None) + assert instantiate_sources() == [] + assert "requirements-ovirt.txt" in caplog.text + + +def test_connection_error_is_reported(ovirt_source, monkeypatch, caplog): + source = ovirt_source() + source.session = None + monkeypatch.setattr(connection.sdk, "Connection", Mock(side_effect=RuntimeError("unavailable"))) + assert source.create_ovirt_session() is False + assert "unavailable" in caplog.text + + +def test_authentication_error_skips_unavailable_source(ovirt_source, monkeypatch, caplog): + source = ovirt_source() + source.session = None + session = Mock() + session.test.side_effect = RuntimeError("access denied") + monkeypatch.setattr(connection.sdk, "Connection", Mock(return_value=session)) + + assert source.create_ovirt_session() is False + assert source.session is None + session.close.assert_called_once() + assert "access denied" in caplog.text + + +def test_apply_queries_related_data_and_closes_session(ovirt_source, inventory): + source = ovirt_source() + vm, host = populate(source) + system = source.session.system_service.return_value + system.disks_service.return_value.list.return_value = list(source.disk_cache.values()) + system.data_centers_service.return_value.list.return_value = [] + system.clusters_service.return_value.list.return_value = [] + system.hosts_service.return_value.list.return_value = [] + system.vms_service.return_value.list.return_value = [vm] + source.apply() + assert len(inventory.get_all_items(NBVM)) == 1 + source.session.test.assert_called_once_with(raise_exception=True) + system.vms_service.return_value.list.assert_called_once_with( + follow="nics,disk_attachments,reported_devices") + source.finish() + source.session.close.assert_called_once()