From 20cea7b0e4f60f60709ff92a0bbbb9b26e324a3f Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Wed, 24 Jun 2020 10:18:42 +0200 Subject: [PATCH 01/18] Add kickstart template for pxe test on rhv44 The test cfme/tests/infrastructure/test_pxe_provisioning.py::test_pxe_provision_from_template[rhv44] was failing because the former RHEL used for installation was RHEL 6.9 -- incompatible with Virtio network in RHV44 --- data/rhel8.cfg | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 data/rhel8.cfg diff --git a/data/rhel8.cfg b/data/rhel8.cfg new file mode 100644 index 0000000000..06439e1433 --- /dev/null +++ b/data/rhel8.cfg @@ -0,0 +1,81 @@ +<% + # Account for some missing values + evm[:root_password] = root_fallback_password if evm[:root_password].blank? + evm[:hostname] = evm[:vm_target_hostname] if evm[:hostname].blank? + evm[:addr_mode] = ['dhcp'] if evm[:ip_addr].blank? || evm[:subnet_mask].blank? || evm[:gateway].blank? + + rhn_activation_key = "" + + # Dynamically create the network string based on values from the dialog + if evm[:addr_mode].first == 'static' + network_string = "network --onboot yes --bootproto=static --noipv6" + ["ip", :ip_addr, "netmask", :subnet_mask, "gateway", :gateway, "hostname", :hostname, "nameserver", :dns_servers].each_slice(2) do |ks_key, evm_key| + network_string << " --#{ks_key} #{evm[evm_key]}" unless evm[evm_key].blank? + end + else + network_string = "network --onboot yes --bootproto=dhcp --noipv6" + network_string << " --#{"hostname"} #{evm[:hostname]}" unless evm[:hostname].blank? + end +%> +# Install OS instead of upgrade +install +# Firewall configuration +firewall --enabled --ssh --service=ssh +# Use network installation +url --url="$url1" +# Network information +network --bootproto=dhcp --device=eth0 +# Root password +rootpw --iscrypted <%=MiqPassword.md5crypt(evm[:root_password]) %> +# System authorization information +auth --useshadow --passalgo=sha512 +# Use text mode install +text +# System keyboard +keyboard us +# System language +lang en_US +# SELinux configuration +selinux --enforcing +# Do not configure the X Window System +skipx +# Installation logging level +logging --level=info +# Power Off after installation - Needed to complete EVM provisioning +shutdown +# System timezone +timezone America/New_York +# System bootloader configuration +# Clear the Master Boot Record +zerombr +# Partition clearing information +clearpart --all --initlabel +# Disk partitioning information +autopart --fstype=ext4 +bootloader --location=mbr --append="rhgb quiet" + +repo --name=rhel-x86_64-server-8 --baseurl=$url1 +repo --name=rhel-x86_64-server-optional-7 --baseurl=$url2 + +%packages +@base +@core +xorg-x11-xauth +nfs-utils +autofs +qemu-guest-agent +wget +%end + +%post --log=/root/kickstart-post.log +set -x + +dhclient + +systemctl enable ovirt-guest-agent.service +systemctl start ovirt-guest-agent.service + + +#Callback to CFME during post-install +wget --no-check-certificate <%=evm[:post_install_callback_url] %> +%end From 3ede072f784272f060badb8ae170c3d6d445f549 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Tue, 7 Jul 2020 19:57:01 +0200 Subject: [PATCH 02/18] Add RHEV-M 4.4 to supportability --- conf/supportability.yaml | 1 + conf/supportability.yaml.template | 2 ++ 2 files changed, 3 insertions(+) diff --git a/conf/supportability.yaml b/conf/supportability.yaml index 67feaaa136..a3016d0334 100644 --- a/conf/supportability.yaml +++ b/conf/supportability.yaml @@ -10,6 +10,7 @@ - 4.1 - 4.2 - 4.3 + - 4.4 - scvmm: - 2012 - 2016 diff --git a/conf/supportability.yaml.template b/conf/supportability.yaml.template index 2e3aeed073..735b593722 100644 --- a/conf/supportability.yaml.template +++ b/conf/supportability.yaml.template @@ -65,6 +65,8 @@ - rhevm: - 4.1 - 4.2 + - 4.3 + - 4.4 - scvmm: - 2012 - 2016 From 00d0feaac8a3ce72c3e170101afd9d3b4a8f18bf Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Mon, 13 Jul 2020 12:38:59 +0200 Subject: [PATCH 03/18] Add BZ 1783355 blocker --- cfme/tests/services/test_iso_service_catalogs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cfme/tests/services/test_iso_service_catalogs.py b/cfme/tests/services/test_iso_service_catalogs.py index 10f72eb913..5d5311c028 100644 --- a/cfme/tests/services/test_iso_service_catalogs.py +++ b/cfme/tests/services/test_iso_service_catalogs.py @@ -7,9 +7,11 @@ from cfme.infrastructure.pxe import get_template_from_config from cfme.infrastructure.pxe import ISODatastore from cfme.services.service_catalogs import ServiceCatalogs +from cfme.utils.blockers import BZ from cfme.utils.generators import random_vm_name from cfme.utils.log import logger + pytestmark = [ pytest.mark.meta(server_roles="+automate"), pytest.mark.usefixtures('uses_infra_providers'), @@ -94,6 +96,9 @@ def catalog_item(appliance, provider, dialog, catalog, provisioning): @test_requirements.rhev +@pytest.mark.meta( + blockers=[BZ(1783355, unblock=lambda provider: provider.version == '4.4')] +) def test_rhev_iso_servicecatalog(appliance, provider, setup_provider, setup_iso_datastore, catalog_item, request): """Tests RHEV ISO service catalog From b1b6a833f5aa7515df8e56e91dbb3a459ef9d83a Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Mon, 13 Jul 2020 21:52:18 +0200 Subject: [PATCH 04/18] Block 4.4 properly --- cfme/tests/services/test_iso_service_catalogs.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cfme/tests/services/test_iso_service_catalogs.py b/cfme/tests/services/test_iso_service_catalogs.py index 5d5311c028..f1ee1a5ec4 100644 --- a/cfme/tests/services/test_iso_service_catalogs.py +++ b/cfme/tests/services/test_iso_service_catalogs.py @@ -7,9 +7,10 @@ from cfme.infrastructure.pxe import get_template_from_config from cfme.infrastructure.pxe import ISODatastore from cfme.services.service_catalogs import ServiceCatalogs -from cfme.utils.blockers import BZ +from cfme.utils.blockers import Blocker from cfme.utils.generators import random_vm_name from cfme.utils.log import logger +from cfme.utils.version import Version pytestmark = [ @@ -95,9 +96,20 @@ def catalog_item(appliance, provider, dialog, catalog, provisioning): ) +class ProviderBlocker(Blocker): + def blocks(self): + return True + + def url(self): + return None + + @test_requirements.rhev @pytest.mark.meta( - blockers=[BZ(1783355, unblock=lambda provider: provider.version == '4.4')] + blockers=[ + ProviderBlocker(unblock=lambda provider: provider.version < Version("4.4")) + # JIRA('RHCFQE-14575') + ] ) def test_rhev_iso_servicecatalog(appliance, provider, setup_provider, setup_iso_datastore, catalog_item, request): From c7ab58da0fef9cfbcc7b24b06261bca0703ec41a Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Tue, 14 Jul 2020 19:31:16 +0200 Subject: [PATCH 05/18] Improve blocking --- cfme/tests/services/test_iso_service_catalogs.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cfme/tests/services/test_iso_service_catalogs.py b/cfme/tests/services/test_iso_service_catalogs.py index f1ee1a5ec4..e4e7f7f4c9 100644 --- a/cfme/tests/services/test_iso_service_catalogs.py +++ b/cfme/tests/services/test_iso_service_catalogs.py @@ -96,10 +96,18 @@ def catalog_item(appliance, provider, dialog, catalog, provisioning): ) -class ProviderBlocker(Blocker): +# There is a Libvirt bug BZ(1783355) on our RHV 4.4. This seem to prevent some tests to run with it. +# I was about to use this blocker: JIRA('RHCFQE-14575'), but this didn't work because we seem to +# lack credentials to read Jira tickets. There seemed to be no nicer way how to mark them other than +# creating EternalBlocker and use the `unblock` kwarg that is handled in special way in the +# `blockers` metaplugin to block only the tests executed against the RHV 4.4. + + +class EternalBlocker(Blocker): def blocks(self): return True + # This needs to be defined, otherwise we are getting some exception. def url(self): return None @@ -107,8 +115,7 @@ def url(self): @test_requirements.rhev @pytest.mark.meta( blockers=[ - ProviderBlocker(unblock=lambda provider: provider.version < Version("4.4")) - # JIRA('RHCFQE-14575') + EternalBlocker(unblock=lambda provider: provider.version < Version("4.4")) ] ) def test_rhev_iso_servicecatalog(appliance, provider, setup_provider, setup_iso_datastore, From e6231d40ff3eba1eed48effa14944f992006c219 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Wed, 15 Jul 2020 15:01:41 +0200 Subject: [PATCH 06/18] Introduce relative_change, stabilize the test_rhv_guest_devices_count --- cfme/tests/infrastructure/test_providers.py | 4 +++- cfme/utils/__init__.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cfme/tests/infrastructure/test_providers.py b/cfme/tests/infrastructure/test_providers.py index 4f5f8c1058..1210a7da17 100644 --- a/cfme/tests/infrastructure/test_providers.py +++ b/cfme/tests/infrastructure/test_providers.py @@ -18,6 +18,7 @@ from cfme.infrastructure.provider.virtualcenter import VMwareProvider from cfme.markers.env_markers.provider import ONE from cfme.markers.env_markers.provider import ONE_PER_VERSION +from cfme.utils import relative_difference from cfme.utils.appliance.implementations.ui import navigate_to from cfme.utils.update import update from cfme.utils.wait import wait_for @@ -365,7 +366,8 @@ def _refresh_provider(): wait_for(_refresh_provider, timeout=300, delay=30) gd_count_after = _gd_count() - assert gd_count_before == gd_count_after, "guest devices count changed after refresh!" + assert abs(relative_difference(gd_count_after, gd_count_before)) > .05, \ + "The guest devices count changed suspiciously after refresh!" @test_requirements.rhev diff --git a/cfme/utils/__init__.py b/cfme/utils/__init__.py index b1c8974122..1f1760e122 100644 --- a/cfme/utils/__init__.py +++ b/cfme/utils/__init__.py @@ -1,4 +1,5 @@ import atexit +import math import os import re import subprocess @@ -411,3 +412,19 @@ def reschedule(): yield finally: timer.cancel() + + +def fraction(numerator: float, denominator: float): + """ + Note this returns -inf or inf when `denominator` is 0.""" + try: + return numerator / denominator + except ZeroDivisionError: + if numerator == denominator == 0.: + return 0. + else: + return math.inf if numerator > 0 else -math.inf + + +def relative_change(before: float, after: float): + return fraction(after, before) - 1 From a0dcd39655b9436116df202ec6452137fd7e476a Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Wed, 1 Jul 2020 14:12:55 +0200 Subject: [PATCH 07/18] Fix test_run_datastore_analysis test. --- .../infrastructure/test_datastore_analysis.py | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/cfme/tests/infrastructure/test_datastore_analysis.py b/cfme/tests/infrastructure/test_datastore_analysis.py index 7f700e10fd..e81e39db4c 100644 --- a/cfme/tests/infrastructure/test_datastore_analysis.py +++ b/cfme/tests/infrastructure/test_datastore_analysis.py @@ -57,26 +57,31 @@ def pytest_generate_tests(metafunc): @pytest.fixture(scope='module') -def datastore(appliance, provider, datastore_type, datastore_name): +def datastore(setup_provider_modscope, appliance, provider, datastore_type, datastore_name): return appliance.collections.datastores.instantiate(name=datastore_name, provider=provider, type=datastore_type) @pytest.fixture(scope='module') -def datastores_hosts_setup(provider, datastore): - hosts = datastore.hosts.all() - for host in hosts: - host_data = [data - for data in provider.data.get("hosts", {}) - if data.get("name") == host.name] - if not host_data: - pytest.skip(f"No host data for provider {provider} and datastore {datastore}") - host.update_credentials_rest(credentials=host_data[0]['credentials']) - else: - pytest.skip(f"No hosts attached to the datastore selected for testing: {datastore}") +def datastores_hosts_setup(setup_provider_modscope, provider, datastore): + updated_hosts = [] + for host in datastore.hosts.all(): + try: + host_data, = [data + for data in provider.data.get("hosts", {}) + if data.get("name") == host.name] + except ValueError as exc: + pytest.skip(f"Data for host {host} in provider {provider} and datastore {datastore} " + f"couldn't be determined: {exc}.") + else: + host.update_credentials_rest(credentials=host_data['credentials']) + updated_hosts.append(host) + + if not updated_hosts: + pytest.skip(f"No hosts attached to the datastore {datastore} was selected for testing.") yield - for host in hosts: + for host in updated_hosts: host.remove_credentials_rest() From 009869bd3dbaefbf1b70130a97e1a87759cb35f1 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Tue, 21 Jul 2020 01:11:50 +0200 Subject: [PATCH 08/18] WIP changes --- cfme/base/ssa.py | 0 cfme/common/datastore_views.py | 172 ++++++++++++++ cfme/common/host_views.py | 8 + cfme/common/provider.py | 39 ++++ cfme/infrastructure/datastore.py | 216 +++--------------- cfme/infrastructure/host.py | 2 +- cfme/modeling/base.py | 19 +- .../cloud_infra_common/test_relationships.py | 4 +- .../infrastructure/test_datastore_analysis.py | 39 +++- cfme/tests/webui/test_general_ui.py | 4 +- cfme/utils/browser.py | 6 +- 11 files changed, 301 insertions(+), 208 deletions(-) create mode 100644 cfme/base/ssa.py create mode 100644 cfme/common/datastore_views.py diff --git a/cfme/base/ssa.py b/cfme/base/ssa.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cfme/common/datastore_views.py b/cfme/common/datastore_views.py new file mode 100644 index 0000000000..171f690d10 --- /dev/null +++ b/cfme/common/datastore_views.py @@ -0,0 +1,172 @@ +from lxml.html import document_fromstring +from widgetastic.widget import Text +from widgetastic.widget import View +from widgetastic_patternfly import Accordion +from widgetastic_patternfly import Dropdown + +from cfme.common import BaseLoggedInPage +from cfme.common.vm_views import VMEntities +from widgetastic_manageiq import BaseEntitiesView +from widgetastic_manageiq import CompareToolBarActionsView +from widgetastic_manageiq import ItemsToolBarViewSelector +from widgetastic_manageiq import JSBaseEntity +from widgetastic_manageiq import ManageIQTree +from widgetastic_manageiq import Search +from widgetastic_manageiq import SummaryTable +from widgetastic_manageiq import Table + + +class DatastoreEntity(JSBaseEntity): + @property + def data(self): + data_dict = super().data + try: + if 'quadicon' in data_dict and data_dict['quadicon']: + quad_data = document_fromstring(data_dict['quadicon']) + data_dict['type'] = quad_data.xpath(self.QUADRANT.format(pos="a"))[0].get('alt') + data_dict['no_vm'] = quad_data.xpath(self.QUADRANT.format(pos="b"))[0].text + data_dict['no_host'] = quad_data.xpath(self.QUADRANT.format(pos="c"))[0].text + return data_dict + except IndexError: + return {} + + +class DatastoreEntities(BaseEntitiesView): + """ + represents central view where all QuadIcons, etc are displayed + """ + @property + def entity_class(self): + return DatastoreEntity + +class DatastoreToolBar(View): + """ + represents datastore toolbar and its controls + """ + configuration = Dropdown(text='Configuration') + policy = Dropdown(text='Policy') + monitoring = Dropdown("Monitoring") + download = Dropdown(text='Download') + view_selector = View.nested(ItemsToolBarViewSelector) + + +class DatastoreSideBar(View): + """ + represents left side bar. it usually contains navigation, filters, etc + """ + @View.nested + class datastores(Accordion): # noqa + ACCORDION_NAME = "Datastores" + tree = ManageIQTree() + + @View.nested + class clusters(Accordion): # noqa + ACCORDION_NAME = "Datastore Clusters" + tree = ManageIQTree() + + +class DatastoresView(BaseLoggedInPage): + """ + represents whole All Datastores page + """ + toolbar = View.nested(DatastoreToolBar) + sidebar = View.nested(DatastoreSideBar) + search = View.nested(Search) + including_entities = View.include(DatastoreEntities, use_parent=True) + + @property + def is_displayed(self): + return (super(BaseLoggedInPage, self).is_displayed and + self.navigation.currently_selected == ['Compute', 'Infrastructure', + 'Datastores'] and + self.entities.title.text == 'All Datastores') + + +class HostAllDatastoresView(DatastoresView): + + @property + def is_displayed(self): + return ( + self.logged_in_as_current_user and + self.navigation.currently_selected == ["Compute", "Infrastructure", "Hosts"] and + self.entities.title.text == "{} (All Datastores)".format(self.context["object"].name) + ) + + +class ProviderAllDatastoresView(DatastoresView): + """ + This view is used in test_provider_relationships + """ + + @property + def is_displayed(self): + msg = "{} (All Datastores)".format(self.context["object"].name) + return ( + self.logged_in_as_current_user and + self.navigation.currently_selected == ["Compute", "Infrastructure", "Providers"] and + self.entities.title.text == msg + ) + + +class DatastoreManagedVMsView(BaseLoggedInPage): + """ + This view represents All VMs and Templates page for datastores + """ + toolbar = View.nested(DatastoreToolBar) + including_entities = View.include(VMEntities, use_parent=True) + + @property + def is_displayed(self): + return ( + super(BaseLoggedInPage, self).is_displayed + and self.navigation.currently_selected == ["Compute", "Infrastructure", "Datastores"] + and self.entities.title.text == f'{self.context["object"].name} (All VMs and Instances)' + and self.context["object"].name in self.breadcrumb.active_location + ) + + +class DatastoreDetailsView(BaseLoggedInPage): + """ + represents Datastore Details page + """ + title = Text('//div[@id="main-content"]//h1') + toolbar = View.nested(DatastoreToolBar) + sidebar = View.nested(DatastoreSideBar) + + @View.nested + class entities(View): # noqa + """ + represents Details page when it is switched to Summary aka Tables view + """ + properties = SummaryTable(title="Properties") + registered_vms = SummaryTable(title="Information for Registered VMs") + relationships = SummaryTable(title="Relationships") + content = SummaryTable(title="Content") + smart_management = SummaryTable(title="Smart Management") + + @property + def is_displayed(self): + return (super(BaseLoggedInPage, self).is_displayed and + self.navigation.currently_selected == ['Compute', 'Infrastructure', + 'Datastores'] and + self.title.text == 'Datastore "{name}"'.format(name=self.context['object'].name)) + + +class DatastoresCompareView(BaseLoggedInPage): + """Compare VM / Template page.""" + # TODO: This table doesn't read properly, fix it. + table = Table('//*[@id="compare-grid"]/table') + title = Text('//*[@id="main-content"]//h1') + + @View.nested + class toolbar(View): + actions = View.nested(CompareToolBarActionsView) + download = Dropdown(text="Download") + + @property + def is_displayed(self): + return ( + self.logged_in_as_current_user + and self.title.text == "Compare VM or Template" + and self.navigation.currently_selected == ["Compute", "Infrastructure", "Datastores"] + ) diff --git a/cfme/common/host_views.py b/cfme/common/host_views.py index 3bafdd3452..11e7d52d0a 100644 --- a/cfme/common/host_views.py +++ b/cfme/common/host_views.py @@ -12,6 +12,7 @@ from cfme.common import BaseLoggedInPage from cfme.common import CompareView from cfme.common import TimelinesView +from cfme.exceptions import displayed_not_implemented from cfme.utils.log import logger from cfme.utils.version import Version from cfme.utils.version import VersionPicker @@ -431,3 +432,10 @@ class HostVmmInfoView(HostsView): def is_displayed(self): active_loc = f"{self.context['object'].name} (VM Monitor Information)" return self.breadcrumb.active_location == active_loc + + +class RegisteredHostsView(HostsView): + """ + represents Hosts related to some datastore + """ + is_displayed = displayed_not_implemented diff --git a/cfme/common/provider.py b/cfme/common/provider.py index 115ad45eea..906ebc072a 100644 --- a/cfme/common/provider.py +++ b/cfme/common/provider.py @@ -19,6 +19,7 @@ from cfme.common import Taggable from cfme.exceptions import AddProviderError from cfme.exceptions import HostStatsNotContains +from cfme.exceptions import ItemNotFound from cfme.exceptions import ProviderHasNoKey from cfme.exceptions import ProviderHasNoProperty from cfme.exceptions import RestLookupError @@ -1166,6 +1167,29 @@ def get_template_guids(self, template_dict): result_list.append(inner_tuple) return result_list + def run_smartstate_analysis_from_provider(self, datastores, wait_for_task_result=False): + """ Runs smartstate analysis on this host + + Note: + The host must have valid credentials already set up for this to work. + """ + view = navigate_to(self.provider, 'DatastoresOfProvider') + datastores = list(datastores) + checked_datastores = list() + + for datastore in datastores: + try: + view.entities.get_entity(name=datastore.name, surf_pages=True).ensure_checked() + checked_datastores.append(datastore) + except ItemNotFound: + raise ValueError(f'Could not find datastore {datastore.name} in the UI') + + view.toolbar.configuration.item_select('Perform SmartState Analysis', handle_alert=True) + for datastore in checked_datastores: + view.flash.assert_success_message( + f'"{datastore.name}": scan successfully initiated') + + class CloudInfraProviderMixin: detail_page_suffix = 'provider' @@ -1335,3 +1359,18 @@ class DefaultEndpointForm(View): change_password = Text(locator='.//a[normalize-space(.)="Change stored password"]') validate = Button('Validate') + + +from cfme.utils.appliance.implementations.ui import CFMENavigateStep +from cfme.common.datastore_views import DatastoresView, ProviderAllDatastoresView + + +@navigator.register(BaseProvider, 'DatastoresOfProvider') +class DatastoresOfProvider(CFMENavigateStep): + VIEW = ProviderAllDatastoresView + + def prerequisite(self): + return navigate_to(self.obj, 'Details') + + def step(self, *args, **kwargs): + self.prerequisite_view.entities.summary('Relationships').click_at('Datastores') diff --git a/cfme/infrastructure/datastore.py b/cfme/infrastructure/datastore.py index 1a5cba327a..bdf9da3f66 100644 --- a/cfme/infrastructure/datastore.py +++ b/cfme/infrastructure/datastore.py @@ -1,23 +1,19 @@ """ A model of an Infrastructure Datastore in CFME """ import attr -from lxml.html import document_fromstring from navmazing import NavigateToAttribute from navmazing import NavigateToSibling -from widgetastic.widget import Text -from widgetastic.widget import View -from widgetastic_patternfly import Accordion -from widgetastic_patternfly import Dropdown -from cfme.common import BaseLoggedInPage from cfme.common import CustomButtonEventsMixin from cfme.common import Taggable from cfme.common.candu_views import DatastoreInfraUtilizationView -from cfme.common.host_views import HostsView -from cfme.common.vm_views import VMEntities -from cfme.exceptions import displayed_not_implemented +from cfme.common.datastore_views import DatastoreDetailsView +from cfme.common.datastore_views import DatastoreManagedVMsView +from cfme.common.datastore_views import DatastoresView +from cfme.common.host_views import RegisteredHostsView from cfme.exceptions import ItemNotFound from cfme.exceptions import MenuItemNotFound +from cfme.infrastructure.provider import InfraProvider from cfme.modeling.base import BaseCollection from cfme.modeling.base import BaseEntity from cfme.optimize.utilization import DatastoreUtilizationTrendsView @@ -29,178 +25,6 @@ from cfme.utils.providers import get_crud_by_name from cfme.utils.wait import TimedOutError from cfme.utils.wait import wait_for -from widgetastic_manageiq import BaseEntitiesView -from widgetastic_manageiq import CompareToolBarActionsView -from widgetastic_manageiq import ItemsToolBarViewSelector -from widgetastic_manageiq import JSBaseEntity -from widgetastic_manageiq import ManageIQTree -from widgetastic_manageiq import Search -from widgetastic_manageiq import SummaryTable -from widgetastic_manageiq import Table - - -class DatastoreToolBar(View): - """ - represents datastore toolbar and its controls - """ - configuration = Dropdown(text='Configuration') - policy = Dropdown(text='Policy') - monitoring = Dropdown("Monitoring") - download = Dropdown(text='Download') - view_selector = View.nested(ItemsToolBarViewSelector) - - -class DatastoreSideBar(View): - """ - represents left side bar. it usually contains navigation, filters, etc - """ - @View.nested - class datastores(Accordion): # noqa - ACCORDION_NAME = "Datastores" - tree = ManageIQTree() - - @View.nested - class clusters(Accordion): # noqa - ACCORDION_NAME = "Datastore Clusters" - tree = ManageIQTree() - - -class DatastoreEntity(JSBaseEntity): - @property - def data(self): - data_dict = super().data - try: - if 'quadicon' in data_dict and data_dict['quadicon']: - quad_data = document_fromstring(data_dict['quadicon']) - data_dict['type'] = quad_data.xpath(self.QUADRANT.format(pos="a"))[0].get('alt') - data_dict['no_vm'] = quad_data.xpath(self.QUADRANT.format(pos="b"))[0].text - data_dict['no_host'] = quad_data.xpath(self.QUADRANT.format(pos="c"))[0].text - return data_dict - except IndexError: - return {} - - -class DatastoreEntities(BaseEntitiesView): - """ - represents central view where all QuadIcons, etc are displayed - """ - @property - def entity_class(self): - return DatastoreEntity - - -class DatastoresView(BaseLoggedInPage): - """ - represents whole All Datastores page - """ - toolbar = View.nested(DatastoreToolBar) - sidebar = View.nested(DatastoreSideBar) - search = View.nested(Search) - including_entities = View.include(DatastoreEntities, use_parent=True) - - @property - def is_displayed(self): - return (super(BaseLoggedInPage, self).is_displayed and - self.navigation.currently_selected == ['Compute', 'Infrastructure', - 'Datastores'] and - self.entities.title.text == 'All Datastores') - - -class HostAllDatastoresView(DatastoresView): - - @property - def is_displayed(self): - return ( - self.logged_in_as_current_user and - self.navigation.currently_selected == ["Compute", "Infrastructure", "Hosts"] and - self.entities.title.text == "{} (All Datastores)".format(self.context["object"].name) - ) - - -class ProviderAllDatastoresView(DatastoresView): - """ - This view is used in test_provider_relationships - """ - - @property - def is_displayed(self): - msg = "{} (All Datastores)".format(self.context["object"].name) - return ( - self.logged_in_as_current_user and - self.navigation.currently_selected == ["Compute", "Infrastructure", "Providers"] and - self.entities.title.text == msg - ) - - -class DatastoreManagedVMsView(BaseLoggedInPage): - """ - This view represents All VMs and Templates page for datastores - """ - toolbar = View.nested(DatastoreToolBar) - including_entities = View.include(VMEntities, use_parent=True) - - @property - def is_displayed(self): - return ( - super(BaseLoggedInPage, self).is_displayed - and self.navigation.currently_selected == ["Compute", "Infrastructure", "Datastores"] - and self.entities.title.text == f'{self.context["object"].name} (All VMs and Instances)' - and self.context["object"].name in self.breadcrumb.active_location - ) - - -class DatastoreDetailsView(BaseLoggedInPage): - """ - represents Datastore Details page - """ - title = Text('//div[@id="main-content"]//h1') - toolbar = View.nested(DatastoreToolBar) - sidebar = View.nested(DatastoreSideBar) - - @View.nested - class entities(View): # noqa - """ - represents Details page when it is switched to Summary aka Tables view - """ - properties = SummaryTable(title="Properties") - registered_vms = SummaryTable(title="Information for Registered VMs") - relationships = SummaryTable(title="Relationships") - content = SummaryTable(title="Content") - smart_management = SummaryTable(title="Smart Management") - - @property - def is_displayed(self): - return (super(BaseLoggedInPage, self).is_displayed and - self.navigation.currently_selected == ['Compute', 'Infrastructure', - 'Datastores'] and - self.title.text == 'Datastore "{name}"'.format(name=self.context['object'].name)) - - -class RegisteredHostsView(HostsView): - """ - represents Hosts related to some datastore - """ - is_displayed = displayed_not_implemented - - -class DatastoresCompareView(BaseLoggedInPage): - """Compare VM / Template page.""" - # TODO: This table doesn't read properly, fix it. - table = Table('//*[@id="compare-grid"]/table') - title = Text('//*[@id="main-content"]//h1') - - @View.nested - class toolbar(View): - actions = View.nested(CompareToolBarActionsView) - download = Dropdown(text="Download") - - @property - def is_displayed(self): - return ( - self.logged_in_as_current_user - and self.title.text == "Compare VM or Template" - and self.navigation.currently_selected == ["Compute", "Infrastructure", "Datastores"] - ) @attr.s @@ -216,7 +40,7 @@ class Datastore(Pretty, BaseEntity, Taggable, CustomButtonEventsMixin): pretty_attrs = ['name', 'provider_key'] _param_name = ParamClassName('name') name = attr.ib() - provider = attr.ib() + provider: InfraProvider = attr.ib() type = attr.ib(default=None) def __attrs_post_init__(self): @@ -343,6 +167,28 @@ def run_smartstate_analysis(self, wait_for_task_result=False): task.wait_for_finished() return task + def run_smartstate_analysis_from_provider(self, wait_for_task_result=False): + """ Runs smartstate analysis on this host + + Note: + The host must have valid credentials already set up for this to work. + """ + view = navigate_to(self.provider, 'DatastoresOfProvider') + try: + view.entities.get_entity(name=self.name, surf_pages=True).ensure_checked() + except ItemNotFound: + raise ValueError(f'Could not find datastore {self.name} in the UI') + + view.toolbar.configuration.item_select('Perform SmartState Analysis', handle_alert=True) + view.flash.assert_success_message( + f'"{self.name}": scan successfully initiated') + + if wait_for_task_result: + task = self.appliance.collections.tasks.instantiate( + name=f"SmartState Analysis for [{self.name}]", tab='MyOtherTasks') + task.wait_for_finished() + return task + def wait_candu_data_available(self, timeout=900): """Waits until C&U data are available for this Datastore @@ -358,7 +204,7 @@ def wait_candu_data_available(self, timeout=900): @attr.s -class DatastoreCollection(BaseCollection): +class DatastoreCollection(BaseCollection[Datastore]): """Collection class for :py:class:`cfme.infrastructure.datastore.Datastore`""" ENTITY = Datastore @@ -421,7 +267,6 @@ def run_smartstate_analysis(self, *datastores): datastores = list(datastores) checked_datastores = list() - view = navigate_to(self, 'All') for datastore in datastores: @@ -432,7 +277,7 @@ def run_smartstate_analysis(self, *datastores): raise ValueError(f'Could not find datastore {datastore.name} in the UI') view.toolbar.configuration.item_select('Perform SmartState Analysis', handle_alert=True) - for datastore in datastores: + for datastore in checked_datastores: view.flash.assert_success_message( f'"{datastore.name}": scan successfully initiated') @@ -469,6 +314,7 @@ class DetailsFromProvider(CFMENavigateStep): VIEW = DatastoreDetailsView def prerequisite(self): + # TODO use DatastoresOfProvider prov_view = navigate_to(self.obj.provider, 'Details') prov_view.entities.summary('Relationships').click_at('Datastores') return self.obj.create_view(DatastoresView) diff --git a/cfme/infrastructure/host.py b/cfme/infrastructure/host.py index 0e83e7b485..1969d999c9 100644 --- a/cfme/infrastructure/host.py +++ b/cfme/infrastructure/host.py @@ -13,6 +13,7 @@ from cfme.common import PolicyProfileAssignable from cfme.common import Taggable from cfme.common.candu_views import HostInfraUtilizationView +from cfme.common.datastore_views import HostAllDatastoresView from cfme.common.host_views import HostAddView from cfme.common.host_views import HostDetailsView from cfme.common.host_views import HostDevicesView @@ -33,7 +34,6 @@ from cfme.common.host_views import ProviderHostsCompareView from cfme.exceptions import ItemNotFound from cfme.exceptions import RestLookupError -from cfme.infrastructure.datastore import HostAllDatastoresView from cfme.modeling.base import BaseCollection from cfme.modeling.base import BaseEntity from cfme.networks.views import OneHostSubnetView diff --git a/cfme/modeling/base.py b/cfme/modeling/base.py index b9d0e3ca61..8e29e2d664 100644 --- a/cfme/modeling/base.py +++ b/cfme/modeling/base.py @@ -87,8 +87,12 @@ def __getattr__(self, name): return self._collection_cache[name] +from typing import Type, TypeVar, Generic + +T = TypeVar('T') + @attr.s -class BaseCollection(NavigatableMixin): +class BaseCollection(NavigatableMixin, Generic[T]): """Class for helping create consistent Collections The BaseCollection class is responsible for ensuring two things: @@ -99,12 +103,14 @@ class BaseCollection(NavigatableMixin): This class works in tandem with the entrypoint loader which ensures that the correct argument names have been used. """ - - ENTITY = None + ENTITY: Type[T] parent = attr.ib(repr=False) filters = attr.ib(default=attr.Factory(dict)) + def instantiate(self, *args, **kwargs) -> T: + return self.ENTITY.from_collection(self, *args, **kwargs) + @property def appliance(self): if isinstance(self.parent, BaseEntity): @@ -124,9 +130,6 @@ def for_entity(cls, obj, *k, **kw): def for_entity_with_filter(cls, obj, filt, *k, **kw): return cls.for_entity(obj, *k, **kw).filter(filt) - def instantiate(self, *args, **kwargs): - return self.ENTITY.from_collection(self, *args, **kwargs) - def filter(self, filter): filters = self.filters.copy() filters.update(filter) @@ -146,7 +149,7 @@ class BaseEntity(NavigatableMixin): argument names have been used. """ - parent = attr.ib(repr=False) # This is the collection or not + parent: BaseCollection = attr.ib(repr=False) # This is the collection or not # TODO This needs removing as we need proper __eq__ on objects, but it is part of a # much larger discussion @@ -157,7 +160,7 @@ def appliance(self): return self.parent.appliance @classmethod - def from_collection(cls, collection, *k, **kw): + def from_collection(cls, collection: BaseCollection, *k, **kw): return cls(collection, *k, **kw) @cached_property diff --git a/cfme/tests/cloud_infra_common/test_relationships.py b/cfme/tests/cloud_infra_common/test_relationships.py index 80921fd7dd..4640fb825c 100644 --- a/cfme/tests/cloud_infra_common/test_relationships.py +++ b/cfme/tests/cloud_infra_common/test_relationships.py @@ -14,14 +14,14 @@ from cfme.cloud.provider.openstack import OpenStackProvider from cfme.cloud.stack import ProviderStackAllView from cfme.cloud.tenant import ProviderTenantAllView +from cfme.common.datastore_views import HostAllDatastoresView +from cfme.common.datastore_views import ProviderAllDatastoresView from cfme.common.host_views import ProviderAllHostsView from cfme.common.provider_views import InfraProviderDetailsView from cfme.common.vm_views import HostAllVMsView from cfme.common.vm_views import ProviderAllVMsView from cfme.infrastructure.cluster import ClusterDetailsView from cfme.infrastructure.cluster import ProviderAllClustersView -from cfme.infrastructure.datastore import HostAllDatastoresView -from cfme.infrastructure.datastore import ProviderAllDatastoresView from cfme.infrastructure.provider import InfraProvider from cfme.infrastructure.provider.rhevm import RHEVMProvider from cfme.infrastructure.provider.virtualcenter import VMwareProvider diff --git a/cfme/tests/infrastructure/test_datastore_analysis.py b/cfme/tests/infrastructure/test_datastore_analysis.py index e81e39db4c..475b9f8c0b 100644 --- a/cfme/tests/infrastructure/test_datastore_analysis.py +++ b/cfme/tests/infrastructure/test_datastore_analysis.py @@ -3,13 +3,17 @@ from cfme import test_requirements from cfme.exceptions import MenuItemNotFound +from cfme.infrastructure.datastore import Datastore +from cfme.infrastructure.datastore import DatastoreCollection from cfme.infrastructure.provider.rhevm import RHEVMProvider from cfme.infrastructure.provider.virtualcenter import VMwareProvider from cfme.utils import testgen from cfme.utils.appliance.implementations.ui import navigate_to +from cfme.utils.blockers import GH from cfme.utils.log import logger from cfme.utils.wait import wait_for + pytestmark = [test_requirements.smartstate] DATASTORE_TYPES = ('vmfs', 'nfs', 'iscsi') @@ -57,12 +61,18 @@ def pytest_generate_tests(metafunc): @pytest.fixture(scope='module') -def datastore(setup_provider_modscope, appliance, provider, datastore_type, datastore_name): +def datastore(setup_provider_modscope, appliance, provider, datastore_type, datastore_name)\ + -> Datastore: return appliance.collections.datastores.instantiate(name=datastore_name, provider=provider, type=datastore_type) +@pytest.fixture(scope='module') +def datastores(appliance, provider) -> DatastoreCollection: + return appliance.collections.datastores + + @pytest.fixture(scope='module') def datastores_hosts_setup(setup_provider_modscope, provider, datastore): updated_hosts = [] @@ -93,8 +103,8 @@ def clear_all_tasks(appliance): @pytest.mark.tier(2) -def test_run_datastore_analysis(setup_provider, datastore, soft_assert, datastores_hosts_setup, - clear_all_tasks, appliance): +def test_run_datastore_analysis(setup_provider, datastore, datastores, soft_assert, + clear_all_tasks, appliance, temp_appliance_preconfig_funcscope): """Tests smarthost analysis Metadata: @@ -106,12 +116,19 @@ def test_run_datastore_analysis(setup_provider, datastore, soft_assert, datastor caseimportance: critical initialEstimate: 1/3h """ + appliance = temp_appliance_preconfig_funcscope # Initiate analysis - try: - datastore.run_smartstate_analysis(wait_for_task_result=True) - except (MenuItemNotFound, DropdownDisabled): - # TODO need to update to cover all detastores - pytest.skip(f'Smart State analysis is disabled for {datastore.name} datastore') + # try: + + # Note that it would be great to test both navigation paths. + if GH(('ManageIQ/manageiq', 20367)).blocks: + datastore.run_smartstate_analysis_from_provider() + else: + datastore.run_smartstate_analysis() + + #except (MenuItemNotFound, DropdownDisabled): + # # TODO need to update to cover all detastores + # pytest.skip(f'Smart State analysis is disabled for {datastore.name} datastore') details_view = navigate_to(datastore, 'DetailsFromProvider') # c_datastore = details_view.entities.properties.get_text_of("Datastore Type") @@ -121,10 +138,14 @@ def test_run_datastore_analysis(setup_provider, datastore, soft_assert, datastor # 'Datastore type does not match the type defined in yaml:' + # 'expected "{}" but was "{}"'.format(datastore.type.upper(), c_datastore)) + if datastore.provider.one_of(RHEVMProvider) and GH(('ManageIQ/manageiq', 20366)).blocks: + return + wait_for(lambda: details_view.entities.content.get_text_of(CONTENT_ROWS_TO_CHECK[0]), - delay=15, timeout="3m", + delay=15, timeout="6m", fail_condition='0', fail_func=appliance.server.browser.refresh) + managed_vms = details_view.entities.relationships.get_text_of('Managed VMs') if managed_vms != '0': for row_name in CONTENT_ROWS_TO_CHECK: diff --git a/cfme/tests/webui/test_general_ui.py b/cfme/tests/webui/test_general_ui.py index e1db273858..fd5c5613c9 100644 --- a/cfme/tests/webui/test_general_ui.py +++ b/cfme/tests/webui/test_general_ui.py @@ -6,6 +6,8 @@ from cfme import test_requirements from cfme.base.ui import LoginPage from cfme.cloud.provider import CloudProvider +from cfme.common.datastore_views import DatastoresCompareView +from cfme.common.datastore_views import ProviderAllDatastoresView from cfme.common.host_views import ProviderAllHostsView from cfme.common.provider import BaseProvider from cfme.common.provider_views import CloudProviderAddView @@ -20,8 +22,6 @@ from cfme.infrastructure.config_management import ConfigManagerProvider from cfme.infrastructure.config_management.ansible_tower import AnsibleTowerProvider from cfme.infrastructure.config_management.satellite import SatelliteProvider -from cfme.infrastructure.datastore import DatastoresCompareView -from cfme.infrastructure.datastore import ProviderAllDatastoresView from cfme.infrastructure.provider import InfraProvider from cfme.infrastructure.provider import ProviderClustersView from cfme.infrastructure.provider import ProviderTemplatesView diff --git a/cfme/utils/browser.py b/cfme/utils/browser.py index a20faf56c7..11bd7dc3cf 100644 --- a/cfme/utils/browser.py +++ b/cfme/utils/browser.py @@ -257,7 +257,11 @@ def from_conf(cls, browser_conf): if browser_conf[ 'webdriver_options'][ 'desired_capabilities']['browserName'].lower() == 'chrome': - browser_kwargs['desired_capabilities']['chromeOptions'] = {} +# from selenium.webdriver.chrome.options import Options +# options = browser_kwargs['desired_capabilities']['chromeOptions'] = Options() +# options.set_capability("acceptInsecureCerts", True) + #browser_kwargs['desired_capabilities']['acceptInsecureCerts'] = True + co = browser_kwargs['desired_capabilities'].setdefault('chromeOptions', {}) browser_kwargs[ 'desired_capabilities']['chromeOptions']['args'] = ['--no-sandbox', '--start-maximized', From 684a5fa5db8d67903466e53bdc65ee6beb86d521 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Tue, 21 Jul 2020 12:34:42 +0200 Subject: [PATCH 09/18] With appliance --- .../infrastructure/test_datastore_analysis.py | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/cfme/tests/infrastructure/test_datastore_analysis.py b/cfme/tests/infrastructure/test_datastore_analysis.py index 475b9f8c0b..8fb702669c 100644 --- a/cfme/tests/infrastructure/test_datastore_analysis.py +++ b/cfme/tests/infrastructure/test_datastore_analysis.py @@ -104,7 +104,7 @@ def clear_all_tasks(appliance): @pytest.mark.tier(2) def test_run_datastore_analysis(setup_provider, datastore, datastores, soft_assert, - clear_all_tasks, appliance, temp_appliance_preconfig_funcscope): + clear_all_tasks, temp_appliance_preconfig_funcscope): """Tests smarthost analysis Metadata: @@ -116,41 +116,41 @@ def test_run_datastore_analysis(setup_provider, datastore, datastores, soft_asse caseimportance: critical initialEstimate: 1/3h """ - appliance = temp_appliance_preconfig_funcscope - # Initiate analysis - # try: - - # Note that it would be great to test both navigation paths. - if GH(('ManageIQ/manageiq', 20367)).blocks: - datastore.run_smartstate_analysis_from_provider() - else: - datastore.run_smartstate_analysis() - - #except (MenuItemNotFound, DropdownDisabled): - # # TODO need to update to cover all detastores - # pytest.skip(f'Smart State analysis is disabled for {datastore.name} datastore') - details_view = navigate_to(datastore, 'DetailsFromProvider') - # c_datastore = details_view.entities.properties.get_text_of("Datastore Type") - - # Check results of the analysis and the datastore type - # TODO need to clarify datastore type difference - # soft_assert(c_datastore == datastore.type.upper(), - # 'Datastore type does not match the type defined in yaml:' + - # 'expected "{}" but was "{}"'.format(datastore.type.upper(), c_datastore)) - - if datastore.provider.one_of(RHEVMProvider) and GH(('ManageIQ/manageiq', 20366)).blocks: - return - - wait_for(lambda: details_view.entities.content.get_text_of(CONTENT_ROWS_TO_CHECK[0]), - delay=15, timeout="6m", - fail_condition='0', - fail_func=appliance.server.browser.refresh) - - managed_vms = details_view.entities.relationships.get_text_of('Managed VMs') - if managed_vms != '0': - for row_name in CONTENT_ROWS_TO_CHECK: - value = details_view.entities.content.get_text_of(row_name) - soft_assert(value != '0', - f'Expected value for {row_name} to be non-empty') - else: - assert details_view.entities.content.get_text_of(CONTENT_ROWS_TO_CHECK[-1]) != '0' + with temp_appliance_preconfig_funcscope as appliance: + # Initiate analysis + # try: + + # Note that it would be great to test both navigation paths. + if GH(('ManageIQ/manageiq', 20367)).blocks: + datastore.run_smartstate_analysis_from_provider() + else: + datastore.run_smartstate_analysis() + + #except (MenuItemNotFound, DropdownDisabled): + # # TODO need to update to cover all detastores + # pytest.skip(f'Smart State analysis is disabled for {datastore.name} datastore') + details_view = navigate_to(datastore, 'DetailsFromProvider') + # c_datastore = details_view.entities.properties.get_text_of("Datastore Type") + + # Check results of the analysis and the datastore type + # TODO need to clarify datastore type difference + # soft_assert(c_datastore == datastore.type.upper(), + # 'Datastore type does not match the type defined in yaml:' + + # 'expected "{}" but was "{}"'.format(datastore.type.upper(), c_datastore)) + + if datastore.provider.one_of(RHEVMProvider) and GH(('ManageIQ/manageiq', 20366)).blocks: + return + + wait_for(lambda: details_view.entities.content.get_text_of(CONTENT_ROWS_TO_CHECK[0]), + delay=15, timeout="6m", + fail_condition='0', + fail_func=appliance.server.browser.refresh) + + managed_vms = details_view.entities.relationships.get_text_of('Managed VMs') + if managed_vms != '0': + for row_name in CONTENT_ROWS_TO_CHECK: + value = details_view.entities.content.get_text_of(row_name) + soft_assert(value != '0', + f'Expected value for {row_name} to be non-empty') + else: + assert details_view.entities.content.get_text_of(CONTENT_ROWS_TO_CHECK[-1]) != '0' From 83a9b8fccd1c7650c75cfbf475c63ca2c2f79e26 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Tue, 21 Jul 2020 13:18:26 +0200 Subject: [PATCH 10/18] setup provider --- cfme/tests/infrastructure/test_datastore_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfme/tests/infrastructure/test_datastore_analysis.py b/cfme/tests/infrastructure/test_datastore_analysis.py index 8fb702669c..4f597daba3 100644 --- a/cfme/tests/infrastructure/test_datastore_analysis.py +++ b/cfme/tests/infrastructure/test_datastore_analysis.py @@ -103,7 +103,7 @@ def clear_all_tasks(appliance): @pytest.mark.tier(2) -def test_run_datastore_analysis(setup_provider, datastore, datastores, soft_assert, +def test_run_datastore_analysis(setup_provider_temp_appliance, datastore, datastores, soft_assert, clear_all_tasks, temp_appliance_preconfig_funcscope): """Tests smarthost analysis From a6d82ca60cce277ed97670bb67145e45781a2a25 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Tue, 21 Jul 2020 15:08:03 +0200 Subject: [PATCH 11/18] funcscope --- .../infrastructure/test_datastore_analysis.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/cfme/tests/infrastructure/test_datastore_analysis.py b/cfme/tests/infrastructure/test_datastore_analysis.py index 4f597daba3..ab4712847a 100644 --- a/cfme/tests/infrastructure/test_datastore_analysis.py +++ b/cfme/tests/infrastructure/test_datastore_analysis.py @@ -60,21 +60,22 @@ def pytest_generate_tests(metafunc): testgen.parametrize(metafunc, argnames, new_argvalues, ids=new_idlist, scope="module") -@pytest.fixture(scope='module') -def datastore(setup_provider_modscope, appliance, provider, datastore_type, datastore_name)\ +@pytest.fixture +def datastore(setup_provider_modscope, temp_appliance_preconfig_funcscope, provider, datastore_type, datastore_name)\ -> Datastore: - return appliance.collections.datastores.instantiate(name=datastore_name, - provider=provider, - type=datastore_type) + with temp_appliance_preconfig_funcscope as appliance: + return appliance.collections.datastores.instantiate(name=datastore_name, + provider=provider, + type=datastore_type) -@pytest.fixture(scope='module') -def datastores(appliance, provider) -> DatastoreCollection: - return appliance.collections.datastores +@pytest.fixture +def datastores(temp_appliance_preconfig_funcscope, provider) -> DatastoreCollection: + return temp_appliance_preconfig_funcscope.collections.datastores -@pytest.fixture(scope='module') -def datastores_hosts_setup(setup_provider_modscope, provider, datastore): +@pytest.fixture +def datastores_hosts_setup(setup_provider_temp_appliance, provider, datastore): updated_hosts = [] for host in datastore.hosts.all(): try: @@ -95,7 +96,7 @@ def datastores_hosts_setup(setup_provider_modscope, provider, datastore): host.remove_credentials_rest() -@pytest.fixture(scope='function') +@pytest.fixture() def clear_all_tasks(appliance): # clear table col = appliance.collections.tasks.filter({'tab': 'AllTasks'}) @@ -116,6 +117,7 @@ def test_run_datastore_analysis(setup_provider_temp_appliance, datastore, datast caseimportance: critical initialEstimate: 1/3h """ + temp_appliance_preconfig_funcscope.browser_steal = True; with temp_appliance_preconfig_funcscope as appliance: # Initiate analysis # try: From d90d5f311c334a5520760c1bf96845201c3afc7a Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Wed, 22 Jul 2020 15:03:15 +0200 Subject: [PATCH 12/18] Workaround a mystery. --- cfme/tests/infrastructure/test_datastore_analysis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cfme/tests/infrastructure/test_datastore_analysis.py b/cfme/tests/infrastructure/test_datastore_analysis.py index ab4712847a..bb95a45109 100644 --- a/cfme/tests/infrastructure/test_datastore_analysis.py +++ b/cfme/tests/infrastructure/test_datastore_analysis.py @@ -140,7 +140,9 @@ def test_run_datastore_analysis(setup_provider_temp_appliance, datastore, datast # 'Datastore type does not match the type defined in yaml:' + # 'expected "{}" but was "{}"'.format(datastore.type.upper(), c_datastore)) - if datastore.provider.one_of(RHEVMProvider) and GH(('ManageIQ/manageiq', 20366)).blocks: + if datastore.provider.one_of(RHEVMProvider) and GH(('ManageIQ/manageiq', 20366)).blocks or + # TODO (jhenner) why is thati? + (datastore.provider.one_of(VMwareProvider) and Version(datastore.provider.version) == '6.5'): return wait_for(lambda: details_view.entities.content.get_text_of(CONTENT_ROWS_TO_CHECK[0]), From 62295e7272a3bbaad99ae83310b96144c6e89818 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Wed, 22 Jul 2020 20:54:32 +0200 Subject: [PATCH 13/18] Cleanup --- .../infrastructure/test_datastore_analysis.py | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/cfme/tests/infrastructure/test_datastore_analysis.py b/cfme/tests/infrastructure/test_datastore_analysis.py index bb95a45109..b27c59e913 100644 --- a/cfme/tests/infrastructure/test_datastore_analysis.py +++ b/cfme/tests/infrastructure/test_datastore_analysis.py @@ -1,10 +1,7 @@ import pytest -from widgetastic_patternfly import DropdownDisabled from cfme import test_requirements -from cfme.exceptions import MenuItemNotFound from cfme.infrastructure.datastore import Datastore -from cfme.infrastructure.datastore import DatastoreCollection from cfme.infrastructure.provider.rhevm import RHEVMProvider from cfme.infrastructure.provider.virtualcenter import VMwareProvider from cfme.utils import testgen @@ -61,7 +58,7 @@ def pytest_generate_tests(metafunc): @pytest.fixture -def datastore(setup_provider_modscope, temp_appliance_preconfig_funcscope, provider, datastore_type, datastore_name)\ +def datastore(temp_appliance_preconfig_funcscope, provider, datastore_type, datastore_name)\ -> Datastore: with temp_appliance_preconfig_funcscope as appliance: return appliance.collections.datastores.instantiate(name=datastore_name, @@ -69,11 +66,6 @@ def datastore(setup_provider_modscope, temp_appliance_preconfig_funcscope, provi type=datastore_type) -@pytest.fixture -def datastores(temp_appliance_preconfig_funcscope, provider) -> DatastoreCollection: - return temp_appliance_preconfig_funcscope.collections.datastores - - @pytest.fixture def datastores_hosts_setup(setup_provider_temp_appliance, provider, datastore): updated_hosts = [] @@ -104,9 +96,9 @@ def clear_all_tasks(appliance): @pytest.mark.tier(2) -def test_run_datastore_analysis(setup_provider_temp_appliance, datastore, datastores, soft_assert, +def test_run_datastore_analysis(setup_provider_temp_appliance, datastore, soft_assert, clear_all_tasks, temp_appliance_preconfig_funcscope): - """Tests smarthost analysis + """Tests SmartState analysis Metadata: test_flag: datastore_analysis @@ -117,34 +109,28 @@ def test_run_datastore_analysis(setup_provider_temp_appliance, datastore, datast caseimportance: critical initialEstimate: 1/3h """ - temp_appliance_preconfig_funcscope.browser_steal = True; + temp_appliance_preconfig_funcscope.browser_steal = True with temp_appliance_preconfig_funcscope as appliance: # Initiate analysis - # try: - # Note that it would be great to test both navigation paths. if GH(('ManageIQ/manageiq', 20367)).blocks: datastore.run_smartstate_analysis_from_provider() else: datastore.run_smartstate_analysis() - #except (MenuItemNotFound, DropdownDisabled): - # # TODO need to update to cover all detastores - # pytest.skip(f'Smart State analysis is disabled for {datastore.name} datastore') - details_view = navigate_to(datastore, 'DetailsFromProvider') # c_datastore = details_view.entities.properties.get_text_of("Datastore Type") - # Check results of the analysis and the datastore type # TODO need to clarify datastore type difference # soft_assert(c_datastore == datastore.type.upper(), # 'Datastore type does not match the type defined in yaml:' + # 'expected "{}" but was "{}"'.format(datastore.type.upper(), c_datastore)) - if datastore.provider.one_of(RHEVMProvider) and GH(('ManageIQ/manageiq', 20366)).blocks or - # TODO (jhenner) why is thati? - (datastore.provider.one_of(VMwareProvider) and Version(datastore.provider.version) == '6.5'): + if datastore.provider.one_of(RHEVMProvider) and GH(('ManageIQ/manageiq', 20366)).blocks: + # or (datastore.provider.one_of(VMwareProvider) and + # Version(datastore.provider.version) == '6.5'): # Why is that needed? return + details_view = navigate_to(datastore, 'DetailsFromProvider') wait_for(lambda: details_view.entities.content.get_text_of(CONTENT_ROWS_TO_CHECK[0]), delay=15, timeout="6m", fail_condition='0', From 7fc34ec341d5d40640fd14ac88eacb96855d12a7 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Wed, 22 Jul 2020 21:13:56 +0200 Subject: [PATCH 14/18] browser lintfix --- cfme/utils/browser.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cfme/utils/browser.py b/cfme/utils/browser.py index 11bd7dc3cf..d2b49969b4 100644 --- a/cfme/utils/browser.py +++ b/cfme/utils/browser.py @@ -257,11 +257,7 @@ def from_conf(cls, browser_conf): if browser_conf[ 'webdriver_options'][ 'desired_capabilities']['browserName'].lower() == 'chrome': -# from selenium.webdriver.chrome.options import Options -# options = browser_kwargs['desired_capabilities']['chromeOptions'] = Options() -# options.set_capability("acceptInsecureCerts", True) - #browser_kwargs['desired_capabilities']['acceptInsecureCerts'] = True - co = browser_kwargs['desired_capabilities'].setdefault('chromeOptions', {}) + browser_kwargs['desired_capabilities'].setdefault('chromeOptions', {}) browser_kwargs[ 'desired_capabilities']['chromeOptions']['args'] = ['--no-sandbox', '--start-maximized', From 9a4c20b3b32e51e1497e5a073972c7b556aca1aa Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Thu, 30 Jul 2020 17:49:22 +0200 Subject: [PATCH 15/18] fix difference --- cfme/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfme/utils/__init__.py b/cfme/utils/__init__.py index 1f1760e122..b598900451 100644 --- a/cfme/utils/__init__.py +++ b/cfme/utils/__init__.py @@ -426,5 +426,5 @@ def fraction(numerator: float, denominator: float): return math.inf if numerator > 0 else -math.inf -def relative_change(before: float, after: float): +def relative_difference(before: float, after: float): return fraction(after, before) - 1 From 14d16cf8ac1c4c51cb2c3f0886cdf731551af1cb Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Thu, 30 Jul 2020 17:50:49 +0200 Subject: [PATCH 16/18] Remove ssa file --- cfme/base/ssa.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 cfme/base/ssa.py diff --git a/cfme/base/ssa.py b/cfme/base/ssa.py deleted file mode 100644 index e69de29bb2..0000000000 From a1475947a8ac4445144b191c547689ec8df9fdc1 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Wed, 22 Jul 2020 21:50:22 +0200 Subject: [PATCH 17/18] Types changes --- cfme/common/datastore_views.py | 1 + cfme/common/provider.py | 7 ++----- cfme/modeling/base.py | 19 +++++++++++-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/cfme/common/datastore_views.py b/cfme/common/datastore_views.py index 171f690d10..56cffc898d 100644 --- a/cfme/common/datastore_views.py +++ b/cfme/common/datastore_views.py @@ -39,6 +39,7 @@ class DatastoreEntities(BaseEntitiesView): def entity_class(self): return DatastoreEntity + class DatastoreToolBar(View): """ represents datastore toolbar and its controls diff --git a/cfme/common/provider.py b/cfme/common/provider.py index 906ebc072a..f933c35ce2 100644 --- a/cfme/common/provider.py +++ b/cfme/common/provider.py @@ -17,6 +17,7 @@ from cfme.base.credential import TokenCredential from cfme.common import CustomButtonEventsMixin from cfme.common import Taggable +from cfme.common.datastore_views import ProviderAllDatastoresView from cfme.exceptions import AddProviderError from cfme.exceptions import HostStatsNotContains from cfme.exceptions import ItemNotFound @@ -27,6 +28,7 @@ from cfme.utils import conf from cfme.utils import ParamClassName from cfme.utils.appliance import Navigatable +from cfme.utils.appliance.implementations.ui import CFMENavigateStep from cfme.utils.appliance.implementations.ui import navigate_to from cfme.utils.appliance.implementations.ui import navigator from cfme.utils.log import logger @@ -1190,7 +1192,6 @@ def run_smartstate_analysis_from_provider(self, datastores, wait_for_task_result f'"{datastore.name}": scan successfully initiated') - class CloudInfraProviderMixin: detail_page_suffix = 'provider' edit_page_suffix = 'provider_edit' @@ -1361,10 +1362,6 @@ class DefaultEndpointForm(View): validate = Button('Validate') -from cfme.utils.appliance.implementations.ui import CFMENavigateStep -from cfme.common.datastore_views import DatastoresView, ProviderAllDatastoresView - - @navigator.register(BaseProvider, 'DatastoresOfProvider') class DatastoresOfProvider(CFMENavigateStep): VIEW = ProviderAllDatastoresView diff --git a/cfme/modeling/base.py b/cfme/modeling/base.py index 8e29e2d664..303b4ed4fd 100644 --- a/cfme/modeling/base.py +++ b/cfme/modeling/base.py @@ -1,4 +1,7 @@ from collections.abc import Callable +from typing import Generic +from typing import Type +from typing import TypeVar import attr from cached_property import cached_property @@ -87,9 +90,8 @@ def __getattr__(self, name): return self._collection_cache[name] -from typing import Type, TypeVar, Generic +T = TypeVar('T', bound='BaseEntity') -T = TypeVar('T') @attr.s class BaseCollection(NavigatableMixin, Generic[T]): @@ -100,7 +102,7 @@ class BaseCollection(NavigatableMixin, Generic[T]): 1) That the API consistently has the first argument passed to it 2) That that first argument is an appliance instance - This class works in tandem with the entrypoint loader which ensures that the correct + This class works in tandem with the entry-point loader which ensures that the correct argument names have been used. """ ENTITY: Type[T] @@ -108,9 +110,6 @@ class BaseCollection(NavigatableMixin, Generic[T]): parent = attr.ib(repr=False) filters = attr.ib(default=attr.Factory(dict)) - def instantiate(self, *args, **kwargs) -> T: - return self.ENTITY.from_collection(self, *args, **kwargs) - @property def appliance(self): if isinstance(self.parent, BaseEntity): @@ -130,6 +129,10 @@ def for_entity(cls, obj, *k, **kw): def for_entity_with_filter(cls, obj, filt, *k, **kw): return cls.for_entity(obj, *k, **kw).filter(filt) + def instantiate(self, *args, **kwargs) -> T: + return self.ENTITY.from_collection(self, *args, **kwargs) + + def filter(self, filter): filters = self.filters.copy() filters.update(filter) @@ -138,14 +141,14 @@ def filter(self, filter): @attr.s class BaseEntity(NavigatableMixin): - """Class for helping create consistent entitys + """Class for helping create consistent entities The BaseEntity class is responsible for ensuring two things: 1) That the API consistently has the first argument passed to it 2) That that first argument is a collection instance - This class works in tandem with the entrypoint loader which ensures that the correct + This class works in tandem with the entry-point loader which ensures that the correct argument names have been used. """ From 5f00e93a3fe1744bc5b399cb35ccba29cf959446 Mon Sep 17 00:00:00 2001 From: Jaroslav Henner Date: Mon, 27 Jul 2020 16:02:07 +0200 Subject: [PATCH 18/18] Messing around with types --- cfme/infrastructure/provider/__init__.py | 7 +++--- cfme/infrastructure/provider/rhevm.py | 6 ++++- cfme/modeling/base.py | 29 ++++++++++++++++-------- cfme/utils/appliance/__init__.py | 4 +++- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/cfme/infrastructure/provider/__init__.py b/cfme/infrastructure/provider/__init__.py index 5fa9040b7c..6000c803ed 100644 --- a/cfme/infrastructure/provider/__init__.py +++ b/cfme/infrastructure/provider/__init__.py @@ -1,5 +1,7 @@ """ A model of an Infrastructure Provider in CFME """ +from typing import Type + import attr from navmazing import NavigateToAttribute from navmazing import NavigateToSibling @@ -33,6 +35,7 @@ from cfme.infrastructure.virtual_machines import InfraTemplateCollection from cfme.infrastructure.virtual_machines import InfraVm from cfme.modeling.base import BaseCollection +from cfme.modeling.base import TBaseEntity from cfme.optimize.utilization import ProviderUtilizationTrendsView from cfme.utils.appliance.implementations.ui import CFMENavigateStep from cfme.utils.appliance.implementations.ui import navigate_to @@ -228,8 +231,6 @@ class InfraProviderCollection(BaseCollection): """Collection object for InfraProvider object """ - ENTITY = InfraProvider - def all(self): view = navigate_to(self, 'All') provs = view.entities.get_all(surf_pages=True) @@ -244,7 +245,7 @@ def _get_class(pid): return [self.instantiate(prov_class=_get_class(p.data['id']), name=p.name) for p in provs] - def instantiate(self, prov_class, *args, **kwargs): + def instantiate(self, prov_class: Type[TBaseEntity], *args, **kwargs) -> TBaseEntity: return prov_class.from_collection(self, *args, **kwargs) def create(self, prov_class, *args, **kwargs): diff --git a/cfme/infrastructure/provider/rhevm.py b/cfme/infrastructure/provider/rhevm.py index 686726fea9..b28ee7678e 100644 --- a/cfme/infrastructure/provider/rhevm.py +++ b/cfme/infrastructure/provider/rhevm.py @@ -112,7 +112,10 @@ def from_config(cls, prov_config, prov_key, appliance=None): end_ip = prov_config['discovery_range']['end'] else: start_ip = end_ip = prov_config.get('ipaddress') - return appliance.collections.infra_providers.instantiate( + + from cfme.infrastructure.provider import InfraProviderCollection + col: InfraProviderCollection = appliance.collections.infra_providers + obj = col.instantiate( prov_class=cls, name=prov_config['name'], endpoints=endpoints, @@ -120,6 +123,7 @@ def from_config(cls, prov_config, prov_key, appliance=None): key=prov_key, start_ip=start_ip, end_ip=end_ip) + return obj # Following methods will only work if the remote console window is open # and if selenium focused on it. These will not work if the selenium is diff --git a/cfme/modeling/base.py b/cfme/modeling/base.py index 303b4ed4fd..d411ffb965 100644 --- a/cfme/modeling/base.py +++ b/cfme/modeling/base.py @@ -1,7 +1,9 @@ from collections.abc import Callable +from typing import ClassVar from typing import Generic from typing import Type from typing import TypeVar +from typing import Union import attr from cached_property import cached_property @@ -14,6 +16,7 @@ from cfme.exceptions import ItemNotFound from cfme.exceptions import KeyPairNotFound from cfme.exceptions import RestLookupError +from cfme.utils.appliance import Appliance from cfme.utils.appliance import NavigatableMixin from cfme.utils.appliance.implementations.ui import navigate_to from cfme.utils.log import logger @@ -90,11 +93,11 @@ def __getattr__(self, name): return self._collection_cache[name] -T = TypeVar('T', bound='BaseEntity') +TBaseEntity = TypeVar('TBaseEntity', bound='BaseEntity') @attr.s -class BaseCollection(NavigatableMixin, Generic[T]): +class BaseCollection(NavigatableMixin, Generic[TBaseEntity]): """Class for helping create consistent Collections The BaseCollection class is responsible for ensuring two things: @@ -105,9 +108,8 @@ class BaseCollection(NavigatableMixin, Generic[T]): This class works in tandem with the entry-point loader which ensures that the correct argument names have been used. """ - ENTITY: Type[T] - - parent = attr.ib(repr=False) + ENTITY: ClassVar[Type[TBaseEntity]] + parent: Union['BaseEntity', Appliance] = attr.ib(repr=False) filters = attr.ib(default=attr.Factory(dict)) @property @@ -118,20 +120,20 @@ def appliance(self): return self.parent @classmethod - def for_appliance(cls, appliance, *k, **kw): + def for_appliance(cls, appliance: Appliance, *k, **kw): return cls(appliance) @classmethod - def for_entity(cls, obj, *k, **kw): + def for_entity(cls, obj: 'BaseEntity', *k, **kw): return cls(obj, *k, **kw) @classmethod def for_entity_with_filter(cls, obj, filt, *k, **kw): return cls.for_entity(obj, *k, **kw).filter(filt) - def instantiate(self, *args, **kwargs) -> T: - return self.ENTITY.from_collection(self, *args, **kwargs) - + def instantiate(self, *args, **kwargs) -> TBaseEntity: + obj = self.ENTITY.from_collection(self, *args, **kwargs) + return obj def filter(self, filter): filters = self.filters.copy() @@ -164,6 +166,13 @@ def appliance(self): @classmethod def from_collection(cls, collection: BaseCollection, *k, **kw): + # TODO (jhenner) What to do with *k and **kw? We seem to need to accept it here to enable + # File "...cfme/infrastructure/provider/virtualcenter.py", line 95, in from_config + # end_ip=end_ip) + # py.test --use-sprout --sprout-group downstream-510z -s --use-provider complete \ + # 'cfme/tests/infrastructure/test_provisioning_dialog.py::\ + # test_provisioning_schedule[ansible_tower-3.4]' \ + # --sprout-user-key jhenner --long-running --pdb return cls(collection, *k, **kw) @cached_property diff --git a/cfme/utils/appliance/__init__.py b/cfme/utils/appliance/__init__.py index dde841b785..1ae46bdbc2 100644 --- a/cfme/utils/appliance/__init__.py +++ b/cfme/utils/appliance/__init__.py @@ -269,6 +269,8 @@ def name(self): @property def server(self): + # TODO(jhenner) This annotation would be great to have: -> cfme.base.Server: + # but importing makes problems sid = self._rest_api_server.id return self.collections.servers.instantiate(sid=sid) @@ -3125,7 +3127,7 @@ def __exit__(self, *args, **kwargs): assert stack.pop() is self, 'Dummy appliance on stack inconsistent' -def find_appliance(obj, require=True): +def find_appliance(obj, require=True) -> IPAppliance: if isinstance(obj, NavigatableMixin): return obj.appliance # duck type - either is the config of pytest, or holds it