Problem with creating expectations suite

[MarekMath]

Hello,

I have set up my GE with following tutorial -
https://docs.greatexpectations.io/docs/deployment_patterns/how_to_use_great_expectations_in_databricks/

I am trying to create my expectation suite based on my spark dataframe but after a while I get error

IndexError: list index out of range

I am wondering whether problem could be that there are some columns which have all null values - I plan to get rid of them but I havent done it yet.

Full error message:

IndexError                                Traceback (most recent call last)
<command-3634743304674036> in <module>
----> 1 suite = profiler.build_suite()

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/profile/user_configurable_profiler.py in build_suite(self)
    233             expectation_suite = self._build_expectation_suite_from_semantic_types_dict()
    234         else:
--> 235             expectation_suite = self._profile_and_build_expectation_suite()
    236 
    237         self._send_usage_stats_message()

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/profile/user_configurable_profiler.py in _profile_and_build_expectation_suite(self)
    372 
    373                 if data_type in ("FLOAT", "INT", "NUMERIC"):
--> 374                     self._build_expectations_numeric(
    375                         profile_dataset=self.profile_dataset,
    376                         column=column_name,

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/profile/user_configurable_profiler.py in _build_expectations_numeric(self, profile_dataset, column)
    952 
    953             quantile_result = (
--> 954                 profile_dataset.expect_column_quantile_values_to_be_between(
    955                     column,
    956                     quantile_ranges={

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/validator/validator.py in inst_expectation(*args, **kwargs)
    412                     )
    413                 else:
--> 414                     raise err
    415             return validation_result
    416 

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/validator/validator.py in inst_expectation(*args, **kwargs)
    368                     )
    369                 else:
--> 370                     validation_result = expectation.validate(
    371                         validator=self,
    372                         evaluation_parameters=self._expectation_suite.evaluation_parameters,

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/expectations/expectation.py in validate(self, validator, configuration, evaluation_parameters, interactive_evaluation, data_context, runtime_configuration)
    892             evaluation_parameters, interactive_evaluation, data_context
    893         )
--> 894         evr = validator.graph_validate(
    895             configurations=[configuration],
    896             runtime_configuration=runtime_configuration,

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/validator/validator.py in graph_validate(self, configurations, metrics, runtime_configuration)
   1000                     )
   1001                 else:
-> 1002                     raise err
   1003 
   1004         return evrs

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/validator/validator.py in graph_validate(self, configurations, metrics, runtime_configuration)
    984         for configuration in processed_configurations:
    985             try:
--> 986                 result = configuration.metrics_validate(
    987                     metrics,
    988                     execution_engine=self._execution_engine,

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/core/expectation_configuration.py in metrics_validate(self, metrics, runtime_configuration, execution_engine, **kwargs)
   1398     ):
   1399         expectation_impl: "Expectation" = self._get_expectation_impl()  # noqa: F821
-> 1400         return expectation_impl(self).metrics_validate(
   1401             metrics,
   1402             runtime_configuration=runtime_configuration,

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/expectations/expectation.py in metrics_validate(self, metrics, configuration, runtime_configuration, execution_engine, **kwargs)
    732         expectation_validation_result: Union[
    733             ExpectationValidationResult, dict
--> 734         ] = self._validate(
    735             configuration=configuration,
    736             metrics=provided_metrics,

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/expectations/core/expect_column_quantile_values_to_be_between.py in _validate(self, configuration, metrics, runtime_configuration, execution_engine)
    741             for (lower_bound, upper_bound) in quantile_value_ranges
    742         ]
--> 743         success_details = [
    744             range_[0] <= quantile_vals[idx] <= range_[1]
    745             for idx, range_ in enumerate(comparison_quantile_ranges)

/local_disk0/.ephemeral_nfs/envs/pythonEnv-e03fc367-a93a-46af-b3a6-b02e85cc3c85/lib/python3.8/site-packages/great_expectations/expectations/core/expect_column_quantile_values_to_be_between.py in <listcomp>(.0)
    742         ]
    743         success_details = [
--> 744             range_[0] <= quantile_vals[idx] <= range_[1]
    745             for idx, range_ in enumerate(comparison_quantile_ranges)
    746         ]

[MarekMath]

I seems that getting rid of all null values solved this problem.

[AFineDayFor]

Howdy @MarekMath, can you share your workflow -- and configuration if applicable -- with us so we can look into this a bit further?