From 85c0a583fddd2c49148441554d179b78b1fb584b Mon Sep 17 00:00:00 2001 From: David Parrini Date: Mon, 16 Oct 2023 19:50:25 -0300 Subject: [PATCH] Fix pandas FutureWarnings about concatenating empty frames on marginal price files --- .../omie_data_importer_from_responses.py | 9 +++++--- .../FileReaders/marginal_price_file_reader.py | 22 +++++++++++++------ .../marginal_pricer_reader_test.py | 2 +- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/OMIEData/DataImport/omie_data_importer_from_responses.py b/OMIEData/DataImport/omie_data_importer_from_responses.py index 66b7d6f..41040f8 100644 --- a/OMIEData/DataImport/omie_data_importer_from_responses.py +++ b/OMIEData/DataImport/omie_data_importer_from_responses.py @@ -20,13 +20,16 @@ def __init__(self, self.date_end = date_end def read_to_dataframe(self, verbose=False) -> pd.DataFrame: - - df = pd.DataFrame(columns=self.fileReader.get_keys()) + df = None for response in self.fileDownloader.url_responses(date_ini=self.date_ini, date_end=self.date_end, verbose=verbose): try: - df = pd.concat([df, self.fileReader.get_data_from_response(response=response)], ignore_index=True) + response_df = self.fileReader.get_data_from_response(response=response) + if df is not None: + df = pd.concat([df, response_df], ignore_index=True) + else: + df = response_df except Exception as exc: print('There was error processing file: ' + response.url) diff --git a/OMIEData/FileReaders/marginal_price_file_reader.py b/OMIEData/FileReaders/marginal_price_file_reader.py index eb36429..d3b5744 100644 --- a/OMIEData/FileReaders/marginal_price_file_reader.py +++ b/OMIEData/FileReaders/marginal_price_file_reader.py @@ -37,7 +37,7 @@ class MarginalPriceFileReader(OMIEFileReader): __key_list_retrieve__ = ['DATE', 'CONCEPT', 'H1', 'H2', 'H3', 'H4','H5', 'H6','H7', 'H8','H9','H10', 'H11', 'H12','H13', 'H14','H15', 'H16','H17', 'H18','H19','H20', - 'H21', 'H22','H23', 'H24', "H25"] + 'H21', 'H22','H23', 'H24'] __dateFormatInFile__ = '%d/%m/%Y' __localeInFile__ = "en_DK.UTF-8" @@ -50,7 +50,7 @@ def get_keys(self): def get_data_from_response(self, response: Response) -> pd.DataFrame: - res = pd.DataFrame(columns=self.get_keys()) + res = None # from first line we get the units and the price date. We just look at the date lines = response.text.split("\n") @@ -76,15 +76,19 @@ def get_data_from_response(self, response: Response) -> pd.DataFrame: if concept_type in self.conceptsToLoad: units = MarginalPriceFileReader.__dic_static_concepts__[first_col][1] - dico = self._process_line(date=date, concept=concept_type, values=splits[1:], multiplier=units) - res = pd.concat([res, pd.DataFrame([dico])], ignore_index=True) + dico = self._process_line(date=date, concept=concept_type, values=splits[1:-1], multiplier=units) + dico_df = pd.DataFrame([dico]) + if res is not None: + res = pd.concat([res, dico_df], ignore_index=True) + else: + res = dico_df return res def get_data_from_file(self, filename: str) -> pd.DataFrame: # Method yield each dictionary one by one - res = pd.DataFrame(columns=self.get_keys()) + res = None file = open(filename, 'r', encoding='latin-1') # from first line we get the units and the price date. We just look at the date @@ -108,8 +112,12 @@ def get_data_from_file(self, filename: str) -> pd.DataFrame: if concept_type in self.conceptsToLoad: units = MarginalPriceFileReader.__dic_static_concepts__[first_col][1] - dico = self._process_line(date=date, concept=concept_type, values=splits[1:], multiplier=units) - res = pd.concat([res, pd.DataFrame([dico])], ignore_index=True) + dico = self._process_line(date=date, concept=concept_type, values=splits[1:-1], multiplier=units) + dico_df = pd.DataFrame([dico]) + if res is not None: + res = pd.concat([res, dico_df], ignore_index=True) + else: + res = dico_df return res diff --git a/tests/filereaders_tests/marginal_pricer_reader_test.py b/tests/filereaders_tests/marginal_pricer_reader_test.py index 9a7d7c4..dccc43a 100644 --- a/tests/filereaders_tests/marginal_pricer_reader_test.py +++ b/tests/filereaders_tests/marginal_pricer_reader_test.py @@ -390,7 +390,7 @@ def run_all_tests(): with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=FutureWarning) warnings.filterwarnings("ignore", category=DeprecationWarning) - for test_func in [test_functions]: + for test_func in test_functions: try: test_func() except (AssertionError, UnicodeDecodeError, TypeError, IndexError) as e: