From 820840d9d46a7683eeba4970b3067ae9049a7c01 Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Mon, 2 Feb 2026 15:06:28 +0000 Subject: [PATCH 01/54] first query for flow sheets --- .gitignore | 1 + sql_scripts/flow_sheet_values.sql | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 sql_scripts/flow_sheet_values.sql diff --git a/.gitignore b/.gitignore index be3f5ce..5fac6cd 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ wheels/ # IDEs .idea/ +.vscode/ # settings files (should not be in the source tree anyway, but just in case) *.env diff --git a/sql_scripts/flow_sheet_values.sql b/sql_scripts/flow_sheet_values.sql new file mode 100644 index 0000000..77d6cc1 --- /dev/null +++ b/sql_scripts/flow_sheet_values.sql @@ -0,0 +1,13 @@ +select observation_datetime as datetime, +value_as_real, +unit, +value_as_text, +comment, +vo.visit_observation_type_id , +(select display_name +from star.visit_observation_type as vt +where vt.visit_observation_type_id = vo.visit_observation_type_id ) as observation +from star.visit_observation as vo +left join star.visit_observation_type as vt +on vo.visit_observation_type_id = vt.visit_observation_type_id +where hospital_visit_id = 'csn' \ No newline at end of file From fb3d6da7edf40527fdbb5df5a8abf161614e1768 Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Wed, 4 Feb 2026 10:30:40 +0000 Subject: [PATCH 02/54] building up scripts --- sql_scripts/flow_sheet_values.sql | 2 +- sql_scripts/lab_results.sql | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 sql_scripts/lab_results.sql diff --git a/sql_scripts/flow_sheet_values.sql b/sql_scripts/flow_sheet_values.sql index 77d6cc1..bbfdfb9 100644 --- a/sql_scripts/flow_sheet_values.sql +++ b/sql_scripts/flow_sheet_values.sql @@ -1,7 +1,7 @@ select observation_datetime as datetime, value_as_real, unit, -value_as_text, +value_as_text,£. comment, vo.visit_observation_type_id , (select display_name diff --git a/sql_scripts/lab_results.sql b/sql_scripts/lab_results.sql new file mode 100644 index 0000000..9160ba6 --- /dev/null +++ b/sql_scripts/lab_results.sql @@ -0,0 +1,14 @@ +select +result_last_modified_datetime as date, +value_as_real as value, +units, +abnormal_flag, +comment + + +from star.lab_result as r +join star.lab_order as o +on r.lab_order_id = o.lab_order_id +where o.hospital_visit_id = 'xxx' +and r.result_status like 'FINAL' +and r.lab_test_definition_id in ('1001', '390793054', '390793057', '390793060', '722790196') From 8476351baada271db1b38a1545ce57096dcd5e90 Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Thu, 19 Feb 2026 14:54:35 +0000 Subject: [PATCH 03/54] adding more files --- sql_scripts/flow_sheet_values.sql | 22 ++++++++++++++++------ sql_scripts/get_hospital_visit_id.sql | 2 ++ sql_scripts/lab_results.sql | 12 +++++++++++- sql_scripts/lab_test_names.sql | 5 +++++ 4 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 sql_scripts/get_hospital_visit_id.sql create mode 100644 sql_scripts/lab_test_names.sql diff --git a/sql_scripts/flow_sheet_values.sql b/sql_scripts/flow_sheet_values.sql index bbfdfb9..2a6c9eb 100644 --- a/sql_scripts/flow_sheet_values.sql +++ b/sql_scripts/flow_sheet_values.sql @@ -1,13 +1,23 @@ +--get the flow sheet values for the particular visit +-- the flow sheet numbers are recorded as id_in_application in the visit_observation_type table +-- Temperature 6 +-- Noradrenalin 3040102622 +-- Metaraminol 12946 +-- Secretion amount 451120 +-- Sputum amount 302600 + + select observation_datetime as datetime, +(select display_name +from star.visit_observation_type as vt +where vt.visit_observation_type_id = vo.visit_observation_type_id ) as observation, value_as_real, unit, -value_as_text,£. +value_as_text, comment, -vo.visit_observation_type_id , -(select display_name -from star.visit_observation_type as vt -where vt.visit_observation_type_id = vo.visit_observation_type_id ) as observation +vo.visit_observation_type_id from star.visit_observation as vo left join star.visit_observation_type as vt on vo.visit_observation_type_id = vt.visit_observation_type_id -where hospital_visit_id = 'csn' \ No newline at end of file +where vt.id_in_application in ('6', '12946', '302600', '451120', '3040102622') +and hospital_visit_id = 'xx' \ No newline at end of file diff --git a/sql_scripts/get_hospital_visit_id.sql b/sql_scripts/get_hospital_visit_id.sql new file mode 100644 index 0000000..5a60221 --- /dev/null +++ b/sql_scripts/get_hospital_visit_id.sql @@ -0,0 +1,2 @@ +select hospital_visit_id from star.hospital_visit as hv +where hv.encounter = 'csn' \ No newline at end of file diff --git a/sql_scripts/lab_results.sql b/sql_scripts/lab_results.sql index 9160ba6..9309722 100644 --- a/sql_scripts/lab_results.sql +++ b/sql_scripts/lab_results.sql @@ -1,5 +1,14 @@ +-- This selects the values are units of lab tests +-- 1011 CRP +-- 722790196 CRP +-- 390793054 WCC +-- 390793057 WCC +-- 390793060 WCC + select result_last_modified_datetime as date, +(select name from star.lab_test_definition as ltd +where ltd.lab_test_definition_id = r.lab_test_definition_id) as name, value_as_real as value, units, abnormal_flag, @@ -9,6 +18,7 @@ comment from star.lab_result as r join star.lab_order as o on r.lab_order_id = o.lab_order_id -where o.hospital_visit_id = 'xxx' +where o.hospital_visit_id = 'xx' and r.result_status like 'FINAL' and r.lab_test_definition_id in ('1001', '390793054', '390793057', '390793060', '722790196') + diff --git a/sql_scripts/lab_test_names.sql b/sql_scripts/lab_test_names.sql new file mode 100644 index 0000000..85ce715 --- /dev/null +++ b/sql_scripts/lab_test_names.sql @@ -0,0 +1,5 @@ +select lab_test_definition_id as id, + name, + standardised_vocabulary as vocab +from star.lab_test_definition as ltd +where ltd.lab_test_definition_id in ('1001', '390793054', '390793057', '390793060', '722790196') \ No newline at end of file From 39611434440923c21c600b6c9b0f83dbe68c393f Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Sun, 9 Aug 2026 16:54:34 +0100 Subject: [PATCH 04/54] Add and refine SQL scripts for EHR data extraction Introduces new queries for airway and sputum/secretions data, and refactors the flow sheet query to consolidate multiple values into a wider format. A new README documents the overall data extraction goal and the current set of scripts. --- sql_scripts/README.md | 33 ++++++++++++++++++ sql_scripts/airway.sql | 20 +++++++++++ sql_scripts/flow_sheet_values.sql | 50 +++++++++++++++++---------- sql_scripts/get_hospital_visit_id.sql | 7 +++- sql_scripts/lab_results.sql | 10 +++--- sql_scripts/sputum_secretions.sql | 25 ++++++++++++++ 6 files changed, 120 insertions(+), 25 deletions(-) create mode 100644 sql_scripts/README.md create mode 100644 sql_scripts/airway.sql create mode 100644 sql_scripts/sputum_secretions.sql diff --git a/sql_scripts/README.md b/sql_scripts/README.md new file mode 100644 index 0000000..98ae5af --- /dev/null +++ b/sql_scripts/README.md @@ -0,0 +1,33 @@ +# Notes on putting together the EHR needed + +## Goal + +The ultimate aim is to have one csv per patient per day which looks roughly like + + | DateTimeRecorded | Temperature | noradrenaline | etc | Secretions | etc | Placementinstant | RemovalInstant | TubeSize | etc |Units | Comments | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 08/08/2026 00:00:15 | 36.4 | | | | | | | | | | +| 08/08/2026 00:00:16 | | 1 | | | | | | | | mg/L | | +| 08/08/2026 00:00:17 | | | | None | | | | | | | +| 08/08/2026 00:02:18 | | | | | | 07/08/2026 | |8mm | | | +| 08/08/2026 00:00:15 | 38.5 | | | | | | | | | | Doctors alerted | + +*Note: The insertion date for a tube may well be earlier than the day on which it is recorded as these seem to get populated during the nightly update to caboodle.* + +## Current scripts + +| script | arguments | record | location of script in repo | database | +|- | --- | --- |- | --- | +| mrn_based_on_bed_and_datetime.sql | location string | csn |waveform-controller/src/sql | star | +| get_hospital_visit_id.sql| csn | hospital_visit_id | waveform-controller/sql_scripts| star | +| flow_sheet_values.sql| hospital_visit_id/today/yesterday | part of table above | waveform-controller/sql_scripts| star | +| airway.sql | csn/today/yesterday | part of the table above | waveform-controller/sql_scripts | caboodle | +| sputum_secretions.sql | csn/today/yesterday | part of the table above | waveform-controller/sql_scripts | caboodle | + +## Unfinished scripts + +lab_results.sql need dealing with in the same way as flow_sheet_values + +lab_test_names.sql forms part of the above query but is useful for exploring + +We need scripts for any of the items in the a tracker that have not yet been covered. diff --git a/sql_scripts/airway.sql b/sql_scripts/airway.sql new file mode 100644 index 0000000..3b0889e --- /dev/null +++ b/sql_scripts/airway.sql @@ -0,0 +1,20 @@ + +SELECT +lda._CreationInstant as DateTimeRecorded, +lda.PlacementInstant, +lda.RemovalInstant, +fvf.Value AS TubeSize + +FROM FilteredAccess.LdaFact lda +JOIN FilteredAccess.FlowsheetValueFact fvf ON fvf.LdaKey = lda.LdaKey +JOIN FilteredAccess.FlowsheetRowDim frd ON frd.FlowsheetRowKey = fvf.FlowsheetRowKey +JOIN FilteredAccess.EncounterFact enc ON enc.EncounterKey = lda.InitialEncounterKey + +WHERE +fvf.FlowsheetRowEpicId ='1120100079' +AND enc.Type != 'Anaesthesia' +AND frd.DisplayName like 'Single Lumen Tube Size' +--and enc.PatientDurableKey = '1782941' + +AND lda._CreationInstant BETWEEN %(yesterday)s AND %(today)s +AND enc.EncounterEpicCsn = %(csn) diff --git a/sql_scripts/flow_sheet_values.sql b/sql_scripts/flow_sheet_values.sql index 2a6c9eb..af9b76f 100644 --- a/sql_scripts/flow_sheet_values.sql +++ b/sql_scripts/flow_sheet_values.sql @@ -1,23 +1,35 @@ ---get the flow sheet values for the particular visit +--get the flow sheet values for the particular visit on a particular day -- the flow sheet numbers are recorded as id_in_application in the visit_observation_type table -- Temperature 6 -- Noradrenalin 3040102622 -- Metaraminol 12946 --- Secretion amount 451120 --- Sputum amount 302600 - - -select observation_datetime as datetime, -(select display_name -from star.visit_observation_type as vt -where vt.visit_observation_type_id = vo.visit_observation_type_id ) as observation, -value_as_real, -unit, -value_as_text, -comment, -vo.visit_observation_type_id -from star.visit_observation as vo -left join star.visit_observation_type as vt -on vo.visit_observation_type_id = vt.visit_observation_type_id -where vt.id_in_application in ('6', '12946', '302600', '451120', '3040102622') -and hospital_visit_id = 'xx' \ No newline at end of file + +SELECT + vo.observation_datetime AS DateTimeRecorded, + + (array_agg(vo.value_as_real) FILTER ( + WHERE vt.id_in_application = '6' + ))[1] AS "Temperature", + + (array_agg(vo.value_as_real) FILTER ( + WHERE vt.id_in_application = '3040102622' + ))[1] AS "Noradrenaline", + + (array_agg(vo.value_as_real) FILTER ( + WHERE vt.id_in_application = '12946' + ))[1] AS "Metaraminol", + + vo.unit AS Units, + vo.comment AS Comments + +FROM star.visit_observation AS vo + +LEFT JOIN star.visit_observation_type AS vt + ON vo.visit_observation_type_id = vt.visit_observation_type_id + +WHERE vt.id_in_application IN ('6', '3040102622', '12946') +AND vo.valid_from BETWEEN %(yesterday)s AND %(today)s +AND vo.hospital_visit_id = %(hospital_visit_id)s + +GROUP BY DateTimeRecorded, Units, vo.comment + diff --git a/sql_scripts/get_hospital_visit_id.sql b/sql_scripts/get_hospital_visit_id.sql index 5a60221..408447f 100644 --- a/sql_scripts/get_hospital_visit_id.sql +++ b/sql_scripts/get_hospital_visit_id.sql @@ -1,2 +1,7 @@ + + + select hospital_visit_id from star.hospital_visit as hv -where hv.encounter = 'csn' \ No newline at end of file +where hv.encounter = %(csn)s -- note the CSN must be in quotes + + diff --git a/sql_scripts/lab_results.sql b/sql_scripts/lab_results.sql index 9309722..aa05264 100644 --- a/sql_scripts/lab_results.sql +++ b/sql_scripts/lab_results.sql @@ -6,13 +6,13 @@ -- 390793060 WCC select -result_last_modified_datetime as date, +r.result_last_modified_datetime as date, (select name from star.lab_test_definition as ltd where ltd.lab_test_definition_id = r.lab_test_definition_id) as name, -value_as_real as value, -units, -abnormal_flag, -comment +r.value_as_real as value, +r.units, +r.abnormal_flag, +r.comment from star.lab_result as r diff --git a/sql_scripts/sputum_secretions.sql b/sql_scripts/sputum_secretions.sql new file mode 100644 index 0000000..8f27ab8 --- /dev/null +++ b/sql_scripts/sputum_secretions.sql @@ -0,0 +1,25 @@ +<- Retrieved the information about sputum and secretions > + +SELECT +fv.TakenInstant AS 'DateTimeRecorded', +CASE +WHEN fsd.FlowsheetRowEpicId = '451120' +THEN fv.Value +END +AS 'Secretions' , +CASE +WHEN fsd.FlowsheetRowEpicId = '302600' +THEN fv.Value +END +AS 'Sputum' , +fv.Comment AS Comments + +FROM FilteredAccess.FlowsheetValueFact fv +INNER JOIN FilteredAccess.FlowsheetRowDim fsd ON fv.FlowsheetRowKey = fsd.FlowsheetRowKey +INNER JOIN FilteredAccess.EncounterFact enc ON fv.EncounterKey = enc.EncounterKey + +WHERE +(fsd.FlowsheetRowEpicId = '451120' OR +fsd.FlowsheetRowEpicId ='302600') +AND fv.TakenInstant BETWEEN %(yesterday)s AND %(today)s +AND enc.EncounterEpicCsn = %(csn) From cdec5caed0749c693984742b417bba6d47c2435a Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 11 Aug 2026 15:42:19 +0100 Subject: [PATCH 05/54] Started on functions to wrap snakemake around sql calls --- src/electronic_health_records/__init__.py | 0 src/electronic_health_records/ehr.py | 180 ++++++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 src/electronic_health_records/__init__.py create mode 100644 src/electronic_health_records/ehr.py diff --git a/src/electronic_health_records/__init__.py b/src/electronic_health_records/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/electronic_health_records/ehr.py b/src/electronic_health_records/ehr.py new file mode 100644 index 0000000..ebd514c --- /dev/null +++ b/src/electronic_health_records/ehr.py @@ -0,0 +1,180 @@ +import argparse +import functools +import json +import logging +from decimal import Decimal +from pathlib import Path +from typing import Any + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import settings + +from locations import ( + CSV_PATTERN, + ORIGINAL_PARQUET_PATTERN, + PSEUDONYMISED_PARQUET_PATTERN, +) +from .hashing import do_hash + +def ehr_for_csv( + *, + date_str: str, + original_csn: str, + hashed_csn: str, +) -> None: + """Extracts electronic healthcare records for a given csn and writes the + results to a pseudonymised csv file for a single day. + + This is a privacy-sensitive area of code. Unhashed CSNs must not appear in + uploaded files. + :param date_str: the date to look up data for + :param original_csn: the csn to base look up on. + :param hashed_csn: the pseudonymised hash to use for file output. + + """ + # will pick up the logger config defined in the snakemake job (ie. log to file) + logger = logging.getLogger(__name__) + + # it's in the csv_path and original_parquet_path, but at least nowhere else! + del original_csn + + logger.info("Turning CSV %s to parquets", csv_path) + csv_path.parent.mkdir(parents=True, exist_ok=True) + original_parquet_path.parent.mkdir(parents=True, exist_ok=True) + df = pd.read_csv( + str(csv_path), + dtype={ + "csn": str, + "mrn": str, + "source_variable_id": str, + "source_channel_id": str, + "units": str, + "sampling_rate": int, + "timestamp": float, + "location": str, + "values": str, + }, + header=0, # the first line is always the header + ) + + def parse_array(x): + # Not sure if this is the most efficient way. Might be able to do something with DecimalArray? + # return [pa.decimal128(i) for i in x.replace(' ', '').split(',')] + return [Decimal(i) for i in x.strip().strip("[]").replace(" ", "").split(",")] + + df["values"] = df["values"].apply(parse_array) + + # Convert pandas DataFrame to pyarrow Table with proper types + schema = pa.schema( + [ + ("csn", pa.string()), + ("mrn", pa.string()), + ("source_variable_id", pa.string()), + ("source_channel_id", pa.string()), + ("units", pa.string()), + ("sampling_rate", pa.int32()), + ("timestamp", pa.float64()), + ("location", pa.string()), + # As per requirements, compactness is important here. + # decimal32 can have a maximum of 9 significant digits and should + # satisfy our needs, but it only exists in pyarrow >= 19. + # We are currently tied to 18.1 because of PIXL core. + # So for now, use decimal128 instead. + # Not yet tested whether the specified precision + # and scale cause it to be equivalent in size to decimal32. + # See issue #31. + ("values", pa.list_(pa.decimal128(9, 4))), + ] + ) + table = pa.Table.from_pandas(df, schema=schema, preserve_index=True) + + # mark the parquet files themselves as production or not. + our_metadata = {"instance_name": settings.INSTANCE_NAME} + + table = add_waveform_metadata_to_table(table, our_metadata) + + pq.write_table( + table, + str(original_parquet_path), + # valid values: {‘NONE’, ‘SNAPPY’, ‘GZIP’, ‘BROTLI’, ‘LZ4’, ‘ZSTD’} + compression="zstd", + use_dictionary=True, + write_statistics=True, # enable indexes/statistics + flavor="spark", + ) + logger.info( + "Done turning CSV %s to original parquet %s", csv_path, original_parquet_path + ) + + df = pseudonymise_relevant_columns(df) + pseudon_table = pa.Table.from_pandas(df, schema=schema, preserve_index=True) + + # Use same metadata for pseudon, must not contain identifiers! + pseudon_table = add_waveform_metadata_to_table(pseudon_table, our_metadata) + + hashed_path = Path( + str(PSEUDONYMISED_PARQUET_PATTERN).format( + date=date_str, + hashed_csn=hashed_csn, + variable_id=variable_id, + channel_id=channel_id, + units=units, + ) + ) + pq.write_table( + pseudon_table, + str(hashed_path), + compression="zstd", + use_dictionary=True, + write_statistics=True, # enable indexes/statistics + flavor="spark", + ) + logger.info( + "Done turning CSV %s to pseudonymised parquet %s", csv_path, hashed_path + ) + + +def add_waveform_metadata_to_table( + existing_table: pa.Table, metadata: dict[str, Any] +) -> pa.Table: + """Replace our metadata in its entirety, leaving untouched metadata we didn't + set.""" + + # Parquet footer metadata is a series of (byte string) key-value pairs. + # Other users of metadata (eg. pandas) convert their metadata to JSON and store it under + # a single key (a namespace, effectively), so we'll do the same under our own key. + waveform_exporter_metadata_key = b"waveform_exporter" + + existing_metadata = existing_table.schema.metadata or {} + json_byte_string = json.dumps(metadata).encode("utf-8") + existing_table = existing_table.replace_schema_metadata( + {**existing_metadata, waveform_exporter_metadata_key: json_byte_string} + ) + return existing_table + + +SAFE_COLUMNS = [ + "sampling_rate", + "source_variable_id", + "source_channel_id", + "timestamp", + "units", + "values", +] + + +def pseudonymise_relevant_columns(df: pd.DataFrame): + """ "csn", "mrn", "location" are examples of columns that must be pseudonymised. + + However, it's safer to list which columns *don't* need to be pseudonymised. Eg. you + add a column but forget to consider whether it's sensitive, OR you rename one of the + known sensitive columns and forget that this will cause privacy to break. This means + that when you add a new column, you have to add it here if you don't want it to be + hashed. + """ + for col in df.columns: + if col not in SAFE_COLUMNS: + df[col] = df[col].apply(functools.partial(do_hash, col)) + return df From 98972596b5ed6865279777b0b1cb177699f41c4b Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 12 Aug 2026 14:13:13 +0100 Subject: [PATCH 06/54] Started to implement caboodle look up code --- src/db.py | 44 ++++++++ src/electronic_health_records/ehr.py | 153 ++++----------------------- 2 files changed, 64 insertions(+), 133 deletions(-) diff --git a/src/db.py b/src/db.py index ebab871..dff5872 100644 --- a/src/db.py +++ b/src/db.py @@ -53,3 +53,47 @@ def get_row(self, location_string: str, observation_datetime: datetime): ) return rows[0] + + +class caboodleDB: + """For querying the caboodle database to extract electronic healthcare records per + patient.""" + + connection_string: str = "dbname={} user={} password={} host={} port={} connect_timeout={} options='-c statement_timeout={}'".format( + settings.CABOODLE_DBNAME, # type:ignore + settings.CABOODLE_USERNAME, # type:ignore + settings.CABOODLE_PASSWORD, # type:ignore + settings.CABOODLE_HOST, # type:ignore + settings.CABOODLE_PORT, # type:ignore + settings.CABOODLE_CONNECT_TIMEOUT, # type:ignore + settings.CABOODLE_QUERY_TIMEOUT, # type:ignore + ) + connection_pool: pool.ThreadedConnectionPool + + def connect(self): + """Set up connection to the database.""" + self.connection_pool = pool.SimpleConnectionPool(1, 1, self.connection_string) + + def get_airflow(self, start_datetime: datetime, end_datetime: datetime, csn: str): + """Retrieve airflow data from database.""" + with open("src/sql/airway.sql", "r") as file: + airway_query = sql.SQL(file.read()) + parameters = { + "start_datetime": start_datetime, + "end_datetime": end_datetime, + "csn": csn, + } + return self._get_rows(airway_query, parameters) + + def _get_rows(self, sql_query: sql.SQL, parameters: dict): + try: + with self.connection_pool.getconn() as db_connection: + with db_connection.cursor() as curs: + curs.execute(sql_query, parameters) + rows = curs.fetchall() + self.connection_pool.putconn(db_connection) + except psycopg2.errors.OperationalError as e: + self.connection_pool.putconn(db_connection) + raise ConnectionError(f"Data base error: {e}") + + return rows diff --git a/src/electronic_health_records/ehr.py b/src/electronic_health_records/ehr.py index ebd514c..a5d063a 100644 --- a/src/electronic_health_records/ehr.py +++ b/src/electronic_health_records/ehr.py @@ -1,158 +1,45 @@ -import argparse import functools -import json import logging -from decimal import Decimal -from pathlib import Path -from typing import Any import pandas as pd -import pyarrow as pa -import pyarrow.parquet as pq -import settings +from datetime import datetime -from locations import ( - CSV_PATTERN, - ORIGINAL_PARQUET_PATTERN, - PSEUDONYMISED_PARQUET_PATTERN, -) from .hashing import do_hash +from db import caboodleDB + def ehr_for_csv( - *, - date_str: str, + datetime: datetime, original_csn: str, hashed_csn: str, + db_connection: caboodleDB, ) -> None: - """Extracts electronic healthcare records for a given csn and writes the - results to a pseudonymised csv file for a single day. + """Extracts electronic healthcare records for a given csn and writes the results to + a pseudonymised csv file for a single day. - This is a privacy-sensitive area of code. Unhashed CSNs must not appear in - uploaded files. + This is a privacy-sensitive area of code. Unhashed CSNs must not appear in uploaded + files. :param date_str: the date to look up data for :param original_csn: the csn to base look up on. :param hashed_csn: the pseudonymised hash to use for file output. - + :param db_connection: connection to the caboodle database. """ # will pick up the logger config defined in the snakemake job (ie. log to file) logger = logging.getLogger(__name__) - # it's in the csv_path and original_parquet_path, but at least nowhere else! - del original_csn - - logger.info("Turning CSV %s to parquets", csv_path) - csv_path.parent.mkdir(parents=True, exist_ok=True) - original_parquet_path.parent.mkdir(parents=True, exist_ok=True) - df = pd.read_csv( - str(csv_path), - dtype={ - "csn": str, - "mrn": str, - "source_variable_id": str, - "source_channel_id": str, - "units": str, - "sampling_rate": int, - "timestamp": float, - "location": str, - "values": str, - }, - header=0, # the first line is always the header - ) - - def parse_array(x): - # Not sure if this is the most efficient way. Might be able to do something with DecimalArray? - # return [pa.decimal128(i) for i in x.replace(' ', '').split(',')] - return [Decimal(i) for i in x.strip().strip("[]").replace(" ", "").split(",")] - - df["values"] = df["values"].apply(parse_array) - - # Convert pandas DataFrame to pyarrow Table with proper types - schema = pa.schema( - [ - ("csn", pa.string()), - ("mrn", pa.string()), - ("source_variable_id", pa.string()), - ("source_channel_id", pa.string()), - ("units", pa.string()), - ("sampling_rate", pa.int32()), - ("timestamp", pa.float64()), - ("location", pa.string()), - # As per requirements, compactness is important here. - # decimal32 can have a maximum of 9 significant digits and should - # satisfy our needs, but it only exists in pyarrow >= 19. - # We are currently tied to 18.1 because of PIXL core. - # So for now, use decimal128 instead. - # Not yet tested whether the specified precision - # and scale cause it to be equivalent in size to decimal32. - # See issue #31. - ("values", pa.list_(pa.decimal128(9, 4))), - ] - ) - table = pa.Table.from_pandas(df, schema=schema, preserve_index=True) - - # mark the parquet files themselves as production or not. - our_metadata = {"instance_name": settings.INSTANCE_NAME} + caboodle = caboodleDB() + caboodle.connect() - table = add_waveform_metadata_to_table(table, our_metadata) + logger.info("Looking for airway data for %s.", hashed_csn) - pq.write_table( - table, - str(original_parquet_path), - # valid values: {‘NONE’, ‘SNAPPY’, ‘GZIP’, ‘BROTLI’, ‘LZ4’, ‘ZSTD’} - compression="zstd", - use_dictionary=True, - write_statistics=True, # enable indexes/statistics - flavor="spark", - ) - logger.info( - "Done turning CSV %s to original parquet %s", csv_path, original_parquet_path - ) + start_datetime = datetime + end_datetime = datetime + airflow = caboodle.get_airflow(start_datetime, end_datetime, original_csn) + airflow = pseudonymise_relevant_columns(airflow) + logger.info(airflow) - df = pseudonymise_relevant_columns(df) - pseudon_table = pa.Table.from_pandas(df, schema=schema, preserve_index=True) - - # Use same metadata for pseudon, must not contain identifiers! - pseudon_table = add_waveform_metadata_to_table(pseudon_table, our_metadata) - - hashed_path = Path( - str(PSEUDONYMISED_PARQUET_PATTERN).format( - date=date_str, - hashed_csn=hashed_csn, - variable_id=variable_id, - channel_id=channel_id, - units=units, - ) - ) - pq.write_table( - pseudon_table, - str(hashed_path), - compression="zstd", - use_dictionary=True, - write_statistics=True, # enable indexes/statistics - flavor="spark", - ) - logger.info( - "Done turning CSV %s to pseudonymised parquet %s", csv_path, hashed_path - ) - - -def add_waveform_metadata_to_table( - existing_table: pa.Table, metadata: dict[str, Any] -) -> pa.Table: - """Replace our metadata in its entirety, leaving untouched metadata we didn't - set.""" - - # Parquet footer metadata is a series of (byte string) key-value pairs. - # Other users of metadata (eg. pandas) convert their metadata to JSON and store it under - # a single key (a namespace, effectively), so we'll do the same under our own key. - waveform_exporter_metadata_key = b"waveform_exporter" - - existing_metadata = existing_table.schema.metadata or {} - json_byte_string = json.dumps(metadata).encode("utf-8") - existing_table = existing_table.replace_schema_metadata( - {**existing_metadata, waveform_exporter_metadata_key: json_byte_string} - ) - return existing_table + # delete csn once we no longer need it + del original_csn SAFE_COLUMNS = [ From 1f2cfbe47d25f50bd5393cbad8a9841eafe9c82a Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 12 Aug 2026 14:51:19 +0100 Subject: [PATCH 07/54] Moved airway sql to sql directory --- {sql_scripts => src/sql}/airway.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename {sql_scripts => src/sql}/airway.sql (81%) diff --git a/sql_scripts/airway.sql b/src/sql/airway.sql similarity index 81% rename from sql_scripts/airway.sql rename to src/sql/airway.sql index 3b0889e..c79c9cb 100644 --- a/sql_scripts/airway.sql +++ b/src/sql/airway.sql @@ -1,4 +1,4 @@ - +-- extract airway data from caboodle for a specific csn and date SELECT lda._CreationInstant as DateTimeRecorded, lda.PlacementInstant, @@ -16,5 +16,5 @@ AND enc.Type != 'Anaesthesia' AND frd.DisplayName like 'Single Lumen Tube Size' --and enc.PatientDurableKey = '1782941' -AND lda._CreationInstant BETWEEN %(yesterday)s AND %(today)s +AND lda._CreationInstant BETWEEN %(start_datetime)s AND %(end_datetime)s AND enc.EncounterEpicCsn = %(csn) From bd72361aadbb9ca867e0d162dd9253fe8ee3d0c7 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 12 Aug 2026 14:52:10 +0100 Subject: [PATCH 08/54] Setting up config and docker bits --- config.EXAMPLE/ehr_lookup.EXAMPLE | 9 +++++++++ docker-compose.yml | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 config.EXAMPLE/ehr_lookup.EXAMPLE diff --git a/config.EXAMPLE/ehr_lookup.EXAMPLE b/config.EXAMPLE/ehr_lookup.EXAMPLE new file mode 100644 index 0000000..022ce93 --- /dev/null +++ b/config.EXAMPLE/ehr_lookup.EXAMPLE @@ -0,0 +1,9 @@ +# This is an EXAMPLE file, do not put real secrets in here. +# Copy it to ../config/ehr_lookup.env and then DELETE THIS COMMENT. +CABOODLE_DBNAME="fakecab" +CABOODLE_USERNAME="inform_user" +CABOODLE_PASSWORD="inform" +CABOODLE_HOST="localhost" +CABOODLE_PORT="5433" +CABOODLE_CONNECT_TIMEOUT="10" # in seconds +CABOODLE_QUERY_TIMEOUT="3000" # in milliseconds diff --git a/docker-compose.yml b/docker-compose.yml index 6c30ca7..eaaa113 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,3 +52,24 @@ services: env_file: - ../config/hasher.env restart: unless-stopped + ehr-lookup: + build: + context: .. + dockerfile: waveform-controller/Dockerfile + target: ehr-lookup + args: + HTTP_PROXY: ${HTTP_PROXY} + http_proxy: ${http_proxy} + HTTPS_PROXY: ${HTTPS_PROXY} + https_proxy: ${https_proxy} + # ideally we'd use docker secrets but it's not enabled currently + env_file: + - ../config/ehr_lookup.env + volumes: + - ../waveform-export:/waveform-export + # because we're launching through cron in the container, which starts + # processes with a clean environment, also mount in the config file so + # it can be read in by snakemake later + - ../config/ehr-lookup.env:/config/ehr-lookup.env:ro + restart: unless-stopped + From 5e303f0edf95ba2fa8e683df5bf8db352984bc69 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 17 Aug 2026 11:44:28 +0100 Subject: [PATCH 09/54] Add a ehr location to snakefile --- src/pipeline/Snakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipeline/Snakefile b/src/pipeline/Snakefile index dbce7db..7dc9a3a 100644 --- a/src/pipeline/Snakefile +++ b/src/pipeline/Snakefile @@ -9,6 +9,7 @@ from locations import ( WAVEFORM_ORIGINAL_CSV, WAVEFORM_SNAKEMAKE_LOGS, WAVEFORM_PSEUDONYMISED_PARQUET, + WAVEFORM_PSEUDONYMISED_EHR, WAVEFORM_FTPS_LOGS, HASH_LOOKUP_JSON, HASH_LOOKUP_JSON_REL, From 89e95269c2642699c29d4706cdb262fef3fb685c Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 17 Aug 2026 15:39:19 +0100 Subject: [PATCH 10/54] Tidied up implementation and rewrote Snakefile logic, with help from Claude Sonnet 5.0 --- src/csv_writer.py | 28 ++++++++++- src/electronic_health_records/ehr.py | 73 ++++++++++++++-------------- src/locations.py | 3 ++ src/pipeline/Snakefile | 38 ++++++++++++++- src/pipeline/utils.py | 6 +++ src/pseudon/pseudon.py | 25 +++++----- 6 files changed, 121 insertions(+), 52 deletions(-) diff --git a/src/csv_writer.py b/src/csv_writer.py index 66948b8..42daeb3 100644 --- a/src/csv_writer.py +++ b/src/csv_writer.py @@ -2,8 +2,15 @@ import csv from datetime import datetime +import pandas as pd -from locations import WAVEFORM_ORIGINAL_CSV, make_file_name, FILE_STEM_PATTERN +from locations import ( + WAVEFORM_ORIGINAL_CSV, + WAVEFORM_PSEUDONYMISED_EHR, + make_file_name, + FILE_STEM_PATTERN, + EHR_STEM_PATTERN_HASHED, +) def create_file_name( @@ -77,3 +84,22 @@ def write_frame( ) return True + + +def write_ehr( + df: pd.DataFrame, + date_str: str, + hashed_csn: str, +) -> bool: + """Writes a frame of electronic healthcare data to a csv file. + + :return: True if write was successful. + """ + subs_dict = dict(date=date_str, hashed_csn=hashed_csn) + stem = make_file_name(EHR_STEM_PATTERN_HASHED, subs_dict) + filename = WAVEFORM_PSEUDONYMISED_EHR / f"{stem}_ehr.csv" + filename.parent.mkdir(exist_ok=True, parents=True) + + df.to_csv(filename) + + return True diff --git a/src/electronic_health_records/ehr.py b/src/electronic_health_records/ehr.py index a5d063a..48fc930 100644 --- a/src/electronic_health_records/ehr.py +++ b/src/electronic_health_records/ehr.py @@ -1,15 +1,31 @@ -import functools import logging -import pandas as pd -from datetime import datetime +from datetime import datetime, timedelta -from .hashing import do_hash from db import caboodleDB +from csv_writer import write_ehr +from pseudon.pseudon import pseudonymise_relevant_columns -def ehr_for_csv( - datetime: datetime, +def ehr_for_csv(date_str: str, original_csn: str, hashed_csn: str) -> None: + """Extracts electronic healthcare records for a given csn and writes the results to + a pseudonymised csv file for a single day. + + This is a privacy-sensitive area of code. Unhashed CSNs must not appear in uploaded + files. + :param date_str: the date to look up data for + :param original_csn: the csn to base look up on. + :param hashed_csn: the pseudonymised hash to use for file output. + """ + + db_connection = caboodleDB() + db_connection.connect() + + _ehr_for_csv(date_str, original_csn, hashed_csn, db_connection) + + +def _ehr_for_csv( + date_str: str, original_csn: str, hashed_csn: str, db_connection: caboodleDB, @@ -27,41 +43,24 @@ def ehr_for_csv( # will pick up the logger config defined in the snakemake job (ie. log to file) logger = logging.getLogger(__name__) - caboodle = caboodleDB() - caboodle.connect() - logger.info("Looking for airway data for %s.", hashed_csn) - start_datetime = datetime - end_datetime = datetime - airflow = caboodle.get_airflow(start_datetime, end_datetime, original_csn) - airflow = pseudonymise_relevant_columns(airflow) - logger.info(airflow) + start_datetime = datetime.strptime(date_str, "%Y-%m-%d") + end_datetime = start_datetime + timedelta(days=1) + airflow = db_connection.get_airflow(start_datetime, end_datetime, original_csn) - # delete csn once we no longer need it - del original_csn + safe_columns = [ + "DateTimeRecorded", + "PlacementInstant", + "RemovalInstant", + "TubeSize", + ] + airflow = pseudonymise_relevant_columns(airflow, safe_columns) -SAFE_COLUMNS = [ - "sampling_rate", - "source_variable_id", - "source_channel_id", - "timestamp", - "units", - "values", -] + write_ehr(airflow, date_str, hashed_csn) + logger.info(airflow) -def pseudonymise_relevant_columns(df: pd.DataFrame): - """ "csn", "mrn", "location" are examples of columns that must be pseudonymised. - - However, it's safer to list which columns *don't* need to be pseudonymised. Eg. you - add a column but forget to consider whether it's sensitive, OR you rename one of the - known sensitive columns and forget that this will cause privacy to break. This means - that when you add a new column, you have to add it here if you don't want it to be - hashed. - """ - for col in df.columns: - if col not in SAFE_COLUMNS: - df[col] = df[col].apply(functools.partial(do_hash, col)) - return df + # delete csn once we no longer need it + del original_csn diff --git a/src/locations.py b/src/locations.py index bb15847..f55fa20 100644 --- a/src/locations.py +++ b/src/locations.py @@ -5,6 +5,7 @@ WAVEFORM_ORIGINAL_PARQUET = WAVEFORM_EXPORT_BASE / "original-parquet" WAVEFORM_HASH_LOOKUPS = WAVEFORM_EXPORT_BASE / "hash-lookups" WAVEFORM_PSEUDONYMISED_PARQUET = WAVEFORM_EXPORT_BASE / "pseudonymised" +WAVEFORM_PSEUDONYMISED_EHR = WAVEFORM_EXPORT_BASE / "pseudonymised_ehr" WAVEFORM_SNAKEMAKE_LOGS = WAVEFORM_EXPORT_BASE / "snakemake-logs" WAVEFORM_FTPS_LOGS = WAVEFORM_EXPORT_BASE / "ftps-logs" @@ -14,6 +15,8 @@ FILE_STEM_PATTERN_HASHED = ( "{date}/{date}.{hashed_csn}.{variable_id}.{channel_id}.{units}" ) +# EHR data is per (date, csn), not per variable/channel/units, so it gets its own stem. +EHR_STEM_PATTERN_HASHED = "{date}/{date}.{hashed_csn}" CSV_PATTERN = WAVEFORM_ORIGINAL_CSV / (FILE_STEM_PATTERN + ".csv") ORIGINAL_PARQUET_PATTERN = WAVEFORM_ORIGINAL_PARQUET / (FILE_STEM_PATTERN + ".parquet") PSEUDONYMISED_PARQUET_PATTERN = WAVEFORM_PSEUDONYMISED_PARQUET / ( diff --git a/src/pipeline/Snakefile b/src/pipeline/Snakefile index 7dc9a3a..71042d9 100644 --- a/src/pipeline/Snakefile +++ b/src/pipeline/Snakefile @@ -15,9 +15,11 @@ from locations import ( HASH_LOOKUP_JSON_REL, FILE_STEM_PATTERN, FILE_STEM_PATTERN_HASHED, + EHR_STEM_PATTERN_HASHED, make_file_name, ) from pseudon.pseudon import csv_to_parquets +from electronic_health_records.ehr import ehr_for_csv from utils import config_bool, determine_eventual_outputs @@ -48,6 +50,7 @@ PROCESS_CSV_FROM_DATE = str(config['PROCESS_CSV_FROM_DATE']) all_outputs, hash_to_csn = determine_eventual_outputs(CSV_AGE_THRESHOLD_MINUTES, ONLY_USE_CSV_FROM_YESTERDAY, PROCESS_CSV_FROM_DATE) ALL_FTPS_UPLOADED = [ao.get_ftps_uploaded_file() for ao in all_outputs] ALL_DAILY_HASH_LOOKUPS = sorted({ao.get_daily_hash_lookup() for ao in all_outputs}) +ALL_EHR_LOOKUPS = sorted({ao.get_ehr_lookup() for ao in all_outputs}) def configure_file_logging(log_file): import logging @@ -65,7 +68,8 @@ def configure_file_logging(log_file): rule all: input: ftps_uploaded = ALL_FTPS_UPLOADED, - daily_hash_lookups = ALL_DAILY_HASH_LOOKUPS + daily_hash_lookups = ALL_DAILY_HASH_LOOKUPS, + ehr_lookups = ALL_EHR_LOOKUPS rule all_ftps_uploaded: input: @@ -75,6 +79,10 @@ rule all_daily_hash_lookups: input: ALL_DAILY_HASH_LOOKUPS +rule all_ehr_lookups: + input: + ALL_EHR_LOOKUPS + def input_file_maker(wc): unhashed_csn = hash_to_csn[wc.hashed_csn] # when using input functions, snakemake doesn't do its normal templating, you have to do it, hence the f-string @@ -116,6 +124,34 @@ def pseudonymised_parquet_files_for_date(wc): return [ao.get_pseudonymised_parquet_path() for ao in all_outputs if ao.date == wc.date] +def pseudonymised_parquet_files_for_date_and_hashed_csn(wc): + return [ + ao.get_pseudonymised_parquet_path() + for ao in all_outputs + if ao.date == wc.date and ao.hashed_csn == wc.hashed_csn + ] + + +rule ehr_lookup: + input: + # As with daily_hash_lookup, we lie to Snakemake that the input is the pseudon + # parquets for this csn/day, purely so this rule is tied into the dependency DAG + # and reruns if the underlying data for this csn/day changes. + pseudonymised_parquets = pseudonymised_parquet_files_for_date_and_hashed_csn + output: + WAVEFORM_PSEUDONYMISED_EHR / (EHR_STEM_PATTERN_HASHED + "_ehr.csv") + log: + WAVEFORM_SNAKEMAKE_LOGS / "ehr_lookup" / (EHR_STEM_PATTERN_HASHED + ".log") + run: + logger = configure_file_logging(log[0]) + original_csn = hash_to_csn[wildcards.hashed_csn] + logger.info("Running EHR look up for csn %s. Hash -> %s", original_csn, wildcards.hashed_csn) + ehr_for_csv( + date_str=wildcards.date, + original_csn=original_csn, + hashed_csn=wildcards.hashed_csn) + + rule daily_hash_lookup: input: # Because we don't declare the original parquets in the output of csv_to_parquet, diff --git a/src/pipeline/utils.py b/src/pipeline/utils.py index 6bf58b4..a17cd89 100644 --- a/src/pipeline/utils.py +++ b/src/pipeline/utils.py @@ -8,10 +8,12 @@ from pseudon.hashing import do_hash from locations import ( WAVEFORM_PSEUDONYMISED_PARQUET, + WAVEFORM_PSEUDONYMISED_EHR, WAVEFORM_FTPS_LOGS, HASH_LOOKUP_JSON, ORIGINAL_PARQUET_PATTERN, FILE_STEM_PATTERN_HASHED, + EHR_STEM_PATTERN_HASHED, CSV_PATTERN, make_file_name, ) @@ -73,6 +75,10 @@ def get_ftps_uploaded_file(self) -> Path: def get_daily_hash_lookup(self) -> Path: return Path(make_file_name(str(HASH_LOOKUP_JSON), self._subs_dict)) + def get_ehr_lookup(self) -> Path: + final_stem = make_file_name(EHR_STEM_PATTERN_HASHED, self._subs_dict) + return WAVEFORM_PSEUDONYMISED_EHR / f"{final_stem}_ehr.csv" + def get_file_age(file_path: Path) -> timedelta: # need to use UTC to avoid DST issues diff --git a/src/pseudon/pseudon.py b/src/pseudon/pseudon.py index 01cc498..485d310 100644 --- a/src/pseudon/pseudon.py +++ b/src/pseudon/pseudon.py @@ -142,7 +142,16 @@ def parse_array(x): "Done turning CSV %s to original parquet %s", csv_path, original_parquet_path ) - df = pseudonymise_relevant_columns(df) + safe_columns = [ + "sampling_rate", + "source_variable_id", + "source_channel_id", + "timestamp", + "units", + "values", + ] + + df = pseudonymise_relevant_columns(df, safe_columns) pseudon_table = pa.Table.from_pandas(df, schema=schema, preserve_index=True) # Use same metadata for pseudon, must not contain identifiers! @@ -189,17 +198,7 @@ def add_waveform_metadata_to_table( return existing_table -SAFE_COLUMNS = [ - "sampling_rate", - "source_variable_id", - "source_channel_id", - "timestamp", - "units", - "values", -] - - -def pseudonymise_relevant_columns(df: pd.DataFrame): +def pseudonymise_relevant_columns(df: pd.DataFrame, safe_columns: [str]): """ "csn", "mrn", "location" are examples of columns that must be pseudonymised. However, it's safer to list which columns *don't* need to be pseudonymised. Eg. you @@ -209,6 +208,6 @@ def pseudonymise_relevant_columns(df: pd.DataFrame): hashed. """ for col in df.columns: - if col not in SAFE_COLUMNS: + if col not in safe_columns: df[col] = df[col].apply(functools.partial(do_hash, col)) return df From e0498a20fd354018f072ffd2e828a3b5978d837a Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 17 Aug 2026 15:53:06 +0100 Subject: [PATCH 11/54] We don't need a separate container for ehr lookup --- docker-compose.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index eaaa113..6c30ca7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,24 +52,3 @@ services: env_file: - ../config/hasher.env restart: unless-stopped - ehr-lookup: - build: - context: .. - dockerfile: waveform-controller/Dockerfile - target: ehr-lookup - args: - HTTP_PROXY: ${HTTP_PROXY} - http_proxy: ${http_proxy} - HTTPS_PROXY: ${HTTPS_PROXY} - https_proxy: ${https_proxy} - # ideally we'd use docker secrets but it's not enabled currently - env_file: - - ../config/ehr_lookup.env - volumes: - - ../waveform-export:/waveform-export - # because we're launching through cron in the container, which starts - # processes with a clean environment, also mount in the config file so - # it can be read in by snakemake later - - ../config/ehr-lookup.env:/config/ehr-lookup.env:ro - restart: unless-stopped - From f7ac2db9f204c448e710ea71c33327c1cb83132d Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 17 Aug 2026 16:00:46 +0100 Subject: [PATCH 12/54] Added ehr variables to settings and added a testing flag. --- config.EXAMPLE/ehr_lookup.EXAMPLE | 5 +++++ src/settings.py | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/config.EXAMPLE/ehr_lookup.EXAMPLE b/config.EXAMPLE/ehr_lookup.EXAMPLE index 022ce93..0c88d12 100644 --- a/config.EXAMPLE/ehr_lookup.EXAMPLE +++ b/config.EXAMPLE/ehr_lookup.EXAMPLE @@ -7,3 +7,8 @@ CABOODLE_HOST="localhost" CABOODLE_PORT="5433" CABOODLE_CONNECT_TIMEOUT="10" # in seconds CABOODLE_QUERY_TIMEOUT="3000" # in milliseconds + +# To avoid having to deploy a fake caboodle for testing we have +# a testing flag for Caboodle. If set TRUE caboodle connection will +# fail silently and ehr file will be created with fake data +CABOODLE_TESTING="FALSE" diff --git a/src/settings.py b/src/settings.py index 3bccd12..0309839 100644 --- a/src/settings.py +++ b/src/settings.py @@ -37,6 +37,15 @@ def get_from_env(env_var, *, default_value=None, setting_name=None, required=Fal get_from_env("HASHER_API_HOSTNAME") get_from_env("HASHER_API_PORT") +get_from_env("CABOODLE_DBNAME") +get_from_env("CABOODLE_USERNAME") +get_from_env("CABOODLE_PASSWORD") +get_from_env("CABOODLE_HOST") +get_from_env("CABOODLE_PORT") +get_from_env("CABOODLE_CONNECT_TIMEOUT") +get_from_env("CABOODLE_QUERY_TIMEOUT") +get_from_env("CABOODLE_TESTING") + get_from_env("LOG_LEVEL", default_value="INFO") get_from_env("INSTANCE_NAME", required=True) From 291be43331de141b05c50f86db05c469d9faa6cf Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 17 Aug 2026 16:18:20 +0100 Subject: [PATCH 13/54] added some caboodle faking logic to enable local running --- src/db.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/db.py b/src/db.py index dff5872..e0fa11f 100644 --- a/src/db.py +++ b/src/db.py @@ -1,4 +1,5 @@ from datetime import datetime +import pandas as pd import psycopg2 from psycopg2 import sql, pool import logging @@ -69,13 +70,24 @@ class caboodleDB: settings.CABOODLE_QUERY_TIMEOUT, # type:ignore ) connection_pool: pool.ThreadedConnectionPool + fake_caboodle: bool def connect(self): """Set up connection to the database.""" - self.connection_pool = pool.SimpleConnectionPool(1, 1, self.connection_string) + self.fake_caboodle = True if settings.CABOODLE_TESTING == "TRUE" else False + if not self.fake_caboodle: + self.connection_pool = pool.SimpleConnectionPool(1, 1, self.connection_string) def get_airflow(self, start_datetime: datetime, end_datetime: datetime, csn: str): """Retrieve airflow data from database.""" + if self.fake_caboodle: + fake_airway = { + "DateTimeRecorded": [0], + "PlacementInstant": [0], + "RemovalInstant": [0], + "TubeSize": [0], + } + return pd.DataFrame(data=fake_airway) with open("src/sql/airway.sql", "r") as file: airway_query = sql.SQL(file.read()) parameters = { From 33e2c08b164aa713c5cdce238dcd0511315aee35 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 18 Aug 2026 15:21:33 +0100 Subject: [PATCH 14/54] Caboodle setup goes into exporter.env --- config.EXAMPLE/ehr_lookup.EXAMPLE | 14 -------------- config.EXAMPLE/exporter.env.EXAMPLE | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) delete mode 100644 config.EXAMPLE/ehr_lookup.EXAMPLE diff --git a/config.EXAMPLE/ehr_lookup.EXAMPLE b/config.EXAMPLE/ehr_lookup.EXAMPLE deleted file mode 100644 index 0c88d12..0000000 --- a/config.EXAMPLE/ehr_lookup.EXAMPLE +++ /dev/null @@ -1,14 +0,0 @@ -# This is an EXAMPLE file, do not put real secrets in here. -# Copy it to ../config/ehr_lookup.env and then DELETE THIS COMMENT. -CABOODLE_DBNAME="fakecab" -CABOODLE_USERNAME="inform_user" -CABOODLE_PASSWORD="inform" -CABOODLE_HOST="localhost" -CABOODLE_PORT="5433" -CABOODLE_CONNECT_TIMEOUT="10" # in seconds -CABOODLE_QUERY_TIMEOUT="3000" # in milliseconds - -# To avoid having to deploy a fake caboodle for testing we have -# a testing flag for Caboodle. If set TRUE caboodle connection will -# fail silently and ehr file will be created with fake data -CABOODLE_TESTING="FALSE" diff --git a/config.EXAMPLE/exporter.env.EXAMPLE b/config.EXAMPLE/exporter.env.EXAMPLE index 39a4119..c49d503 100644 --- a/config.EXAMPLE/exporter.env.EXAMPLE +++ b/config.EXAMPLE/exporter.env.EXAMPLE @@ -32,3 +32,17 @@ ONLY_USE_CSV_FROM_YESTERDAY=TRUE # specify a date to process format YYYY-MM-DD also accepts a regular # expression to match multiple date PROCESS_CSV_FROM_DATE= + +# We query Caboodle to get electronic healthcare record date per patient per day +CABOODLE_DBNAME="fakecab" +CABOODLE_USERNAME="inform_user" +CABOODLE_PASSWORD="inform" +CABOODLE_HOST="localhost" +CABOODLE_PORT="5433" +CABOODLE_CONNECT_TIMEOUT="10" # in seconds +CABOODLE_QUERY_TIMEOUT="3000" # in milliseconds + +# To avoid having to deploy a fake caboodle for testing we have +# a testing flag for Caboodle. If set TRUE caboodle connection will +# fail silently and ehr file will be created with fake data +CABOODLE_TESTING="FALSE" From 9b291cb257a86c51802fcf8bdfeb17efe3e53b8d Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 18 Aug 2026 15:51:40 +0100 Subject: [PATCH 15/54] Format fix --- src/db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/db.py b/src/db.py index e0fa11f..04eb37a 100644 --- a/src/db.py +++ b/src/db.py @@ -76,7 +76,9 @@ def connect(self): """Set up connection to the database.""" self.fake_caboodle = True if settings.CABOODLE_TESTING == "TRUE" else False if not self.fake_caboodle: - self.connection_pool = pool.SimpleConnectionPool(1, 1, self.connection_string) + self.connection_pool = pool.SimpleConnectionPool( + 1, 1, self.connection_string + ) def get_airflow(self, start_datetime: datetime, end_datetime: datetime, csn: str): """Retrieve airflow data from database.""" From d4fde5905926162f3ab8fdd978556b93c175d4a0 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 18 Aug 2026 16:06:02 +0100 Subject: [PATCH 16/54] Beginning of test for ehr integration --- src/csv_writer.py | 2 +- tests/helpers.py | 3 +++ tests/test_snakemake_integration.py | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/csv_writer.py b/src/csv_writer.py index 42daeb3..11588b1 100644 --- a/src/csv_writer.py +++ b/src/csv_writer.py @@ -100,6 +100,6 @@ def write_ehr( filename = WAVEFORM_PSEUDONYMISED_EHR / f"{stem}_ehr.csv" filename.parent.mkdir(exist_ok=True, parents=True) - df.to_csv(filename) + df.to_csv(filename, index = False) return True diff --git a/tests/helpers.py b/tests/helpers.py index 63d82f2..68df5cc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -51,6 +51,9 @@ def get_orig_parquet(self): def get_pseudon_parquet(self): return f"{self.date}/{self.date}.{self.get_hashed_csn()}.{self.variable_id}.{self.channel_id}.{self.units}.parquet" + def get_pseudon_ehr(self): + return f"{self.date}/{self.date}.{self.get_hashed_csn()}_ehr.csv" + def get_hashes(self): return f"{self.date}/{self.date}.hashes.json" diff --git a/tests/test_snakemake_integration.py b/tests/test_snakemake_integration.py index 50d7b9c..3bf8a4d 100644 --- a/tests/test_snakemake_integration.py +++ b/tests/test_snakemake_integration.py @@ -207,9 +207,11 @@ def test_snakemake_pipeline(tmp_path: Path, background_hasher): tmp_path / "original-parquet" / filename.get_orig_parquet() ) pseudon_path = tmp_path / "pseudonymised" / filename.get_pseudon_parquet() + ehr_path = tmp_path / "pseudonymised_ehr" / filename.get_pseudon_ehr() assert original_parquet_path.exists() assert pseudon_path.exists() + assert ehr_path.exists() _compare_original_parquet_to_expected(original_parquet_path, expected_data) _compare_parquets(original_parquet_path, pseudon_path) From d910831441fe7bb29bfb76706e184f9921bdbe0b Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 18 Aug 2026 16:25:30 +0100 Subject: [PATCH 17/54] SQL formatting --- sql_scripts/flow_sheet_values.sql | 1 - sql_scripts/get_hospital_visit_id.sql | 2 -- sql_scripts/lab_results.sql | 1 - sql_scripts/lab_test_names.sql | 2 +- 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/sql_scripts/flow_sheet_values.sql b/sql_scripts/flow_sheet_values.sql index af9b76f..3be8624 100644 --- a/sql_scripts/flow_sheet_values.sql +++ b/sql_scripts/flow_sheet_values.sql @@ -32,4 +32,3 @@ AND vo.valid_from BETWEEN %(yesterday)s AND %(today)s AND vo.hospital_visit_id = %(hospital_visit_id)s GROUP BY DateTimeRecorded, Units, vo.comment - diff --git a/sql_scripts/get_hospital_visit_id.sql b/sql_scripts/get_hospital_visit_id.sql index 408447f..862a67f 100644 --- a/sql_scripts/get_hospital_visit_id.sql +++ b/sql_scripts/get_hospital_visit_id.sql @@ -3,5 +3,3 @@ select hospital_visit_id from star.hospital_visit as hv where hv.encounter = %(csn)s -- note the CSN must be in quotes - - diff --git a/sql_scripts/lab_results.sql b/sql_scripts/lab_results.sql index aa05264..81783ce 100644 --- a/sql_scripts/lab_results.sql +++ b/sql_scripts/lab_results.sql @@ -21,4 +21,3 @@ on r.lab_order_id = o.lab_order_id where o.hospital_visit_id = 'xx' and r.result_status like 'FINAL' and r.lab_test_definition_id in ('1001', '390793054', '390793057', '390793060', '722790196') - diff --git a/sql_scripts/lab_test_names.sql b/sql_scripts/lab_test_names.sql index 85ce715..1838b72 100644 --- a/sql_scripts/lab_test_names.sql +++ b/sql_scripts/lab_test_names.sql @@ -2,4 +2,4 @@ select lab_test_definition_id as id, name, standardised_vocabulary as vocab from star.lab_test_definition as ltd -where ltd.lab_test_definition_id in ('1001', '390793054', '390793057', '390793060', '722790196') \ No newline at end of file +where ltd.lab_test_definition_id in ('1001', '390793054', '390793057', '390793060', '722790196') From e8f29a73f7d15fa9c07578bafc2d8dc5deb98b1a Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 18 Aug 2026 16:44:13 +0100 Subject: [PATCH 18/54] Test with ehr lookup --- tests/test_snakemake_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_snakemake_integration.py b/tests/test_snakemake_integration.py index 3bf8a4d..843ca1c 100644 --- a/tests/test_snakemake_integration.py +++ b/tests/test_snakemake_integration.py @@ -268,7 +268,7 @@ def _run_snakemake(tmp_path): tmp_exporter_env_path = tmp_path / "config/exporter.env" tmp_exporter_env_path.parent.mkdir(exist_ok=True) tmp_exporter_env_path.write_text( - "SNAKEMAKE_RULE_UNTIL=all_daily_hash_lookups\n" + "SNAKEMAKE_RULE_UNTIL=all_ehr_lookups\n" "SNAKEMAKE_CORES=1\n" "INSTANCE_NAME=pytest\n" "CSV_AGE_THRESHOLD_MINUTES=5\n" From 86d5dea4f8beb05b22ef1a5089ee8a25d03b67ad Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 18 Aug 2026 16:55:20 +0100 Subject: [PATCH 19/54] use caboodle testing --- tests/test_snakemake_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_snakemake_integration.py b/tests/test_snakemake_integration.py index 843ca1c..908fed5 100644 --- a/tests/test_snakemake_integration.py +++ b/tests/test_snakemake_integration.py @@ -274,6 +274,7 @@ def _run_snakemake(tmp_path): "CSV_AGE_THRESHOLD_MINUTES=5\n" "ONLY_USE_CSV_FROM_YESTERDAY=False\n" "PROCESS_CSV_FROM_DATE=\n" + "CABOODLE_TESTING=TRUE\n" ) # run system under test (exporter container) in foreground compose_args = [ From 7fbcfc043fc685f0446011f7614c574195570773 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 18 Aug 2026 17:18:10 +0100 Subject: [PATCH 20/54] Trying to test ehr and hash lookup with ftps --- src/pipeline/Snakefile | 6 ++++++ tests/test_snakemake_integration.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pipeline/Snakefile b/src/pipeline/Snakefile index 71042d9..30eb607 100644 --- a/src/pipeline/Snakefile +++ b/src/pipeline/Snakefile @@ -83,6 +83,12 @@ rule all_ehr_lookups: input: ALL_EHR_LOOKUPS +# a rule combining ehr and hash look ups to enable testing without ftps upload +rule all_ehr_and_hash_lookups: + input: + ALL_EHR_LOOKUPS, + ALL_DAILY_HASH_LOOKUPS + def input_file_maker(wc): unhashed_csn = hash_to_csn[wc.hashed_csn] # when using input functions, snakemake doesn't do its normal templating, you have to do it, hence the f-string diff --git a/tests/test_snakemake_integration.py b/tests/test_snakemake_integration.py index 908fed5..522bb76 100644 --- a/tests/test_snakemake_integration.py +++ b/tests/test_snakemake_integration.py @@ -268,7 +268,7 @@ def _run_snakemake(tmp_path): tmp_exporter_env_path = tmp_path / "config/exporter.env" tmp_exporter_env_path.parent.mkdir(exist_ok=True) tmp_exporter_env_path.write_text( - "SNAKEMAKE_RULE_UNTIL=all_ehr_lookups\n" + "SNAKEMAKE_RULE_UNTIL=all_ehr_and_hash_lookups\n" "SNAKEMAKE_CORES=1\n" "INSTANCE_NAME=pytest\n" "CSV_AGE_THRESHOLD_MINUTES=5\n" From af487e6690af65dbed5089fbd2b6d87934730a31 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 19 Aug 2026 14:42:36 +0100 Subject: [PATCH 21/54] Moved sql --- {sql_scripts => src/sql}/flow_sheet_values.sql | 0 {sql_scripts => src/sql}/get_hospital_visit_id.sql | 0 {sql_scripts => src/sql}/sputum_secretions.sql | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {sql_scripts => src/sql}/flow_sheet_values.sql (100%) rename {sql_scripts => src/sql}/get_hospital_visit_id.sql (100%) rename {sql_scripts => src/sql}/sputum_secretions.sql (100%) diff --git a/sql_scripts/flow_sheet_values.sql b/src/sql/flow_sheet_values.sql similarity index 100% rename from sql_scripts/flow_sheet_values.sql rename to src/sql/flow_sheet_values.sql diff --git a/sql_scripts/get_hospital_visit_id.sql b/src/sql/get_hospital_visit_id.sql similarity index 100% rename from sql_scripts/get_hospital_visit_id.sql rename to src/sql/get_hospital_visit_id.sql diff --git a/sql_scripts/sputum_secretions.sql b/src/sql/sputum_secretions.sql similarity index 100% rename from sql_scripts/sputum_secretions.sql rename to src/sql/sputum_secretions.sql From b5351913977f9666270cd2d32b346f10d8dac040 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 19 Aug 2026 15:43:45 +0100 Subject: [PATCH 22/54] refactored db to make adding ehr lookup implementation clearer --- src/controller.py | 7 ++-- src/db.py | 49 ++++++++++++++--------- src/sql/mrn_based_on_bed_and_datetime.sql | 8 ++-- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/controller.py b/src/controller.py index 3dc3099..956be16 100644 --- a/src/controller.py +++ b/src/controller.py @@ -43,7 +43,6 @@ def reject_message(ch, delivery_tag, requeue): class WaveformController: def __init__(self): self.emap_db = db.starDB() - self.emap_db.init_query() self.emap_db.connect() def waveform_callback(self, ch, method_frame, _header_frame, body): @@ -76,7 +75,9 @@ def waveform_callback(self, ch, method_frame, _header_frame, body): ) lookup_success = True try: - matched_mrn = self.emap_db.get_row(location_string, observation_time) + matched_mrn = self.emap_db.get_matched_mrn( + location_string, observation_time + ) except ValueError: lookup_success = False logger.error( @@ -86,6 +87,7 @@ def waveform_callback(self, ch, method_frame, _header_frame, body): exc_info=True, ) matched_mrn = ("unmatched_mrn", "unmatched_nhs", "unmatched_csn", False) + # matched_mrn = ("1234568", "12345678", "12345678", False) except ConnectionError: logger.error("Database error, will try again", exc_info=True) reject_message(ch, method_frame.delivery_tag, True) @@ -96,7 +98,6 @@ def waveform_callback(self, ch, method_frame, _header_frame, body): logger.info("Research opt-out is set for mrn %s, not writing.", mrn) reject_message(ch, method_frame.delivery_tag, False) return - if writer.write_frame( waveform_data, source_variable_id, diff --git a/src/db.py b/src/db.py index 04eb37a..f6d78f6 100644 --- a/src/db.py +++ b/src/db.py @@ -11,7 +11,7 @@ class starDB: - sql_query: str = "" + mrn_lookup_query: str = "" connection_string: str = "dbname={} user={} password={} host={} port={} connect_timeout={} options='-c statement_timeout={}'".format( settings.UDS_DBNAME, # type:ignore settings.UDS_USERNAME, # type:ignore @@ -21,27 +21,32 @@ class starDB: settings.UDS_CONNECT_TIMEOUT, # type:ignore settings.UDS_QUERY_TIMEOUT, # type:ignore ) - connection_pool: pool.ThreadedConnectionPool + connection_pool: pool.SimpleConnectionPool - def connect(self): + def connect(self) -> None: self.connection_pool = pool.SimpleConnectionPool(1, 1, self.connection_string) - def init_query(self): + def _init_mrn_lookup_query(self) -> None: with open("src/sql/mrn_based_on_bed_and_datetime.sql", "r") as file: - self.sql_query = sql.SQL(file.read()) - self.sql_query = self.sql_query.format( + self.mrn_lookup_query = sql.SQL(file.read()) # type:ignore + + self.mrn_lookup_query = self.mrn_lookup_query.format( schema_name=sql.Identifier(settings.SCHEMA_NAME) ) - def get_row(self, location_string: str, observation_datetime: datetime): + def get_matched_mrn( + self, location_string: str, observation_datetime: datetime + ) -> pd.DataFrame: parameters = { "location_string": location_string, "observation_datetime": observation_datetime, } + if self.mrn_lookup_query == "": + self._init_mrn_lookup_query() try: with self.connection_pool.getconn() as db_connection: with db_connection.cursor() as curs: - curs.execute(self.sql_query, parameters) + curs.execute(self.mrn_lookup_query, parameters) rows = curs.fetchall() self.connection_pool.putconn(db_connection) except psycopg2.errors.OperationalError as e: @@ -55,6 +60,9 @@ def get_row(self, location_string: str, observation_datetime: datetime): return rows[0] + def get_hospital_visit_from_csn(self, csn: str) -> str: + return "not implemented yet" + class caboodleDB: """For querying the caboodle database to extract electronic healthcare records per @@ -69,10 +77,10 @@ class caboodleDB: settings.CABOODLE_CONNECT_TIMEOUT, # type:ignore settings.CABOODLE_QUERY_TIMEOUT, # type:ignore ) - connection_pool: pool.ThreadedConnectionPool + connection_pool: pool.SimpleConnectionPool fake_caboodle: bool - def connect(self): + def connect(self) -> None: """Set up connection to the database.""" self.fake_caboodle = True if settings.CABOODLE_TESTING == "TRUE" else False if not self.fake_caboodle: @@ -80,8 +88,19 @@ def connect(self): 1, 1, self.connection_string ) - def get_airflow(self, start_datetime: datetime, end_datetime: datetime, csn: str): + def get_airflow( + self, start_datetime: datetime, end_datetime: datetime, csn: str + ) -> pd.DataFrame: """Retrieve airflow data from database.""" + + with open("src/sql/airway.sql", "r") as file: + airway_query = sql.SQL(file.read()) + parameters = { + "start_datetime": start_datetime, + "end_datetime": end_datetime, + "csn": csn, + } + if self.fake_caboodle: fake_airway = { "DateTimeRecorded": [0], @@ -90,13 +109,7 @@ def get_airflow(self, start_datetime: datetime, end_datetime: datetime, csn: str "TubeSize": [0], } return pd.DataFrame(data=fake_airway) - with open("src/sql/airway.sql", "r") as file: - airway_query = sql.SQL(file.read()) - parameters = { - "start_datetime": start_datetime, - "end_datetime": end_datetime, - "csn": csn, - } + return self._get_rows(airway_query, parameters) def _get_rows(self, sql_query: sql.SQL, parameters: dict): diff --git a/src/sql/mrn_based_on_bed_and_datetime.sql b/src/sql/mrn_based_on_bed_and_datetime.sql index 7eccf5e..4494d21 100644 --- a/src/sql/mrn_based_on_bed_and_datetime.sql +++ b/src/sql/mrn_based_on_bed_and_datetime.sql @@ -1,7 +1,7 @@ -/* Find a medical record number (MRN), NHS number, and contact serial number (CSN) based on location -string and date time. Returns a list of MRN, NHS numbers, and CSN with the -first entry being the most recent. -*/ +-- Find a medical record number (MRN), NHS number, and contact serial number (CSN) based on location +-- string and date time. Returns a list of MRN, NHS numbers, and CSN with the +-- first entry being the most recent. +-- SELECT mn.mrn as mrn, mn.nhs_number as nhs_number, From 4f0ba6f57446119d44f3af146667756b81ad9cd4 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 19 Aug 2026 16:22:42 +0100 Subject: [PATCH 23/54] Started on logic to get flowsheets, first need hospital visit number --- src/db.py | 33 +++++++++++++++++++-------- src/electronic_health_records/ehr.py | 34 +++++++++++++++------------- src/sql/get_hospital_visit_id.sql | 5 ++-- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/src/db.py b/src/db.py index f6d78f6..13d8a3c 100644 --- a/src/db.py +++ b/src/db.py @@ -43,15 +43,8 @@ def get_matched_mrn( } if self.mrn_lookup_query == "": self._init_mrn_lookup_query() - try: - with self.connection_pool.getconn() as db_connection: - with db_connection.cursor() as curs: - curs.execute(self.mrn_lookup_query, parameters) - rows = curs.fetchall() - self.connection_pool.putconn(db_connection) - except psycopg2.errors.OperationalError as e: - self.connection_pool.putconn(db_connection) - raise ConnectionError(f"Data base error: {e}") + + rows = self._get_rows(self.mrn_lookup_query, parameters) # type: ignore if len(rows) != 1: raise ValueError( @@ -61,7 +54,27 @@ def get_matched_mrn( return rows[0] def get_hospital_visit_from_csn(self, csn: str) -> str: - return "not implemented yet" + with open("src/sql/get_hospital_visit_id.sql", "r") as file: + hv_query = sql.SQL(file.read()) + + parameters = { + "schema": settings.SCHEMA_NAME, + "csn": csn, + } + + return self._get_rows(hv_query, parameters) + + def _get_rows(self, sql_query: sql.SQL, parameters: dict): + try: + with self.connection_pool.getconn() as db_connection: + with db_connection.cursor() as curs: + curs.execute(sql_query, parameters) + rows = curs.fetchall() + self.connection_pool.putconn(db_connection) + except psycopg2.errors.OperationalError as e: + self.connection_pool.putconn(db_connection) + raise ConnectionError(f"Data base error: {e}") + return rows class caboodleDB: diff --git a/src/electronic_health_records/ehr.py b/src/electronic_health_records/ehr.py index 48fc930..7470282 100644 --- a/src/electronic_health_records/ehr.py +++ b/src/electronic_health_records/ehr.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta -from db import caboodleDB +from db import caboodleDB, starDB from csv_writer import write_ehr from pseudon.pseudon import pseudonymise_relevant_columns @@ -18,28 +18,24 @@ def ehr_for_csv(date_str: str, original_csn: str, hashed_csn: str) -> None: :param hashed_csn: the pseudonymised hash to use for file output. """ - db_connection = caboodleDB() - db_connection.connect() + caboodle_connection = caboodleDB() + caboodle_connection.connect() - _ehr_for_csv(date_str, original_csn, hashed_csn, db_connection) + star_connection = starDB() + star_connection.connect() + + _ehr_for_csv( + date_str, original_csn, hashed_csn, caboodle_connection, star_connection + ) def _ehr_for_csv( date_str: str, original_csn: str, hashed_csn: str, - db_connection: caboodleDB, + caboodle_connection: caboodleDB, + star_connection: starDB, ) -> None: - """Extracts electronic healthcare records for a given csn and writes the results to - a pseudonymised csv file for a single day. - - This is a privacy-sensitive area of code. Unhashed CSNs must not appear in uploaded - files. - :param date_str: the date to look up data for - :param original_csn: the csn to base look up on. - :param hashed_csn: the pseudonymised hash to use for file output. - :param db_connection: connection to the caboodle database. - """ # will pick up the logger config defined in the snakemake job (ie. log to file) logger = logging.getLogger(__name__) @@ -47,7 +43,13 @@ def _ehr_for_csv( start_datetime = datetime.strptime(date_str, "%Y-%m-%d") end_datetime = start_datetime + timedelta(days=1) - airflow = db_connection.get_airflow(start_datetime, end_datetime, original_csn) + airflow = caboodle_connection.get_airflow( + start_datetime, end_datetime, original_csn + ) + + hospital_visit = star_connection.get_hospital_visit_from_csn(original_csn) + + logger.info(hospital_visit) safe_columns = [ "DateTimeRecorded", diff --git a/src/sql/get_hospital_visit_id.sql b/src/sql/get_hospital_visit_id.sql index 862a67f..15af7a0 100644 --- a/src/sql/get_hospital_visit_id.sql +++ b/src/sql/get_hospital_visit_id.sql @@ -1,5 +1,4 @@ - +-- Retrieve the hospital_visit_id associated with the csn value applied to this function -- - -select hospital_visit_id from star.hospital_visit as hv +select hospital_visit_id from %(schema)s.hospital_visit as hv where hv.encounter = %(csn)s -- note the CSN must be in quotes From 88ba017e6ddfa890aa098758b07636d66b22475b Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 19 Aug 2026 16:47:14 +0100 Subject: [PATCH 24/54] The exporter needs to access uds, do duplicated some of the config from controller --- config.EXAMPLE/exporter.env.EXAMPLE | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config.EXAMPLE/exporter.env.EXAMPLE b/config.EXAMPLE/exporter.env.EXAMPLE index c49d503..ed8efeb 100644 --- a/config.EXAMPLE/exporter.env.EXAMPLE +++ b/config.EXAMPLE/exporter.env.EXAMPLE @@ -46,3 +46,14 @@ CABOODLE_QUERY_TIMEOUT="3000" # in milliseconds # a testing flag for Caboodle. If set TRUE caboodle connection will # fail silently and ehr file will be created with fake data CABOODLE_TESTING="FALSE" + +# The following is duplicated from controller.env +# the exporter needs access to uds +UDS_DBNAME="fakeuds" +UDS_USERNAME="inform_user" +UDS_PASSWORD="inform" +UDS_HOST="172.17.0.1" +UDS_PORT="5433" +UDS_CONNECT_TIMEOUT="10" +UDS_QUERY_TIMEOUT="3000" +SCHEMA_NAME="schemaname" From 72f4935dfb5f616133550a42f5faa795543e7449 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 20 Aug 2026 08:22:42 +0100 Subject: [PATCH 25/54] Use enviroment to manage sql locations as it varies between containers --- config.EXAMPLE/controller.env.EXAMPLE | 2 ++ config.EXAMPLE/exporter.env.EXAMPLE | 2 ++ src/db.py | 6 +++--- src/settings.py | 1 + 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/config.EXAMPLE/controller.env.EXAMPLE b/config.EXAMPLE/controller.env.EXAMPLE index e055216..66d9672 100644 --- a/config.EXAMPLE/controller.env.EXAMPLE +++ b/config.EXAMPLE/controller.env.EXAMPLE @@ -13,3 +13,5 @@ RABBITMQ_PASSWORD="my_pw" RABBITMQ_HOST="localhost" RABBITMQ_PORT=5672 RABBITMQ_QUEUE="waveform" + +SQL_PATH="./src/sql/" diff --git a/config.EXAMPLE/exporter.env.EXAMPLE b/config.EXAMPLE/exporter.env.EXAMPLE index ed8efeb..e8ed0f9 100644 --- a/config.EXAMPLE/exporter.env.EXAMPLE +++ b/config.EXAMPLE/exporter.env.EXAMPLE @@ -57,3 +57,5 @@ UDS_PORT="5433" UDS_CONNECT_TIMEOUT="10" UDS_QUERY_TIMEOUT="3000" SCHEMA_NAME="schemaname" + +SQL_PATH="/app/src/sql/" diff --git a/src/db.py b/src/db.py index 13d8a3c..1d62798 100644 --- a/src/db.py +++ b/src/db.py @@ -27,7 +27,7 @@ def connect(self) -> None: self.connection_pool = pool.SimpleConnectionPool(1, 1, self.connection_string) def _init_mrn_lookup_query(self) -> None: - with open("src/sql/mrn_based_on_bed_and_datetime.sql", "r") as file: + with open(settings.SQL_PATH + "mrn_based_on_bed_and_datetime.sql", "r") as file: self.mrn_lookup_query = sql.SQL(file.read()) # type:ignore self.mrn_lookup_query = self.mrn_lookup_query.format( @@ -54,7 +54,7 @@ def get_matched_mrn( return rows[0] def get_hospital_visit_from_csn(self, csn: str) -> str: - with open("src/sql/get_hospital_visit_id.sql", "r") as file: + with open(settings.SQL_PATH + "get_hospital_visit_id.sql", "r") as file: hv_query = sql.SQL(file.read()) parameters = { @@ -106,7 +106,7 @@ def get_airflow( ) -> pd.DataFrame: """Retrieve airflow data from database.""" - with open("src/sql/airway.sql", "r") as file: + with open(settings.SQL_PATH + "airway.sql", "r") as file: airway_query = sql.SQL(file.read()) parameters = { "start_datetime": start_datetime, diff --git a/src/settings.py b/src/settings.py index 0309839..df0322c 100644 --- a/src/settings.py +++ b/src/settings.py @@ -49,3 +49,4 @@ def get_from_env(env_var, *, default_value=None, setting_name=None, required=Fal get_from_env("LOG_LEVEL", default_value="INFO") get_from_env("INSTANCE_NAME", required=True) +get_from_env("SQL_PATH") From 29865e01ade17f6c2fa9a4b324016d5f9db1cfa2 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 20 Aug 2026 08:24:50 +0100 Subject: [PATCH 26/54] path into test --- tests/test_snakemake_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_snakemake_integration.py b/tests/test_snakemake_integration.py index 522bb76..68a023c 100644 --- a/tests/test_snakemake_integration.py +++ b/tests/test_snakemake_integration.py @@ -275,6 +275,7 @@ def _run_snakemake(tmp_path): "ONLY_USE_CSV_FROM_YESTERDAY=False\n" "PROCESS_CSV_FROM_DATE=\n" "CABOODLE_TESTING=TRUE\n" + "SQL_PATH=/app/src/sql/\n" ) # run system under test (exporter container) in foreground compose_args = [ From de9442254430c1eeb0729b72a339bc6be3b7a253 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 20 Aug 2026 08:32:59 +0100 Subject: [PATCH 27/54] Use format rather than parameters for schema name --- src/db.py | 3 ++- src/sql/get_hospital_visit_id.sql | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/db.py b/src/db.py index 1d62798..d826231 100644 --- a/src/db.py +++ b/src/db.py @@ -57,8 +57,9 @@ def get_hospital_visit_from_csn(self, csn: str) -> str: with open(settings.SQL_PATH + "get_hospital_visit_id.sql", "r") as file: hv_query = sql.SQL(file.read()) + hv_query = hv_query.format(schema_name=sql.Identifier(settings.SCHEMA_NAME)) # type: ignore + parameters = { - "schema": settings.SCHEMA_NAME, "csn": csn, } diff --git a/src/sql/get_hospital_visit_id.sql b/src/sql/get_hospital_visit_id.sql index 15af7a0..8cd2f79 100644 --- a/src/sql/get_hospital_visit_id.sql +++ b/src/sql/get_hospital_visit_id.sql @@ -1,4 +1,4 @@ -- Retrieve the hospital_visit_id associated with the csn value applied to this function -- -select hospital_visit_id from %(schema)s.hospital_visit as hv +select hospital_visit_id from {schema_name}.hospital_visit as hv where hv.encounter = %(csn)s -- note the CSN must be in quotes From 518060b2f5deae0303aef533de4c7d355fcdda1e Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 20 Aug 2026 09:00:32 +0100 Subject: [PATCH 28/54] Default for sql path to simplify testing --- src/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/settings.py b/src/settings.py index df0322c..151306b 100644 --- a/src/settings.py +++ b/src/settings.py @@ -49,4 +49,4 @@ def get_from_env(env_var, *, default_value=None, setting_name=None, required=Fal get_from_env("LOG_LEVEL", default_value="INFO") get_from_env("INSTANCE_NAME", required=True) -get_from_env("SQL_PATH") +get_from_env("SQL_PATH", default_value="./src/sql/") From 03dbb608f1c8d201fcb667a4600e7477b8bb991d Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 20 Aug 2026 09:14:56 +0100 Subject: [PATCH 29/54] Updated mocked function name in test controller --- src/settings.py | 2 +- tests/test_controller.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/settings.py b/src/settings.py index 151306b..df0322c 100644 --- a/src/settings.py +++ b/src/settings.py @@ -49,4 +49,4 @@ def get_from_env(env_var, *, default_value=None, setting_name=None, required=Fal get_from_env("LOG_LEVEL", default_value="INFO") get_from_env("INSTANCE_NAME", required=True) -get_from_env("SQL_PATH", default_value="./src/sql/") +get_from_env("SQL_PATH") diff --git a/tests/test_controller.py b/tests/test_controller.py index d4fd611..3bee1e2 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -22,9 +22,11 @@ def test_controller_callback(monkeypatch, opt_out, db_connect_failure, bad_data): emap_db_mock = Mock() if db_connect_failure: - emap_db_mock.get_row.side_effect = ConnectionError("mock database error") + emap_db_mock.get_matched_mrn.side_effect = ConnectionError( + "mock database error" + ) else: - emap_db_mock.get_row.return_value = ("mrn", "nhsno", "csn", opt_out) + emap_db_mock.get_matched_mrn.return_value = ("mrn", "nhsno", "csn", opt_out) monkeypatch.setattr("controller.db.starDB", Mock(return_value=emap_db_mock)) write_frame_mock = Mock(return_value=True) @@ -56,12 +58,12 @@ def test_controller_callback(monkeypatch, opt_out, db_connect_failure, bad_data) if not bad_data: # we at least tried to query the DB - emap_db_mock.get_row.assert_called_once() + emap_db_mock.get_matched_mrn.assert_called_once() if bad_data: write_frame_mock.assert_not_called() # db should not even have been queried if data was bad - emap_db_mock.get_row.assert_not_called() + emap_db_mock.get_matched_mrn.assert_not_called() channel_mock.basic_reject.assert_called_once_with(delivery_tag, False) channel_mock.basic_ack.assert_not_called() elif db_connect_failure: From c08785c6d84b433106eb72be4ec1bd6d4c2ba397 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 24 Aug 2026 09:07:45 +0100 Subject: [PATCH 30/54] Enable a fake star db shortcut so that snakemake integration test can run --- src/db.py | 12 +++++++++++- src/settings.py | 1 + tests/test_snakemake_integration.py | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/db.py b/src/db.py index d826231..1d2459b 100644 --- a/src/db.py +++ b/src/db.py @@ -22,8 +22,16 @@ class starDB: settings.UDS_QUERY_TIMEOUT, # type:ignore ) connection_pool: pool.SimpleConnectionPool + fake_star: bool = False def connect(self) -> None: + self.fake_star = True if settings.STARDB_TESTING == "TRUE" else False + if not self.fake_star: + self.connection_pool = pool.SimpleConnectionPool( + 1, 1, self.connection_string + ) + + self.connection_pool = pool.SimpleConnectionPool(1, 1, self.connection_string) def _init_mrn_lookup_query(self) -> None: @@ -62,6 +70,8 @@ def get_hospital_visit_from_csn(self, csn: str) -> str: parameters = { "csn": csn, } + if self.fake_star: + return '12345678' return self._get_rows(hv_query, parameters) @@ -92,7 +102,7 @@ class caboodleDB: settings.CABOODLE_QUERY_TIMEOUT, # type:ignore ) connection_pool: pool.SimpleConnectionPool - fake_caboodle: bool + fake_caboodle: bool = False def connect(self) -> None: """Set up connection to the database.""" diff --git a/src/settings.py b/src/settings.py index df0322c..c70c05a 100644 --- a/src/settings.py +++ b/src/settings.py @@ -22,6 +22,7 @@ def get_from_env(env_var, *, default_value=None, setting_name=None, required=Fal get_from_env("UDS_PORT") get_from_env("UDS_CONNECT_TIMEOUT") get_from_env("UDS_QUERY_TIMEOUT") +get_from_env("STARDB_TESTING") get_from_env("SCHEMA_NAME") get_from_env("RABBITMQ_USERNAME") get_from_env("RABBITMQ_PASSWORD") diff --git a/tests/test_snakemake_integration.py b/tests/test_snakemake_integration.py index 68a023c..aa00818 100644 --- a/tests/test_snakemake_integration.py +++ b/tests/test_snakemake_integration.py @@ -274,6 +274,7 @@ def _run_snakemake(tmp_path): "CSV_AGE_THRESHOLD_MINUTES=5\n" "ONLY_USE_CSV_FROM_YESTERDAY=False\n" "PROCESS_CSV_FROM_DATE=\n" + "STARDB_TESTING=TRUE\n" "CABOODLE_TESTING=TRUE\n" "SQL_PATH=/app/src/sql/\n" ) From cccf07804689c49753a4b715049928c571845ef0 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 24 Aug 2026 09:37:10 +0100 Subject: [PATCH 31/54] Don't connect for fake star --- src/db.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/db.py b/src/db.py index 1d2459b..59cdcae 100644 --- a/src/db.py +++ b/src/db.py @@ -31,9 +31,6 @@ def connect(self) -> None: 1, 1, self.connection_string ) - - self.connection_pool = pool.SimpleConnectionPool(1, 1, self.connection_string) - def _init_mrn_lookup_query(self) -> None: with open(settings.SQL_PATH + "mrn_based_on_bed_and_datetime.sql", "r") as file: self.mrn_lookup_query = sql.SQL(file.read()) # type:ignore @@ -71,7 +68,7 @@ def get_hospital_visit_from_csn(self, csn: str) -> str: "csn": csn, } if self.fake_star: - return '12345678' + return "12345678" return self._get_rows(hv_query, parameters) From c84ca84abb6a7687578cbb3d7ff777ed8aeef3f5 Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Mon, 24 Aug 2026 11:13:57 +0100 Subject: [PATCH 32/54] Refactor lab results query for wide format output Transforms specific lab test results (e.g., CRP, WCC) from a long format into distinct columns using `MAX() FILTER`. This enables direct consumption of lab data in a wider format, simplifying downstream analysis. Additionally, the query is updated to use parameters for `hospital_visit_id` and to include date range filtering, improving its flexibility. --- sql_scripts/lab_results.sql | 41 +++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/sql_scripts/lab_results.sql b/sql_scripts/lab_results.sql index aa05264..21b8b5d 100644 --- a/sql_scripts/lab_results.sql +++ b/sql_scripts/lab_results.sql @@ -1,24 +1,35 @@ --- This selects the values are units of lab tests +-- This selects the values of lab tests -- 1011 CRP -- 722790196 CRP -- 390793054 WCC -- 390793057 WCC -- 390793060 WCC -select -r.result_last_modified_datetime as date, -(select name from star.lab_test_definition as ltd -where ltd.lab_test_definition_id = r.lab_test_definition_id) as name, -r.value_as_real as value, -r.units, -r.abnormal_flag, -r.comment +SELECT + r.result_last_modified_datetime AS DateTimeRecorded, + MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '1001') AS "C-reactive protein", + MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '390793054') AS "CSF WCC TUBE 1", + MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '390793057') AS "CSF WCC TUBE 2", + MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '390793060') AS "CSF WCC TUBE 3", + MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '722790196') AS "C-reactive protein" -from star.lab_result as r -join star.lab_order as o -on r.lab_order_id = o.lab_order_id -where o.hospital_visit_id = 'xx' -and r.result_status like 'FINAL' -and r.lab_test_definition_id in ('1001', '390793054', '390793057', '390793060', '722790196') + r.units AS Units, + r.abnormal_flag AS Abnormal_result, + r.comment AS Comments +FROM star.lab_result AS r +LEFT JOIN star.lab_order AS o + ON r.lab_order_id = o.lab_order_id + +WHERE r.result_status like 'FINAL' +AND +r.lab_test_definition_id IN ('1001', + '390793054', + '390793057', + '390793060', + '722790196') +AND vo.valid_from BETWEEN %(yesterday)s AND %(today)s +AND o.hospital_visit_id = %(hospital_visit_id)s + +GROUP BY DateTimeRecorded, Units, Abnormal_result, Comments From 9ae3715aa4419b08cb79643a42ab439ef5f57c39 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 26 Aug 2026 08:59:09 +0100 Subject: [PATCH 33/54] Tidied up getting for hospital visit number --- src/db.py | 8 +++++++- src/settings.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/db.py b/src/db.py index 59cdcae..25c8e0d 100644 --- a/src/db.py +++ b/src/db.py @@ -70,7 +70,13 @@ def get_hospital_visit_from_csn(self, csn: str) -> str: if self.fake_star: return "12345678" - return self._get_rows(hv_query, parameters) + hospital_visit_id = self._get_rows(hv_query, parameters) + + # fetchall returns a list of tuples. We want the first element of the first tuple + if not isinstance(hospital_visit_id[0][0], str): + raise TypeError(f"hospital_visit_id is not string {hospital_visit_id}") + + return hospital_visit_id[0][0] def _get_rows(self, sql_query: sql.SQL, parameters: dict): try: diff --git a/src/settings.py b/src/settings.py index c70c05a..8f6cc91 100644 --- a/src/settings.py +++ b/src/settings.py @@ -50,4 +50,4 @@ def get_from_env(env_var, *, default_value=None, setting_name=None, required=Fal get_from_env("LOG_LEVEL", default_value="INFO") get_from_env("INSTANCE_NAME", required=True) -get_from_env("SQL_PATH") +get_from_env("SQL_PATH", default_value="./src/sql/") From bf90f902edefc9d10cff4bf8f53d550e31d47862 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 26 Aug 2026 11:12:40 +0100 Subject: [PATCH 34/54] return integer hospital visit id --- src/db.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/db.py b/src/db.py index 25c8e0d..acbd855 100644 --- a/src/db.py +++ b/src/db.py @@ -58,7 +58,7 @@ def get_matched_mrn( return rows[0] - def get_hospital_visit_from_csn(self, csn: str) -> str: + def get_hospital_visit_from_csn(self, csn: str) -> int: with open(settings.SQL_PATH + "get_hospital_visit_id.sql", "r") as file: hv_query = sql.SQL(file.read()) @@ -68,13 +68,15 @@ def get_hospital_visit_from_csn(self, csn: str) -> str: "csn": csn, } if self.fake_star: - return "12345678" + return 12345678 hospital_visit_id = self._get_rows(hv_query, parameters) # fetchall returns a list of tuples. We want the first element of the first tuple - if not isinstance(hospital_visit_id[0][0], str): - raise TypeError(f"hospital_visit_id is not string {hospital_visit_id}") + if not isinstance(hospital_visit_id[0][0], int): + logger.warning( + f"hospital_visit_id[0][0] is not integer {hospital_visit_id}" + ) return hospital_visit_id[0][0] From aa8f86ab666e32efb14d757845593c8f926d1234 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 26 Aug 2026 15:02:11 +0100 Subject: [PATCH 35/54] Added flowsheet query to ehr --- src/db.py | 24 ++++++++++++++++++++++++ src/electronic_health_records/ehr.py | 20 +++++++++++++++----- src/sql/flow_sheet_values.sql | 2 +- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/db.py b/src/db.py index acbd855..abaf3dd 100644 --- a/src/db.py +++ b/src/db.py @@ -141,6 +141,30 @@ def get_airflow( return self._get_rows(airway_query, parameters) + def get_flowsheets( + self, start_datetime: datetime, end_datetime: datetime, hospital_visit_id: int + ) -> pd.DataFrame: + """Retrieve airflow data from database.""" + + with open(settings.SQL_PATH + "flow_sheet_values.sql", "r") as file: + flowsheet_query = sql.SQL(file.read()) + parameters = { + "start_datetime": start_datetime, + "end_datetime": end_datetime, + "hospital_visit_id": hospital_visit_id, + } + + if self.fake_caboodle: + fake_flowsheet = { + "DateTimeRecorded": [0], + "Temperature": [0], + "Noradrenaline": [0], + "Metaraminol": [0], + } + return pd.DataFrame(data=fake_flowsheet) + + return self._get_rows(flowsheet_query, parameters) + def _get_rows(self, sql_query: sql.SQL, parameters: dict): try: with self.connection_pool.getconn() as db_connection: diff --git a/src/electronic_health_records/ehr.py b/src/electronic_health_records/ehr.py index 7470282..eefc830 100644 --- a/src/electronic_health_records/ehr.py +++ b/src/electronic_health_records/ehr.py @@ -1,6 +1,7 @@ import logging from datetime import datetime, timedelta +import pandas as pd from db import caboodleDB, starDB from csv_writer import write_ehr @@ -47,22 +48,31 @@ def _ehr_for_csv( start_datetime, end_datetime, original_csn ) - hospital_visit = star_connection.get_hospital_visit_from_csn(original_csn) + hospital_visit_id = star_connection.get_hospital_visit_from_csn(original_csn) - logger.info(hospital_visit) + logger.info(hospital_visit_id) + + flowsheet_values = caboodle_connection.get_flowsheets( + start_datetime, end_datetime, hospital_visit_id + ) + + ehr_data = pd.concat([airflow, flowsheet_values]) safe_columns = [ "DateTimeRecorded", "PlacementInstant", "RemovalInstant", "TubeSize", + "Temperature", + "Noradrenaline", + "Metaraminol", ] - airflow = pseudonymise_relevant_columns(airflow, safe_columns) + ehr_data = pseudonymise_relevant_columns(ehr_data, safe_columns) - write_ehr(airflow, date_str, hashed_csn) + write_ehr(ehr_data, date_str, hashed_csn) - logger.info(airflow) + logger.info(ehr_data) # delete csn once we no longer need it del original_csn diff --git a/src/sql/flow_sheet_values.sql b/src/sql/flow_sheet_values.sql index 3be8624..b66b626 100644 --- a/src/sql/flow_sheet_values.sql +++ b/src/sql/flow_sheet_values.sql @@ -28,7 +28,7 @@ LEFT JOIN star.visit_observation_type AS vt ON vo.visit_observation_type_id = vt.visit_observation_type_id WHERE vt.id_in_application IN ('6', '3040102622', '12946') -AND vo.valid_from BETWEEN %(yesterday)s AND %(today)s +AND vo.valid_from BETWEEN %(start_datetime)s AND %(end_datetime)s AND vo.hospital_visit_id = %(hospital_visit_id)s GROUP BY DateTimeRecorded, Units, vo.comment From 5d06eccf13f9e155d661049c70b9176cfe964510 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 27 Aug 2026 14:47:37 +0100 Subject: [PATCH 36/54] moved sql and implemented lab results --- src/db.py | 29 ++++++++++++++++++++++++ src/electronic_health_records/ehr.py | 27 ++++++++++++++++------ {sql_scripts => src/sql}/lab_results.sql | 6 ++--- 3 files changed, 52 insertions(+), 10 deletions(-) rename {sql_scripts => src/sql}/lab_results.sql (88%) diff --git a/src/db.py b/src/db.py index abaf3dd..f468881 100644 --- a/src/db.py +++ b/src/db.py @@ -165,6 +165,35 @@ def get_flowsheets( return self._get_rows(flowsheet_query, parameters) + def get_lab_results( + self, start_datetime: datetime, end_datetime: datetime, hospital_visit_id: int + ) -> pd.DataFrame: + """Retrieve lab result data from caboodle.""" + + with open(settings.SQL_PATH + "lab_results.sql", "r") as file: + flowsheet_query = sql.SQL(file.read()) + parameters = { + "start_datetime": start_datetime, + "end_datetime": end_datetime, + "hospital_visit_id": hospital_visit_id, + } + + if self.fake_caboodle: + fake_flowsheet = { + "DateTimeRecorded": [0], + "Units": ["None"], + "Abnormal_result": ["No"], + "Comments": ["None"], + "C-reactive protein 1": ["-"], + "CSF WCC TUBE 1": ["-"], + "CSF WCC TUBE 2": ["-"], + "CSF WCC TUBE 3": ["-"], + "C-reactive protein 2": ["-"], + } + return pd.DataFrame(data=fake_flowsheet) + + return self._get_rows(flowsheet_query, parameters) + def _get_rows(self, sql_query: sql.SQL, parameters: dict): try: with self.connection_pool.getconn() as db_connection: diff --git a/src/electronic_health_records/ehr.py b/src/electronic_health_records/ehr.py index eefc830..0f6ebc8 100644 --- a/src/electronic_health_records/ehr.py +++ b/src/electronic_health_records/ehr.py @@ -44,20 +44,27 @@ def _ehr_for_csv( start_datetime = datetime.strptime(date_str, "%Y-%m-%d") end_datetime = start_datetime + timedelta(days=1) - airflow = caboodle_connection.get_airflow( - start_datetime, end_datetime, original_csn - ) + # we need hospital visit id for flowsheet and lab_result queries hospital_visit_id = star_connection.get_hospital_visit_from_csn(original_csn) - logger.info(hospital_visit_id) + # fetch data from caboodle + airflow = caboodle_connection.get_airflow( + start_datetime, end_datetime, original_csn + ) flowsheet_values = caboodle_connection.get_flowsheets( start_datetime, end_datetime, hospital_visit_id ) - ehr_data = pd.concat([airflow, flowsheet_values]) + lab_results = caboodle_connection.get_lab_results( + start_datetime, end_datetime, hospital_visit_id + ) + ehr_data = pd.concat([airflow, flowsheet_values, lab_results]) + + # we can pseudonymise to safe, although at the moment all columns + # are considered safe safe_columns = [ "DateTimeRecorded", "PlacementInstant", @@ -66,13 +73,19 @@ def _ehr_for_csv( "Temperature", "Noradrenaline", "Metaraminol", + "Units", + "Abnormal_result", + "C-reactive protein", + "CSF WCC TUBE 1", + "CSF WCC TUBE 2", + "CSF WCC TUBE 3", + "C-reactive protein", + "Comments", # Free text comments could contain sensitive information. Should we hash it? ] ehr_data = pseudonymise_relevant_columns(ehr_data, safe_columns) write_ehr(ehr_data, date_str, hashed_csn) - logger.info(ehr_data) - # delete csn once we no longer need it del original_csn diff --git a/sql_scripts/lab_results.sql b/src/sql/lab_results.sql similarity index 88% rename from sql_scripts/lab_results.sql rename to src/sql/lab_results.sql index 21b8b5d..eb2d0cb 100644 --- a/sql_scripts/lab_results.sql +++ b/src/sql/lab_results.sql @@ -8,11 +8,11 @@ SELECT r.result_last_modified_datetime AS DateTimeRecorded, - MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '1001') AS "C-reactive protein", + MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '1001') AS "C-reactive protein 1", MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '390793054') AS "CSF WCC TUBE 1", MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '390793057') AS "CSF WCC TUBE 2", MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '390793060') AS "CSF WCC TUBE 3", - MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '722790196') AS "C-reactive protein" + MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '722790196') AS "C-reactive protein 2" r.units AS Units, r.abnormal_flag AS Abnormal_result, @@ -29,7 +29,7 @@ r.lab_test_definition_id IN ('1001', '390793057', '390793060', '722790196') -AND vo.valid_from BETWEEN %(yesterday)s AND %(today)s + AND vo.valid_from BETWEEN %(start_datetime)s AND %(end_datetime)s AND o.hospital_visit_id = %(hospital_visit_id)s GROUP BY DateTimeRecorded, Units, Abnormal_result, Comments From c61164c0478f0be4d32b633a5c4156078e56393d Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 27 Aug 2026 16:34:12 +0100 Subject: [PATCH 37/54] Fixed column names in pseudon --- src/electronic_health_records/ehr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/electronic_health_records/ehr.py b/src/electronic_health_records/ehr.py index 0f6ebc8..51e1dde 100644 --- a/src/electronic_health_records/ehr.py +++ b/src/electronic_health_records/ehr.py @@ -75,11 +75,11 @@ def _ehr_for_csv( "Metaraminol", "Units", "Abnormal_result", - "C-reactive protein", + "C-reactive protein 1", "CSF WCC TUBE 1", "CSF WCC TUBE 2", "CSF WCC TUBE 3", - "C-reactive protein", + "C-reactive protein 2", "Comments", # Free text comments could contain sensitive information. Should we hash it? ] From a450faffa591cb1568049ee5a66425ad110d832e Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 27 Aug 2026 16:53:58 +0100 Subject: [PATCH 38/54] Flowsheets and labresults are star, not caboodle --- src/db.py | 116 +++++++++++++++------------ src/electronic_health_records/ehr.py | 4 +- src/sql/flow_sheet_values.sql | 4 +- src/sql/lab_results.sql | 4 +- 4 files changed, 69 insertions(+), 59 deletions(-) diff --git a/src/db.py b/src/db.py index f468881..7f199b3 100644 --- a/src/db.py +++ b/src/db.py @@ -80,6 +80,69 @@ def get_hospital_visit_from_csn(self, csn: str) -> int: return hospital_visit_id[0][0] + def get_flowsheets( + self, start_datetime: datetime, end_datetime: datetime, hospital_visit_id: int + ) -> pd.DataFrame: + """Retrieve airflow data from database.""" + + with open(settings.SQL_PATH + "flow_sheet_values.sql", "r") as file: + flowsheet_query = sql.SQL(file.read()) + + flowsheet_query = flowsheet_query.format( + schema_name=sql.Identifier(settings.SCHEMA_NAME) + ) # type: ignore + + parameters = { + "start_datetime": start_datetime, + "end_datetime": end_datetime, + "hospital_visit_id": hospital_visit_id, + } + + if self.fake_star: + fake_flowsheet = { + "DateTimeRecorded": [0], + "Temperature": [0], + "Noradrenaline": [0], + "Metaraminol": [0], + } + return pd.DataFrame(data=fake_flowsheet) + + return self._get_rows(flowsheet_query, parameters) + + def get_lab_results( + self, start_datetime: datetime, end_datetime: datetime, hospital_visit_id: int + ) -> pd.DataFrame: + """Retrieve lab result data from caboodle.""" + + with open(settings.SQL_PATH + "lab_results.sql", "r") as file: + lab_result_query = sql.SQL(file.read()) + + lab_result_query = lab_result_query.format( + schema_name=sql.Identifier(settings.SCHEMA_NAME) + ) # type: ignore + + parameters = { + "start_datetime": start_datetime, + "end_datetime": end_datetime, + "hospital_visit_id": hospital_visit_id, + } + + if self.fake_star: + fake_lab_result = { + "DateTimeRecorded": [0], + "Units": ["None"], + "Abnormal_result": ["No"], + "Comments": ["None"], + "C-reactive protein 1": ["-"], + "CSF WCC TUBE 1": ["-"], + "CSF WCC TUBE 2": ["-"], + "CSF WCC TUBE 3": ["-"], + "C-reactive protein 2": ["-"], + } + return pd.DataFrame(data=fake_lab_result) + + return self._get_rows(lab_result_query, parameters) + def _get_rows(self, sql_query: sql.SQL, parameters: dict): try: with self.connection_pool.getconn() as db_connection: @@ -141,59 +204,6 @@ def get_airflow( return self._get_rows(airway_query, parameters) - def get_flowsheets( - self, start_datetime: datetime, end_datetime: datetime, hospital_visit_id: int - ) -> pd.DataFrame: - """Retrieve airflow data from database.""" - - with open(settings.SQL_PATH + "flow_sheet_values.sql", "r") as file: - flowsheet_query = sql.SQL(file.read()) - parameters = { - "start_datetime": start_datetime, - "end_datetime": end_datetime, - "hospital_visit_id": hospital_visit_id, - } - - if self.fake_caboodle: - fake_flowsheet = { - "DateTimeRecorded": [0], - "Temperature": [0], - "Noradrenaline": [0], - "Metaraminol": [0], - } - return pd.DataFrame(data=fake_flowsheet) - - return self._get_rows(flowsheet_query, parameters) - - def get_lab_results( - self, start_datetime: datetime, end_datetime: datetime, hospital_visit_id: int - ) -> pd.DataFrame: - """Retrieve lab result data from caboodle.""" - - with open(settings.SQL_PATH + "lab_results.sql", "r") as file: - flowsheet_query = sql.SQL(file.read()) - parameters = { - "start_datetime": start_datetime, - "end_datetime": end_datetime, - "hospital_visit_id": hospital_visit_id, - } - - if self.fake_caboodle: - fake_flowsheet = { - "DateTimeRecorded": [0], - "Units": ["None"], - "Abnormal_result": ["No"], - "Comments": ["None"], - "C-reactive protein 1": ["-"], - "CSF WCC TUBE 1": ["-"], - "CSF WCC TUBE 2": ["-"], - "CSF WCC TUBE 3": ["-"], - "C-reactive protein 2": ["-"], - } - return pd.DataFrame(data=fake_flowsheet) - - return self._get_rows(flowsheet_query, parameters) - def _get_rows(self, sql_query: sql.SQL, parameters: dict): try: with self.connection_pool.getconn() as db_connection: diff --git a/src/electronic_health_records/ehr.py b/src/electronic_health_records/ehr.py index 51e1dde..5cd5ef4 100644 --- a/src/electronic_health_records/ehr.py +++ b/src/electronic_health_records/ehr.py @@ -53,11 +53,11 @@ def _ehr_for_csv( start_datetime, end_datetime, original_csn ) - flowsheet_values = caboodle_connection.get_flowsheets( + flowsheet_values = star_connection.get_flowsheets( start_datetime, end_datetime, hospital_visit_id ) - lab_results = caboodle_connection.get_lab_results( + lab_results = star_connection.get_lab_results( start_datetime, end_datetime, hospital_visit_id ) diff --git a/src/sql/flow_sheet_values.sql b/src/sql/flow_sheet_values.sql index b66b626..9cb4288 100644 --- a/src/sql/flow_sheet_values.sql +++ b/src/sql/flow_sheet_values.sql @@ -22,9 +22,9 @@ SELECT vo.unit AS Units, vo.comment AS Comments -FROM star.visit_observation AS vo +FROM {schema_name}.visit_observation AS vo -LEFT JOIN star.visit_observation_type AS vt +LEFT JOIN {schema_name}.visit_observation_type AS vt ON vo.visit_observation_type_id = vt.visit_observation_type_id WHERE vt.id_in_application IN ('6', '3040102622', '12946') diff --git a/src/sql/lab_results.sql b/src/sql/lab_results.sql index eb2d0cb..426f48d 100644 --- a/src/sql/lab_results.sql +++ b/src/sql/lab_results.sql @@ -18,8 +18,8 @@ SELECT r.abnormal_flag AS Abnormal_result, r.comment AS Comments -FROM star.lab_result AS r -LEFT JOIN star.lab_order AS o +FROM {schema_name}.lab_result AS r +LEFT JOIN {schema_name}.lab_order AS o ON r.lab_order_id = o.lab_order_id WHERE r.result_status like 'FINAL' From 7b60b7d28c9b78b249d5629b0e3d8d15daf66a6b Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Fri, 28 Aug 2026 10:35:02 +0100 Subject: [PATCH 39/54] removed queries from proprietary database --- src/sql/airway.sql | 20 -------------------- src/sql/sputum_secretions.sql | 25 ------------------------- 2 files changed, 45 deletions(-) delete mode 100644 src/sql/airway.sql delete mode 100644 src/sql/sputum_secretions.sql diff --git a/src/sql/airway.sql b/src/sql/airway.sql deleted file mode 100644 index c79c9cb..0000000 --- a/src/sql/airway.sql +++ /dev/null @@ -1,20 +0,0 @@ --- extract airway data from caboodle for a specific csn and date -SELECT -lda._CreationInstant as DateTimeRecorded, -lda.PlacementInstant, -lda.RemovalInstant, -fvf.Value AS TubeSize - -FROM FilteredAccess.LdaFact lda -JOIN FilteredAccess.FlowsheetValueFact fvf ON fvf.LdaKey = lda.LdaKey -JOIN FilteredAccess.FlowsheetRowDim frd ON frd.FlowsheetRowKey = fvf.FlowsheetRowKey -JOIN FilteredAccess.EncounterFact enc ON enc.EncounterKey = lda.InitialEncounterKey - -WHERE -fvf.FlowsheetRowEpicId ='1120100079' -AND enc.Type != 'Anaesthesia' -AND frd.DisplayName like 'Single Lumen Tube Size' ---and enc.PatientDurableKey = '1782941' - -AND lda._CreationInstant BETWEEN %(start_datetime)s AND %(end_datetime)s -AND enc.EncounterEpicCsn = %(csn) diff --git a/src/sql/sputum_secretions.sql b/src/sql/sputum_secretions.sql deleted file mode 100644 index 8f27ab8..0000000 --- a/src/sql/sputum_secretions.sql +++ /dev/null @@ -1,25 +0,0 @@ -<- Retrieved the information about sputum and secretions > - -SELECT -fv.TakenInstant AS 'DateTimeRecorded', -CASE -WHEN fsd.FlowsheetRowEpicId = '451120' -THEN fv.Value -END -AS 'Secretions' , -CASE -WHEN fsd.FlowsheetRowEpicId = '302600' -THEN fv.Value -END -AS 'Sputum' , -fv.Comment AS Comments - -FROM FilteredAccess.FlowsheetValueFact fv -INNER JOIN FilteredAccess.FlowsheetRowDim fsd ON fv.FlowsheetRowKey = fsd.FlowsheetRowKey -INNER JOIN FilteredAccess.EncounterFact enc ON fv.EncounterKey = enc.EncounterKey - -WHERE -(fsd.FlowsheetRowEpicId = '451120' OR -fsd.FlowsheetRowEpicId ='302600') -AND fv.TakenInstant BETWEEN %(yesterday)s AND %(today)s -AND enc.EncounterEpicCsn = %(csn) From 83908c70655ac50f0cc13011f5a39ba6f03c6926 Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Fri, 28 Aug 2026 10:45:48 +0100 Subject: [PATCH 40/54] removed queries from proprietary database --- sql_scripts/airway.sql | 20 -------------------- sql_scripts/sputum_secretions.sql | 25 ------------------------- 2 files changed, 45 deletions(-) delete mode 100644 sql_scripts/airway.sql delete mode 100644 sql_scripts/sputum_secretions.sql diff --git a/sql_scripts/airway.sql b/sql_scripts/airway.sql deleted file mode 100644 index 3b0889e..0000000 --- a/sql_scripts/airway.sql +++ /dev/null @@ -1,20 +0,0 @@ - -SELECT -lda._CreationInstant as DateTimeRecorded, -lda.PlacementInstant, -lda.RemovalInstant, -fvf.Value AS TubeSize - -FROM FilteredAccess.LdaFact lda -JOIN FilteredAccess.FlowsheetValueFact fvf ON fvf.LdaKey = lda.LdaKey -JOIN FilteredAccess.FlowsheetRowDim frd ON frd.FlowsheetRowKey = fvf.FlowsheetRowKey -JOIN FilteredAccess.EncounterFact enc ON enc.EncounterKey = lda.InitialEncounterKey - -WHERE -fvf.FlowsheetRowEpicId ='1120100079' -AND enc.Type != 'Anaesthesia' -AND frd.DisplayName like 'Single Lumen Tube Size' ---and enc.PatientDurableKey = '1782941' - -AND lda._CreationInstant BETWEEN %(yesterday)s AND %(today)s -AND enc.EncounterEpicCsn = %(csn) diff --git a/sql_scripts/sputum_secretions.sql b/sql_scripts/sputum_secretions.sql deleted file mode 100644 index 8f27ab8..0000000 --- a/sql_scripts/sputum_secretions.sql +++ /dev/null @@ -1,25 +0,0 @@ -<- Retrieved the information about sputum and secretions > - -SELECT -fv.TakenInstant AS 'DateTimeRecorded', -CASE -WHEN fsd.FlowsheetRowEpicId = '451120' -THEN fv.Value -END -AS 'Secretions' , -CASE -WHEN fsd.FlowsheetRowEpicId = '302600' -THEN fv.Value -END -AS 'Sputum' , -fv.Comment AS Comments - -FROM FilteredAccess.FlowsheetValueFact fv -INNER JOIN FilteredAccess.FlowsheetRowDim fsd ON fv.FlowsheetRowKey = fsd.FlowsheetRowKey -INNER JOIN FilteredAccess.EncounterFact enc ON fv.EncounterKey = enc.EncounterKey - -WHERE -(fsd.FlowsheetRowEpicId = '451120' OR -fsd.FlowsheetRowEpicId ='302600') -AND fv.TakenInstant BETWEEN %(yesterday)s AND %(today)s -AND enc.EncounterEpicCsn = %(csn) From 613fa3351021379d1e4593dab0a951e3c98c52ec Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Mon, 31 Aug 2026 12:28:05 +0100 Subject: [PATCH 41/54] Auto stash before merge of "sk/queries" and "st/queries-to-snakemake" --- sql_scripts/lab_test_names.sql | 10 ++++++++++ src/sql/flow_sheet_values.sql | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/sql_scripts/lab_test_names.sql b/sql_scripts/lab_test_names.sql index 1838b72..cbd6070 100644 --- a/sql_scripts/lab_test_names.sql +++ b/sql_scripts/lab_test_names.sql @@ -3,3 +3,13 @@ select lab_test_definition_id as id, standardised_vocabulary as vocab from star.lab_test_definition as ltd where ltd.lab_test_definition_id in ('1001', '390793054', '390793057', '390793060', '722790196') + +id name vocab +1001 C-reactive protein +390793054 CSF WCC TUBE 1 +390793057 CSF WCC TUBE 2 +390793060 CSF WCC TUBE 3 +722790196 C-reactive protein + +PaCO2 39947 - in star +PaO2 40191 \ No newline at end of file diff --git a/src/sql/flow_sheet_values.sql b/src/sql/flow_sheet_values.sql index 9cb4288..f035b1b 100644 --- a/src/sql/flow_sheet_values.sql +++ b/src/sql/flow_sheet_values.sql @@ -31,4 +31,4 @@ WHERE vt.id_in_application IN ('6', '3040102622', '12946') AND vo.valid_from BETWEEN %(start_datetime)s AND %(end_datetime)s AND vo.hospital_visit_id = %(hospital_visit_id)s -GROUP BY DateTimeRecorded, Units, vo.comment +GROUP BY DateTimeRecorded, Units, Comments From 07636099014e74e7eafcd4df26622064644fcd59 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Fri, 4 Sep 2026 15:26:32 +0100 Subject: [PATCH 42/54] First go at janitor script. Only doing waveform processing files for now, and hard-coded as dry-run. --- config.EXAMPLE/janitoring.env.EXAMPLE | 12 + docker-compose.yml | 15 + docs/janitoring.md | 37 +++ monitoring/Dockerfile | 17 +- monitoring/janitor.py | 126 ++++++++ monitoring/janitor.py.lock | 280 ++++++++++++++++++ monitoring/janitoring_entrypoint.sh | 14 + monitoring/monitor.py | 46 +-- ...entrypoint.sh => monitoring_entrypoint.sh} | 0 monitoring/utils.py | 47 +++ 10 files changed, 547 insertions(+), 47 deletions(-) create mode 100644 config.EXAMPLE/janitoring.env.EXAMPLE create mode 100644 docs/janitoring.md create mode 100644 monitoring/janitor.py create mode 100644 monitoring/janitor.py.lock create mode 100755 monitoring/janitoring_entrypoint.sh rename monitoring/{entrypoint.sh => monitoring_entrypoint.sh} (100%) create mode 100644 monitoring/utils.py diff --git a/config.EXAMPLE/janitoring.env.EXAMPLE b/config.EXAMPLE/janitoring.env.EXAMPLE new file mode 100644 index 0000000..565bcd2 --- /dev/null +++ b/config.EXAMPLE/janitoring.env.EXAMPLE @@ -0,0 +1,12 @@ +# This is an EXAMPLE file, do not put real secrets in here. +# Copy it to ../config/janitoring.env and then DELETE THIS COMMENT. +# When does the janitoring job run +JANITORING_CRON_SCHEDULE="*/15 * * * *" +# OpenTelemetry OTLP/HTTP endpoint of the LGTM collector. +OTEL_EXPORTER_OTLP_ENDPOINT="http://lgtm:4318" +OTEL_SERVICE_NAME=waveform-janitoring + +HL7_BZ2_ARCHIVE_RETENTION_DAYS= +ORIGINAL_CSV_RETENTION_DAYS=30 +ORIGINAL_PARQUET_RETENTION_DAYS=30 +PSEUDONYMISED_RETENTION_DAYS=30 diff --git a/docker-compose.yml b/docker-compose.yml index 638169b..316bd0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,6 +70,7 @@ services: build: context: . dockerfile: monitoring/Dockerfile + target: supercronic_monitoring args: <<: *proxy-common env_file: @@ -79,3 +80,17 @@ services: - ../../waveform-saved-messages:/waveform-saved-messages:ro - ../waveform-export:/waveform-export:ro restart: unless-stopped + waveform-janitoring: + build: + context: . + dockerfile: monitoring/Dockerfile + target: supercronic_janitoring + args: + <<: *proxy-common + env_file: + - ../config/janitoring.env + volumes: + # we are assuming this FS layout to emap saved messages + - ../../waveform-saved-messages:/waveform-saved-messages:ro + - ../waveform-export:/waveform-export:ro + restart: unless-stopped diff --git a/docs/janitoring.md b/docs/janitoring.md new file mode 100644 index 0000000..f220681 --- /dev/null +++ b/docs/janitoring.md @@ -0,0 +1,37 @@ +# File tidy up (waveform-janitoring) + +## Suggested retention times + +| Data | Size | Replaceability | Comment | Suggested deletion policy | +|--------|--------|--------|--------|--------| +| compressed HL7 | Large | Cannot be replaced | | Never delete unless under duress. If disk gets low, try to find more disk, or delete old stuff first if we have confidence that our uploaded data will not need to be re-processed. Archives are indexed by time and bed, but ALL variables are mixed together, so targeted deletion is only possible by time and bed. | +| original CSV | Large but see #15 | Can be reprocessed from HL7 | can be useful for debugging | Delete after 30 days or if disk gets low; consider targeted deletion (see #79) | +| original parquet | Small | Can be reprocessed from CSV | parquets are generated in a single step from CSV | Delete after 30 days or if disk gets low; consider targeted deletion (see #79) | +| pseduonymised parquet | Small | Can be reprocessed from CSV | parquets are generated in a single step from CSV | Delete after 30 days or if disk gets low; consider targeted deletion (see #79) | +| pre-upload tar files | Small | Is an intermediate file for upload | Not needed except for (short term) debugging | Delete after 30 days or if disk gets low; consider deleting immediately after upload | + +Retention times for file types that are only useful for debugging could be shortened when we have more confidence in the pipeline. + +## Design decisions + +In the first iteration of this feature, retention times are evaluated against +file modification times. Ie. processing time. + +It might, in future, be reasonable to also take observation time into account +(ie. using the dirname "2024-10-01" to determine when the data relates to). + +For live data, these time stamps will be very similar so it doesn't matter which we use. + +But if we reprocess some data from stored HL7 archives, +the modified times could be quite new vs the observation times. +If we only use modification times, a large amount of data could be produced quite quickly +that won't get cleared up, possibly leading to a full disk. +If we only use observation times and the data is sufficiently old, we could see CSV files being +deleted before we have a chance to convert and upload them. +Therefore it seems likely that to solve those problems we'd have to somehow take both into account. + +There shouldn't be a scenario where observation times are newer than modification times +(bar synthetic data). + +Snakemake has the ability to mark files as temporary. They are immediately deleted after they are needed. +However, we want to keep files for a certain time after snakemake has finished. diff --git a/monitoring/Dockerfile b/monitoring/Dockerfile index eda6db5..023ad6c 100644 --- a/monitoring/Dockerfile +++ b/monitoring/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.13-slim-bookworm@sha256:8092ae2ef67061f9db412458dbdce44dbf16748fb3cae5cdbd020f467a9712d0 +FROM python:3.13-slim-bookworm@sha256:8092ae2ef67061f9db412458dbdce44dbf16748fb3cae5cdbd020f467a9712d0 AS supercronic_base LABEL authors="Stephen Thompson, Jeremy Stein" RUN export DEBIAN_FRONTEND=noninteractive && \ @@ -29,12 +29,15 @@ COPY --from=ghcr.io/astral-sh/uv@sha256:e85be844203885286c60ffad8a858d48afb6c5a5 ARG UVCACHE=/root/.cache/uv WORKDIR /app -COPY monitoring/monitor.py monitoring/monitor.py.lock /app/ -RUN uv lock --check --script monitor.py +COPY monitoring/* /app/ +RUN chmod +x /app/*.sh +FROM supercronic_base AS supercronic_monitoring +RUN uv lock --check --script monitor.py RUN --mount=type=cache,target=${UVCACHE} uv export --script monitor.py --locked | uv pip install --system -r - +ENTRYPOINT ["/app/monitoring_entrypoint.sh"] -COPY monitoring/entrypoint.sh /app/entrypoint.sh -RUN chmod +x /app/entrypoint.sh - -ENTRYPOINT ["/app/entrypoint.sh"] +FROM supercronic_base AS supercronic_janitoring +RUN uv lock --check --script janitor.py +RUN --mount=type=cache,target=${UVCACHE} uv export --script janitor.py --locked | uv pip install --system -r - +ENTRYPOINT ["/app/janitoring_entrypoint.sh"] diff --git a/monitoring/janitor.py b/monitoring/janitor.py new file mode 100644 index 0000000..82c08ca --- /dev/null +++ b/monitoring/janitor.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Scan saved HL7 messages and emit OpenTelemetry metrics. + +Run in this command in dev to update the lockfile: `uv lock --script monitoring/janitor.py` +""" + +import logging +import sys +from datetime import timedelta, datetime, timezone +from pathlib import Path +from time import perf_counter +from typing import Optional + +from opentelemetry import metrics + +import utils + +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "opentelemetry-exporter-otlp-proto-http==1.42.0", +# ] +# /// + +INSTRUMENTATION_SCOPE = "waveform-janitoring.meter" +SAVED_MESSAGES_DIR = Path("/waveform-saved-messages") +WAVEFORM_EXPORT_DIR = Path("/waveform-export") + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, +) +logger = logging.getLogger(__name__) + + +def scan_waveform_exporter_files(meter, dry_run): + scan_time_hist = meter.create_histogram( + "waveform.janitoring.exporter.disk_cleanup_time", + unit="s", + description="Duration of cleanup in exporter directory", + ) + start_time = perf_counter() + # Dirs that contain large files where we need to clean up. + # Missing/blank env means do not clean up at all. + big_top_level_dirs: dict[str, Optional[float]] = { + "original-csv": utils.get_env("ORIGINAL_CSV_RETENTION_DAYS", None, float), + "original-parquet": utils.get_env( + "ORIGINAL_PARQUET_RETENTION_DAYS", None, float + ), + "pseudonymised": utils.get_env("PSEUDONYMISED_RETENTION_DAYS", None, float), + } + bytes_deleted_histo = meter.create_histogram( + "waveform.janitoring.deleted_bytes", + unit="By", + description="Bytes deleted by the janitoring process", + ) + for tld_name, retention_days in big_top_level_dirs.items(): + logger.info(f"Scanning {tld_name} for items older than {retention_days} days") + if not retention_days: + logger.info("Skipping %s due to empty/missing retention value", tld_name) + continue + tld = WAVEFORM_EXPORT_DIR / tld_name + tld_meter_name = tld_name.replace("-", "_") + byte_count = _delete_old_files(tld, retention_days, dry_run) + bytes_deleted_histo.record( + byte_count, + attributes={ + "directory": tld_meter_name, + "dry_run": bool(dry_run), + }, + ) + time_taken = perf_counter() - start_time + scan_time_hist.record(time_taken) + logger.info("Scanned %s in %ss", WAVEFORM_EXPORT_DIR, time_taken) + + +def _delete_old_files(tld: Path, retention_days: float, dry_run) -> int: + retention_threshold = datetime.now(timezone.utc) - timedelta(days=retention_days) + retention_threshold_timestamp = retention_threshold.timestamp() + deleted_byte_count = 0 + for dn, _, files in tld.walk(): + for f in files: + f_path = dn / f + if f_path.is_file(): + stat = f_path.stat() + actual_mtime = stat.st_mtime + if actual_mtime < retention_threshold_timestamp: + if not dry_run: + f_path.unlink() + logger.info( + "%sDeleting file [%s bytes] %s", + "[DRY RUN] " if dry_run else "", + stat.st_size, + f_path, + ) + deleted_byte_count += stat.st_size + return deleted_byte_count + + +def main(args) -> int: + service_name = utils.get_env("OTEL_SERVICE_NAME") + otlp_endpoint = utils.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") + + # setup + utils.setup_metrics(service_name, otlp_endpoint) + meter = metrics.get_meter(INSTRUMENTATION_SCOPE) + + # things to clean up + scan_waveform_exporter_files(meter, args.dry_run) + + # shutdown, flush data + provider = metrics.get_meter_provider() + provider.force_flush(timeout_millis=15000) + + return 0 + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + raise SystemExit(main(args)) diff --git a/monitoring/janitor.py.lock b/monitoring/janitor.py.lock new file mode 100644 index 0000000..84cd2ea --- /dev/null +++ b/monitoring/janitor.py.lock @@ -0,0 +1,280 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[manifest] +requirements = [{ name = "opentelemetry-exporter-otlp-proto-http", specifier = "==1.42.0" }] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" }, + { url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" }, + { url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" }, + { url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" }, + { url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" }, + { url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" }, + { url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" }, + { url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" }, + { url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" }, + { url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" }, + { url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" }, + { url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" }, + { url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" }, + { url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" }, + { url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" }, + { url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" }, + { url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" }, + { url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" }, + { url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" }, + { url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" }, + { url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" }, + { url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" }, + { url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" }, + { url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" }, + { url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" }, + { url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" }, + { url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" }, + { url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" }, + { url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" }, + { url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" }, + { url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" }, + { url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" }, + { url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" }, + { url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" }, + { url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" }, + { url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" }, + { url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" }, + { url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" }, + { url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" }, + { url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" }, + { url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" }, + { url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" }, + { url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" }, + { url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" }, + { url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" }, + { url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" }, + { url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" }, + { url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" }, + { url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" }, + { url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" }, + { url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" }, + { url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" }, + { url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" }, + { url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" }, + { url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" }, + { url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" }, + { url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.75.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/90/fb8f1c84537fbf210c1f53a53ae473a805f6599c5a40b93c1bbadd211f7a/googleapis_common_protos-1.75.2.tar.gz", hash = "sha256:8829a3d1e4508c5b7b9a6b9525f7fccff611f8531644579a76466c29295d4bb2", size = 154083, upload-time = "2026-08-25T19:19:13.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/5b/1c9e55363c3b1890a98cae813de5b4ea327845756cd8fb7ee690140c7eac/googleapis_common_protos-1.75.2-py3-none-any.whl", hash = "sha256:6b83302f554ea93a0f48409c7fc2050f954bcbcddb7e3a9c76d4a823cb22920e", size = 307002, upload-time = "2026-08-25T19:18:08.927Z" }, +] + +[[package]] +name = "idna" +version = "3.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/1c/125e1c936c0873796771b7f04f6c93b9f1bf5d424cea90fda94a99f61da8/opentelemetry_api-1.42.1.tar.gz", hash = "sha256:56c63bea9f77b62856be8c47600474acad853b2924b99b1687c4cb6297166716", size = 72296, upload-time = "2026-05-21T16:32:49.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/ca/9520cc1f3dfbbd03ac5903bbf55833e257bc64b1cf30fa8b0d6df374d821/opentelemetry_api-1.42.1-py3-none-any.whl", hash = "sha256:51a69edacadbc03a8950ace1c4c21099cacc538820ac2c9e36277e78cebba714", size = 61311, upload-time = "2026-05-21T16:32:28.822Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.42.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/a9/1496f27ecdfc7d504eac80f5e16474ee9d47cd08cda1f2917b58cf1c299c/opentelemetry_exporter_otlp_proto_common-1.42.0.tar.gz", hash = "sha256:c7a1a61f3a4c4dfa83127353edb1c75b873289d9ee42379db46eb835963b72e3", size = 21430, upload-time = "2026-05-19T09:46:32.838Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/7b/1542eb6e3d941a7dd93648d485b7c8495bc2841a2bb7dd5f394f370cf607/opentelemetry_exporter_otlp_proto_common-1.42.0-py3-none-any.whl", hash = "sha256:92de67f096c9200770f16fbdb63b96fb6061d604b4bc266726d8355caeb864e8", size = 17328, upload-time = "2026-05-19T09:46:11.291Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.42.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/da/be1a80ca82f155e6067213243f5bae5c10985d3c7603eadaf8c750ebe9eb/opentelemetry_exporter_otlp_proto_http-1.42.0.tar.gz", hash = "sha256:640013aacdbbe01bfb187c66dd331b24249517b7f36c0cd994ef75544d1dbd0b", size = 25405, upload-time = "2026-05-19T09:46:34.356Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/df/a5b8df72a4ef59519170d6f5fc4bd06528fed42e593b9b16b027cb477e50/opentelemetry_exporter_otlp_proto_http-1.42.0-py3-none-any.whl", hash = "sha256:f2cb9ea5dfac29e91b7b04034ca5a4bfae14b51cb334172c5b42b86a48843324", size = 21792, upload-time = "2026-05-19T09:46:13.67Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.42.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/2c/7c56a19498b46da4c54dc4e765c95d17f8fec2ba86bec1817b41ae635360/opentelemetry_proto-1.42.0.tar.gz", hash = "sha256:5d56a9067b631ea931a135d7b86428ae99649f591d4db69b9fc8c8e0465fce65", size = 45841, upload-time = "2026-05-19T09:46:42.058Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/ad/ff5f619a04cddb4936ead0dd8f590c5b373c5b4b9f2eef555e9d3d951ccb/opentelemetry_proto-1.42.0-py3-none-any.whl", hash = "sha256:2c0716a37e5c12efef37cbd01906d649b7fb85c85ac687518d0bd28527c6498e", size = 71779, upload-time = "2026-05-19T09:46:24.536Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f7/b390bd9bfd703bf98a68fea1f27786c6872331fd617164a54b8a59bdc008/opentelemetry_sdk-1.42.1.tar.gz", hash = "sha256:8c834e8f8c9ba4171d4ec843d0cb8a67e4c7394d3f9e9297e582cbd9456ddbf7", size = 239262, upload-time = "2026-05-21T16:33:04.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/6b/4287766cfbde577ae2272e8884abac325aeaac0d64f41c61d5b8cc595105/opentelemetry_sdk-1.42.1-py3-none-any.whl", hash = "sha256:083cd4bbfaa5aa7b5a9e552430d9951219967cfb27aa61feb13a77aba1fc839d", size = 170907, upload-time = "2026-05-21T16:32:45.894Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/99/4d7dd6df64795951413ce6e815f8cf1eb191daf7196ae86574589643d5f3/opentelemetry_semantic_conventions-0.63b1.tar.gz", hash = "sha256:3daf963611334b365e98a57438183eb012d3bfb40b2d931a9af613476b8701a9", size = 148340, upload-time = "2026-05-21T16:33:05.455Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7a/7fe66f5f3682b1dd47d88cc4e11f1c6c0966b737de2d16671146e23c39a5/opentelemetry_semantic_conventions-0.63b1-py3-none-any.whl", hash = "sha256:dfe5ef4dee82586b746f522b818ceb298d00b3d59f660042bd79404bff8d0682", size = 203713, upload-time = "2026-05-21T16:32:47.016Z" }, +] + +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] diff --git a/monitoring/janitoring_entrypoint.sh b/monitoring/janitoring_entrypoint.sh new file mode 100755 index 0000000..c69e6be --- /dev/null +++ b/monitoring/janitoring_entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -eo pipefail + +if [ -z "$JANITORING_CRON_SCHEDULE" ]; then + echo "You must set JANITORING_CRON_SCHEDULE when running this container" + exit 1 +fi + +cat > /etc/crontab < str: - value = os.environ.get(name) - if value is None or value == "": - if default is not None: - return default - else: - raise RuntimeError(f"Environment variable {name} not set") - return value - - def _scan_directory_ages(path: Path) -> tuple[int, float | None, float | None]: """Return file count, newest age (seconds), oldest age (seconds).""" now = time.time() @@ -67,25 +53,6 @@ def _scan_directory_ages(path: Path) -> tuple[int, float | None, float | None]: return len(mtimes), now - newest_mtime, now - oldest_mtime -def _setup_metrics(service_name: str, otlp_endpoint: str | None) -> None: - if not otlp_endpoint: - logger.error( - "OTEL_EXPORTER_OTLP_ENDPOINT not set; metrics will not be exported" - ) - return - - metrics.set_meter_provider( - MeterProvider( - resource=Resource.create({SERVICE_NAME: service_name}), - metric_readers=[ - PeriodicExportingMetricReader( - OTLPMetricExporter(), export_interval_millis=15000 - ) - ], - ) - ) - - def scan_hl7_bz2(meter: Meter): file_count = meter.create_up_down_counter( "waveform.monitoring.hl7bz2_files.count", @@ -147,7 +114,6 @@ def scan_waveform_exporter_files(meter): start_time = perf_counter() # dirs that contain large files where we need to track disk usage big_top_level_dirs = ["original-csv", "original-parquet", "pseudonymised"] - # dirs that won't get too large but do have other info we'll want to track for tld_name in big_top_level_dirs: tld = WAVEFORM_EXPORT_DIR / tld_name tld_meter_name = tld_name.replace("-", "_") @@ -187,11 +153,11 @@ def report_disk_free_space(meter: Meter): def main() -> int: - service_name = _env("OTEL_SERVICE_NAME") - otlp_endpoint = _env("OTEL_EXPORTER_OTLP_ENDPOINT") + service_name = utils.get_env("OTEL_SERVICE_NAME") + otlp_endpoint = utils.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") # setup - _setup_metrics(service_name, otlp_endpoint) + utils.setup_metrics(service_name, otlp_endpoint) meter = metrics.get_meter(INSTRUMENTATION_SCOPE) # things to measure diff --git a/monitoring/entrypoint.sh b/monitoring/monitoring_entrypoint.sh similarity index 100% rename from monitoring/entrypoint.sh rename to monitoring/monitoring_entrypoint.sh diff --git a/monitoring/utils.py b/monitoring/utils.py new file mode 100644 index 0000000..42071c4 --- /dev/null +++ b/monitoring/utils.py @@ -0,0 +1,47 @@ +import logging +import os +from typing import Optional, Any + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, +) + +logger = logging.getLogger(__name__) + + +def get_env( + name: str, default: str | None = None, as_type: Optional[type] = None +) -> Any: + value = os.environ.get(name) + if value is None or value == "": + if default is not None: + return default + else: + raise RuntimeError(f"Environment variable {name} not set") + if as_type: + return as_type(value) + else: + return value + + +def setup_metrics(service_name: str, otlp_endpoint: str | None) -> None: + if not otlp_endpoint: + logger.error( + "OTEL_EXPORTER_OTLP_ENDPOINT not set; metrics will not be exported" + ) + return + + metrics.set_meter_provider( + MeterProvider( + resource=Resource.create({SERVICE_NAME: service_name}), + metric_readers=[ + PeriodicExportingMetricReader( + OTLPMetricExporter(), export_interval_millis=15000 + ) + ], + ) + ) From 6057e70afd471b4344adae0fa91aae1b7caf39f1 Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Mon, 7 Sep 2026 10:15:39 +0100 Subject: [PATCH 43/54] star flow sheet query --- src/sql/flow_sheet_values.sql | 48 ++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/sql/flow_sheet_values.sql b/src/sql/flow_sheet_values.sql index f035b1b..7076491 100644 --- a/src/sql/flow_sheet_values.sql +++ b/src/sql/flow_sheet_values.sql @@ -1,34 +1,46 @@ ---get the flow sheet values for the particular visit on a particular day --- the flow sheet numbers are recorded as id_in_application in the visit_observation_type table +-- get the flow sheet values for the particular visit on a particular day +-- the flow sheet numbers are recorded as id_in_application +-- in the visit_observation_type table -- Temperature 6 --- Noradrenalin 3040102622 +-- Noradrenaline 3040102622 -- Metaraminol 12946 +-- PaO2 40191 +-- PaCO2 39947 + SELECT - vo.observation_datetime AS DateTimeRecorded, + vo.observation_datetime AS "DateTimeRecorded", - (array_agg(vo.value_as_real) FILTER ( + MAX(vo.value_as_real) FILTER ( WHERE vt.id_in_application = '6' - ))[1] AS "Temperature", + ) AS "Temperature", - (array_agg(vo.value_as_real) FILTER ( + MAX(vo.value_as_real) FILTER ( WHERE vt.id_in_application = '3040102622' - ))[1] AS "Noradrenaline", + ) AS "Noradrenaline", - (array_agg(vo.value_as_real) FILTER ( + MAX(vo.value_as_real) FILTER ( WHERE vt.id_in_application = '12946' - ))[1] AS "Metaraminol", - - vo.unit AS Units, - vo.comment AS Comments - + ) AS "Metaraminol", + + MAX(vo.value_as_real) FILTER ( + WHERE vt.id_in_application = '40191' + ) AS "PaO2", + + MAX(vo.value_as_real) FILTER ( + WHERE vt.id_in_application = '39947' + ) AS "PaCO2", + + vo.unit AS "Units" + FROM {schema_name}.visit_observation AS vo LEFT JOIN {schema_name}.visit_observation_type AS vt ON vo.visit_observation_type_id = vt.visit_observation_type_id -WHERE vt.id_in_application IN ('6', '3040102622', '12946') -AND vo.valid_from BETWEEN %(start_datetime)s AND %(end_datetime)s -AND vo.hospital_visit_id = %(hospital_visit_id)s +WHERE + vt.id_in_application IN ('6', '3040102622', '12946', '40191', '39947') + AND vo.valid_from >= %(start_datetime)s AND vo.valid_from < %(end$_datetime)s + AND vo.hospital_visit_id = %(hospital_visit_id)s -GROUP BY DateTimeRecorded, Units, Comments +GROUP BY "DateTimeRecorded", "Units" From fbfc355fd7541077771497af0a78e67ceaaa62fe Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Mon, 7 Sep 2026 10:37:47 +0100 Subject: [PATCH 44/54] finished star lab queries --- src/sql/lab_results.sql | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/sql/lab_results.sql b/src/sql/lab_results.sql index 426f48d..e013af7 100644 --- a/src/sql/lab_results.sql +++ b/src/sql/lab_results.sql @@ -1,35 +1,27 @@ -- This selects the values of lab tests --- 1011 CRP --- 722790196 CRP --- 390793054 WCC --- 390793057 WCC --- 390793060 WCC +-- 1011 C REACTIVE PROTEIN +-- 686 WHITE CELL COUNT SELECT - r.result_last_modified_datetime AS DateTimeRecorded, + r.result_last_modified_datetime AS "DateTimeRecorded", - MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '1001') AS "C-reactive protein 1", - MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '390793054') AS "CSF WCC TUBE 1", - MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '390793057') AS "CSF WCC TUBE 2", - MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '390793060') AS "CSF WCC TUBE 3", - MAX(r.value_as_real) FILTER (WHERE r.lab_test_definition_id = '722790196') AS "C-reactive protein 2" + MAX(r.value_as_real) FILTER + (WHERE r.lab_test_definition_id = '1001') AS "CRP", - r.units AS Units, - r.abnormal_flag AS Abnormal_result, - r.comment AS Comments + MAX(r.value_as_real) FILTER + (WHERE r.lab_test_definition_id = '686') AS "WCC", + + r.units AS "Units" FROM {schema_name}.lab_result AS r LEFT JOIN {schema_name}.lab_order AS o ON r.lab_order_id = o.lab_order_id -WHERE r.result_status like 'FINAL' -AND -r.lab_test_definition_id IN ('1001', - '390793054', - '390793057', - '390793060', - '722790196') - AND vo.valid_from BETWEEN %(start_datetime)s AND %(end_datetime)s -AND o.hospital_visit_id = %(hospital_visit_id)s +WHERE + r.result_status LIKE 'FINAL' + AND r.lab_test_definition_id IN ('1001', '686') + AND r.result_last_modified_datetime >= %(start_datetime)s + AND r.result_last_modified_datetime < %(end_datetime)s + AND o.hospital_visit_id = %(hospital_visit_id)s -GROUP BY DateTimeRecorded, Units, Abnormal_result, Comments +GROUP BY "DateTimeRecorded", "Units" From 93fe4e7140c2fdc8b780bb0f8254b067bead345e Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Mon, 7 Sep 2026 11:25:23 +0100 Subject: [PATCH 45/54] Add test for janitor, and add ability to clear up save HL7 --- .dockerignore | 6 ++ .github/workflows/pytest.yml | 2 +- docker-compose.yml | 2 +- monitoring/janitor.py | 24 ++++-- monitoring/test_janitor.py | 162 +++++++++++++++++++++++++++++++++++ monitoring/utils.py | 13 +-- 6 files changed, 194 insertions(+), 15 deletions(-) create mode 100644 .dockerignore create mode 100644 monitoring/test_janitor.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2a0c6b6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +**/.venv +**/*.venv +.idea +.git +**/.mypy_cache +**/.pytest_tmp diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 21a9aaf..db5cb1d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -67,7 +67,7 @@ jobs: - name: Run the tests working-directory: waveform-controller - run: uv run pytest tests --cov=src --cov-report=term-missing --cov-report=xml + run: uv run pytest --cov=src --cov-report=term-missing --cov-report=xml - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 diff --git a/docker-compose.yml b/docker-compose.yml index 316bd0e..f5e1fc7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,7 +90,7 @@ services: env_file: - ../config/janitoring.env volumes: - # we are assuming this FS layout to emap saved messages + # This will need to be rw if we ever enable deletion for real - ../../waveform-saved-messages:/waveform-saved-messages:ro - ../waveform-export:/waveform-export:ro restart: unless-stopped diff --git a/monitoring/janitor.py b/monitoring/janitor.py index 82c08ca..9e01aa1 100644 --- a/monitoring/janitor.py +++ b/monitoring/janitor.py @@ -43,12 +43,19 @@ def scan_waveform_exporter_files(meter, dry_run): start_time = perf_counter() # Dirs that contain large files where we need to clean up. # Missing/blank env means do not clean up at all. - big_top_level_dirs: dict[str, Optional[float]] = { - "original-csv": utils.get_env("ORIGINAL_CSV_RETENTION_DAYS", None, float), - "original-parquet": utils.get_env( - "ORIGINAL_PARQUET_RETENTION_DAYS", None, float + big_top_level_dirs: dict[Path, Optional[float]] = { + WAVEFORM_EXPORT_DIR / "original-csv": utils.get_env( + "ORIGINAL_CSV_RETENTION_DAYS", as_type=float + ), + WAVEFORM_EXPORT_DIR / "original-parquet": utils.get_env( + "ORIGINAL_PARQUET_RETENTION_DAYS", as_type=float + ), + WAVEFORM_EXPORT_DIR / "pseudonymised": utils.get_env( + "PSEUDONYMISED_RETENTION_DAYS", as_type=float + ), + SAVED_MESSAGES_DIR: utils.get_env( + "HL7_BZ2_ARCHIVE_RETENTION_DAYS", as_type=float ), - "pseudonymised": utils.get_env("PSEUDONYMISED_RETENTION_DAYS", None, float), } bytes_deleted_histo = meter.create_histogram( "waveform.janitoring.deleted_bytes", @@ -61,7 +68,7 @@ def scan_waveform_exporter_files(meter, dry_run): logger.info("Skipping %s due to empty/missing retention value", tld_name) continue tld = WAVEFORM_EXPORT_DIR / tld_name - tld_meter_name = tld_name.replace("-", "_") + tld_meter_name = tld_name.name.replace("-", "_") byte_count = _delete_old_files(tld, retention_days, dry_run) bytes_deleted_histo.record( byte_count, @@ -111,7 +118,10 @@ def main(args) -> int: # shutdown, flush data provider = metrics.get_meter_provider() - provider.force_flush(timeout_millis=15000) + + # (blank endpoint can return a provider that doesn't contain force_flush) + if hasattr(provider, "force_flush"): + provider.force_flush(timeout_millis=15000) return 0 diff --git a/monitoring/test_janitor.py b/monitoring/test_janitor.py new file mode 100644 index 0000000..07e1ca2 --- /dev/null +++ b/monitoring/test_janitor.py @@ -0,0 +1,162 @@ +import logging +import os +from collections import namedtuple +from datetime import timezone, datetime, timedelta +from pathlib import Path + +import pytest +from unittest.mock import Mock +import janitor + +logger = logging.getLogger(__name__) +TestFile = namedtuple("TestFile", ["id", "base", "path", "mtime"]) + + +@pytest.fixture(scope="module") +def maybe_stale_files(): + now = datetime.now(timezone.utc) + test_files = [ + TestFile( + id=10, + base="saved", + path=Path( + "20240912T08/UCHT03ICURM06/UCHT03ICURM06_20240912T0815Z_blah1.hl7archive.bz2" + ), + mtime=(now - timedelta(days=2.49)).timestamp(), + ), + TestFile( + id=11, + base="saved", + path=Path( + "20240912T08/UCHT03ICURM06/UCHT03ICURM06_20240912T0815Z_blah2.hl7archive.bz2" + ), + mtime=(now - timedelta(days=2.51)).timestamp(), + ), + TestFile( + id=20, + base="exp", + path=Path("original-csv/2024-09-12/foo.parquet"), + mtime=(now - timedelta(days=5.99)).timestamp(), + ), + TestFile( + id=21, + base="exp", + path=Path("original-csv/2024-10-12/foo.parquet"), + mtime=(now - timedelta(days=6.01)).timestamp(), + ), + TestFile( + id=30, + base="exp", + path=Path("original-parquet/2024-09-12/foo.parquet"), + mtime=(now - timedelta(days=7.99)).timestamp(), + ), + TestFile( + id=31, + base="exp", + path=Path("original-parquet/2024-10-12/foo.parquet"), + mtime=(now - timedelta(days=8.01)).timestamp(), + ), + TestFile( + id=40, + base="exp", + path=Path("pseudonymised/2024-09-25/foo.parquet"), + mtime=(now - timedelta(days=11.99)).timestamp(), + ), + TestFile( + id=41, + base="exp", + path=Path("pseudonymised/2024-10-25/foo.parquet"), + mtime=(now - timedelta(days=12.01)).timestamp(), + ), + ] + # check files are not overlapping + all_ids = [f.id for f in test_files] + all_paths = [(f.base, f.path) for f in test_files] + assert len(set(all_ids)) == len(all_ids) + assert len(set(all_paths)) == len(all_paths) + return test_files + + +@pytest.mark.parametrize( + "dry_run", + [True, False], +) +@pytest.mark.parametrize( + [ + "hl7_thresh", + "csv_thresh", + "original_pq_thresh", + "pseudo_thresh", + "expected_files", + ], + [ + ("2.5", "6", "8.0", "12", {10, 20, 30, 40}), + # empty variable means do not clean up that directory + ("", "6", "8.0", "12", {10, 11, 20, 30, 40}), + ("2.5", "", "8.0", "12", {10, 20, 21, 30, 40}), + ("2.5", "6", "", "12", {10, 20, 30, 31, 40}), + ("2.5", "6", "8.0", "", {10, 20, 30, 40, 41}), + ("", "", "", "", {10, 11, 20, 21, 30, 31, 40, 41}), + ], +) +def test_janitor( + tmp_path_factory, + maybe_stale_files, + monkeypatch, + dry_run, + hl7_thresh, + csv_thresh, + original_pq_thresh, + pseudo_thresh, + expected_files: set[int], +): + # check that all IDs expected are real, otherwise the test is not well-formed, + assert expected_files.issubset(set([f.id for f in maybe_stale_files])) + + # setup + os.environ["OTEL_SERVICE_NAME"] = "test-only" + os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "" + os.environ["ORIGINAL_CSV_RETENTION_DAYS"] = csv_thresh + os.environ["ORIGINAL_PARQUET_RETENTION_DAYS"] = original_pq_thresh + os.environ["PSEUDONYMISED_RETENTION_DAYS"] = pseudo_thresh + os.environ["HL7_BZ2_ARCHIVE_RETENTION_DAYS"] = hl7_thresh + export_dir = tmp_path_factory.mktemp("export") + saved_hl7_dir = tmp_path_factory.mktemp("saved") + + def get_base_path(code): + # these return values don't exist until the test has started + if code == "exp": + return export_dir + elif code == "saved": + return saved_hl7_dir + else: + raise AssertionError() + + for test_file in maybe_stale_files: + base_dir = get_base_path(test_file.base) + test_file_path = base_dir / test_file.path + test_file_path.parent.mkdir(parents=True, exist_ok=True) + test_file_path.write_text("whatever") + os.utime(str(test_file_path), (test_file.mtime, test_file.mtime)) + + monkeypatch.setattr(janitor, "WAVEFORM_EXPORT_DIR", export_dir) + monkeypatch.setattr(janitor, "SAVED_MESSAGES_DIR", saved_hl7_dir) + + # run the janitor + args = Mock() + args.dry_run = dry_run + janitor.main(args) + + # assert files got removed or not + for test_file in maybe_stale_files: + base_dir = get_base_path(test_file.base) + test_file_full = base_dir / test_file.path + file_actually_exists = test_file_full.exists() + if dry_run: + # dry run, so all files should still exist + assert file_actually_exists + else: + if file_actually_exists: + assert test_file.id in expected_files + else: + assert test_file.id not in expected_files diff --git a/monitoring/utils.py b/monitoring/utils.py index 42071c4..562806b 100644 --- a/monitoring/utils.py +++ b/monitoring/utils.py @@ -16,12 +16,13 @@ def get_env( name: str, default: str | None = None, as_type: Optional[type] = None ) -> Any: - value = os.environ.get(name) - if value is None or value == "": - if default is not None: - return default - else: - raise RuntimeError(f"Environment variable {name} not set") + # missing -> use default, "" -> None + try: + value = os.environ.get(name) + except KeyError: + value = default + if not value: + return None if as_type: return as_type(value) else: From 0560106b431904af89f0f55159f13606da236d49 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Mon, 7 Sep 2026 11:55:46 +0100 Subject: [PATCH 46/54] Docs tweak --- config.EXAMPLE/janitoring.env.EXAMPLE | 1 + docs/janitoring.md | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config.EXAMPLE/janitoring.env.EXAMPLE b/config.EXAMPLE/janitoring.env.EXAMPLE index 565bcd2..d405a6e 100644 --- a/config.EXAMPLE/janitoring.env.EXAMPLE +++ b/config.EXAMPLE/janitoring.env.EXAMPLE @@ -6,6 +6,7 @@ JANITORING_CRON_SCHEDULE="*/15 * * * *" OTEL_EXPORTER_OTLP_ENDPOINT="http://lgtm:4318" OTEL_SERVICE_NAME=waveform-janitoring +# Blank means do not delete HL7_BZ2_ARCHIVE_RETENTION_DAYS= ORIGINAL_CSV_RETENTION_DAYS=30 ORIGINAL_PARQUET_RETENTION_DAYS=30 diff --git a/docs/janitoring.md b/docs/janitoring.md index f220681..f755af2 100644 --- a/docs/janitoring.md +++ b/docs/janitoring.md @@ -34,4 +34,9 @@ There shouldn't be a scenario where observation times are newer than modificatio (bar synthetic data). Snakemake has the ability to mark files as temporary. They are immediately deleted after they are needed. -However, we want to keep files for a certain time after snakemake has finished. +However, we want to keep files for a certain time after snakemake has finished, so we can't use it. + + +## Configuration of janitoring service + +See [janitoring config example file](../config.EXAMPLE/janitoring.env.EXAMPLE) for config variables. \ No newline at end of file From 392d579e39b4c8484b3ae84bc0c9c34e59170f13 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Mon, 7 Sep 2026 17:03:30 +0100 Subject: [PATCH 47/54] Linting fix --- docs/janitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/janitoring.md b/docs/janitoring.md index f755af2..7503fc7 100644 --- a/docs/janitoring.md +++ b/docs/janitoring.md @@ -39,4 +39,4 @@ However, we want to keep files for a certain time after snakemake has finished, ## Configuration of janitoring service -See [janitoring config example file](../config.EXAMPLE/janitoring.env.EXAMPLE) for config variables. \ No newline at end of file +See [janitoring config example file](../config.EXAMPLE/janitoring.env.EXAMPLE) for config variables. From b1394c71f14345b8acda398895ab58d12adbcce4 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Tue, 8 Sep 2026 12:20:33 +0100 Subject: [PATCH 48/54] Rename namedtuple so pytest doesn't think it's a test class --- monitoring/test_janitor.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/monitoring/test_janitor.py b/monitoring/test_janitor.py index 07e1ca2..47b6aa4 100644 --- a/monitoring/test_janitor.py +++ b/monitoring/test_janitor.py @@ -9,14 +9,14 @@ import janitor logger = logging.getLogger(__name__) -TestFile = namedtuple("TestFile", ["id", "base", "path", "mtime"]) +InputTestFile = namedtuple("InputTestFile", ["id", "base", "path", "mtime"]) @pytest.fixture(scope="module") def maybe_stale_files(): now = datetime.now(timezone.utc) test_files = [ - TestFile( + InputTestFile( id=10, base="saved", path=Path( @@ -24,7 +24,7 @@ def maybe_stale_files(): ), mtime=(now - timedelta(days=2.49)).timestamp(), ), - TestFile( + InputTestFile( id=11, base="saved", path=Path( @@ -32,37 +32,37 @@ def maybe_stale_files(): ), mtime=(now - timedelta(days=2.51)).timestamp(), ), - TestFile( + InputTestFile( id=20, base="exp", path=Path("original-csv/2024-09-12/foo.parquet"), mtime=(now - timedelta(days=5.99)).timestamp(), ), - TestFile( + InputTestFile( id=21, base="exp", path=Path("original-csv/2024-10-12/foo.parquet"), mtime=(now - timedelta(days=6.01)).timestamp(), ), - TestFile( + InputTestFile( id=30, base="exp", path=Path("original-parquet/2024-09-12/foo.parquet"), mtime=(now - timedelta(days=7.99)).timestamp(), ), - TestFile( + InputTestFile( id=31, base="exp", path=Path("original-parquet/2024-10-12/foo.parquet"), mtime=(now - timedelta(days=8.01)).timestamp(), ), - TestFile( + InputTestFile( id=40, base="exp", path=Path("pseudonymised/2024-09-25/foo.parquet"), mtime=(now - timedelta(days=11.99)).timestamp(), ), - TestFile( + InputTestFile( id=41, base="exp", path=Path("pseudonymised/2024-10-25/foo.parquet"), From 7376d32176067fa76b069df3eccf8c11c36216a0 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Tue, 8 Sep 2026 12:23:58 +0100 Subject: [PATCH 49/54] Copy the janitoring config file --- .github/workflows/pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b3b265c..adc4604 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -54,6 +54,8 @@ jobs: cp config.EXAMPLE/hasher.env.EXAMPLE ../config/hasher.env cp config.EXAMPLE/controller.env.EXAMPLE ../config/controller.env cp config.EXAMPLE/monitoring.env.EXAMPLE ../config/monitoring.env + cp config.EXAMPLE/janitoring.env.EXAMPLE ../config/janitoring.env + # (lgtm is not involved in this test so its config file is not copied) { echo "" echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" From b919c14a174e49ddd14dfb03d2c6b3cc8d146599 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Tue, 8 Sep 2026 12:57:28 +0100 Subject: [PATCH 50/54] Was overriding coverage directories specified in config file on the command line. Specify in just one place and exclude tests from coverage. --- .github/workflows/pytest.yml | 2 +- pyproject.toml | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index adc4604..670b270 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -69,7 +69,7 @@ jobs: - name: Run the tests working-directory: waveform-controller - run: uv run pytest --cov=src --cov-report=term-missing --cov-report=xml + run: uv run pytest --cov --cov-report=term-missing --cov-report=xml - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 diff --git a/pyproject.toml b/pyproject.toml index 47929fc..6c15d11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,8 +51,12 @@ parallel = true relative_files = true # Relative "src" resolves to ./src on the host and /app/src in the image (WORKDIR /app). source = ["src", "monitoring"] -# Snakemake DSL, not Python -omit = ["**/Snakefile"] +omit = [ + # Snakemake DSL, not Python + "**/Snakefile", + "**/test_*.py", + "**/tests/**", +] # Map in-container / installed paths onto the local src/ tree for reports. [tool.coverage.paths] From 87d9ae9f1eb3f9fead9ffd1d4ef1fd354864e7b4 Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Tue, 8 Sep 2026 15:34:04 +0100 Subject: [PATCH 51/54] removed un new scripts and moved read me to current script location --- sql_scripts/lab_test_names.sql | 15 --------------- {sql_scripts => src/sql}/README.md | 30 +++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 24 deletions(-) delete mode 100644 sql_scripts/lab_test_names.sql rename {sql_scripts => src/sql}/README.md (58%) diff --git a/sql_scripts/lab_test_names.sql b/sql_scripts/lab_test_names.sql deleted file mode 100644 index cbd6070..0000000 --- a/sql_scripts/lab_test_names.sql +++ /dev/null @@ -1,15 +0,0 @@ -select lab_test_definition_id as id, - name, - standardised_vocabulary as vocab -from star.lab_test_definition as ltd -where ltd.lab_test_definition_id in ('1001', '390793054', '390793057', '390793060', '722790196') - -id name vocab -1001 C-reactive protein -390793054 CSF WCC TUBE 1 -390793057 CSF WCC TUBE 2 -390793060 CSF WCC TUBE 3 -722790196 C-reactive protein - -PaCO2 39947 - in star -PaO2 40191 \ No newline at end of file diff --git a/sql_scripts/README.md b/src/sql/README.md similarity index 58% rename from sql_scripts/README.md rename to src/sql/README.md index 98ae5af..38e248c 100644 --- a/sql_scripts/README.md +++ b/src/sql/README.md @@ -1,5 +1,16 @@ # Notes on putting together the EHR needed +## Private scripts + +This is a public repository and so we cannot include any scripts that are proprietary from the hospital system EPIC. +These are included in a separate private repository named waveform-private-queries. This has a directory structure + +[top-level]/src/sql + +so that it can be copied directly onto the directory structure of this repository and thus all scripts will be contained in the same place upon deployment. + + + ## Goal The ultimate aim is to have one csv per patient per day which looks roughly like @@ -19,15 +30,16 @@ The ultimate aim is to have one csv per patient per day which looks roughly like | script | arguments | record | location of script in repo | database | |- | --- | --- |- | --- | | mrn_based_on_bed_and_datetime.sql | location string | csn |waveform-controller/src/sql | star | -| get_hospital_visit_id.sql| csn | hospital_visit_id | waveform-controller/sql_scripts| star | -| flow_sheet_values.sql| hospital_visit_id/today/yesterday | part of table above | waveform-controller/sql_scripts| star | -| airway.sql | csn/today/yesterday | part of the table above | waveform-controller/sql_scripts | caboodle | -| sputum_secretions.sql | csn/today/yesterday | part of the table above | waveform-controller/sql_scripts | caboodle | +| get_hospital_visit_id.sql| csn | hospital_visit_id | waveform-controller/src/sql| star | +| flow_sheet_values.sql| hospital_visit_id/today/yesterday | part of table above | waveform-controller/src/sql| star | +| lab_results.sql | csn/today/yesterday | part of the table above | waveform-controller/src/sql | star | +| sputum_secretions.sql | csn/today/yesterday | part of the table above | waveform-private-queries/src/sql | caboodle | +| reposition.sql | csn/today/yesterday | part of the table above | waveform-private-queries/src/sql | caboodle | +--- ## Unfinished scripts -lab_results.sql need dealing with in the same way as flow_sheet_values - -lab_test_names.sql forms part of the above query but is useful for exploring - -We need scripts for any of the items in the a tracker that have not yet been covered. +| script | arguments | record | location of script in repo | database | +|- | --- | --- |- | --- | +| airway.sql | csn/today/yesterday | part of the table above | waveform-private-queries/src/sql | caboodle | +--- From dfbb05206c4c7fc2c8e63e4c9e6fd3536c63591b Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Tue, 8 Sep 2026 15:40:40 +0100 Subject: [PATCH 52/54] added note about getting scripts from the waveform-private-queries repository as part of deployment --- docs/deployment.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/deployment.md b/docs/deployment.md index 2a7f8ba..aaac4a5 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -35,6 +35,7 @@ sledgehammer approach which is rather similar to * Delete all Emap tables in `star_dev` as per Emap deployment instructions. * Waveform: `docker compose down` to bring everything down * git pull and rebuild containers for the two repos. +* **REMEMBER: that the files from waveform-private-queries repository need to be copied to the waveform-controller/src/sql directory.** * Change config if necessary * Bring it all up again From 2be6191acb7690b5623ea07aa6e95bb92143ddc3 Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Tue, 8 Sep 2026 16:25:16 +0100 Subject: [PATCH 53/54] Files that it looks like didn't get merged properly --- src/controller.py | 6 +- src/csv_writer.py | 28 +----- src/db.py | 191 +++------------------------------------ src/locations.py | 3 - src/pipeline/Snakefile | 44 +-------- src/pipeline/utils.py | 5 - src/pseudon/pseudon.py | 27 +++--- src/settings.py | 11 --- tests/helpers.py | 3 - tests/test_controller.py | 10 +- 10 files changed, 36 insertions(+), 292 deletions(-) diff --git a/src/controller.py b/src/controller.py index 32fd79f..c5c84fc 100644 --- a/src/controller.py +++ b/src/controller.py @@ -108,6 +108,7 @@ def finalise_message(outcome: MessageOutcome): class WaveformController: def __init__(self): self.emap_db = db.starDB() + self.emap_db.init_query() self.emap_db.connect() def waveform_callback( @@ -209,9 +210,7 @@ def outcome( ) lookup_success = True try: - matched_mrn = self.emap_db.get_matched_mrn( - location_string, observation_time - ) + matched_mrn = self.emap_db.get_row(location_string, observation_time) except ValueError: lookup_success = False logger.error( @@ -221,7 +220,6 @@ def outcome( exc_info=True, ) matched_mrn = ("unmatched_mrn", "unmatched_nhs", "unmatched_csn", False) - # matched_mrn = ("1234568", "12345678", "12345678", False) except ConnectionError: logger.error("Database error, will try again", exc_info=True) return outcome("reject", reason="db_conn_err", requeue=True) diff --git a/src/csv_writer.py b/src/csv_writer.py index aa780d3..a762273 100644 --- a/src/csv_writer.py +++ b/src/csv_writer.py @@ -3,16 +3,9 @@ import csv import json from datetime import datetime -import pandas as pd from typing import Optional -from locations import ( - WAVEFORM_ORIGINAL_CSV, - WAVEFORM_PSEUDONYMISED_EHR, - make_file_name, - FILE_STEM_PATTERN, - EHR_STEM_PATTERN_HASHED, -) +from locations import WAVEFORM_ORIGINAL_CSV, make_file_name, FILE_STEM_PATTERN def create_file_name( @@ -100,22 +93,3 @@ def write_frame( ] wv_writer.writerow(row_array) - - -def write_ehr( - df: pd.DataFrame, - date_str: str, - hashed_csn: str, -) -> bool: - """Writes a frame of electronic healthcare data to a csv file. - - :return: True if write was successful. - """ - subs_dict = dict(date=date_str, hashed_csn=hashed_csn) - stem = make_file_name(EHR_STEM_PATTERN_HASHED, subs_dict) - filename = WAVEFORM_PSEUDONYMISED_EHR / f"{stem}_ehr.csv" - filename.parent.mkdir(exist_ok=True, parents=True) - - df.to_csv(filename, index=False) - - return True diff --git a/src/db.py b/src/db.py index 7f199b3..ebab871 100644 --- a/src/db.py +++ b/src/db.py @@ -1,5 +1,4 @@ from datetime import datetime -import pandas as pd import psycopg2 from psycopg2 import sql, pool import logging @@ -11,7 +10,7 @@ class starDB: - mrn_lookup_query: str = "" + sql_query: str = "" connection_string: str = "dbname={} user={} password={} host={} port={} connect_timeout={} options='-c statement_timeout={}'".format( settings.UDS_DBNAME, # type:ignore settings.UDS_USERNAME, # type:ignore @@ -21,198 +20,36 @@ class starDB: settings.UDS_CONNECT_TIMEOUT, # type:ignore settings.UDS_QUERY_TIMEOUT, # type:ignore ) - connection_pool: pool.SimpleConnectionPool - fake_star: bool = False + connection_pool: pool.ThreadedConnectionPool - def connect(self) -> None: - self.fake_star = True if settings.STARDB_TESTING == "TRUE" else False - if not self.fake_star: - self.connection_pool = pool.SimpleConnectionPool( - 1, 1, self.connection_string - ) - - def _init_mrn_lookup_query(self) -> None: - with open(settings.SQL_PATH + "mrn_based_on_bed_and_datetime.sql", "r") as file: - self.mrn_lookup_query = sql.SQL(file.read()) # type:ignore + def connect(self): + self.connection_pool = pool.SimpleConnectionPool(1, 1, self.connection_string) - self.mrn_lookup_query = self.mrn_lookup_query.format( + def init_query(self): + with open("src/sql/mrn_based_on_bed_and_datetime.sql", "r") as file: + self.sql_query = sql.SQL(file.read()) + self.sql_query = self.sql_query.format( schema_name=sql.Identifier(settings.SCHEMA_NAME) ) - def get_matched_mrn( - self, location_string: str, observation_datetime: datetime - ) -> pd.DataFrame: + def get_row(self, location_string: str, observation_datetime: datetime): parameters = { "location_string": location_string, "observation_datetime": observation_datetime, } - if self.mrn_lookup_query == "": - self._init_mrn_lookup_query() - - rows = self._get_rows(self.mrn_lookup_query, parameters) # type: ignore - - if len(rows) != 1: - raise ValueError( - f"Wrong number of rows returned from database. {len(rows)} != 1, for {location_string}:{observation_datetime}" - ) - - return rows[0] - - def get_hospital_visit_from_csn(self, csn: str) -> int: - with open(settings.SQL_PATH + "get_hospital_visit_id.sql", "r") as file: - hv_query = sql.SQL(file.read()) - - hv_query = hv_query.format(schema_name=sql.Identifier(settings.SCHEMA_NAME)) # type: ignore - - parameters = { - "csn": csn, - } - if self.fake_star: - return 12345678 - - hospital_visit_id = self._get_rows(hv_query, parameters) - - # fetchall returns a list of tuples. We want the first element of the first tuple - if not isinstance(hospital_visit_id[0][0], int): - logger.warning( - f"hospital_visit_id[0][0] is not integer {hospital_visit_id}" - ) - - return hospital_visit_id[0][0] - - def get_flowsheets( - self, start_datetime: datetime, end_datetime: datetime, hospital_visit_id: int - ) -> pd.DataFrame: - """Retrieve airflow data from database.""" - - with open(settings.SQL_PATH + "flow_sheet_values.sql", "r") as file: - flowsheet_query = sql.SQL(file.read()) - - flowsheet_query = flowsheet_query.format( - schema_name=sql.Identifier(settings.SCHEMA_NAME) - ) # type: ignore - - parameters = { - "start_datetime": start_datetime, - "end_datetime": end_datetime, - "hospital_visit_id": hospital_visit_id, - } - - if self.fake_star: - fake_flowsheet = { - "DateTimeRecorded": [0], - "Temperature": [0], - "Noradrenaline": [0], - "Metaraminol": [0], - } - return pd.DataFrame(data=fake_flowsheet) - - return self._get_rows(flowsheet_query, parameters) - - def get_lab_results( - self, start_datetime: datetime, end_datetime: datetime, hospital_visit_id: int - ) -> pd.DataFrame: - """Retrieve lab result data from caboodle.""" - - with open(settings.SQL_PATH + "lab_results.sql", "r") as file: - lab_result_query = sql.SQL(file.read()) - - lab_result_query = lab_result_query.format( - schema_name=sql.Identifier(settings.SCHEMA_NAME) - ) # type: ignore - - parameters = { - "start_datetime": start_datetime, - "end_datetime": end_datetime, - "hospital_visit_id": hospital_visit_id, - } - - if self.fake_star: - fake_lab_result = { - "DateTimeRecorded": [0], - "Units": ["None"], - "Abnormal_result": ["No"], - "Comments": ["None"], - "C-reactive protein 1": ["-"], - "CSF WCC TUBE 1": ["-"], - "CSF WCC TUBE 2": ["-"], - "CSF WCC TUBE 3": ["-"], - "C-reactive protein 2": ["-"], - } - return pd.DataFrame(data=fake_lab_result) - - return self._get_rows(lab_result_query, parameters) - - def _get_rows(self, sql_query: sql.SQL, parameters: dict): try: with self.connection_pool.getconn() as db_connection: with db_connection.cursor() as curs: - curs.execute(sql_query, parameters) + curs.execute(self.sql_query, parameters) rows = curs.fetchall() self.connection_pool.putconn(db_connection) except psycopg2.errors.OperationalError as e: self.connection_pool.putconn(db_connection) raise ConnectionError(f"Data base error: {e}") - return rows - -class caboodleDB: - """For querying the caboodle database to extract electronic healthcare records per - patient.""" - - connection_string: str = "dbname={} user={} password={} host={} port={} connect_timeout={} options='-c statement_timeout={}'".format( - settings.CABOODLE_DBNAME, # type:ignore - settings.CABOODLE_USERNAME, # type:ignore - settings.CABOODLE_PASSWORD, # type:ignore - settings.CABOODLE_HOST, # type:ignore - settings.CABOODLE_PORT, # type:ignore - settings.CABOODLE_CONNECT_TIMEOUT, # type:ignore - settings.CABOODLE_QUERY_TIMEOUT, # type:ignore - ) - connection_pool: pool.SimpleConnectionPool - fake_caboodle: bool = False - - def connect(self) -> None: - """Set up connection to the database.""" - self.fake_caboodle = True if settings.CABOODLE_TESTING == "TRUE" else False - if not self.fake_caboodle: - self.connection_pool = pool.SimpleConnectionPool( - 1, 1, self.connection_string + if len(rows) != 1: + raise ValueError( + f"Wrong number of rows returned from database. {len(rows)} != 1, for {location_string}:{observation_datetime}" ) - def get_airflow( - self, start_datetime: datetime, end_datetime: datetime, csn: str - ) -> pd.DataFrame: - """Retrieve airflow data from database.""" - - with open(settings.SQL_PATH + "airway.sql", "r") as file: - airway_query = sql.SQL(file.read()) - parameters = { - "start_datetime": start_datetime, - "end_datetime": end_datetime, - "csn": csn, - } - - if self.fake_caboodle: - fake_airway = { - "DateTimeRecorded": [0], - "PlacementInstant": [0], - "RemovalInstant": [0], - "TubeSize": [0], - } - return pd.DataFrame(data=fake_airway) - - return self._get_rows(airway_query, parameters) - - def _get_rows(self, sql_query: sql.SQL, parameters: dict): - try: - with self.connection_pool.getconn() as db_connection: - with db_connection.cursor() as curs: - curs.execute(sql_query, parameters) - rows = curs.fetchall() - self.connection_pool.putconn(db_connection) - except psycopg2.errors.OperationalError as e: - self.connection_pool.putconn(db_connection) - raise ConnectionError(f"Data base error: {e}") - - return rows + return rows[0] diff --git a/src/locations.py b/src/locations.py index 53c4e94..a3c867f 100644 --- a/src/locations.py +++ b/src/locations.py @@ -5,7 +5,6 @@ WAVEFORM_ORIGINAL_PARQUET = WAVEFORM_EXPORT_BASE / "original-parquet" WAVEFORM_HASH_LOOKUPS = WAVEFORM_EXPORT_BASE / "hash-lookups" WAVEFORM_PSEUDONYMISED_PARQUET = WAVEFORM_EXPORT_BASE / "pseudonymised" -WAVEFORM_PSEUDONYMISED_EHR = WAVEFORM_EXPORT_BASE / "pseudonymised_ehr" WAVEFORM_SNAKEMAKE_LOGS = WAVEFORM_EXPORT_BASE / "snakemake-logs" WAVEFORM_FTPS_LOGS = WAVEFORM_EXPORT_BASE / "ftps-logs" @@ -15,8 +14,6 @@ FILE_STEM_PATTERN_HASHED = ( "{date}/{date}.{hashed_csn}.{variable_id}.{channel_id}.{units}" ) -# EHR data is per (date, csn), not per variable/channel/units, so it gets its own stem. -EHR_STEM_PATTERN_HASHED = "{date}/{date}.{hashed_csn}" CSV_PATTERN = WAVEFORM_ORIGINAL_CSV / (FILE_STEM_PATTERN + ".csv") ORIGINAL_PARQUET_PATTERN = WAVEFORM_ORIGINAL_PARQUET / (FILE_STEM_PATTERN + ".parquet") PSEUDONYMISED_PARQUET_PATTERN = WAVEFORM_PSEUDONYMISED_PARQUET / ( diff --git a/src/pipeline/Snakefile b/src/pipeline/Snakefile index 925ec73..3876b86 100644 --- a/src/pipeline/Snakefile +++ b/src/pipeline/Snakefile @@ -12,14 +12,12 @@ from locations import ( HASH_LOOKUP_JSON_REL, FILE_STEM_PATTERN, FILE_STEM_PATTERN_HASHED, - EHR_STEM_PATTERN_HASHED, make_file_name, ALL_UPLOADED_JSON, ALL_FTPS_LOG, ) from pipeline.utils import config_bool, determine_eventual_outputs, timestamp_for_paths from pseudon.pseudon import csv_to_parquets -from electronic_health_records.ehr import ehr_for_csv # How long before we assume that no more data will be written to the file, and @@ -48,7 +46,6 @@ PROCESS_CSV_FROM_DATE = str(config['PROCESS_CSV_FROM_DATE']) all_outputs, hash_to_csn = determine_eventual_outputs(CSV_AGE_THRESHOLD_MINUTES, ONLY_USE_CSV_FROM_YESTERDAY, PROCESS_CSV_FROM_DATE) ALL_FTPS_UPLOADED = sorted({ao.get_ftps_uploaded_all_file() for ao in all_outputs}) ALL_DAILY_HASH_LOOKUPS = sorted({ao.get_daily_hash_lookup() for ao in all_outputs}) -ALL_EHR_LOOKUPS = sorted({ao.get_ehr_lookup() for ao in all_outputs}) def configure_file_logging(log_file): import logging @@ -66,8 +63,7 @@ def configure_file_logging(log_file): rule all: input: ftps_uploaded = ALL_FTPS_UPLOADED, - daily_hash_lookups = ALL_DAILY_HASH_LOOKUPS, - ehr_lookups = ALL_EHR_LOOKUPS + daily_hash_lookups = ALL_DAILY_HASH_LOOKUPS rule all_ftps_uploaded: input: @@ -77,16 +73,6 @@ rule all_daily_hash_lookups: input: ALL_DAILY_HASH_LOOKUPS -rule all_ehr_lookups: - input: - ALL_EHR_LOOKUPS - -# a rule combining ehr and hash look ups to enable testing without ftps upload -rule all_ehr_and_hash_lookups: - input: - ALL_EHR_LOOKUPS, - ALL_DAILY_HASH_LOOKUPS - def input_file_maker(wc): unhashed_csn = hash_to_csn[wc.hashed_csn] # when using input functions, snakemake doesn't do its normal templating, you have to do it, hence the f-string @@ -128,34 +114,6 @@ def pseudonymised_parquet_files_for_date(wc): return [ao.get_pseudonymised_parquet_path() for ao in all_outputs if ao.date == wc.date] -def pseudonymised_parquet_files_for_date_and_hashed_csn(wc): - return [ - ao.get_pseudonymised_parquet_path() - for ao in all_outputs - if ao.date == wc.date and ao.hashed_csn == wc.hashed_csn - ] - - -rule ehr_lookup: - input: - # As with daily_hash_lookup, we lie to Snakemake that the input is the pseudon - # parquets for this csn/day, purely so this rule is tied into the dependency DAG - # and reruns if the underlying data for this csn/day changes. - pseudonymised_parquets = pseudonymised_parquet_files_for_date_and_hashed_csn - output: - WAVEFORM_PSEUDONYMISED_EHR / (EHR_STEM_PATTERN_HASHED + "_ehr.csv") - log: - WAVEFORM_SNAKEMAKE_LOGS / "ehr_lookup" / (EHR_STEM_PATTERN_HASHED + ".log") - run: - logger = configure_file_logging(log[0]) - original_csn = hash_to_csn[wildcards.hashed_csn] - logger.info("Running EHR look up for csn %s. Hash -> %s", original_csn, wildcards.hashed_csn) - ehr_for_csv( - date_str=wildcards.date, - original_csn=original_csn, - hashed_csn=wildcards.hashed_csn) - - rule daily_hash_lookup: input: # Because we don't declare the original parquets in the output of csv_to_parquet, diff --git a/src/pipeline/utils.py b/src/pipeline/utils.py index c1d564c..c16f0f1 100644 --- a/src/pipeline/utils.py +++ b/src/pipeline/utils.py @@ -12,7 +12,6 @@ HASH_LOOKUP_JSON, ORIGINAL_PARQUET_PATTERN, FILE_STEM_PATTERN_HASHED, - EHR_STEM_PATTERN_HASHED, CSV_PATTERN, make_file_name, ALL_UPLOADED_JSON, @@ -74,10 +73,6 @@ def get_ftps_uploaded_all_file(self) -> Path: def get_daily_hash_lookup(self) -> Path: return Path(make_file_name(str(HASH_LOOKUP_JSON), self._subs_dict)) - def get_ehr_lookup(self) -> Path: - final_stem = make_file_name(EHR_STEM_PATTERN_HASHED, self._subs_dict) - return WAVEFORM_PSEUDONYMISED_EHR / f"{final_stem}_ehr.csv" - def get_file_age(file_path: Path) -> timedelta: # need to use UTC to avoid DST issues diff --git a/src/pseudon/pseudon.py b/src/pseudon/pseudon.py index 41af2a2..27d653c 100644 --- a/src/pseudon/pseudon.py +++ b/src/pseudon/pseudon.py @@ -157,17 +157,7 @@ def csv_to_parquets( "Done turning CSV %s to original parquet %s", csv_path, original_parquet_path ) - safe_columns = [ - "sampling_rate", - "source_variable_id", - "source_channel_id", - "timestamp", - "units", - "numeric_values", - "string_values", - ] - - df = pseudonymise_relevant_columns(df, safe_columns) + df = pseudonymise_relevant_columns(df) pseudon_table = pa.Table.from_pandas(df, schema=schema, preserve_index=True) # Use same metadata for pseudon, must not contain identifiers! @@ -215,7 +205,18 @@ def add_waveform_metadata_to_table( return existing_table -def pseudonymise_relevant_columns(df: pd.DataFrame, safe_columns: list[str]): +SAFE_COLUMNS = [ + "sampling_rate", + "source_variable_id", + "source_channel_id", + "timestamp", + "units", + "numeric_values", + "string_values", +] + + +def pseudonymise_relevant_columns(df: pd.DataFrame): """ "csn", "mrn", "location" are examples of columns that must be pseudonymised. However, it's safer to list which columns *don't* need to be pseudonymised. Eg. you @@ -225,6 +226,6 @@ def pseudonymise_relevant_columns(df: pd.DataFrame, safe_columns: list[str]): hashed. """ for col in df.columns: - if col not in safe_columns: + if col not in SAFE_COLUMNS: df[col] = df[col].apply(functools.partial(do_hash, col)) return df diff --git a/src/settings.py b/src/settings.py index 08a7bfd..75f6908 100644 --- a/src/settings.py +++ b/src/settings.py @@ -22,7 +22,6 @@ def get_from_env(env_var, *, default_value=None, setting_name=None, required=Fal get_from_env("UDS_PORT") get_from_env("UDS_CONNECT_TIMEOUT") get_from_env("UDS_QUERY_TIMEOUT") -get_from_env("STARDB_TESTING") get_from_env("SCHEMA_NAME") get_from_env("RABBITMQ_USERNAME") get_from_env("RABBITMQ_PASSWORD") @@ -38,19 +37,9 @@ def get_from_env(env_var, *, default_value=None, setting_name=None, required=Fal get_from_env("HASHER_API_HOSTNAME") get_from_env("HASHER_API_PORT") -get_from_env("CABOODLE_DBNAME") -get_from_env("CABOODLE_USERNAME") -get_from_env("CABOODLE_PASSWORD") -get_from_env("CABOODLE_HOST") -get_from_env("CABOODLE_PORT") -get_from_env("CABOODLE_CONNECT_TIMEOUT") -get_from_env("CABOODLE_QUERY_TIMEOUT") -get_from_env("CABOODLE_TESTING") - get_from_env("LOG_LEVEL", default_value="INFO") get_from_env("INSTANCE_NAME", required=True) -get_from_env("SQL_PATH", default_value="./src/sql/") # OpenTelemetry: OTLP/HTTP base URL of the LGTM collector, e.g. http://lgtm:4318 get_from_env("OTEL_EXPORTER_OTLP_ENDPOINT") diff --git a/tests/helpers.py b/tests/helpers.py index 18b720a..0ceaff0 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -69,9 +69,6 @@ def get_orig_parquet(self): def get_pseudon_parquet(self): return f"{self.date}/{self.date}.{self.get_hashed_csn()}.{self.variable_id}.{self.channel_id}.{self.units}.parquet" - def get_pseudon_ehr(self): - return f"{self.date}/{self.date}.{self.get_hashed_csn()}_ehr.csv" - def get_hashes(self): return f"{self.date}/{self.date}.hashes.json" diff --git a/tests/test_controller.py b/tests/test_controller.py index 9845288..092b356 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -171,11 +171,9 @@ def test_controller_callback( emap_db_mock = Mock() if db_connect_failure: - emap_db_mock.get_matched_mrn.side_effect = ConnectionError( - "mock database error" - ) + emap_db_mock.get_row.side_effect = ConnectionError("mock database error") else: - emap_db_mock.get_matched_mrn.return_value = ("mrn", "nhsno", "csn", opt_out) + emap_db_mock.get_row.return_value = ("mrn", "nhsno", "csn", opt_out) monkeypatch.setattr("controller.db.starDB", Mock(return_value=emap_db_mock)) write_frame_mock = Mock() @@ -209,12 +207,12 @@ def test_controller_callback( was_bad_data = bad_data_type or lf_value_type == "both" if not was_bad_data: # we at least tried to query the DB - emap_db_mock.get_matched_mrn.assert_called_once() + emap_db_mock.get_row.assert_called_once() if was_bad_data: write_frame_mock.assert_not_called() # db should not even have been queried if data was bad - emap_db_mock.get_matched_mrn.assert_not_called() + emap_db_mock.get_row.assert_not_called() channel_mock.basic_reject.assert_called_once_with(delivery_tag, False) channel_mock.basic_ack.assert_not_called() elif db_connect_failure: From 89825d336a0bc7aa0168026fbfc903e01784ffb5 Mon Sep 17 00:00:00 2001 From: Sarah Keating Date: Tue, 8 Sep 2026 16:35:33 +0100 Subject: [PATCH 54/54] I missed one --- tests/test_snakemake_integration.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_snakemake_integration.py b/tests/test_snakemake_integration.py index d28eacb..6fbc866 100644 --- a/tests/test_snakemake_integration.py +++ b/tests/test_snakemake_integration.py @@ -283,11 +283,9 @@ def test_snakemake_pipeline(tmp_path: Path, background_hasher, monkeypatch): tmp_path / "original-parquet" / filename.get_orig_parquet() ) pseudon_path = tmp_path / "pseudonymised" / filename.get_pseudon_parquet() - ehr_path = tmp_path / "pseudonymised_ehr" / filename.get_pseudon_ehr() assert original_parquet_path.exists() assert pseudon_path.exists() - assert ehr_path.exists() _compare_original_parquet_to_expected(original_parquet_path, expected_data) _compare_parquets(original_parquet_path, pseudon_path) @@ -369,15 +367,12 @@ def _run_snakemake(tmp_path): tmp_exporter_env_path = tmp_path / "config/exporter.env" tmp_exporter_env_path.parent.mkdir(exist_ok=True) tmp_exporter_env_path.write_text( - "SNAKEMAKE_RULE_UNTIL=all_ehr_and_hash_lookups\n" + "SNAKEMAKE_RULE_UNTIL=all_daily_hash_lookups\n" "SNAKEMAKE_CORES=1\n" "INSTANCE_NAME=pytest\n" "CSV_AGE_THRESHOLD_MINUTES=5\n" "ONLY_USE_CSV_FROM_YESTERDAY=False\n" "PROCESS_CSV_FROM_DATE=\n" - "STARDB_TESTING=TRUE\n" - "CABOODLE_TESTING=TRUE\n" - "SQL_PATH=/app/src/sql/\n" ) # Collect coverage from Python processes inside the exporter container